Package io.crate.integrationtests

Source Code of io.crate.integrationtests.ColumnPolicyIntegrationTest

/*
* Licensed to CRATE Technology GmbH ("Crate") under one or more contributor
* license agreements.  See the NOTICE file distributed with this work for
* additional information regarding copyright ownership.  Crate licenses
* this file to you under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.  You may
* obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
* License for the specific language governing permissions and limitations
* under the License.
*
* However, if you have executed another commercial license agreement
* with Crate these terms will supersede the license and you may use the
* software solely pursuant to the terms of the relevant commercial agreement.
*/

package io.crate.integrationtests;

import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import io.crate.Constants;
import io.crate.PartitionName;
import io.crate.action.sql.SQLActionException;
import io.crate.metadata.table.ColumnPolicy;
import io.crate.test.integration.CrateIntegrationTest;
import io.crate.testing.TestingHelpers;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse;
import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.compress.CompressedString;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.hamcrest.Matchers;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import javax.annotation.Nullable;
import java.io.File;
import java.util.Arrays;
import java.util.Map;

import static org.hamcrest.Matchers.*;

@CrateIntegrationTest.ClusterScope(scope = CrateIntegrationTest.Scope.GLOBAL)
public class ColumnPolicyIntegrationTest extends SQLTransportIntegrationTest {

    static {
        ClassLoader.getSystemClassLoader().setDefaultAssertionStatus(true);
    }

    private String copyFilePath = getClass().getResource("/essetup/data/copy").getPath();

    @Rule
    public ExpectedException expectedException = ExpectedException.none();


    @Test
    public void testInsertNewColumnTableStrictColumnPolicy() throws Exception {
        execute("create table strict_table (" +
                "  id integer primary key, " +
                "  name string" +
                ") with (column_policy='strict', number_of_replicas=0)");
        ensureGreen();
        execute("insert into strict_table (id, name) values (1, 'Ford')");
        execute("refresh table strict_table");

        execute("select * from strict_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("id", "name")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(1, "Ford")));

        expectedException.expect(SQLActionException.class);
        expectedException.expectMessage("Column 'boo' unknown");
        execute("insert into strict_table (id, name, boo) values (2, 'Trillian', true)");
    }

    @Test
    public void testUpdateNewColumnTableStrictColumnPolicy() throws Exception {
        execute("create table strict_table (" +
                "  id integer primary key, " +
                "  name string" +
                ") with (column_policy='strict', number_of_replicas=0)");
        ensureGreen();
        execute("insert into strict_table (id, name) values (1, 'Ford')");
        execute("refresh table strict_table");

        execute("select * from strict_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("id", "name")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(1, "Ford")));

        expectedException.expect(SQLActionException.class);
        expectedException.expectMessage("Column 'boo' unknown");
        execute("update strict_table set name='Trillian', boo=true where id=1");
    }

    @Test
    public void testCopyFromFileStrictTable() throws Exception {
        execute("create table quotes (id int primary key) with (column_policy='strict', number_of_replicas = 0)");
        ensureGreen();

        String filePath = Joiner.on(File.separator).join(copyFilePath, "test_copy_from.json");

        execute("copy quotes from ?", new Object[]{filePath});
        assertThat(response.rowCount(), is(0L));
    }

    @Test
    public void testInsertNewColumnTableDynamic() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer primary key, " +
                "  name string" +
                ") with (column_policy='dynamic', number_of_replicas=0)");
        ensureGreen();
        execute("insert into dynamic_table (id, name) values (1, 'Ford')");
        execute("refresh table dynamic_table");

        execute("select * from dynamic_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("id", "name")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(1, "Ford")));

        execute("insert into dynamic_table (id, name, boo) values (2, 'Trillian', true)");
        execute("refresh table dynamic_table");
        waitNoPendingTasksOnAll();

        execute("select * from dynamic_table order by id");
        assertThat(response.rowCount(), is(2L));
        assertThat(response.cols(), is(arrayContaining("boo", "id", "name")));
        assertThat(TestingHelpers.printedTable(response.rows()), is(
                "NULL| 1| Ford\n" +
                "true| 2| Trillian\n"));
    }

    @Test
    public void testUpdateNewColumnTableDynamic() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer primary key, " +
                "  name string" +
                ") with (column_policy='dynamic', number_of_replicas=0)");
        ensureGreen();
        execute("insert into dynamic_table (id, name) values (1, 'Ford')");
        execute("refresh table dynamic_table");

        execute("select * from dynamic_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("id", "name")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(1, "Ford")));

        execute("update dynamic_table set name='Trillian', boo=true where name='Ford'");
        execute("refresh table dynamic_table");
        waitNoPendingTasksOnAll();

        execute("select * from dynamic_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("boo", "id", "name")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(true, 1, "Trillian")));
    }

    @Test
    public void testInsertNewColumnTableDefault() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer primary key, " +
                "  score double" +
                ") with (number_of_replicas=0)");
        ensureGreen();
        execute("insert into dynamic_table (id, score) values (1, 42.24)");
        execute("refresh table dynamic_table");

        execute("select * from dynamic_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("id", "score")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(1, 42.24D)));

        execute("insert into dynamic_table (id, score, good) values (2, -0.01, false)");
        execute("refresh table dynamic_table");
        waitNoPendingTasksOnAll();

        execute("select * from dynamic_table order by id");
        assertThat(response.rowCount(), is(2L));
        assertThat(response.cols(), is(arrayContaining("good", "id", "score")));
        assertThat(TestingHelpers.printedTable(response.rows()), is(
                "NULL| 1| 42.24\n" +
                "false| 2| -0.01\n"));
    }

    @Test
    public void testUpdateNewColumnTableDefault() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer primary key, " +
                "  score double" +
                ") with (number_of_replicas=0)");
        ensureGreen();
        execute("insert into dynamic_table (id, score) values (1, 4656234.345)");
        execute("refresh table dynamic_table");

        execute("select * from dynamic_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("id", "score")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(1, 4656234.345D)));

        execute("update dynamic_table set name='Trillian', good=true where score > 0.0");
        execute("refresh table dynamic_table");
        waitNoPendingTasksOnAll();

        execute("select * from dynamic_table");
        assertThat(response.rowCount(), is(1L));
        assertThat(response.cols(), is(arrayContaining("good", "id", "name", "score")));
        assertThat(response.rows()[0], is(Matchers.<Object>arrayContaining(true, 1, "Trillian", 4656234.345D)));
    }

    @Test
    public void testStrictPartitionedTableInsert() throws Exception {
        execute("create table numbers (" +
                "  num int, " +
                "  odd boolean," +
                "  prime boolean" +
                ") partitioned by (odd) with (column_policy='strict', number_of_replicas=0)");
        ensureGreen();

        GetIndexTemplatesResponse response = client().admin().indices()
                .prepareGetTemplates(PartitionName.templateName("numbers"))
                .execute().actionGet();
        assertThat(response.getIndexTemplates().size(), is(1));
        IndexTemplateMetaData template = response.getIndexTemplates().get(0);
        CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(mappingStr, is(notNullValue()));
        Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(), false);
        @SuppressWarnings("unchecked")
        Map<String, Object> mapping = (Map<String, Object>)typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(mapping.get("dynamic")), is(ColumnPolicy.STRICT.value()));

        execute("insert into numbers (num, odd, prime) values (?, ?, ?)",
                new Object[]{6, true, false});
        execute("refresh table numbers");

        MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("numbers", Arrays.asList(new BytesRef("true"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(ColumnPolicy.STRICT.value()));

        expectedException.expect(SQLActionException.class);
        expectedException.expectMessage("Column 'perfect' unknown");
        execute("insert into numbers (num, odd, prime, perfect) values (?, ?, ?, ?)",
                new Object[]{28, true, false, true});
    }

    @Test
    public void testStrictPartitionedTableUpdate() throws Exception {
        execute("create table numbers (" +
                "  num int, " +
                "  odd boolean," +
                "  prime boolean" +
                ") partitioned by (odd) with (column_policy='strict', number_of_replicas=0)");
        ensureGreen();

        GetIndexTemplatesResponse response = client().admin().indices()
                .prepareGetTemplates(PartitionName.templateName("numbers"))
                .execute().actionGet();
        assertThat(response.getIndexTemplates().size(), is(1));
        IndexTemplateMetaData template = response.getIndexTemplates().get(0);
        CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(mappingStr, is(notNullValue()));
        Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(), false);
        @SuppressWarnings("unchecked")
        Map<String, Object> mapping = (Map<String, Object>)typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(mapping.get("dynamic")), is(ColumnPolicy.STRICT.value()));

        execute("insert into numbers (num, odd, prime) values (?, ?, ?)",
                new Object[]{6, true, false});
        execute("refresh table numbers");

        MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("numbers", Arrays.asList(new BytesRef("true"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(ColumnPolicy.STRICT.value()));

        expectedException.expect(SQLActionException.class);
        expectedException.expectMessage("Column 'perfect' unknown");
        execute("update numbers set num=?, perfect=? where num=6",
                new Object[]{28, true});
    }

    @Test
    public void testDynamicPartitionedTable() throws Exception {
        execute("create table numbers (" +
                "  num int, " +
                "  odd boolean," +
                "  prime boolean" +
                ") partitioned by (odd) with (column_policy='dynamic', number_of_replicas=0)");
        ensureGreen();

        GetIndexTemplatesResponse templateResponse = client().admin().indices()
                .prepareGetTemplates(PartitionName.templateName("numbers"))
                .execute().actionGet();
        assertThat(templateResponse.getIndexTemplates().size(), is(1));
        IndexTemplateMetaData template = templateResponse.getIndexTemplates().get(0);
        CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(mappingStr, is(notNullValue()));
        Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(), false);
        @SuppressWarnings("unchecked")
        Map<String, Object> mapping = (Map<String, Object>)typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(mapping.get("dynamic")), is("true"));

        execute("insert into numbers (num, odd, prime) values (?, ?, ?)",
                new Object[]{6, true, false});
        execute("refresh table numbers");

        MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("numbers", Arrays.asList(new BytesRef("true"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is("true"));

        execute("insert into numbers (num, odd, prime, perfect) values (?, ?, ?, ?)",
                new Object[]{28, true, false, true});
        ensureGreen();
        execute("refresh table numbers");
        waitNoPendingTasksOnAll();

        execute("select * from numbers order by num");
        assertThat(response.rowCount(), is(2L));
        assertThat(response.cols(), arrayContaining("num", "odd", "perfect", "prime"));
        assertThat(TestingHelpers.printedTable(response.rows()), is(
                "6| true| NULL| false\n" +
                "28| true| true| false\n"));

        execute("update numbers set prime=true, changed='2014-10-23T10:20', author='troll' where num=28");
        assertThat(response.rowCount(), is(1L));
        execute("refresh table numbers");

        // mapping update is asynchronous, so wait for it
        awaitBusy(new Predicate<Object>() {
            @Override
            public boolean apply(@Nullable Object input) {
                execute("select * from numbers order by num");
                return     response.rowCount() == 2L
                        && response.cols().length == 6
                        && Joiner.on(", ").join(Arrays.asList(response.cols()))
                            .equals("author, changed, num, odd, perfect, prime")
                        && TestingHelpers.printedTable(response.rows()).equals(
                                "NULL| NULL| 6| true| NULL| false\n" +
                                "troll| 1414059600000| 28| true| true| true\n");
            }
        });
    }

    @Test
    public void testAlterDynamicTable() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer primary key, " +
                "  score double" +
                ") with (number_of_replicas=0)");
        ensureGreen();
        execute("alter table dynamic_table set (column_policy = 'strict')");
        waitNoPendingTasksOnAll();
        expectedException.expect(SQLActionException.class);
        expectedException.expectMessage("Column 'new_col' unknown");
        execute("insert into dynamic_table (id, score, new_col) values (1, 4656234.345, 'hello')");
    }

    @Test
    public void testAlterTable() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer primary key, " +
                "  score double" +
                ") with (number_of_replicas=0, column_policy='strict')");
        ensureGreen();
        execute("insert into dynamic_table (id, score) values (1, 42)");
        execute("alter table dynamic_table set (column_policy = 'dynamic')");
        waitNoPendingTasksOnAll();
        execute("insert into dynamic_table (id, score, new_col) values (2, 4656234.345, 'hello')");
    }

    @Test
    public void testResetColumnPolicy() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer, " +
                "  score double" +
                ") with (number_of_replicas=0)");
        ensureGreen();
        execute("alter table dynamic_table set (column_policy = 'strict')");
        waitNoPendingTasksOnAll();
        MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
                .get("dynamic_table")
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(ColumnPolicy.STRICT.value()));
        execute("alter table dynamic_table reset (column_policy)");
        waitNoPendingTasksOnAll();
        partitionMetaData = clusterService().state().metaData().indices()
                .get("dynamic_table")
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));
    }

    @Test
    public void testResetColumnPolicyPartitioned() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer, " +
                "  score double" +
                ") partitioned by (score) with (number_of_replicas=0)");
        ensureGreen();
        execute("insert into dynamic_table (id, score) values (1, 10)");
        execute("refresh table dynamic_table");
        execute("alter table dynamic_table set (column_policy = 'strict')");
        waitNoPendingTasksOnAll();
        MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("10.0"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(ColumnPolicy.STRICT.value()));
        execute("alter table dynamic_table reset (column_policy)");
        waitNoPendingTasksOnAll();
        partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("10.0"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));
    }

    @Test
    public void testAlterPartitionedTable() throws Exception {
        execute("create table dynamic_table (" +
                "  id integer, " +
                "  score double" +
                ") partitioned by (score) with (number_of_replicas=0, column_policy='strict')");
        ensureGreen();
        execute("insert into dynamic_table (id, score) values (1, 10)");
        execute("refresh table dynamic_table");
        ensureGreen();
        execute("alter table dynamic_table set (column_policy= 'dynamic')");
        waitNoPendingTasksOnAll();
        // After changing the column_policy it's possible to add new columns to existing and new
        // partitions
        execute("insert into dynamic_table (id, score, comment) values (2,10,'this is a new column')");
        execute("insert into dynamic_table (id, score, new_comment) values (2,5,'this is a new column on a new partition')");
        execute("refresh table dynamic_table");
        ensureGreen();
        GetIndexTemplatesResponse response = client().admin().indices()
                .prepareGetTemplates(PartitionName.templateName("dynamic_table"))
                .execute().actionGet();
        assertThat(response.getIndexTemplates().size(), is(1));
        IndexTemplateMetaData template = response.getIndexTemplates().get(0);
        CompressedString mappingStr = template.mappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(mappingStr, is(notNullValue()));
        Tuple<XContentType, Map<String, Object>> typeAndMap = XContentHelper.convertToMap(mappingStr.uncompressed(), false);
        @SuppressWarnings("unchecked")
        Map<String, Object> mapping = (Map<String, Object>)typeAndMap.v2().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(mapping.get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));

        execute("insert into dynamic_table (id, score, new_col) values (?, ?, ?)",
                new Object[]{6, 3, "hello"});
        execute("refresh table dynamic_table");
        ensureGreen();

        MappingMetaData partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("10.0"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));

        partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("5.0"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));

        partitionMetaData = clusterService().state().metaData().indices()
                .get(new PartitionName("dynamic_table", Arrays.asList(new BytesRef("3.0"))).stringValue())
                .getMappings().get(Constants.DEFAULT_MAPPING_TYPE);
        assertThat(String.valueOf(partitionMetaData.getSourceAsMap().get("dynamic")), is(String.valueOf(ColumnPolicy.DYNAMIC.mappingValue())));
    }

}
TOP

Related Classes of io.crate.integrationtests.ColumnPolicyIntegrationTest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.