Package org.apache.hcatalog.data.schema

Examples of org.apache.hcatalog.data.schema.HCatSchema


            context.write(NullWritable.get(), record);
        }
    }

    private HCatSchema getSchema() throws HCatException {
        HCatSchema schema = new HCatSchema(new ArrayList<HCatFieldSchema>());
        schema.append(new HCatFieldSchema("a0", HCatFieldSchema.Type.INT,
                ""));
        schema.append(new HCatFieldSchema("a1",
                HCatFieldSchema.Type.STRING, ""));
        schema.append(new HCatFieldSchema("a2",
                HCatFieldSchema.Type.STRING, ""));
        return schema;
    }
View Full Code Here


            context.write(NullWritable.get(), record);
        }
    }

    private HCatSchema getSchema() throws HCatException {
        HCatSchema schema = new HCatSchema(new ArrayList<HCatFieldSchema>());
        schema.append(new HCatFieldSchema("a0", HCatFieldSchema.Type.INT,
                ""));
        schema.append(new HCatFieldSchema("a1",
                HCatFieldSchema.Type.STRING, ""));
        schema.append(new HCatFieldSchema("a2",
                HCatFieldSchema.Type.STRING, ""));
        return schema;
    }
View Full Code Here

        job.setMapOutputKeyClass(BytesWritable.class);
        job.setMapOutputValueClass(DefaultHCatRecord.class);

        job.setNumReduceTasks(0);

        HCatOutputFormat.setSchema(job, new HCatSchema(partitionColumns));

        boolean success = job.waitForCompletion(true);

        // Ensure counters are set when data has actually been read.
        if (partitionValues != null) {
View Full Code Here


    //test that new columns gets added to table schema
    private void tableSchemaTest() throws Exception {

        HCatSchema tableSchema = getTableSchema();

        assertEquals(4, tableSchema.getFields().size());

        //Update partition schema to have 3 fields
        partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", serdeConstants.STRING_TYPE_NAME, "")));

        writeRecords = new ArrayList<HCatRecord>();

        for (int i = 0; i < 20; i++) {
            List<Object> objList = new ArrayList<Object>();

            objList.add(i);
            objList.add("strvalue" + i);
            objList.add("str2value" + i);

            writeRecords.add(new DefaultHCatRecord(objList));
        }

        Map<String, String> partitionMap = new HashMap<String, String>();
        partitionMap.put("part1", "p1value5");
        partitionMap.put("part0", "p0value5");

        runMRCreate(partitionMap, partitionColumns, writeRecords, 10, true);

        tableSchema = getTableSchema();

        //assert that c3 has got added to table schema
        assertEquals(5, tableSchema.getFields().size());
        assertEquals("c1", tableSchema.getFields().get(0).getName());
        assertEquals("c2", tableSchema.getFields().get(1).getName());
        assertEquals("c3", tableSchema.getFields().get(2).getName());
        assertEquals("part1", tableSchema.getFields().get(3).getName());
        assertEquals("part0", tableSchema.getFields().get(4).getName());

        //Test that changing column data type fails
        partitionMap.clear();
        partitionMap.put("part1", "p1value6");
        partitionMap.put("part0", "p0value6");
View Full Code Here

    }

    //check behavior while change the order of columns
    private void columnOrderChangeTest() throws Exception {

        HCatSchema tableSchema = getTableSchema();

        assertEquals(5, tableSchema.getFields().size());

        partitionColumns = new ArrayList<HCatFieldSchema>();
        partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c1", serdeConstants.INT_TYPE_NAME, "")));
        partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c3", serdeConstants.STRING_TYPE_NAME, "")));
        partitionColumns.add(HCatSchemaUtils.getHCatFieldSchema(new FieldSchema("c2", serdeConstants.STRING_TYPE_NAME, "")));
View Full Code Here

        bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
        ResourceSchema expected = new ResourceSchema().setFields(bagSubFieldSchemas);

        // Get the actual converted schema.
        HCatSchema hCatSchema = new HCatSchema(Lists.newArrayList(
            new HCatFieldSchema("innerLlama", HCatFieldSchema.Type.STRING, null)));
        HCatFieldSchema hCatFieldSchema =
            new HCatFieldSchema("llama", HCatFieldSchema.Type.ARRAY, hCatSchema, null);
        ResourceSchema actual = PigHCatUtil.getBagSubSchema(hCatFieldSchema);
View Full Code Here

        bagSubFieldSchemas[0].setSchema(new ResourceSchema().setFields(innerTupleFieldSchemas));
        ResourceSchema expected = new ResourceSchema().setFields(bagSubFieldSchemas);

        // Get the actual converted schema.
        HCatSchema actualHCatSchema = new HCatSchema(Lists.newArrayList(
            new HCatFieldSchema("innerLlama", HCatFieldSchema.Type.STRING, null)));
        HCatFieldSchema actualHCatFieldSchema =
            new HCatFieldSchema("llama", HCatFieldSchema.Type.ARRAY, actualHCatSchema, null);
        ResourceSchema actual = PigHCatUtil.getBagSubSchema(actualHCatFieldSchema);
View Full Code Here


    static HCatTableSnapshot convertSnapshot(TableSnapshot hbaseSnapshot,
                                             HCatTableInfo hcatTableInfo) throws IOException {

        HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns();
        Map<String, String> hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo);
        HashMap<String, Long> revisionMap = new HashMap<String, Long>();

        for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) {
            if (hcatHbaseColMap.containsKey(fSchema.getName())) {
                String colFamily = hcatHbaseColMap.get(fSchema.getName());
                long revisionID = hbaseSnapshot.getRevision(colFamily);
                revisionMap.put(fSchema.getName(), revisionID);
            }
View Full Code Here

    }

    static TableSnapshot convertSnapshot(HCatTableSnapshot hcatSnapshot,
                                         HCatTableInfo hcatTableInfo) throws IOException {

        HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns();
        Map<String, Long> revisionMap = new HashMap<String, Long>();
        Map<String, String> hcatHbaseColMap = getHCatHBaseColumnMapping(hcatTableInfo);
        for (HCatFieldSchema fSchema : hcatTableSchema.getFields()) {
            String colFamily = hcatHbaseColMap.get(fSchema.getName());
            if (hcatSnapshot.containsColumn(fSchema.getName())) {
                long revision = hcatSnapshot.getRevision(fSchema.getName());
                revisionMap.put(colFamily, revision);
            }
View Full Code Here

    }

    private static Map<String, String> getHCatHBaseColumnMapping(HCatTableInfo hcatTableInfo)
        throws IOException {

        HCatSchema hcatTableSchema = hcatTableInfo.getDataColumns();
        StorerInfo storeInfo = hcatTableInfo.getStorerInfo();
        String hbaseColumnMapping = storeInfo.getProperties().getProperty(
            HBaseSerDe.HBASE_COLUMNS_MAPPING);

        Map<String, String> hcatHbaseColMap = new HashMap<String, String>();
        List<String> columnFamilies = new ArrayList<String>();
        List<String> columnQualifiers = new ArrayList<String>();
        HBaseUtil.parseColumnMapping(hbaseColumnMapping, columnFamilies,
            null, columnQualifiers, null);

        for (HCatFieldSchema column : hcatTableSchema.getFields()) {
            int fieldPos = hcatTableSchema.getPosition(column.getName());
            String colFamily = columnFamilies.get(fieldPos);
            if (colFamily.equals(HBaseSerDe.HBASE_KEY_COL) == false) {
                hcatHbaseColMap.put(column.getName(), colFamily);
            }
        }
View Full Code Here

TOP

Related Classes of org.apache.hcatalog.data.schema.HCatSchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.