Package org.apache.howl.data.schema

Examples of org.apache.howl.data.schema.HowlSchema


  public void testConversion() throws IOException {
    Configuration conf = new Configuration();
    JobContext jc = new JobContext(conf, new JobID());

    HowlSchema schema = buildHiveSchema();
    HowlInputStorageDriver isd = new RCFileInputDriver();

    isd.setOriginalSchema(jc, schema);
    isd.setOutputSchema(jc, schema);
    isd.initialize(jc, new Properties());
View Full Code Here


    fields.add(new FieldSchema("adouble", "double", ""));
    fields.add(new FieldSchema("astring", "string", ""));
    fields.add(new FieldSchema("anullint", "int", ""));
    fields.add(new FieldSchema("anullstring", "string", ""));

    return new HowlSchema(HowlUtil.getHowlFieldSchemaList(fields));
  }
View Full Code Here


  //test that new columns gets added to table schema
  private void tableSchemaTest() throws Exception {

    HowlSchema tableSchema = getTableSchema();

    assertEquals(3, tableSchema.getFields().size());

    //Update partition schema to have 3 fields
    partitionColumns.add(HowlSchemaUtils.getHowlFieldSchema(new FieldSchema("c3", Constants.STRING_TYPE_NAME, "")));

    writeRecords = new ArrayList<HowlRecord>();

    for(int i = 0;i < 20;i++) {
      List<Object> objList = new ArrayList<Object>();

      objList.add(i);
      objList.add("strvalue" + i);
      objList.add("str2value" + i);

      writeRecords.add(new DefaultHowlRecord(objList));
    }

    Map<String, String> partitionMap = new HashMap<String, String>();
    partitionMap.put("part1", "p1value5");

    runMRCreate(partitionMap, partitionColumns, writeRecords, 10);

    tableSchema = getTableSchema();

    //assert that c3 has got added to table schema
    assertEquals(4, tableSchema.getFields().size());
    assertEquals("c1", tableSchema.getFields().get(0).getName());
    assertEquals("c2", tableSchema.getFields().get(1).getName());
    assertEquals("c3", tableSchema.getFields().get(2).getName());
    assertEquals("part1", tableSchema.getFields().get(3).getName());

    //Test that changing column data type fails
    partitionMap.clear();
    partitionMap.put("part1", "p1value6");
View Full Code Here

  }

  //check behavior while change the order of columns
  private void columnOrderChangeTest() throws Exception {

    HowlSchema tableSchema = getTableSchema();

    assertEquals(4, tableSchema.getFields().size());

    partitionColumns = new ArrayList<HowlFieldSchema>();
    partitionColumns.add(HowlSchemaUtils.getHowlFieldSchema(new FieldSchema("c1", Constants.INT_TYPE_NAME, "")));
    partitionColumns.add(HowlSchemaUtils.getHowlFieldSchema(new FieldSchema("c3", Constants.STRING_TYPE_NAME, "")));
    partitionColumns.add(HowlSchemaUtils.getHowlFieldSchema(new FieldSchema("c2", Constants.STRING_TYPE_NAME, "")));
View Full Code Here

    }
    writer.append(bytes2);
    writer.close();
    BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2};

    HowlSchema schema = buildHiveSchema();
    RCFileInputDriver sd = new RCFileInputDriver();
    JobContext jc = new JobContext(conf, new JobID());
    sd.setInputPath(jc, file.toString());
    InputFormat<?,?> iF = sd.getInputFormat(null);
    InputSplit split = iF.getSplits(jc).get(0);
View Full Code Here

    fields.add(new FieldSchema("adouble", "double", ""));
    fields.add(new FieldSchema("astring", "string", ""));
    fields.add(new FieldSchema("anullint", "int", ""));
    fields.add(new FieldSchema("anullstring", "string", ""));

    return new HowlSchema(HowlUtil.getHowlFieldSchemaList(fields));
  }
View Full Code Here

    fields.add(new FieldSchema("aint", "int", ""));
    fields.add(new FieldSchema("adouble", "double", ""));
    fields.add(new FieldSchema("astring", "string", ""));
    fields.add(new FieldSchema("anullint", "int", ""));

    return new HowlSchema(HowlUtil.getHowlFieldSchemaList(fields));
  }
View Full Code Here

    fields.add(new FieldSchema("astring", "string", ""));
    fields.add(new FieldSchema("atinyint", "tinyint", ""));
    fields.add(new FieldSchema("anullint", "int", ""));


    return new HowlSchema(HowlUtil.getHowlFieldSchemaList(fields));
  }
View Full Code Here

    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(DefaultHowlRecord.class);

    job.setNumReduceTasks(0);

    HowlOutputFormat.setSchema(job, new HowlSchema(partitionColumns));

    //new HowlOutputCommitter(null).setupJob(job);
    job.waitForCompletion(true);
    new HowlOutputCommitter(null).cleanupJob(job);
    Assert.assertEquals(writeCount, MapCreate.writeCount);
View Full Code Here

  public static HowlSchema extractSchemaFromStorageDescriptor(StorageDescriptor sd) throws HowlException {
      if (sd == null){
          throw new HowlException("Cannot construct partition info from an empty storage descriptor.");
        }
        HowlSchema schema = new HowlSchema(HowlUtil.getHowlFieldSchemaList(sd.getCols()));
        return schema;
  }
View Full Code Here

TOP

Related Classes of org.apache.howl.data.schema.HowlSchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.