Examples of org.kitesdk.data.hbase.avro.GenericAvroDao

org.kitesdk.data.hbase.avro.GenericAvroDao
A Dao for Avro's GenericRecords. In this Dao implementation, both the underlying key record type, and the entity type are GenericRecords. This Dao allows us to persist and fetch these GenericRecords to and from HBase.

    String entityName = HBaseMetadataProvider.getEntityName(name);
    Dao dao;
    if (isSpecific(descriptor)) {
      dao = new SpecificAvroDao(tablePool, tableName, entityName, schemaManager);
    } else {
      dao = new GenericAvroDao(tablePool, tableName, entityName, schemaManager);
    }
    return new DaoDataset(namespace, name, dao, descriptor,
        new URIBuilder(repositoryUri, namespace, name).build(), type);
  }

View Full Code Here


  }


  @Test
  public void testPassAllRegexMatchFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    for (int i = 0; i < 10; i++) {
      possibleValues.add("field1_3" + Integer.toString(i));
    }


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder();
    builder.addRegexMatchFilter("field1", "field1_3\\d");
    builder.addRegexMatchFilter("field2", "field2_3\\d");
    builder.setPassAllFilters(true);
    checkScannerYieldValues(builder.build(), possibleValues);
  }

View Full Code Here

    checkScannerYieldValues(builder.build(), possibleValues);
  }


  @Test
  public void testPassAllNotNullFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    for (int i = 0; i < 100; i++) {
      possibleValues.add("field1_" + Integer.toString(i));
    }
    possibleValues.add("field1_MISSING_FIELD2");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder();
    builder.addNotNullFilter("field1");
    checkScannerYieldValues(builder.build(), possibleValues);
  }

View Full Code Here

    checkScannerYieldValues(builder.build(), possibleValues);
  }


  @Test
  public void testPassIsNullFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    possibleValues.add("");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder();
    builder.addIsNullFilter("field1");
    checkScannerYieldValues(builder.build(), possibleValues);
  }

View Full Code Here

    checkScannerYieldValues(builder.build(), possibleValues);
  }


  @Test
  public void testPassIsMissingFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    possibleValues.add("field1_MISSING_FIELD2");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder();
    builder.addIsMissingFilter("field2");
    checkScannerYieldValues(builder.build(), possibleValues);
  }

View Full Code Here

    checkScannerYieldValues(builder.build(), possibleValues);
  }


  @Test
  public void testPassIfNotEqualFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    for (int i = 6; i < 100; i++) {
      possibleValues.add("field1_" + Integer.toString(i));
    }
    possibleValues.add("field1_MISSING_FIELD2");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder()
        .addNotEqualFilter("field1", "field1_0")
        .addNotEqualFilter("field1", "field1_1")
        .addNotEqualFilter("field1", "field1_2")
        .addNotEqualFilter("field1", "field1_3")
        .addNotEqualFilter("field1", "field1_4")

View Full Code Here

    checkScannerYieldValues(builder.build(), possibleValues);
  }


  @Test
  public void testStartRowScan() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    possibleValues.add("field1_20");
    possibleValues.add("field1_21");
    possibleValues.add("field1_22");
    possibleValues.add("field1_23");
    possibleValues.add("field1_24");
    PartitionKey startKey = new PartitionKey("part1_20", "part2_20");
    PartitionKey stopKey = new PartitionKey("part1_25", "part2_25");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder()
        .setStartKey(startKey).setStopKey(stopKey);


    checkScannerYieldValues(builder.build(), possibleValues);
  }

View Full Code Here

  @Before
  public void beforeTest() throws Exception {
    HBaseTestUtils.util.truncateTable(Bytes.toBytes(tableName));
    tablePool = new HTablePool(HBaseTestUtils.getConf(), 10);


    Dao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    for (int i = 0; i < 100; ++i) {
      @SuppressWarnings("deprecation")
      GenericRecord entity = new GenericData.Record(Schema.parse(recordString));
      entity.put("keyPart1", "part1_" + i);
      entity.put("keyPart2", "part2_" + i);
      entity.put("field1", "field1_" + Integer.toString(i));
      entity.put("field2", "field2_" + Integer.toString(i));
      dao.put(entity);
    }


    // Add a few NULL values
    @SuppressWarnings("deprecation")
    GenericRecord entity = new GenericData.Record(Schema.parse(recordString));
    entity.put("keyPart1", "part1_NULL");
    entity.put("keyPart2", "part2_NULL");
    entity.put("field1", "");
    entity.put("field2", "");
    dao.put(entity);


    @SuppressWarnings("deprecation")
    GenericRecord entityMissing = new GenericData.Record(
        Schema.parse(recordString));
    entityMissing.put("keyPart1", "part1_MISSING");
    entityMissing.put("keyPart2", "part2_MISSING");
    entityMissing.put("field1", "field1_MISSING_FIELD2");
    dao.put(entityMissing);
  }

View Full Code Here

    }
  }


  @Test
  public void testPassAllEqualityFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder()
        .addEqualFilter("field1", "field1_2")
        .addEqualFilter("field2", "field2_2").setPassAllFilters(true);


    Set<String> possibleValues = new HashSet<String>();
    possibleValues.add("field1_2");

View Full Code Here

    checkScannerYieldValues(builder.build(), possibleValues);
  }


  @Test
  public void testPassOneEqualityFilter() throws Exception {
    BaseDao<GenericRecord> dao = new GenericAvroDao(tablePool, tableName,
        recordString);


    Set<String> possibleValues = new HashSet<String>();
    possibleValues.add("field1_1");
    possibleValues.add("field1_62");
    possibleValues.add("field1_54");
    possibleValues.add("field1_29");
    possibleValues.add("field1_18");


    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder();
    for (String possibleValue : possibleValues) {
      builder.addEqualFilter("field1", possibleValue);
    }


    builder.setPassAllFilters(false);

View Full Code Here

0 1

TOP

Related Classes of org.kitesdk.data.hbase.avro.GenericAvroDao

org.kitesdk.data.hbase.filters.ScannerFilterTest

org.kitesdk.data.hbase.HBaseDatasetRepository

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.