Package com.cloudera.cdk.data

Examples of com.cloudera.cdk.data.PartitionKey

90
91
92
93
94
95
96
97
98
99
100
    return returnList;
  }

  @Override
  public ManagedSchema getManagedSchema(String tableName, String entityName) {
    PartitionKey key = managedSchemaDao.getPartitionStrategy().partitionKey(
        tableName, entityName);
    ManagedSchema managedSchemaRecord = managedSchemaDao.get(key);
    if (managedSchemaRecord == null) {
      return null;
    } else {
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
        .setSubRecord1(subRecord1).setSubRecord2(subRecord2).build();

    // Test put and get
    dao.put(compositeRecord);

    PartitionKey key = dao.getPartitionStrategy().partitionKey("1", "1");
    CompositeRecord returnedCompositeRecord = dao.get(key);
    assertEquals("field1_1", returnedCompositeRecord.getSubRecord1()
        .getField1());
    assertEquals("field1_2", returnedCompositeRecord.getSubRecord1()
        .getField2());
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
    compositeRecord.put("SubRecord1", subRecord1);
    compositeRecord.put("SubRecord2", subRecord2);

    // Test put and get
    dao.put(compositeRecord);
    PartitionKey key = dao.getPartitionStrategy().partitionKey("1", "1");
    Map<String, SpecificRecord> returnedCompositeRecord = dao.get(key);
    assertEquals("field1_1", ((SubRecord1) returnedCompositeRecord.get("SubRecord1"))
        .getField1());
    assertEquals("field1_2", ((SubRecord1) returnedCompositeRecord.get("SubRecord1"))
        .getField2());
View Full Code Here
281
282
283
284
285
286
287
288
289
290
291
292
293
    possibleValues.add("field1_20");
    possibleValues.add("field1_21");
    possibleValues.add("field1_22");
    possibleValues.add("field1_23");
    possibleValues.add("field1_24");
    PartitionKey startKey = dao.getPartitionStrategy().partitionKey("part1_20",
        "part2_20");
    PartitionKey stopKey = dao.getPartitionStrategy().partitionKey("part1_25",
        "part2_25");

    EntityScannerBuilder<GenericRecord> builder = dao.getScannerBuilder()
        .setStartKey(startKey).setStopKey(stopKey);
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
    Dataset<Record> outputDataset = repo.create("out", new DatasetDescriptor.Builder()
        .schema(USER_SCHEMA).partitionStrategy(partitionStrategy).build());

    writeTestUsers(inputDataset, 10);

    PartitionKey key = partitionStrategy.partitionKey(0);
    Dataset<Record> inputPart0 = inputDataset.getPartition(key, false);
    Dataset<Record> outputPart0 = outputDataset.getPartition(key, true);

    Pipeline pipeline = new MRPipeline(TestCrunchDatasets.class);
    PCollection<GenericData.Record> data = pipeline.read(
View Full Code Here
41
42
43
44
45
46
47
48
49
50
51
    Dataset<GenericRecord> users = repo.load("users");

    // Get the partition strategy and use it to construct a partition key for
    // hash(username)=0
    PartitionStrategy partitionStrategy = users.getDescriptor().getPartitionStrategy();
    PartitionKey partitionKey = partitionStrategy.partitionKey(0);

    // Get the dataset partition for the partition key
    Dataset<GenericRecord> partition = users.getPartition(partitionKey, false);

    // Get a reader for the partition and read all the users
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
    }
    return ds;
  }

  private <E> Dataset<E> getPartitionForURI(Dataset<E> eventsDataset, String uri) {
    PartitionKey partitionKey = FileSystemDatasetRepository.partitionKeyForPath(
        eventsDataset, URI.create(uri));
    Dataset<E> partition = eventsDataset.getPartition(partitionKey, false);
    if (partition == null) {
      throw new IllegalArgumentException("Partition not found: " + uri);
    }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
    final DatasetWriter<GenericRecord> writer = persistent.newWriter();
    writer.open();

    // the source dataset: yesterday's partition in the staging area
    final Dataset<GenericRecord> staging = repo.load("logs-staging");
    final PartitionKey yesterday = getPartitionKey(staging, yesterdayTimestamp);
    final DatasetReader<GenericRecord> reader = staging
        .getPartition(yesterday, false).newReader();

    try {
      reader.open();
View Full Code Here
134
135
136
137
138
139
140
141
142
143
144
    }
    return ds;
  }

  private <E> Dataset<E> getPartitionForURI(Dataset<E> eventsDataset, String uri) {
    PartitionKey partitionKey = FileSystemDatasetRepository.partitionKeyForPath(
        eventsDataset, URI.create(uri));
    Dataset<E> partition = eventsDataset.getPartition(partitionKey, false);
    if (partition == null) {
      throw new IllegalArgumentException("Partition not found: " + uri);
    }
View Full Code Here
TOP

Related Classes of com.cloudera.cdk.data.PartitionKey

  • com.cloudera.cdk.data.crunch.TestCrunchDatasets
  • com.cloudera.cdk.data.filesystem.FileSystemDataset
  • com.cloudera.cdk.data.filesystem.TestFileSystemDataset
  • com.cloudera.cdk.data.filesystem.TestFileSystemDatasetPartitionKeyForPath
  • com.cloudera.cdk.data.hbase.avro.AvroDaoTest
  • com.cloudera.cdk.data.hbase.avro.CompositeDaoTest
  • com.cloudera.cdk.data.hbase.avro.example.UserProfileExample
  • com.cloudera.cdk.data.hbase.avro.ManagedDaoTest
  • com.cloudera.cdk.data.hbase.filters.ScannerFilterTest
  • com.cloudera.cdk.data.hbase.impl.BaseDao
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.