Package org.apache.hadoop.hive.accumulo.columns

Examples of org.apache.hadoop.hive.accumulo.columns.ColumnMapper

155
156
157
158
159
160
161
162
163
164
165
166
167
    baos.reset();
    out.writeInt(72);
    hiveRow.add("personal", "height", baos.toByteArray());

    ColumnMapper columnMapper = params.getColumnMapper();

    lazyRow.init(hiveRow, columnMapper.getColumnMappings(), rowIdFactory);

    Object o = lazyRow.getField(0);
    Assert.assertNotNull(o);
    Assert.assertEquals(LazyString.class, o.getClass());
    Assert.assertEquals("1", ((LazyString) o).toString());
View Full Code Here
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    AccumuloSerDeParameters params = new AccumuloSerDeParameters(new Configuration(), props,
        AccumuloSerDe.class.getName());

    rowIdFactory.init(params, props);

    ColumnMapper columnMapper = params.getColumnMapper();

    LazyAccumuloRow lazyRow = new LazyAccumuloRow(objectInspector);
    AccumuloHiveRow hiveRow = new AccumuloHiveRow("1");
    hiveRow.add("cf", "cq1", "foo".getBytes());
    hiveRow.add("cf", "cq3", "bar".getBytes());

    lazyRow.init(hiveRow, columnMapper.getColumnMappings(), rowIdFactory);

    // Noticed that we also suffer from the same issue as HIVE-3179
    // Only want to call a field init'ed when it's non-NULL
    // Check it twice, make sure we get null both times
    Assert.assertEquals("{'row':'1','1':'foo','2':null,'3':'bar'}".replace('\'', '"'),
View Full Code Here
104
105
106
107
108
109
110
111
    conf.set(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columnNames));
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,string");

    String columnMappingStr = "cf:f1,:rowID";
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, columnMappingStr);
    columnMapper = new ColumnMapper(columnMappingStr, ColumnEncoding.STRING.getName(), columnNames,
        columnTypes);
  }
View Full Code Here
440
441
442
443
444
445
446
447
448
449
450
    conf.set(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columnNames));
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,int,string");

    String columnMappingStr = "cf:f1,cf:f2,:rowID";
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, columnMappingStr);
    columnMapper = new ColumnMapper(columnMappingStr, ColumnEncoding.STRING.getName(), columnNames,
        columnTypes);

    ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "field1", null,
        false);
    ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "aaa");
View Full Code Here
491
492
493
494
495
496
497
498
499
500
501
        TypeInfoFactory.intTypeInfo, TypeInfoFactory.stringTypeInfo);
    conf.set(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columnNames));
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,int,string");
    String columnMappingStr = "cf:f1,cf:f2,:rowID";
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, columnMappingStr);
    columnMapper = new ColumnMapper(columnMappingStr, ColumnEncoding.STRING.getName(), columnNames,
        columnTypes);

    ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.stringTypeInfo, "field1", null,
        false);
    ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.stringTypeInfo, "aaa");
View Full Code Here
685
686
687
688
689
690
691
692
693
694
695
    conf.set(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columnNames));
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,string");

    String columnMappingStr = ":rowID,cf:f1";
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, columnMappingStr);
    columnMapper = new ColumnMapper(columnMappingStr, ColumnEncoding.STRING.getName(), columnNames,
        columnTypes);

    // 100 < key
    ExprNodeDesc column = new ExprNodeColumnDesc(TypeInfoFactory.intTypeInfo, "key", null, false);
    ExprNodeDesc constant = new ExprNodeConstantDesc(TypeInfoFactory.intTypeInfo, 100);
View Full Code Here
717
718
719
720
721
722
723
724
725
726
727
    conf.set(serdeConstants.LIST_COLUMNS, Joiner.on(',').join(columnNames));
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "int,string");

    String columnMappingStr = ":rowID#b,cf:f1";
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, columnMappingStr);
    columnMapper = new ColumnMapper(columnMappingStr, ColumnEncoding.STRING.getName(), columnNames,
        columnTypes);

    int intValue = 100;

    // Make binary integer value in the bytearray
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
  @Override
  public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException {
    final AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(jobConf);
    final Instance instance = accumuloParams.getInstance();
    final ColumnMapper columnMapper;
    try {
      columnMapper = getColumnMapper(jobConf);
    } catch (TooManyAccumuloColumnsException e) {
      throw new IOException(e);
    }

    JobContext context = ShimLoader.getHadoopShims().newJobContext(Job.getInstance(jobConf));
    Path[] tablePaths = FileInputFormat.getInputPaths(context);

    try {
      final Connector connector = accumuloParams.getConnector(instance);
      final List<ColumnMapping> columnMappings = columnMapper.getColumnMappings();
      final List<IteratorSetting> iterators = predicateHandler.getIterators(jobConf, columnMapper);
      final Collection<Range> ranges = predicateHandler.getRanges(jobConf, columnMapper);

      // Setting an empty collection of ranges will, unexpectedly, scan all data
      // We don't want that.
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
   * @throws IOException
   */
  @Override
  public RecordReader<Text,AccumuloHiveRow> getRecordReader(InputSplit inputSplit,
      final JobConf jobConf, final Reporter reporter) throws IOException {
    final ColumnMapper columnMapper;
    try {
      columnMapper = getColumnMapper(jobConf);
    } catch (TooManyAccumuloColumnsException e) {
      throw new IOException(e);
    }
View Full Code Here
224
225
226
227
228
229
230
231
      throw new IOException(
          "Hive column types must be provided to InputFormat in the Configuration");
    }
    ArrayList<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(serializedTypes);

    return new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS), defaultStorageType,
        columnNames, columnTypes);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.accumulo.columns.ColumnMapper

  • org.apache.hadoop.hive.accumulo.mr.HiveAccumuloTableInputFormat
  • org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat
  • org.apache.hadoop.hive.accumulo.predicate.TestAccumuloPredicateHandler
  • org.apache.hadoop.hive.accumulo.serde.AccumuloSerDeParameters
  • org.apache.hadoop.hive.accumulo.serde.TestDefaultAccumuloRowIdFactory
  • org.apache.hadoop.hive.accumulo.TestLazyAccumuloRow
  • org.apache.hadoop.hive.accumulo.serde.TooManyAccumuloColumnsException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.