Package org.apache.hadoop.hive.accumulo

Examples of org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters

461
462
463
464
465
466
467
468
469
470
471
    assertEquals(columns, inputformat.getPairCollection(mappings));
  }

  @Test
  public void testConfigureMockAccumuloInputFormat() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
        conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text,Text>> cfCqPairs = inputformat
        .getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = Collections.emptyList();
View Full Code Here
495
496
497
498
499
500
501
502
503
504
505
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
  }

  @Test
  public void testConfigureAccumuloInputFormat() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
        conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text,Text>> cfCqPairs = inputformat
        .getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = Collections.emptyList();
View Full Code Here
536
537
538
539
540
541
542
543
544
545
546
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
  }

  @Test
  public void testConfigureAccumuloInputFormatWithAuthorizations() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    conf.set(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo,bar");
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
        conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text,Text>> cfCqPairs = inputformat
        .getPairCollection(columnMapper.getColumnMappings());
View Full Code Here
577
578
579
580
581
582
583
584
585
586
587
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
  }

  @Test
  public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
        conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    Set<Pair<Text,Text>> cfCqPairs = inputformat
        .getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
View Full Code Here
632
633
634
635
636
637
638
639
640
641
642
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
  }

  @Test
  public void testConfigureAccumuloInputFormatWithEmptyColumns() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper = new ColumnMapper(conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
        conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes);
    HashSet<Pair<Text,Text>> cfCqPairs = Sets.newHashSet();
    List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
    Set<Range> ranges = Collections.singleton(new Range());
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  protected AccumuloRowInputFormat accumuloInputFormat = new AccumuloRowInputFormat();
  protected AccumuloPredicateHandler predicateHandler = AccumuloPredicateHandler.getInstance();

  @Override
  public InputSplit[] getSplits(JobConf jobConf, int numSplits) throws IOException {
    final AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(jobConf);
    final Instance instance = accumuloParams.getInstance();
    final ColumnMapper columnMapper;
    try {
      columnMapper = getColumnMapper(jobConf);
    } catch (TooManyAccumuloColumnsException e) {
      throw new IOException(e);
    }

    JobContext context = ShimLoader.getHadoopShims().newJobContext(Job.getInstance(jobConf));
    Path[] tablePaths = FileInputFormat.getInputPaths(context);

    try {
      final Connector connector = accumuloParams.getConnector(instance);
      final List<ColumnMapping> columnMappings = columnMapper.getColumnMappings();
      final List<IteratorSetting> iterators = predicateHandler.getIterators(jobConf, columnMapper);
      final Collection<Range> ranges = predicateHandler.getRanges(jobConf, columnMapper);

      // Setting an empty collection of ranges will, unexpectedly, scan all data
View Full Code Here
191
192
193
194
195
196
197
198
199
200
201
202
203
204
      }

      // ACCUMULO-3015 Like the above, RangeInputSplit should have the table name
      // but we want it to, so just re-set it if it's null.
      if (null == getTableName(rangeSplit)) {
        final AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(
            jobConf);
        log.debug("Re-setting table name on InputSplit due to Accumulo bug.");
        setTableName(rangeSplit, accumuloParams.getAccumuloTableName());
      }

      final RecordReader<Text,PeekingIterator<Map.Entry<Key,Value>>> recordReader = accumuloInputFormat
          .getRecordReader(rangeSplit, jobConf, reporter);
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
    return super.getRecordWriter(ignored, job, name, progress);
  }

  protected void configureAccumuloOutputFormat(JobConf job) throws IOException {
    AccumuloConnectionParameters cnxnParams = new AccumuloConnectionParameters(job);

    final String tableName = job.get(AccumuloSerDeParameters.TABLE_NAME);

    // Make sure we actually go the table name
    Preconditions.checkNotNull(tableName,
        "Expected Accumulo table name to be provided in job configuration");

    // Set the necessary Accumulo information
    try {
      // Username/passwd for Accumulo
      setAccumuloConnectorInfo(job, cnxnParams.getAccumuloUserName(),
          new PasswordToken(cnxnParams.getAccumuloPassword()));

      if (cnxnParams.useMockInstance()) {
        setAccumuloMockInstance(job, cnxnParams.getAccumuloInstanceName());
      } else {
        // Accumulo instance name with ZK quorum
        setAccumuloZooKeeperInstance(job, cnxnParams.getAccumuloInstanceName(),
            cnxnParams.getZooKeepers());
      }

      // Set the table where we're writing this data
      setDefaultAccumuloTableName(job, tableName);
    } catch (AccumuloSecurityException e) {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.accumulo.AccumuloConnectionParameters

  • org.apache.accumulo.core.client.Instance
  • org.apache.accumulo.core.client.mock.MockInstance
  • org.apache.accumulo.core.client.security.tokens.PasswordToken
  • org.apache.accumulo.core.client.ZooKeeperInstance
  • org.apache.hadoop.hive.accumulo.mr.HiveAccumuloTableInputFormat
  • org.apache.hadoop.hive.accumulo.mr.HiveAccumuloTableOutputFormat
  • org.apache.hadoop.hive.accumulo.mr.TestHiveAccumuloTableInputFormat
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.