Examples of org.apache.hadoop.hive.common.ValidTxnListImpl

org.apache.hadoop.hive.common.ValidTxnListImpl

      // Based on the split we're passed we go instantiate the real reader and then iterate on it
      // until it finishes.
      AcidInputFormat aif =
          instantiate(AcidInputFormat.class, jobConf.get(INPUT_FORMAT_CLASS_NAME));
      ValidTxnList txnList =
          new ValidTxnListImpl(jobConf.get(ValidTxnList.VALID_TXNS_KEY));


      AcidInputFormat.RawReader<V> reader =
          aif.getRawReader(jobConf, jobConf.getBoolean(IS_MAJOR, false), split.getBucket(),
              txnList, split.getBaseDir(), split.getDeltaDirs());
      RecordIdentifier identifier = reader.createKey();

View Full Code Here

    assertEquals(null, pair.nextRecord);
    Mockito.verify(recordReader).close();
  }


  private static ValidTxnList createMaximalTxnList() {
    return new ValidTxnListImpl(Long.MAX_VALUE + ":");
  }

View Full Code Here

    AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
        .inspector(inspector).bucket(BUCKET).writingBase(true)
        .maximumTransactionId(100);
    of.getRecordUpdater(root, options).close(false);


    ValidTxnList txnList = new ValidTxnListImpl("200:");
    AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList);


    Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(),
        BUCKET);
    Reader baseReader = OrcFile.createReader(basePath,

View Full Code Here

    ru.update(200, 0, 3, new MyRow("update 3"));
    ru.delete(200, 0, 7);
    ru.delete(200, 0, 8);
    ru.close(false);


    ValidTxnList txnList = new ValidTxnListImpl("200:");
    AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList);


    assertEquals(new Path(root, "base_0000100"), directory.getBaseDirectory());
    assertEquals(new Path(root, "delta_0000200_0000200"),
        directory.getCurrentDirectories().get(0).getPath());


    Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(),
        BUCKET);
    Reader baseReader = OrcFile.createReader(basePath,
        OrcFile.readerOptions(conf));
    OrcRawRecordMerger merger =
        new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
            createMaximalTxnList(), new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));
    assertEquals(null, merger.getMinKey());
    assertEquals(null, merger.getMaxKey());
    RecordIdentifier id = merger.createKey();
    OrcStruct event = merger.createValue();


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
    assertEquals("update 1", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
    assertEquals("second", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 200), id);
    assertEquals("update 2", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 200), id);
    assertEquals("update 3", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 4, 0), id);
    assertEquals("fifth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 5, 0), id);
    assertEquals("sixth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 6, 0), id);
    assertEquals("seventh", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 9, 0), id);
    assertEquals("tenth", getValue(event));


    assertEquals(false, merger.next(id, event));
    merger.close();


    // make a merger that doesn't collapse events
    merger = new OrcRawRecordMerger(conf, false, baseReader, false, BUCKET,
            createMaximalTxnList(), new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
    assertEquals("update 1", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 0), id);
    assertEquals("first", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
    assertEquals("second", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 200), id);
    assertEquals("update 2", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 0), id);
    assertEquals("third", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 200), id);
    assertEquals("update 3", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 0), id);
    assertEquals("fourth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 4, 0), id);
    assertEquals("fifth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 5, 0), id);
    assertEquals("sixth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 6, 0), id);
    assertEquals("seventh", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 0), id);
    assertEquals("eighth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 0), id);
    assertEquals("ninth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 9, 0), id);
    assertEquals("tenth", getValue(event));


    assertEquals(false, merger.next(id, event));
    merger.close();


    // try ignoring the 200 transaction and make sure it works still
    ValidTxnList txns = new ValidTxnListImpl("2000:200");
    merger =
        new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
            txns, new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));
    for(int i=0; i < values.length; ++i) {

View Full Code Here

    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(1L, 1L));
    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(5L, 10L));


    validTxns = new ValidTxnListImpl("10:4:5:6");
    Assert.assertEquals(ValidTxnList.RangeResponse.NONE,
        validTxns.isTxnRangeCommitted(4,6));
    Assert.assertEquals(ValidTxnList.RangeResponse.ALL,
        validTxns.isTxnRangeCommitted(7, 10));
    Assert.assertEquals(ValidTxnList.RangeResponse.SOME,

View Full Code Here

  }


  @Test
  public void stringifyValidTxns() throws Exception {
    // Test with just high water mark
    ValidTxnList validTxns = new ValidTxnListImpl("1:");
    String asString = validTxns.toString();
    Assert.assertEquals("1:", asString);
    validTxns = new ValidTxnListImpl(asString);
    Assert.assertEquals(1, validTxns.getHighWatermark());
    Assert.assertNotNull(validTxns.getOpenTransactions());
    Assert.assertEquals(0, validTxns.getOpenTransactions().length);
    asString = validTxns.toString();
    Assert.assertEquals("1:", asString);
    validTxns = new ValidTxnListImpl(asString);
    Assert.assertEquals(1, validTxns.getHighWatermark());
    Assert.assertNotNull(validTxns.getOpenTransactions());
    Assert.assertEquals(0, validTxns.getOpenTransactions().length);


    // Test with open transactions
    validTxns = new ValidTxnListImpl("10:5:3");
    asString = validTxns.toString();
    if (!asString.equals("10:3:5") && !asString.equals("10:5:3")) {
      Assert.fail("Unexpected string value " + asString);
    }
    validTxns = new ValidTxnListImpl(asString);
    Assert.assertEquals(10, validTxns.getHighWatermark());
    Assert.assertNotNull(validTxns.getOpenTransactions());
    Assert.assertEquals(2, validTxns.getOpenTransactions().length);
    boolean sawThree = false, sawFive = false;
    for (long tid : validTxns.getOpenTransactions()) {
      if (tid == 3)  sawThree = true;
      else if (tid == 5) sawFive = true;
      else  Assert.fail("Unexpected value " + tid);
    }
    Assert.assertTrue(sawThree);

View Full Code Here

    // No-op
  }


  @Override
  public ValidTxnList getValidTxns() throws LockException {
    return new ValidTxnListImpl();
  }

View Full Code Here

      bucket = (int) split.getStart();
      reader = null;
    }
    String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY,
                                Long.MAX_VALUE + ":");
    ValidTxnList validTxnList = new ValidTxnListImpl(txnString);
    final OrcRawRecordMerger records =
        new OrcRawRecordMerger(conf, true, reader, split.isOriginal(), bucket,
            validTxnList, readOptions, deltas);
    return new RowReader<OrcStruct>() {
      OrcStruct innerRecord = records.createValue();

View Full Code Here

              .initialCapacity(cacheStripeDetailsSize).softValues().build();
        }
      }
      String value = conf.get(ValidTxnList.VALID_TXNS_KEY,
                              Long.MAX_VALUE + ":");
      transactionList = new ValidTxnListImpl(value);
    }

View Full Code Here

      final String location = sd.getLocation();


      // Create a bogus validTxnList with a high water mark set to MAX_LONG and no open
      // transactions.  This assures that all deltas are treated as valid and all we return are
      // obsolete files.
      final ValidTxnList txnList = new ValidTxnListImpl();


      if (runJobAsSelf(ci.runAs)) {
        removeFiles(location, txnList);
      } else {
        LOG.info("Cleaning as user " + ci.runAs);

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.hive.common.ValidTxnListImpl

org.apache.hadoop.hive.metastore.TestHiveMetaStoreTxns

org.apache.hadoop.hive.metastore.txn.TxnHandler

org.apache.hadoop.hive.ql.exec.TestFileSinkOperator

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat$Context

org.apache.hadoop.hive.ql.io.orc.TestOrcRawRecordMerger

org.apache.hadoop.hive.ql.io.TestAcidUtils

org.apache.hadoop.hive.ql.lockmgr.DummyTxnManager

org.apache.hadoop.hive.ql.txn.compactor.Cleaner

org.apache.hadoop.hive.ql.txn.compactor.CompactorMR$CompactorMap

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.