Examples of org.apache.hadoop.hive.common.ValidTxnList

org.apache.hadoop.hive.common.ValidTxnList
Models the list of transactions that should be included in a snapshot. It is modelled as a high water mark, which is the largest transaction id that has been committed and a list of transactions that are not included.

      bucket = (int) split.getStart();
      reader = null;
    }
    String txnString = conf.get(ValidTxnList.VALID_TXNS_KEY,
                                Long.MAX_VALUE + ":");
    ValidTxnList validTxnList = new ValidTxnListImpl(txnString);
    final OrcRawRecordMerger records =
        new OrcRawRecordMerger(conf, true, reader, split.isOriginal(), bucket,
            validTxnList, readOptions, deltas);
    return new RowReader<OrcStruct>() {
      OrcStruct innerRecord = records.createValue();

View Full Code Here

          txnHandler.markCleaned(ci);
          continue;
        }


        final boolean isMajor = (ci.type == CompactionType.MAJOR);
        final ValidTxnList txns =
            TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
        final StringBuffer jobName = new StringBuffer(name);
        jobName.append("-compactor-");
        jobName.append(ci.getFullPartitionName());

View Full Code Here


        // Wrap the inner parts of the loop in a catch throwable so that any errors in the loop
        // don't doom the entire thread.
        try {
          ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest());
          ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns());
          Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreashold);
          LOG.debug("Found " + potentials.size() + " potential compactions, " +
              "checking to see if we should compact any of them");
          for (CompactionInfo ci : potentials) {
            LOG.debug("Checking to see if we should compact " + ci.getFullPartitionName());

View Full Code Here

      final String location = sd.getLocation();


      // Create a bogus validTxnList with a high water mark set to MAX_LONG and no open
      // transactions.  This assures that all deltas are treated as valid and all we return are
      // obsolete files.
      final ValidTxnList txnList = new ValidTxnListImpl();


      if (runJobAsSelf(ci.runAs)) {
        removeFiles(location, txnList);
      } else {
        LOG.info("Cleaning as user " + ci.runAs);

View Full Code Here

      // the input split.
      // Based on the split we're passed we go instantiate the real reader and then iterate on it
      // until it finishes.
      AcidInputFormat aif =
          instantiate(AcidInputFormat.class, jobConf.get(INPUT_FORMAT_CLASS_NAME));
      ValidTxnList txnList =
          new ValidTxnListImpl(jobConf.get(ValidTxnList.VALID_TXNS_KEY));


      AcidInputFormat.RawReader<V> reader =
          aif.getRawReader(jobConf, jobConf.getBoolean(IS_MAJOR, false), split.getBucket(),
              txnList, split.getBaseDir(), split.getDeltaDirs());

View Full Code Here

    AcidOutputFormat.Options options = new AcidOutputFormat.Options(conf)
        .inspector(inspector).bucket(BUCKET).writingBase(true)
        .maximumTransactionId(100);
    of.getRecordUpdater(root, options).close(false);


    ValidTxnList txnList = new ValidTxnListImpl("200:");
    AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList);


    Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(),
        BUCKET);
    Reader baseReader = OrcFile.createReader(basePath,

View Full Code Here

    ru.update(200, 0, 3, new MyRow("update 3"));
    ru.delete(200, 0, 7);
    ru.delete(200, 0, 8);
    ru.close(false);


    ValidTxnList txnList = new ValidTxnListImpl("200:");
    AcidUtils.Directory directory = AcidUtils.getAcidState(root, conf, txnList);


    assertEquals(new Path(root, "base_0000100"), directory.getBaseDirectory());
    assertEquals(new Path(root, "delta_0000200_0000200"),
        directory.getCurrentDirectories().get(0).getPath());


    Path basePath = AcidUtils.createBucketFile(directory.getBaseDirectory(),
        BUCKET);
    Reader baseReader = OrcFile.createReader(basePath,
        OrcFile.readerOptions(conf));
    OrcRawRecordMerger merger =
        new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
            createMaximalTxnList(), new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));
    assertEquals(null, merger.getMinKey());
    assertEquals(null, merger.getMaxKey());
    RecordIdentifier id = merger.createKey();
    OrcStruct event = merger.createValue();


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
    assertEquals("update 1", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
    assertEquals("second", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 200), id);
    assertEquals("update 2", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 200), id);
    assertEquals("update 3", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 4, 0), id);
    assertEquals("fifth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 5, 0), id);
    assertEquals("sixth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 6, 0), id);
    assertEquals("seventh", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 9, 0), id);
    assertEquals("tenth", getValue(event));


    assertEquals(false, merger.next(id, event));
    merger.close();


    // make a merger that doesn't collapse events
    merger = new OrcRawRecordMerger(conf, false, baseReader, false, BUCKET,
            createMaximalTxnList(), new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 200), id);
    assertEquals("update 1", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 0, 0), id);
    assertEquals("first", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 1, 0), id);
    assertEquals("second", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 200), id);
    assertEquals("update 2", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 2, 0), id);
    assertEquals("third", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.UPDATE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 200), id);
    assertEquals("update 3", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 3, 0), id);
    assertEquals("fourth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 4, 0), id);
    assertEquals("fifth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 5, 0), id);
    assertEquals("sixth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 6, 0), id);
    assertEquals("seventh", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 7, 0), id);
    assertEquals("eighth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.DELETE_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 200), id);
    assertEquals(null, OrcRecordUpdater.getRow(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 8, 0), id);
    assertEquals("ninth", getValue(event));


    assertEquals(true, merger.next(id, event));
    assertEquals(OrcRecordUpdater.INSERT_OPERATION,
        OrcRecordUpdater.getOperation(event));
    assertEquals(new ReaderKey(0, BUCKET, 9, 0), id);
    assertEquals("tenth", getValue(event));


    assertEquals(false, merger.next(id, event));
    merger.close();


    // try ignoring the 200 transaction and make sure it works still
    ValidTxnList txns = new ValidTxnListImpl("2000:200");
    merger =
        new OrcRawRecordMerger(conf, true, baseReader, false, BUCKET,
            txns, new Reader.Options(),
            AcidUtils.getPaths(directory.getCurrentDirectories()));
    for(int i=0; i < values.length; ++i) {

View Full Code Here

          txnHandler.markCleaned(ci);
          continue;
        }


        final boolean isMajor = ci.isMajorCompaction();
        final ValidTxnList txns =
            TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
        final StringBuffer jobName = new StringBuffer(name);
        jobName.append("-compactor-");
        jobName.append(ci.getFullPartitionName());

View Full Code Here


        // Wrap the inner parts of the loop in a catch throwable so that any errors in the loop
        // don't doom the entire thread.
        try {
          ShowCompactResponse currentCompactions = txnHandler.showCompact(new ShowCompactRequest());
          ValidTxnList txns = TxnHandler.createValidTxnList(txnHandler.getOpenTxns(), 0);
          Set<CompactionInfo> potentials = txnHandler.findPotentialCompactions(abortedThreshold);
          LOG.debug("Found " + potentials.size() + " potential compactions, " +
              "checking to see if we should compact any of them");
          for (CompactionInfo ci : potentials) {
            LOG.debug("Checking to see if we should compact " + ci.getFullPartitionName());

View Full Code Here

    Assert.assertEquals(1L, (long) tids.get(0));
    Assert.assertEquals(2L, (long) tids.get(1));
    Assert.assertEquals(3L, (long) tids.get(2));
    client.rollbackTxn(1);
    client.commitTxn(2);
    ValidTxnList validTxns = client.getValidTxns();
    Assert.assertFalse(validTxns.isTxnCommitted(1));
    Assert.assertTrue(validTxns.isTxnCommitted(2));
    Assert.assertFalse(validTxns.isTxnCommitted(3));
    Assert.assertFalse(validTxns.isTxnCommitted(4));
  }

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.hive.common.ValidTxnList

org.apache.hadoop.hive.metastore.TestHiveMetaStoreTxns

org.apache.hadoop.hive.ql.Driver

org.apache.hadoop.hive.ql.io.orc.OrcInputFormat

org.apache.hadoop.hive.ql.io.orc.TestOrcRawRecordMerger

org.apache.hadoop.hive.ql.txn.compactor.Cleaner

org.apache.hadoop.hive.ql.txn.compactor.CompactorMR$CompactorMap

org.apache.hadoop.hive.ql.txn.compactor.Initiator

org.apache.hadoop.hive.ql.txn.compactor.Worker

org.apache.hive.hcatalog.streaming.TestStreaming

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.