Examples of RevisionManager

org.apache.hcatalog.hbase.snapshot.RevisionManager
This interface provides APIs for implementing revision management.

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

    resultItr = this.scanner.iterator();
  }


  private Set<Long> getAbortedTransactions(String tableName, Scan scan) throws IOException {
    Set<Long> abortedTransactions = new HashSet<Long>();
    RevisionManager rm = null;
    try {
      rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
      byte[][] families = scan.getFamilies();
      for (byte[] familyKey : families) {
        String family = Bytes.toString(familyKey);
        List<FamilyRevision> abortedWriteTransactions = rm.getAbortedWriteTransactions(
          tableName, family);
        if (abortedWriteTransactions != null) {
          for (FamilyRevision revision : abortedWriteTransactions) {
            abortedTransactions.add(revision.getRevision());
          }

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager


      // ensure the table is online
      new HTable(hbaseConf, tableDesc.getName());


      //Set up table in revision manager.
      RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf);
      rm.createTable(tableName, new ArrayList<String>(uniqueColumnFamilies));


    } catch (MasterNotRunningException mnre) {
      throw new MetaException(StringUtils.stringifyException(mnre));
    } catch (IOException ie) {
      throw new MetaException(StringUtils.stringifyException(ie));

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

  }


  private void checkDeleteTable(Table table) throws MetaException {
    boolean isExternal = MetaStoreUtils.isExternalTable(table);
    String tableName = getFullyQualifiedHBaseTableName(table);
    RevisionManager rm = null;
    try {
      if (!isExternal && getHBaseAdmin().tableExists(tableName)) {
        // we have created an HBase table, so we delete it to roll back;
        if (getHBaseAdmin().isTableEnabled(tableName)) {
          getHBaseAdmin().disableTable(tableName);
        }
        getHBaseAdmin().deleteTable(tableName);


        //Drop table in revision manager.
        rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hbaseConf);
        rm.dropTable(tableName);
      }
    } catch (IOException ie) {
      throw new MetaException(StringUtils.stringifyException(ie));
    } finally {
      HBaseRevisionManagerUtil.closeRevisionManagerQuietly(rm);

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

    job.setOutputFormat(HBaseBulkOutputFormat.class);
    org.apache.hadoop.mapred.SequenceFileOutputFormat.setOutputPath(job, interPath);
    job.setOutputCommitter(HBaseBulkOutputCommitter.class);


    //manually create transaction
    RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
    try {
      OutputJobInfo outputJobInfo = OutputJobInfo.create("default", tableName, null);
      Transaction txn = rm.beginWriteTransaction(tableName, Arrays.asList(familyName));
      outputJobInfo.getProperties().setProperty(HBaseConstants.PROPERTY_WRITE_TXN_KEY,
        HCatUtil.serialize(txn));
      job.set(HCatConstants.HCAT_KEY_OUTPUT_INFO,
        HCatUtil.serialize(outputJobInfo));
    } finally {
      rm.close();
    }


    job.setMapOutputKeyClass(ImmutableBytesWritable.class);
    job.setMapOutputValueClass(HCatRecord.class);

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

    job.setOutputValueClass(HCatRecord.class);


    job.setNumReduceTasks(0);


    assertTrue(job.waitForCompletion(true));
    RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
    try {
      TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName);
      for (String el : snapshot.getColumnFamilies()) {
        assertEquals(1, snapshot.getRevision(el));
      }
    } finally {
      rm.close();
    }


    //verify
    HTable table = new HTable(conf, databaseName + "." + tableName);
    Scan scan = new Scan();

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

      conf, workingDir, MapWriteAbortTransaction.class,
      outputJobInfo, inputPath);
    assertFalse(job.waitForCompletion(true));


    // verify that revision manager has it as aborted transaction
    RevisionManager rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
    try {
      TableSnapshot snapshot = rm.createSnapshot(databaseName + "." + tableName);
      for (String family : snapshot.getColumnFamilies()) {
        assertEquals(1, snapshot.getRevision(family));
        List<FamilyRevision> abortedWriteTransactions = rm.getAbortedWriteTransactions(
          databaseName + "." + tableName, family);
        assertEquals(1, abortedWriteTransactions.size());
        assertEquals(1, abortedWriteTransactions.get(0).getRevision());
      }
    } finally {
      rm.close();
    }


    //verify that hbase does not have any of the records.
    //Since records are only written during commitJob,
    //hbase should not have any records.

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

  }


  private List<Put> generatePuts(int num, String tableName) throws IOException {


    List<String> columnFamilies = Arrays.asList("testFamily");
    RevisionManager rm = null;
    List<Put> myPuts;
    try {
      rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
      rm.open();
      myPuts = new ArrayList<Put>();
      for (int i = 1; i <= num; i++) {
        Put put = new Put(Bytes.toBytes("testRow"));
        put.add(FAMILY, QUALIFIER1, i, Bytes.toBytes("textValue-" + i));
        put.add(FAMILY, QUALIFIER2, i, Bytes.toBytes("textValue-" + i));
        myPuts.add(put);
        Transaction tsx = rm.beginWriteTransaction(tableName,
            columnFamilies);
        rm.commitWriteTransaction(tsx);
      }
    } finally {
      if (rm != null)
        rm.close();
    }


    return myPuts;
  }

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

  }


  private long populateHBaseTableQualifier1(String tName, int value, Boolean commit)
    throws IOException {
    List<String> columnFamilies = Arrays.asList("testFamily");
    RevisionManager rm = null;
    List<Put> myPuts = new ArrayList<Put>();
    long revision;
    try {
      rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(hcatConf);
      rm.open();
      Transaction tsx = rm.beginWriteTransaction(tName, columnFamilies);


      Put put = new Put(Bytes.toBytes("testRow"));
      revision = tsx.getRevisionNumber();
      put.add(FAMILY, QUALIFIER1, revision,
          Bytes.toBytes("textValue-" + value));
      myPuts.add(put);


      // If commit is null it is left as a running transaction
      if (commit != null) {
        if (commit) {
          rm.commitWriteTransaction(tsx);
        } else {
          rm.abortWriteTransaction(tsx);
        }
      }
    } finally {
      if (rm != null)
        rm.close();
    }
    HTable table = new HTable(getHbaseConf(), Bytes.toBytes(tName));
    table.put(myPuts);
    return revision;
  }

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager


    @Override
    public void abortJob(JobContext jobContext, int status)
      throws IOException {
      baseOutputCommitter.abortJob(jobContext, status);
      RevisionManager rm = null;
      try {
        rm = HBaseRevisionManagerUtil
          .getOpenedRevisionManager(jobContext.getConfiguration());
        rm.abortWriteTransaction(HBaseRevisionManagerUtil
          .getWriteTransaction(jobContext.getConfiguration()));
      } finally {
        cleanIntermediate(jobContext);
        if (rm != null)
          rm.close();
      }
    }

View Full Code Here

Examples of org.apache.hcatalog.hbase.snapshot.RevisionManager

    }


    @Override
    public void commitJob(JobContext jobContext) throws IOException {
      baseOutputCommitter.commitJob(jobContext);
      RevisionManager rm = null;
      try {
        Configuration conf = jobContext.getConfiguration();
        Path srcPath = FileOutputFormat.getOutputPath(jobContext.getJobConf());
        if (!FileSystem.get(conf).exists(srcPath)) {
          throw new IOException("Failed to bulk import hfiles. " +
            "Intermediate data directory is cleaned up or missing. " +
            "Please look at the bulk import job if it exists for failure reason");
        }
        Path destPath = new Path(srcPath.getParent(), srcPath.getName() + "_hfiles");
        boolean success = ImportSequenceFile.runJob(jobContext,
          conf.get(HBaseConstants.PROPERTY_OUTPUT_TABLE_NAME_KEY),
          srcPath,
          destPath);
        if (!success) {
          cleanIntermediate(jobContext);
          throw new IOException("Failed to bulk import hfiles." +
            " Please look at the bulk import job for failure reason");
        }
        rm = HBaseRevisionManagerUtil.getOpenedRevisionManager(conf);
        rm.commitWriteTransaction(HBaseRevisionManagerUtil.getWriteTransaction(conf));
        cleanIntermediate(jobContext);
      } finally {
        if (rm != null)
          rm.close();
      }
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.