Examples of org.apache.hadoop.hbase.client.replication.ReplicationAdmin

Package org.apache.hadoop.hbase.client.replication

Examples of org.apache.hadoop.hbase.client.replication.ReplicationAdmin

org.apache.hadoop.hbase.client.replication.ReplicationAdmin

This class provides the administrative interface to HBase cluster replication. In order to use it, the cluster and the client using ReplicationAdmin must be configured with hbase.replication set to true.

Adding a new peer results in creating new outbound connections from every region server to a subset of region servers on the slave cluster. Each new stream of replication will start replicating from the beginning of the current HLog, meaning that edits from that past will be replicated.

Removing a peer is a destructive and irreversible operation that stops all the replication streams for the given cluster and deletes the metadata used to keep track of the replication state.

Enabling and disabling peers is currently not supported.

As cluster replication is still experimental, a kill switch is provided in order to stop all replication-related operations, see {@link #setReplicating(boolean)}. When setting it back to true, the new state of all the replication streams will be unknown and may have holes. Use at your own risk.

To see which commands are available in the shell, type replication.

 }


  @Test(timeout=300000)
  public void testPerTableCFReplication() throws Exception {
    LOG.info("testPerTableCFReplication");
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);


    new HBaseAdmin(conf1).createTable(tabA);
    new HBaseAdmin(conf1).createTable(tabB);
    new HBaseAdmin(conf1).createTable(tabC);
    new HBaseAdmin(conf2).createTable(tabA);
    new HBaseAdmin(conf2).createTable(tabB);
    new HBaseAdmin(conf2).createTable(tabC);
    new HBaseAdmin(conf3).createTable(tabA);
    new HBaseAdmin(conf3).createTable(tabB);
    new HBaseAdmin(conf3).createTable(tabC);


    Table htab1A = new HTable(conf1, tabAName);
    Table htab2A = new HTable(conf2, tabAName);
    Table htab3A = new HTable(conf3, tabAName);


    Table htab1B = new HTable(conf1, tabBName);
    Table htab2B = new HTable(conf2, tabBName);
    Table htab3B = new HTable(conf3, tabBName);


    Table htab1C = new HTable(conf1, tabCName);
    Table htab2C = new HTable(conf2, tabCName);
    Table htab3C = new HTable(conf3, tabCName);


    // A. add cluster2/cluster3 as peers to cluster1
    admin1.addPeer("2", utility2.getClusterKey(), "TC;TB:f1,f3");
    admin1.addPeer("3", utility3.getClusterKey(), "TA;TB:f1,f2");


    // A1. tableA can only replicated to cluster3
    putAndWaitWithFamily(row1, f1Name, htab1A, htab3A);
    ensureRowNotReplicated(row1, f1Name, htab2A);
    deleteAndWaitWithFamily(row1, f1Name, htab1A, htab3A);


    putAndWaitWithFamily(row1, f2Name, htab1A, htab3A);
    ensureRowNotReplicated(row1, f2Name, htab2A);
    deleteAndWaitWithFamily(row1, f2Name, htab1A, htab3A);


    putAndWaitWithFamily(row1, f3Name, htab1A, htab3A);
    ensureRowNotReplicated(row1, f3Name, htab2A);
    deleteAndWaitWithFamily(row1, f3Name, htab1A, htab3A);


    // A2. cf 'f1' of tableB can replicated to both cluster2 and cluster3
    putAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);
    deleteAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);


    //  cf 'f2' of tableB can only replicated to cluster3
    putAndWaitWithFamily(row1, f2Name, htab1B, htab3B);
    ensureRowNotReplicated(row1, f2Name, htab2B);
    deleteAndWaitWithFamily(row1, f2Name, htab1B, htab3B);


    //  cf 'f3' of tableB can only replicated to cluster2
    putAndWaitWithFamily(row1, f3Name, htab1B, htab2B);
    ensureRowNotReplicated(row1, f3Name, htab3B);
    deleteAndWaitWithFamily(row1, f3Name, htab1B, htab2B);


    // A3. tableC can only replicated to cluster2
    putAndWaitWithFamily(row1, f1Name, htab1C, htab2C);
    ensureRowNotReplicated(row1, f1Name, htab3C);
    deleteAndWaitWithFamily(row1, f1Name, htab1C, htab2C);


    putAndWaitWithFamily(row1, f2Name, htab1C, htab2C);
    ensureRowNotReplicated(row1, f2Name, htab3C);
    deleteAndWaitWithFamily(row1, f2Name, htab1C, htab2C);


    putAndWaitWithFamily(row1, f3Name, htab1C, htab2C);
    ensureRowNotReplicated(row1, f3Name, htab3C);
    deleteAndWaitWithFamily(row1, f3Name, htab1C, htab2C);


    // B. change peers' replicable table-cf config
    admin1.setPeerTableCFs("2", "TA:f1,f2; TC:f2,f3");
    admin1.setPeerTableCFs("3", "TB; TC:f3");


    // B1. cf 'f1' of tableA can only replicated to cluster2
    putAndWaitWithFamily(row2, f1Name, htab1A, htab2A);
    ensureRowNotReplicated(row2, f1Name, htab3A);
    deleteAndWaitWithFamily(row2, f1Name, htab1A, htab2A);

View Full Code Here

    HTU2.setZkCluster(miniZK);
    HTU2.startMiniCluster(NB_SERVERS);
    LOG.info("Setup second Zk");
    HTU2.getHBaseAdmin().createTable(hdt, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);


    ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
    admin.addPeer("2", HTU2.getClusterKey());
    admin.close();


    Put p = new Put(row);
    p.add(row, row, row);
    final Table table = new HTable(HTU.getConfiguration(), hdt.getTableName());
    table.put(p);

View Full Code Here

  public void testMultiSlaveReplication() throws Exception {
    LOG.info("testCyclicReplication");
    MiniHBaseCluster master = utility1.startMiniCluster();
    utility2.startMiniCluster();
    utility3.startMiniCluster();
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);


    new HBaseAdmin(conf1).createTable(table);
    new HBaseAdmin(conf2).createTable(table);
    new HBaseAdmin(conf3).createTable(table);
    Table htable1 = new HTable(conf1, tableName);
    htable1.setWriteBufferSize(1024);
    Table htable2 = new HTable(conf2, tableName);
    htable2.setWriteBufferSize(1024);
    Table htable3 = new HTable(conf3, tableName);
    htable3.setWriteBufferSize(1024);
    
    admin1.addPeer("1", utility2.getClusterKey());


    // put "row" and wait 'til it got around, then delete
    putAndWait(row, famName, htable1, htable2);
    deleteAndWait(row, htable1, htable2);
    // check it wasn't replication to cluster 3
    checkRow(row,0,htable3);


    putAndWait(row2, famName, htable1, htable2);


    // now roll the region server's logs
    new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString());
    // after the log was rolled put a new row
    putAndWait(row3, famName, htable1, htable2);


    admin1.addPeer("2", utility3.getClusterKey());


    // put a row, check it was replicated to all clusters
    putAndWait(row1, famName, htable1, htable2, htable3);
    // delete and verify
    deleteAndWait(row1, htable1, htable2, htable3);

View Full Code Here

    mimicSyncUpAfterPut();


  }


  private void setupReplication() throws Exception {
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);


    Admin ha = new HBaseAdmin(conf1);
    ha.createTable(t1_syncupSource);
    ha.createTable(t2_syncupSource);
    ha.close();


    ha = new HBaseAdmin(conf2);
    ha.createTable(t1_syncupTarget);
    ha.createTable(t2_syncupTarget);
    ha.close();


    // Get HTable from Master
    ht1Source = new HTable(conf1, t1_su);
    ht1Source.setWriteBufferSize(1024);
    ht2Source = new HTable(conf1, t2_su);
    ht1Source.setWriteBufferSize(1024);


    // Get HTable from Peer1
    ht1TargetAtPeer1 = new HTable(conf2, t1_su);
    ht1TargetAtPeer1.setWriteBufferSize(1024);
    ht2TargetAtPeer1 = new HTable(conf2, t2_su);
    ht2TargetAtPeer1.setWriteBufferSize(1024);


    /**
     * set M-S : Master: utility1 Slave1: utility2
     */
    admin1.addPeer("1", utility2.getClusterKey());


    admin1.close();
    admin2.close();
  }

View Full Code Here

   */
  public static void setupRegionReplicaReplication(Configuration conf) throws IOException {
    if (!conf.getBoolean(REGION_REPLICA_REPLICATION_CONF_KEY, DEFAULT_REGION_REPLICA_REPLICATION)) {
      return;
    }
    ReplicationAdmin repAdmin = new ReplicationAdmin(conf);
    try {
      if (repAdmin.getPeerConfig(REGION_REPLICA_REPLICATION_PEER) == null) {
        ReplicationPeerConfig peerConfig = new ReplicationPeerConfig();
        peerConfig.setClusterKey(ZKUtil.getZooKeeperClusterKey(conf));
        peerConfig.setReplicationEndpointImpl(RegionReplicaReplicationEndpoint.class.getName());
        repAdmin.addPeer(REGION_REPLICA_REPLICATION_PEER, peerConfig, null);
      }
    } catch (ReplicationException ex) {
      throw new IOException(ex);
    } finally {
      repAdmin.close();
    }
  }

View Full Code Here

    }
  }


  private void addPeer(String id, int masterClusterNumber,
      int slaveClusterNumber) throws Exception {
    ReplicationAdmin replicationAdmin = null;
    try {
      replicationAdmin = new ReplicationAdmin(
          configurations[masterClusterNumber]);
      replicationAdmin.addPeer(id,
          utilities[slaveClusterNumber].getClusterKey());
    } finally {
      close(replicationAdmin);
    }
  }

View Full Code Here

      close(replicationAdmin);
    }
  }


  private void disablePeer(String id, int masterClusterNumber) throws Exception {
    ReplicationAdmin replicationAdmin = null;
    try {
      replicationAdmin = new ReplicationAdmin(
          configurations[masterClusterNumber]);
      replicationAdmin.disablePeer(id);
    } finally {
      close(replicationAdmin);
    }
  }

View Full Code Here

      close(replicationAdmin);
    }
  }


  private void enablePeer(String id, int masterClusterNumber) throws Exception {
    ReplicationAdmin replicationAdmin = null;
    try {
      replicationAdmin = new ReplicationAdmin(
          configurations[masterClusterNumber]);
      replicationAdmin.enablePeer(id);
    } finally {
      close(replicationAdmin);
    }
  }

View Full Code Here

  @Ignore("To be fixed before 1.0")
  @Test
  public void testRegionReplicaReplicationPeerIsCreated() throws IOException, ReplicationException {
    // create a table with region replicas. Check whether the replication peer is created
    // and replication started.
    ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
    String peerId = "region_replica_replication";


    if (admin.getPeerConfig(peerId) != null) {
      admin.removePeer(peerId);
    }


    HTableDescriptor htd = HTU.createTableDescriptor(
      "testReplicationPeerIsCreated_no_region_replicas");
    HTU.getHBaseAdmin().createTable(htd);
    ReplicationPeerConfig peerConfig = admin.getPeerConfig(peerId);
    assertNull(peerConfig);


    htd = HTU.createTableDescriptor("testReplicationPeerIsCreated");
    htd.setRegionReplication(2);
    HTU.getHBaseAdmin().createTable(htd);


    // assert peer configuration is correct
    peerConfig = admin.getPeerConfig(peerId);
    assertNotNull(peerConfig);
    assertEquals(peerConfig.getClusterKey(), ZKUtil.getZooKeeperClusterKey(HTU.getConfiguration()));
    assertEquals(peerConfig.getReplicationEndpointImpl(),
      RegionReplicaReplicationEndpoint.class.getName());
    admin.close();
  }

View Full Code Here

    htd = HTU.createTableDescriptor(toBeDisabledTable.toString());
    htd.setRegionReplication(regionReplication);
    HTU.getHBaseAdmin().createTable(htd);


    // both tables are created, now pause replication
    ReplicationAdmin admin = new ReplicationAdmin(HTU.getConfiguration());
    admin.disablePeer(ServerRegionReplicaUtil.getReplicationPeerId());


    // now that the replication is disabled, write to the table to be dropped, then drop the table.


    HConnection connection = HConnectionManager.createConnection(HTU.getConfiguration());
    Table table = connection.getTable(tableName);
    Table tableToBeDisabled = connection.getTable(toBeDisabledTable);


    HTU.loadNumericRows(tableToBeDisabled, HBaseTestingUtility.fam1, 6000, 7000);


    AtomicLong skippedEdits = new AtomicLong();
    RegionReplicaReplicationEndpoint.RegionReplicaOutputSink sink =
        mock(RegionReplicaReplicationEndpoint.RegionReplicaOutputSink.class);
    when(sink.getSkippedEditsCounter()).thenReturn(skippedEdits);
    RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter sinkWriter =
        new RegionReplicaReplicationEndpoint.RegionReplicaSinkWriter(sink,
          (ClusterConnection) connection,
          Executors.newSingleThreadExecutor(), 1000);


    HRegionLocation hrl = connection.locateRegion(toBeDisabledTable, HConstants.EMPTY_BYTE_ARRAY);
    byte[] encodedRegionName = hrl.getRegionInfo().getEncodedNameAsBytes();


    HLog.Entry entry = new HLog.Entry(
      new HLogKey(encodedRegionName, toBeDisabledTable, 1),
      new WALEdit());


    HTU.getHBaseAdmin().disableTable(toBeDisabledTable); // disable the table
    if (dropTable) {
      HTU.getHBaseAdmin().deleteTable(toBeDisabledTable);
    }


    sinkWriter.append(toBeDisabledTable, encodedRegionName,
      HConstants.EMPTY_BYTE_ARRAY, Lists.newArrayList(entry, entry));


    assertEquals(2, skippedEdits.get());


    try {
      // load some data to the to-be-dropped table


      // load the data to the table
      HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000);


      // now enable the replication
      admin.enablePeer(ServerRegionReplicaUtil.getReplicationPeerId());


      verifyReplication(tableName, regionReplication, 0, 6000);


    } finally {
      admin.close();
      table.close();
      tableToBeDisabled.close();
      HTU.deleteTableIfAny(toBeDisabledTable);
      connection.close();
    }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.hbase.client.replication.ReplicationAdmin

com.ngdata.hbaseindexer.indexer.IndexerIT

com.ngdata.sep.impl.SepModelImpl

org.apache.hadoop.hbase.Abortable

org.apache.hadoop.hbase.client.TestReplicaWithCluster

org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint

org.apache.hadoop.hbase.replication.ReplicationPeerConfig

org.apache.hadoop.hbase.replication.ReplicationZookeeper

org.apache.hadoop.hbase.replication.TestMasterReplication

org.apache.hadoop.hbase.replication.TestMultiSlaveReplication

org.apache.hadoop.hbase.replication.TestPerTableCFReplication

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.