Examples of org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient

Package org.apache.hadoop.hbase.client.coprocessor

Examples of org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient

org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient
Client proxy for SecureBulkLoadProtocol used in conjunction with SecureBulkLoadEndpoint

        //This condition is here for unit testing
        //Since delegation token doesn't work in mini cluster
        if(User.isSecurityEnabled()) {
         userToken = fs.getDelegationToken("renewer");
        }
        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getName());
      }


      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }


        int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;


        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);


        bulkLoadPhase(table, conn, pool, queue, regionGroups);


        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }


    } finally {
      if(useSecure) {
        if(userToken != null) {
          try {
            userToken.cancel(cfg);
          } catch (Exception e) {
            LOG.warn("Failed to cancel HDFS delegation token.", e);
          }
        }
        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();

View Full Code Here


    final RegionServerCallable<Boolean> svrCallable =
        new RegionServerCallable<Boolean>(conn, tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;


        try {
          LOG.debug("Going to connect to server " + getLocation() + " for row "
              + Bytes.toStringBinary(getRow()) + " with hfile group " + famPaths);
          byte[] regionName = getLocation().getRegionInfo().getRegionName();
          if(!useSecure) {
            success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), getTableName());
            secureClient = new SecureBulkLoadClient(table);
            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken,
              getLocation().getRegionInfo().getStartKey());
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem fs = FileSystem.get(cfg);
            for(Pair<byte[], String> el : famPaths) {
              Path hfileStagingPath = null;
              Path hfileOrigPath = new Path(el.getSecond());
              try {
                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
                      hfileStagingPath);
                } else if(fs.exists(hfileStagingPath)){

View Full Code Here

          } else {
            hasForwardedToken = true;
            LOG.info("Use the existing token: " + userToken);
          }
        }
        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getName());
      }


      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }


        int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;


        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);


        bulkLoadPhase(table, conn, pool, queue, regionGroups);


        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }


    } finally {
      if (userProvider.isHBaseSecurityEnabled()) {
        if (userToken != null && !hasForwardedToken) {
          try {
            userToken.cancel(cfg);
          } catch (Exception e) {
            LOG.warn("Failed to cancel HDFS delegation token.", e);
          }
        }
        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();

View Full Code Here


    final RegionServerCallable<Boolean> svrCallable =
        new RegionServerCallable<Boolean>(conn, tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;


        try {
          LOG.debug("Going to connect to server " + getLocation() + " for row "
              + Bytes.toStringBinary(getRow()) + " with hfile group " + famPaths);
          byte[] regionName = getLocation().getRegionInfo().getRegionName();
          if(!userProvider.isHBaseSecurityEnabled()) {
            success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), getTableName());
            secureClient = new SecureBulkLoadClient(table);
            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken,
              getLocation().getRegionInfo().getStartKey());
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem fs = FileSystem.get(cfg);
            for(Pair<byte[], String> el : famPaths) {
              Path hfileStagingPath = null;
              Path hfileOrigPath = new Path(el.getSecond());
              try {
                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
                      hfileStagingPath);
                } else if(fs.exists(hfileStagingPath)){

View Full Code Here

      //prepare staging directory and token
      if (userProvider.isHBaseSecurityEnabled()) {
        // fs is the source filesystem
        fsDelegationToken.acquireDelegationToken(fs);


        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getName());
      }


      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }


        int maxRetries = getConf().getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;


        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);


        if (!checkHFilesCountPerRegionPerFamily(regionGroups)) {
          // Error is logged inside checkHFilesCountPerRegionPerFamily.
          throw new IOException("Trying to load more than " + maxFilesPerRegionPerFamily
            + " hfiles to one family of one region");
        }


        bulkLoadPhase(table, conn, pool, queue, regionGroups);


        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }


    } finally {
      if (userProvider.isHBaseSecurityEnabled()) {
        fsDelegationToken.releaseDelegationToken();


        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();

View Full Code Here


    final RegionServerCallable<Boolean> svrCallable =
        new RegionServerCallable<Boolean>(conn, tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;


        try {
          LOG.debug("Going to connect to server " + getLocation() + " for row "
              + Bytes.toStringBinary(getRow()) + " with hfile group " + famPaths);
          byte[] regionName = getLocation().getRegionInfo().getRegionName();
          if(!userProvider.isHBaseSecurityEnabled()) {
            success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), getTableName());
            secureClient = new SecureBulkLoadClient(table);
            success = secureClient.bulkLoadHFiles(famPaths, fsDelegationToken.getUserToken(),
              bulkToken, getLocation().getRegionInfo().getStartKey());
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem targetFs = FileSystem.get(getConf());
            // Check to see if the source and target filesystems are the same
            // If they are the same filesystem, we will try move the files back
            // because previously we moved them to the staging directory.
            if (FSHDFSUtils.isSameHdfs(getConf(), fs, targetFs)) {
              for(Pair<byte[], String> el : famPaths) {
                Path hfileStagingPath = null;
                Path hfileOrigPath = new Path(el.getSecond());
                try {
                  hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                  if(targetFs.rename(hfileStagingPath, hfileOrigPath)) {
                    LOG.debug("Moved back file " + hfileOrigPath + " from " +
                        hfileStagingPath);
                  } else if(targetFs.exists(hfileStagingPath)){

View Full Code Here

      //prepare staging directory and token
      if (userProvider.isHBaseSecurityEnabled()) {
        FileSystem fs = FileSystem.get(getConf());
        fsDelegationToken.acquireDelegationToken(fs);


        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getName());
      }


      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }


        int maxRetries = getConf().getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;


        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);


        if (!checkHFilesCountPerRegionPerFamily(regionGroups)) {
          // Error is logged inside checkHFilesCountPerRegionPerFamily.
          throw new IOException("Trying to load more than " + maxFilesPerRegionPerFamily
            + " hfiles to one family of one region");
        }


        bulkLoadPhase(table, conn, pool, queue, regionGroups);


        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }


    } finally {
      if (userProvider.isHBaseSecurityEnabled()) {
        fsDelegationToken.releaseDelegationToken();


        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();

View Full Code Here


    final RegionServerCallable<Boolean> svrCallable =
        new RegionServerCallable<Boolean>(conn, tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;


        try {
          LOG.debug("Going to connect to server " + getLocation() + " for row "
              + Bytes.toStringBinary(getRow()) + " with hfile group " + famPaths);
          byte[] regionName = getLocation().getRegionInfo().getRegionName();
          if(!userProvider.isHBaseSecurityEnabled()) {
            success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), getTableName());
            secureClient = new SecureBulkLoadClient(table);
            success = secureClient.bulkLoadHFiles(famPaths, fsDelegationToken.getUserToken(),
              bulkToken, getLocation().getRegionInfo().getStartKey());
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem fs = FileSystem.get(getConf());
            for(Pair<byte[], String> el : famPaths) {
              Path hfileStagingPath = null;
              Path hfileOrigPath = new Path(el.getSecond());
              try {
                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
                      hfileStagingPath);
                } else if(fs.exists(hfileStagingPath)){

View Full Code Here

        //This condition is here for unit testing
        //Since delegation token doesn't work in mini cluster
        if(User.isSecurityEnabled()) {
         userToken = fs.getDelegationToken("renewer");
        }
        bulkToken = new SecureBulkLoadClient(table).prepareBulkLoad(table.getName());
      }


      // Assumes that region splits can happen while this occurs.
      while (!queue.isEmpty()) {
        // need to reload split keys each iteration.
        final Pair<byte[][], byte[][]> startEndKeys = table.getStartEndKeys();
        if (count != 0) {
          LOG.info("Split occured while grouping HFiles, retry attempt " +
              + count + " with " + queue.size() + " files remaining to group or split");
        }


        int maxRetries = cfg.getInt("hbase.bulkload.retries.number", 0);
        if (maxRetries != 0 && count >= maxRetries) {
          LOG.error("Retry attempted " + count +  " times without completing, bailing out");
          return;
        }
        count++;


        // Using ByteBuffer for byte[] equality semantics
        Multimap<ByteBuffer, LoadQueueItem> regionGroups = groupOrSplitPhase(table,
            pool, queue, startEndKeys);


        bulkLoadPhase(table, conn, pool, queue, regionGroups);


        // NOTE: The next iteration's split / group could happen in parallel to
        // atomic bulkloads assuming that there are splits and no merges, and
        // that we can atomically pull out the groups we want to retry.
      }


    } finally {
      if(useSecure) {
        if(userToken != null) {
          try {
            userToken.cancel(cfg);
          } catch (Exception e) {
            LOG.warn("Failed to cancel HDFS delegation token.", e);
          }
        }
        if(bulkToken != null) {
          new SecureBulkLoadClient(table).cleanupBulkLoad(bulkToken);
        }
      }
      pool.shutdown();
      if (queue != null && !queue.isEmpty()) {
        StringBuilder err = new StringBuilder();

View Full Code Here


    final RegionServerCallable<Boolean> svrCallable =
        new RegionServerCallable<Boolean>(conn, tableName, first) {
      @Override
      public Boolean call() throws Exception {
        SecureBulkLoadClient secureClient = null;
        boolean success = false;


        try {
          LOG.debug("Going to connect to server " + getLocation() + " for row "
              + Bytes.toStringBinary(getRow()));
          byte[] regionName = getLocation().getRegionInfo().getRegionName();
          if(!useSecure) {
            success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, assignSeqIds);
          } else {
            HTable table = new HTable(conn.getConfiguration(), getTableName());
            secureClient = new SecureBulkLoadClient(table);
            success = secureClient.bulkLoadHFiles(famPaths, userToken, bulkToken,
              getLocation().getRegionInfo().getStartKey());
          }
          return success;
        } finally {
          //Best effort copying of files that might not have been imported
          //from the staging directory back to original location
          //in user directory
          if(secureClient != null && !success) {
            FileSystem fs = FileSystem.get(cfg);
            for(Pair<byte[], String> el : famPaths) {
              Path hfileStagingPath = null;
              Path hfileOrigPath = new Path(el.getSecond());
              try {
                hfileStagingPath= new Path(secureClient.getStagingPath(bulkToken, el.getFirst()),
                    hfileOrigPath.getName());
                if(fs.rename(hfileStagingPath, hfileOrigPath)) {
                  LOG.debug("Moved back file " + hfileOrigPath + " from " +
                      hfileStagingPath);
                } else if(fs.exists(hfileStagingPath)){

View Full Code Here

0 1 2

TOP

Related Classes of org.apache.hadoop.hbase.client.coprocessor.SecureBulkLoadClient

org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel

org.apache.hadoop.hbase.ipc.ServerRpcController

org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles

java.io.IOException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.