Examples of Shard


Examples of com.amazonaws.services.kinesis.model.Shard

     * @throws KinesisClientLibIOException
     */
    private static void assertAllParentShardsAreClosed(Map<String, Set<String>> shardIdToChildShardIdsMap,
            Map<String, Shard> shardIdToShardMap) throws KinesisClientLibIOException {
        for (String parentShardId : shardIdToChildShardIdsMap.keySet()) {
            Shard parentShard = shardIdToShardMap.get(parentShardId);
            if ((parentShardId == null) || (parentShard.getSequenceNumberRange().getEndingSequenceNumber() == null)) {
                throw new KinesisClientLibIOException("Parent shardId " + parentShardId + " is not closed. "
                        + "This can happen due to a race condition between describeStream and a reshard operation.");
            }
        }
    }
View Full Code Here

Examples of com.amazonaws.services.kinesis.model.Shard

            Set<String> shardIdsOfClosedShards) throws KinesisClientLibIOException {       
        String exceptionMessageSuffix = "This can happen if we constructed the list of shards "
                        + " while a reshard operation was in progress.";
       
        for (String shardId : shardIdsOfClosedShards) {
            Shard shard = shardIdToShardMap.get(shardId);
            if (shard == null) {
                LOG.info("Shard " + shardId + " is not present in Kinesis anymore.");
                continue;
            }
           
            String endingSequenceNumber = shard.getSequenceNumberRange().getEndingSequenceNumber();
            if (endingSequenceNumber == null) {
                throw new KinesisClientLibIOException("Shard " + shardIdsOfClosedShards
                        + " is not closed. " + exceptionMessageSuffix);
            }
View Full Code Here

Examples of com.amazonaws.services.kinesis.model.Shard

        BigInteger endingHashKeyOfClosedShard = new BigInteger(closedShard.getHashKeyRange().getEndingHashKey());
        BigInteger minStartingHashKeyOfChildren = null;
        BigInteger maxEndingHashKeyOfChildren = null;

        for (String childShardId : childShardIds) {
            Shard childShard = shardIdToShardMap.get(childShardId);
            BigInteger startingHashKey = new BigInteger(childShard.getHashKeyRange().getStartingHashKey());
            if ((minStartingHashKeyOfChildren == null)
                    || (startingHashKey.compareTo(minStartingHashKeyOfChildren) < 0)) {
                minStartingHashKeyOfChildren = startingHashKey;
            }
            BigInteger endingHashKey = new BigInteger(childShard.getHashKeyRange().getEndingHashKey());
            if ((maxEndingHashKeyOfChildren == null)
                    || (endingHashKey.compareTo(maxEndingHashKeyOfChildren) > 0)) {
                maxEndingHashKeyOfChildren = endingHashKey;
            }
        }
View Full Code Here

Examples of com.amazonaws.services.kinesis.model.Shard

    static Map<String, Set<String>> constructShardIdToChildShardIdsMap(
            Map<String, Shard> shardIdToShardMap) {
        Map<String, Set<String>> shardIdToChildShardIdsMap = new HashMap<>();
        for (Map.Entry<String, Shard> entry : shardIdToShardMap.entrySet()) {
            String shardId = entry.getKey();
            Shard shard = entry.getValue();
            String parentShardId = shard.getParentShardId();
            if ((parentShardId != null) && (shardIdToShardMap.containsKey(parentShardId))) {
                Set<String> childShardIds = shardIdToChildShardIdsMap.get(parentShardId);
                if (childShardIds == null) {
                    childShardIds = new HashSet<String>();
                    shardIdToChildShardIdsMap.put(parentShardId, childShardIds);
                }
                childShardIds.add(shardId);
            }
           
            String adjacentParentShardId = shard.getAdjacentParentShardId();
            if ((adjacentParentShardId != null) && (shardIdToShardMap.containsKey(adjacentParentShardId))) {
                Set<String> childShardIds = shardIdToChildShardIdsMap.get(adjacentParentShardId);
                if (childShardIds == null) {
                    childShardIds = new HashSet<String>();
                    shardIdToChildShardIdsMap.put(adjacentParentShardId, childShardIds);
View Full Code Here

Examples of com.amazonaws.services.kinesis.model.Shard

        if (previousValue != null) {
            return previousValue;
        }

        boolean isDescendant = false;
        Shard shard;
        Set<String> parentShardIds;
        Set<String> descendantParentShardIds = new HashSet<String>();

        if ((shardId != null) && (shardIdToShardMapOfAllKinesisShards.containsKey(shardId))) {
            if (shardIdsOfCurrentLeases.contains(shardId)) {
View Full Code Here

Examples of com.amazonaws.services.kinesis.model.Shard

        @Override
        public int compare(KinesisClientLease lease1, KinesisClientLease lease2) {
            int result = 0;
            String shardId1 = lease1.getLeaseKey();
            String shardId2 = lease2.getLeaseKey();
            Shard shard1 = shardIdToShardMap.get(shardId1);
            Shard shard2 = shardIdToShardMap.get(shardId2);
           
            // If we found shards for the two leases, use comparison of the starting sequence numbers
            if ((shard1 != null) && (shard2 != null)) {
                BigInteger sequenceNumber1 =
                        new BigInteger(shard1.getSequenceNumberRange().getStartingSequenceNumber());
                BigInteger sequenceNumber2 =
                        new BigInteger(shard2.getSequenceNumberRange().getStartingSequenceNumber());
                result = sequenceNumber1.compareTo(sequenceNumber2);               
            }
           
            if (result == 0) {
                result = shardId1.compareTo(shardId2);
View Full Code Here

Examples of com.senseidb.indexing.hadoop.keyvalueformat.Shard

        Arrays.sort(shardNames, 0, count);

        Shard[] shards = new Shard[count >= numShards ? count : numShards];
        for (int i = 0; i < count; i++) {
          shards[i] =
              new Shard(versionNumber, parent + shardNames[i], generation);
        }

        int number = count;
        for (int i = count; i < numShards; i++) {
          String shardPath;
          while (true) {
            shardPath = parent + indexSubDirPrefix + NUMBER_FORMAT.format(number++);
            if (!fs.exists(new Path(shardPath))) {
              break;
            }
          }
          shards[i] = new Shard(versionNumber, shardPath, generation);
        }
        return shards;
      } else {
        Shard[] shards = new Shard[numShards];
        for (int i = 0; i < shards.length; i++) {
          shards[i] =
              new Shard(versionNumber, parent + indexSubDirPrefix + NUMBER_FORMAT.format(i),
                  generation);
        }
        return shards;
      }
    }
View Full Code Here

Examples of com.senseidb.indexing.hadoop.keyvalueformat.Shard

          }

          if (generation != shards[i].getGeneration()) {
            // set the starting generation for the shard
            shards[i] =
                new Shard(shards[i].getVersion(), shards[i].getDirectory(),
                    generation);
          }
        }
      }
View Full Code Here

Examples of com.volantis.mcs.dissection.impl.Shard

                     "Shard count incorrect",
                     expectedAreaStats.shardCount,
                     actualShardCount);

        for (int i = 0; i < actualShardCount; i += 1) {
            Shard shard = area.retrieveShard(context, i, null);
            ShardStats shardStats = expectedAreaStats.getShardStats(i);
            checkSharedContentUsages(failures,
                                     daDescription + " Shard " + i,
                                     details,
                                     shardStats.getSharedContentUsages(),
                                     shard.getSharedContentUsages());
        }
    }
View Full Code Here

Examples of com.volantis.mcs.dissection.impl.Shard

        int shardIndex = requestedShards.getShard(area.getIndex());

        // Retrieve the appropriate shard from the area. If the shard index
        // is invalid for some reason then we may not get back exactly the
        // one that we asked for but we will get one back.
        Shard shard
            = area.retrieveShard(dissectionContext, shardIndex,
                                 (AvailableShardsImpl) availableShards);
        if (shard == null) {
            throw new DissectionException(
                        exceptionLocalizer.format("shard-not-found"));
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.