Package com.alibaba.wasp

Examples of com.alibaba.wasp.EntityGroupInfo


      // account for the out-of-band entityGroups which were assigned to this server
      // after some other entityGroup server crashed
      Collections.sort(entityGroups, riComparator);
      int numTaken = 0;
      for (int i = 0; i <= numToOffload;) {
        EntityGroupInfo egInfo = entityGroups.get(i); // fetch from head
        if (fetchFromTail) {
          egInfo = entityGroups.get(entityGroups.size() - 1 - i);
        }
        i++;
        entityGroupsToMove.add(new EntityGroupPlan(egInfo, sal.getServerName(),
            null));
        numTaken++;
        if (numTaken >= numToOffload)
          break;
        // fetch in alternate order if there is new entityGroup server
        if (emptyFServerPresent) {
          fetchFromTail = !fetchFromTail;
        }
      }
      serverBalanceInfo.put(sal.getServerName(), new BalanceInfo(numToOffload,
          (-1) * numTaken));
    }
    int totalNumMoved = entityGroupsToMove.size();

    // Walk down least loaded, filling each to the min
    int neededEntityGroups = 0; // number of entityGroups needed to bring all up to min
    fetchFromTail = false;

    Map<ServerName, Integer> underloadedServers = new HashMap<ServerName, Integer>();
    for (Map.Entry<ServerAndLoad, List<EntityGroupInfo>> server : serversByLoad
        .entrySet()) {
      int entityGroupCount = server.getKey().getLoad();
      if (entityGroupCount >= min) {
        break;
      }
      underloadedServers
          .put(server.getKey().getServerName(), min - entityGroupCount);
    }
    // number of servers that get new entityGroups
    int serversUnderloaded = underloadedServers.size();
    int incr = 1;
    List<ServerName> sns = Arrays.asList(underloadedServers.keySet().toArray(
        new ServerName[serversUnderloaded]));
    Collections.shuffle(sns, RANDOM);
    while (entityGroupsToMove.size() > 0) {
      int cnt = 0;
      int i = incr > 0 ? 0 : underloadedServers.size() - 1;
      for (; i >= 0 && i < underloadedServers.size(); i += incr) {
        if (entityGroupsToMove.isEmpty())
          break;
        ServerName si = sns.get(i);
        int numToTake = underloadedServers.get(si);
        if (numToTake == 0)
          continue;

        addEntityGroupPlan(entityGroupsToMove, fetchFromTail, si, entityGroupsToReturn);
        if (emptyFServerPresent) {
          fetchFromTail = !fetchFromTail;
        }

        underloadedServers.put(si, numToTake - 1);
        cnt++;
        BalanceInfo bi = serverBalanceInfo.get(si);
        if (bi == null) {
          bi = new BalanceInfo(0, 0);
          serverBalanceInfo.put(si, bi);
        }
        bi.setNumEntityGroupsAdded(bi.getNumEntityGroupsAdded() + 1);
      }
      if (cnt == 0)
        break;
      // iterates underloadedServers in the other direction
      incr = -incr;
    }
    for (Integer i : underloadedServers.values()) {
      // If we still want to take some, increment needed
      neededEntityGroups += i;
    }

    // If none needed to fill all to min and none left to drain all to max,
    // we are done
    if (neededEntityGroups == 0 && entityGroupsToMove.isEmpty()) {
      long endTime = System.currentTimeMillis();
      LOG.info("Calculated a load balance in " + (endTime - startTime) + "ms. "
          + "Moving " + totalNumMoved + " entityGroups off of " + serversOverloaded
          + " overloaded servers onto " + serversUnderloaded
          + " less loaded servers");
      return entityGroupsToReturn;
    }

    // Need to do a second pass.
    // Either more entityGroups to assign out or servers that are still underloaded

    // If we need more to fill min, grab one from each most loaded until enough
    if (neededEntityGroups != 0) {
      // Walk down most loaded, grabbing one from each until we get enough
      for (Map.Entry<ServerAndLoad, List<EntityGroupInfo>> server : serversByLoad
          .descendingMap().entrySet()) {
        BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey()
            .getServerName());
        int idx = balanceInfo == null ? 0 : balanceInfo
            .getNextEntityGroupForUnload();
        if (idx >= server.getValue().size())
          break;
        EntityGroupInfo entityGroup = server.getValue().get(idx);
        entityGroupsToMove.add(new EntityGroupPlan(entityGroup, server.getKey()
            .getServerName(), null));
        totalNumMoved++;
        if (--neededEntityGroups == 0) {
          // No more entityGroups needed, done shedding
View Full Code Here


    Set<String> oldHostsNoLongerPresent = Sets.newTreeSet();

    int numRandomAssignments = 0;
    int numRetainedAssigments = 0;
    for (Map.Entry<EntityGroupInfo, ServerName> entry : entityGroups.entrySet()) {
      EntityGroupInfo entityGroup = entry.getKey();
      ServerName oldServerName = entry.getValue();
      List<ServerName> localServers = new ArrayList<ServerName>();
      if (oldServerName != null) {
        localServers = serversByHostname.get(oldServerName.getHostname());
      }
View Full Code Here

      return false;
    }
    // Split key can be null if this entityGroup is unsplittable; i.e. has refs.
    if (this.splitrow == null)
      return false;
    EntityGroupInfo egi = this.parent.getEntityGroupInfo();
    // Check splitrow.
    byte[] startKey = egi.getStartKey();
    byte[] endKey = egi.getEndKey();
    if (Bytes.equals(startKey, splitrow)
        || !this.parent.getEntityGroupInfo().containsRow(splitrow)) {
      LOG.info("Split row is not inside entityGroup key range or is equal to "
          + "startkey: " + Bytes.toStringBinary(this.splitrow));
      return false;
    }
    long egid = getDaughterEntityGroupIdTimestamp(egi);
    this.egi_a = new EntityGroupInfo(egi.getTableName(), startKey,
        this.splitrow, false, egid);
    this.egi_b = new EntityGroupInfo(egi.getTableName(), this.splitrow, endKey,
        false, egid);
    return true;
  }
View Full Code Here

        rt = EntityGroupTransaction.parseFrom(data);
      } catch (DeserializationException e) {
        LOG.warn("Failed parse znode data", e);
        return false;
      }
      EntityGroupInfo egInfo = entityGroupInfo;
      if (egInfo == null) {
        egInfo = entityGroupStates.getEntityGroupInfo(rt.getEntityGroupName());
        if (egInfo == null)
          return false;
      }
View Full Code Here

        + egTransition.getServerName() + ", entityGroup=" + encodedName);
    EntityGroupState entityGroupState = entityGroupStates
        .getEntityGroupTransitionState(encodedName);
    switch (egTransition.getEventType()) {
    case M_ZK_ENTITYGROUP_OFFLINE:
      EntityGroupInfo entityGroupInfo = null;
      if (entityGroupState != null) {
        entityGroupInfo = entityGroupState.getEntityGroup();
      } else {
        try {
          byte[] name = egTransition.getEntityGroupName();
          Pair<EntityGroupInfo, ServerName> p = FMetaReader
              .getEntityGroupAndLocation(server.getConfiguration(), name);
          entityGroupInfo = p.getFirst();
        } catch (IOException e) {
          LOG.info("Exception reading META doing HBCK repair operation", e);
          return;
        }
      }
      LOG.info("HBCK repair is triggering assignment of entityGroup="
          + entityGroupInfo.getEntityGroupNameAsString());
      // trigger assign, node is already in OFFLINE so don't need to update ZK
      assign(entityGroupInfo, false);
      break;

    default:
View Full Code Here

            EntityGroupState egState = entityGroupStates
                .getEntityGroupTransitionState(entityGroupName);
            if (egState == null)
              return;

            EntityGroupInfo entityGroupInfo = egState.getEntityGroup();
            if (egState.isSplit()) {
              LOG.debug("Ephemeral node deleted, entityGroupserver crashed?, "
                  + "clearing from EGIT; egState=" + egState);
              entityGroupOffline(egState.getEntityGroup());
            } else {
              LOG.debug("The znode of entityGroup "
                  + entityGroupInfo.getEntityGroupNameAsString()
                  + " has been deleted.");
              if (egState.isOpened()) {
                ServerName serverName = egState.getServerName();
                entityGroupOnline(entityGroupInfo, serverName);
                LOG.info("The master has opened the entityGroup "
                    + entityGroupInfo.getEntityGroupNameAsString()
                    + " that was online on " + serverName);
                if (getZKTable().isDisablingOrDisabledTable(
                    entityGroupInfo.getTableNameAsString())) {
                  LOG.debug("Opened entityGroup "
                      + entityGroupInfo.getEntityGroupNameAsString()
                      + " but "
                      + "this table is disabled, triggering close of entityGroup");
                  unassign(entityGroupInfo);
                }
              }
View Full Code Here

      this.addPlans(plans);

      List<EntityGroupInfo> entityGroupOpenInfos = new ArrayList<EntityGroupInfo>(
          states.size());
      for (EntityGroupState state : states) {
        EntityGroupInfo entityGroup = state.getEntityGroup();
        String encodedEntityGroupName = entityGroup.getEncodedName();
        Integer nodeVersion = offlineNodesVersions.get(encodedEntityGroupName);
        if (nodeVersion == null || nodeVersion.intValue() == -1) {
          LOG.warn("failed to offline in zookeeper: " + entityGroup);
          failedToOpenEntityGroups.add(entityGroup); // assign individually
                                                     // later
          Lock lock = locks.remove(encodedEntityGroupName);
          lock.unlock();
        } else {
          entityGroupStates.updateEntityGroupState(entityGroup,
              EntityGroupState.State.PENDING_OPEN, destination);
          entityGroupOpenInfos.add(entityGroup);
        }
      }

      // Move on to open entityGroups.
      try {
        // Send OPEN RPC. If it fails on a IOE or RemoteException, the
        // TimeoutMonitor will pick up the pieces.
        long maxWaitTime = System.currentTimeMillis()
            + this.server.getConfiguration().getLong(
                "wasp.entityGroupserver.rpc.startup.waittime", 60000);
        for (int i = 1; i <= maximumAttempts && !server.isStopped(); i++) {
          try {
            List<EntityGroupOpeningState> entityGroupOpeningStateList = serverManager
                .sendEntityGroupsOpen(destination, entityGroupOpenInfos);
            if (entityGroupOpeningStateList == null) {
              // Failed getting RPC connection to this server
              return false;
            }
            for (int k = 0, n = entityGroupOpeningStateList.size(); k < n; k++) {
              EntityGroupOpeningState openingState = entityGroupOpeningStateList
                  .get(k);
              if (openingState != EntityGroupOpeningState.OPENED) {
                EntityGroupInfo entityGroup = entityGroupOpenInfos.get(k);
                if (openingState == EntityGroupOpeningState.ALREADY_OPENED) {
                  processAlreadyOpenedEntityGroup(entityGroup, destination);
                } else if (openingState == EntityGroupOpeningState.FAILED_OPENING) {
                  // Failed opening this entityGroup, reassign it later
                  failedToOpenEntityGroups.add(entityGroup);
View Full Code Here

      final boolean forceNewPlan) {
    EntityGroupState currentState = state;
    int versionOfOfflineNode = -1;
    EntityGroupPlan plan = null;
    long maxEntityGroupServerStartupWaitTime = -1;
    EntityGroupInfo entityGroup = state.getEntityGroup();
    for (int i = 1; i <= maximumAttempts && !server.isStopped(); i++) {
      if (plan == null) { // Get a server for the entityGroup at first
        plan = getEntityGroupPlan(entityGroup, forceNewPlan);
      }
      if (plan == null) {
        LOG.debug("Unable to determine a plan to assign " + entityGroup);
        this.timeoutMonitor.setAllEntityGroupServersOffline(true);
        return; // Should get reassigned later when EGIT times out.
      }
      if (setOfflineInZK && versionOfOfflineNode == -1) {
        // get the version of the znode after setting it to OFFLINE.
        // versionOfOfflineNode will be -1 if the znode was not set to OFFLINE
        versionOfOfflineNode = setOfflineInZooKeeper(currentState,
            plan.getDestination());
        if (versionOfOfflineNode != -1) {
          if (isDisabledorDisablingEntityGroupInEGIT(entityGroup)) {
            return;
          }
          // In case of assignment from EnableTableHandler table state is
          // ENABLING. Any how
          // EnableTableHandler will set ENABLED after assigning all the table
          // entityGroups. If we
          // try to set to ENABLED directly then client API may think table is
          // enabled.
          // When we have a case such as all the entityGroups are added directly
          // into .META. and we call
          // assignEntityGroup then we need to make the table ENABLED. Hence in
          // such case the table
          // will not be in ENABLING or ENABLED state.
          String tableName = entityGroup.getTableNameAsString();
          if (!zkTable.isEnablingTable(tableName)
              && !zkTable.isEnabledTable(tableName)) {
            LOG.debug("Setting table " + tableName + " to ENABLED state.");
            setEnabledTable(tableName);
          }
        }
      }
      if (setOfflineInZK && versionOfOfflineNode == -1) {
        return;
      }
      if (this.server.isStopped()) {
        LOG.debug("Server stopped; skipping assign of " + entityGroup);
        return;
      }
      try {
        LOG.info("Assigning entityGroup "
            + entityGroup.getEntityGroupNameAsString() + " to "
            + plan.getDestination().toString());
        // Transition EntityGroupState to PENDING_OPEN
        currentState = entityGroupStates.updateEntityGroupState(entityGroup,
            EntityGroupState.State.PENDING_OPEN, plan.getDestination());
        // Send OPEN RPC. This can fail if the server on other end is is not up.
        // Pass the version that was obtained while setting the node to OFFLINE.
        EntityGroupOpeningState entityGroupOpenState = serverManager
            .sendEntityGroupOpen(plan.getDestination(), entityGroup,
                versionOfOfflineNode);
        if (entityGroupOpenState == EntityGroupOpeningState.ALREADY_OPENED) {
          processAlreadyOpenedEntityGroup(entityGroup, plan.getDestination());
        } else if (entityGroupOpenState == EntityGroupOpeningState.FAILED_OPENING) {
          // Failed opening this entityGroup
          throw new Exception("Get entityGroupOpeningState="
              + entityGroupOpenState);
        }
        break;
      } catch (Throwable t) {
        if (t instanceof RemoteException) {
          t = ((RemoteException) t).unwrapRemoteException();
        }
        boolean entityGroupAlreadyInTransitionException = false;
        boolean serverNotRunningYet = false;
        boolean socketTimedOut = false;
        if (t instanceof EntityGroupAlreadyInTransitionException) {
          entityGroupAlreadyInTransitionException = true;
          if (LOG.isDebugEnabled()) {
            LOG.debug("Failed assignment in: " + plan.getDestination()
                + " due to " + t.getMessage());
          }
        } else if (t instanceof ServerNotRunningYetException) {
          if (maxEntityGroupServerStartupWaitTime < 0) {
            maxEntityGroupServerStartupWaitTime = System.currentTimeMillis()
                + this.server.getConfiguration().getLong(
                    "wasp.entityGroupserver.rpc.startup.waittime", 60000);
          }
          try {
            long now = System.currentTimeMillis();
            if (now < maxEntityGroupServerStartupWaitTime) {
              LOG.debug("Server is not yet up; waiting up to "
                  + (maxEntityGroupServerStartupWaitTime - now) + "ms", t);
              serverNotRunningYet = true;
              Thread.sleep(100);
              i--; // reset the try count
            } else {
              LOG.debug("Server is not up for a while; try a new one", t);
            }
          } catch (InterruptedException ie) {
            LOG.warn(
                "Failed to assign " + entityGroup.getEntityGroupNameAsString()
                    + " since interrupted", ie);
            Thread.currentThread().interrupt();
            return;
          }
        } else if (t instanceof java.net.SocketTimeoutException
            && this.serverManager.isServerOnline(plan.getDestination())) {
          // In case socket is timed out and the entityGroup server is still
          // online,
          // the openEntityGroup RPC could have been accepted by the server and
          // just the response didn't go through. So we will retry to
          // open the entityGroup on the same server to avoid possible
          // double assignment.
          socketTimedOut = true;
          if (LOG.isDebugEnabled()) {
            LOG.debug(
                "Call openEntityGroup() to " + plan.getDestination()
                    + " has timed out when trying to assign "
                    + entityGroup.getEntityGroupNameAsString()
                    + ", but the entityGroup might already be opened on "
                    + plan.getDestination() + ".", t);
          }
        }

        LOG.warn(
            "Failed assignment of "
                + entityGroup.getEntityGroupNameAsString()
                + " to "
                + plan.getDestination()
                + ", trying to assign "
                + (entityGroupAlreadyInTransitionException
                    || serverNotRunningYet || socketTimedOut ? "to the same entityGroup server because of EntityGroupAlreadyInTransitionException"
                    + "/ServerNotRunningYetException/SocketTimeoutException;"
                    : "elsewhere instead; ") + "try=" + i + " of "
                + this.maximumAttempts, t);

        if (i == this.maximumAttempts) {
          // Don't reset the entityGroup state or get a new plan any more.
          // This is the last try.
          continue;
        }

        // If entityGroup opened on destination of present plan, reassigning to
        // new
        // FSERVER may cause double assignments. In case of
        // EntityGroupAlreadyInTransitionException
        // reassigning to same FSERVER.
        EntityGroupPlan newPlan = plan;
        if (!(entityGroupAlreadyInTransitionException || serverNotRunningYet || socketTimedOut)) {
          // Force a new plan and reassign. Will return null if no servers.
          // The new plan could be the same as the existing plan since we don't
          // exclude the server of the original plan, which should not be
          // excluded since it could be the only server up now.
          newPlan = getEntityGroupPlan(entityGroup, true);
        }
        if (newPlan == null) {
          this.timeoutMonitor.setAllEntityGroupServersOffline(true);
          LOG.warn("Unable to find a viable location to assign entityGroup "
              + entityGroup.getEntityGroupNameAsString());
          return;
        }
        if (plan != newPlan
            && !plan.getDestination().equals(newPlan.getDestination())) {
          // Clean out plan we failed execute and one that doesn't look like
View Full Code Here

      for (Result result : results) {
        Pair<EntityGroupInfo, ServerName> entityGroup = EntityGroupInfo
            .getEntityGroupInfoAndServerName(result);
        if (entityGroup == null)
          continue;
        EntityGroupInfo entityGroupInfo = entityGroup.getFirst();
        ServerName entityGroupLocation = entityGroup.getSecond();
        if (entityGroupInfo == null)
          continue;
        entityGroupStates.createEntityGroupState(entityGroupInfo);
        String tableName = entityGroupInfo.getTableNameAsString();
        if (entityGroupLocation == null) {
          // entityGroupLocation could be null if createTable didn't finish
          // properly.
          // When createTable is in progress, HMaster restarts.
          // Some entityGroups have been added to .META., but have not been
          // assigned.
          // When this happens, the entityGroup's table must be in ENABLING
          // state.
          // It can't be in ENABLED state as that is set when all entityGroups
          // are
          // assigned.
          // It can't be in DISABLING state, because DISABLING state transitions
          // from ENABLED state when application calls disableTable.
          // It can't be in DISABLED state, because DISABLED states transitions
          // from DISABLING state.
          if (!enablingTables.contains(tableName)) {
            LOG.warn("EntityGroup " + entityGroupInfo.getEncodedName()
                + " has null entityGroupLocation." + " But its table "
                + tableName + " isn't in ENABLING state.");
          }
        } else if (!onlineServers.contains(entityGroupLocation)) {
          // EntityGroup is located on a server that isn't online
          List<EntityGroupInfo> offlineEntityGroups = offlineServers
              .get(entityGroupLocation);
          if (offlineEntityGroups == null) {
            offlineEntityGroups = new ArrayList<EntityGroupInfo>(1);
            offlineServers.put(entityGroupLocation, offlineEntityGroups);
          }
          offlineEntityGroups.add(entityGroupInfo);
          // need to enable the table if not disabled or disabling or enabling
          // this will be used in rolling restarts
          if (!disabledOrDisablingOrEnabling.contains(tableName)
              && !getZKTable().isEnabledTable(tableName)) {
            setEnabledTable(tableName);
          }
        } else {
          // If entityGroup is in offline and split state check the ZKNode
          if (entityGroupInfo.isOffline() && entityGroupInfo.isSplit()) {
            String node = ZKAssign.getNodeName(this.watcher,
                entityGroupInfo.getEncodedName());
            Stat stat = new Stat();
            byte[] data = ZKUtil.getDataNoWatch(this.watcher, node, stat);
            // If znode does not exist, don't consider this entityGroup
            if (data == null) {
              LOG.debug("EntityGroup "
                  + entityGroupInfo.getEntityGroupNameAsString()
                  + " split is completed. Hence need not add to entityGroups list");
              continue;
            }
          }
          // EntityGroup is being served and on an active server
View Full Code Here

    if (entityGroupState == null) {
      throw new ServiceException(new UnknownEntityGroupException(
          Bytes.toStringBinary(encodedEntityGroupName)));
    }

    EntityGroupInfo egInfo = entityGroupState.getEntityGroup();
    ServerName dest;
    if (destServerName == null || destServerName.length == 0) {
      LOG.info("Passed destination servername is null/empty so "
          + "choosing a server at random");
      final List<ServerName> destServers = this.serverManager
          .createDestinationServersList(entityGroupState.getServerName());
      dest = balancer.randomAssignment(egInfo, destServers);
    } else {
      dest = new ServerName(Bytes.toString(destServerName));
      if (dest.equals(entityGroupState.getServerName())) {
        LOG.debug("Skipping move of entityGroup "
            + egInfo.getEntityGroupNameAsString()
            + " because entityGroup already assigned to the same server "
            + dest + ".");
        return megr;
      }
    }
View Full Code Here

TOP

Related Classes of com.alibaba.wasp.EntityGroupInfo

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.