* Licensed to the Apache Software Foundation (ASF) under one or more

* contributor license agreements.  See the NOTICE file distributed with

* this work for additional information regarding copyright ownership.

* The ASF licenses this file to You under the Apache License, Version 2.0

* (the "License"); you may not use this file except in compliance with

* the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* The Master is responsible for assigning and balancing tablets to tablet servers.

*

* The master will also coordinate log recoveries and reports general status.

*/

 

  final private static Logger log = Logger.getLogger(Master.class);

 

  final private static int ONE_SECOND = 1000;

  final private static Text METADATA_TABLE_ID = new Text(Constants.METADATA_TABLE_ID);

  final private static long TIME_TO_WAIT_BETWEEN_SCANS = 60 * ONE_SECOND;

  final private static long TIME_BETWEEN_MIGRATION_CLEANUPS = 5 * 60 * ONE_SECOND;

  final private static long WAIT_BETWEEN_ERRORS = ONE_SECOND;

  final private static long DEFAULT_WAIT_FOR_WATCHER = 10 * ONE_SECOND;

  final private static int MAX_CLEANUP_WAIT_TIME = 1000;

  final private static int TIME_TO_WAIT_BETWEEN_LOCK_CHECKS = 1000;

  final private static int MAX_TSERVER_WORK_CHUNK = 5000;

  final private static int MAX_BAD_STATUS_COUNT = 3;

 

  final private FileSystem fs;

  final private Instance instance;

  final private String hostname;

  final private LiveTServerSet tserverSet;

  final private List<TabletGroupWatcher> watchers = new ArrayList<TabletGroupWatcher>();

  final private SecurityOperation security;

  final private Map<TServerInstance,AtomicInteger> badServers = Collections.synchronizedMap(new DefaultMap<TServerInstance,AtomicInteger>(new AtomicInteger()));

  final private Set<TServerInstance> serversToShutdown = Collections.synchronizedSet(new HashSet<TServerInstance>());

  final private SortedMap<KeyExtent,TServerInstance> migrations = Collections.synchronizedSortedMap(new TreeMap<KeyExtent,TServerInstance>());

  final private EventCoordinator nextEvent = new EventCoordinator();

  final private Object mergeLock = new Object();

  private RecoveryManager recoveryManager = null;

 

  private ZooLock masterLock = null;

  private TServer clientService = null;

  private TabletBalancer tabletBalancer;

 

  private MasterState state = MasterState.INITIAL;

 

  private Fate<Master> fate;

 

  volatile private SortedMap<TServerInstance,TabletServerStatus> tserverStatus = Collections

      .unmodifiableSortedMap(new TreeMap<TServerInstance,TabletServerStatus>());

 

  private final Set<String> recoveriesInProgress = Collections.synchronizedSet(new HashSet<String>());

 

  synchronized private MasterState getMasterState() {

    return state;

  }

 

  public boolean stillMaster() {

    return getMasterState() != MasterState.STOP;

  }

 

  static final boolean X = true;

  static final boolean _ = false;

  // @formatter:off

  static final boolean transitionOK[][] = {

      //                              INITIAL HAVE_LOCK SAFE_MODE NORMAL UNLOAD_META UNLOAD_ROOT STOP

      /* INITIAL */                   {X,     X,        _,        _,      _,         _,          X},

      /* HAVE_LOCK */                 {_,     X,        X,        X,      _,         _,          X},

      /* SAFE_MODE */                 {_,     _,        X,        X,      X,         _,          X},

      /* NORMAL */                    {_,     _,        X,        X,      X,         _,          X},

      /* UNLOAD_METADATA_TABLETS */   {_,     _,        X,        X,      X,         X,          X},

      /* UNLOAD_ROOT_TABLET */        {_,     _,        _,        X,      _,         X,          X},

      /* STOP */                      {_,     _,        _,        _,      _,         _,          X}};

  //@formatter:on

  synchronized private void setMasterState(MasterState newState) {

    if (state.equals(newState))

      return;

    if (!transitionOK[state.ordinal()][newState.ordinal()]) {

      log.error("Programmer error: master should not transition from " + state + " to " + newState);

    }

    MasterState oldState = state;

    state = newState;

    nextEvent.event("State changed from %s to %s", oldState, newState);

    if (newState == MasterState.STOP) {

      // Give the server a little time before shutdown so the client

      // thread requesting the stop can return

      SimpleTimer.getInstance().schedule(new Runnable() {

        @Override

        public void run() {

          // This frees the main thread and will cause the master to exit

          clientService.stop();

          Master.this.nextEvent.event("stopped event loop");

        }

       

      }, 100l, 1000l);

    }

   

    if (oldState != newState && (newState == MasterState.HAVE_LOCK)) {

      upgradeZookeeper();

    }

   

    if (oldState != newState && (newState == MasterState.NORMAL)) {

      upgradeMetadata();

    }

  }

 

  private void upgradeZookeeper() {

    // 1.5.1 and 1.6.0 both do some state checking after obtaining the zoolock for the

    // monitor and before starting up. It's not tied to the data version at all (and would

    // introduce unnecessary complexity to try to make the master do it), but be aware

    // that the master is not the only thing that may alter zookeeper before starting.

    if (Accumulo.getAccumuloPersistentVersion(fs) == Constants.PREV_DATA_VERSION) {

      try {

        log.info("Upgrading zookeeper");

       

        IZooReaderWriter zoo = ZooReaderWriter.getInstance();

       

        zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/loggers", NodeMissingPolicy.SKIP);

        zoo.recursiveDelete(ZooUtil.getRoot(instance) + "/dead/loggers", NodeMissingPolicy.SKIP);

        zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZRECOVERY, new byte[] {'0'}, NodeExistsPolicy.SKIP);

       

        for (String id : Tables.getIdToNameMap(instance).keySet()) {

         

          zoo.putPersistentData(ZooUtil.getRoot(instance) + Constants.ZTABLES + "/" + id + Constants.ZTABLE_COMPACT_CANCEL_ID, "0".getBytes(Constants.UTF8),

              NodeExistsPolicy.SKIP);

        }

      } catch (Exception ex) {

        log.fatal("Error performing upgrade", ex);

        System.exit(1);

      }

    }

  }

 

  private final AtomicBoolean upgradeMetadataRunning = new AtomicBoolean(false);

 

  private final ServerConfiguration serverConfig;

 

  private void upgradeMetadata() {

    if (Accumulo.getAccumuloPersistentVersion(fs) == Constants.PREV_DATA_VERSION) {

      if (upgradeMetadataRunning.compareAndSet(false, true)) {

        Runnable upgradeTask = new Runnable() {

          @Override

          public void run() {

            try {

              MetadataTable.moveMetaDeleteMarkers(instance, SecurityConstants.getSystemCredentials());

              Accumulo.updateAccumuloVersion(fs);

             

              log.info("Upgrade complete");

             

            } catch (Exception ex) {

              log.fatal("Error performing upgrade", ex);

              System.exit(1);

            }

           

          }

        };

       

        // need to run this in a separate thread because a lock is held that prevents !METADATA tablets from being assigned and this task writes to the

        // !METADATA table

        new Thread(upgradeTask).start();

      }

    }

  }

 

  private int assignedOrHosted(Text tableId) {

    int result = 0;

    for (TabletGroupWatcher watcher : watchers) {

      TableCounts count = watcher.getStats(tableId);

      result += count.hosted() + count.assigned();

    }

    return result;

  }

 

  private int totalAssignedOrHosted() {

    int result = 0;

    for (TabletGroupWatcher watcher : watchers) {

      for (TableCounts counts : watcher.getStats().values()) {

        result += counts.assigned() + counts.hosted();

      }

    }

    return result;

  }

 

  private int nonMetaDataTabletsAssignedOrHosted() {

    return totalAssignedOrHosted() - assignedOrHosted(new Text(Constants.METADATA_TABLE_ID));

  }

 

  private int notHosted() {

    int result = 0;

    for (TabletGroupWatcher watcher : watchers) {

      for (TableCounts counts : watcher.getStats().values()) {

        result += counts.assigned() + counts.assignedToDeadServers();

      }

    }

    return result;

  }

 

  // The number of unassigned tablets that should be assigned: displayed on the monitor page

  private int displayUnassigned() {

    int result = 0;

    Text meta = new Text(Constants.METADATA_TABLE_ID);

    switch (getMasterState()) {

      case NORMAL:

        // Count offline tablets for online tables

        for (TabletGroupWatcher watcher : watchers) {

          TableManager manager = TableManager.getInstance();

          for (Entry<Text,TableCounts> entry : watcher.getStats().entrySet()) {

            Text tableId = entry.getKey();

            TableCounts counts = entry.getValue();

            TableState tableState = manager.getTableState(tableId.toString());

            if (tableState != null && tableState.equals(TableState.ONLINE)) {

              result += counts.unassigned() + counts.assignedToDeadServers() + counts.assigned();

            }

          }

        }

        break;

      case SAFE_MODE:

        // Count offline tablets for the METADATA table

        for (TabletGroupWatcher watcher : watchers) {

          result += watcher.getStats(meta).unassigned();

        }

        break;

      case UNLOAD_METADATA_TABLETS:

      case UNLOAD_ROOT_TABLET:

        for (TabletGroupWatcher watcher : watchers) {

          result += watcher.getStats(meta).unassigned();

        }

        break;

      default:

        break;

    }

    return result;

  }

 

  private void checkNotMetadataTable(String tableName, TableOperation operation) throws ThriftTableOperationException {

    if (tableName.compareTo(Constants.METADATA_TABLE_NAME) == 0) {

      String why = "Table names cannot be == " + Constants.METADATA_TABLE_NAME;

      log.warn(why);

      throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.OTHER, why);

    }

  }

 

  private void checkTableName(String tableName, TableOperation operation) throws ThriftTableOperationException {

    if (!tableName.matches(Constants.VALID_TABLE_NAME_REGEX)) {

      String why = "Table names must only contain word characters (letters, digits, and underscores): " + tableName;

      log.warn(why);

      throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.OTHER, why);

    }

    if (Tables.getNameToIdMap(HdfsZooInstance.getInstance()).containsKey(tableName)) {

      String why = "Table name already exists: " + tableName;

      throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.EXISTS, why);

    }

   

  }

 

  public void mustBeOnline(final String tableId) throws ThriftTableOperationException {

    Tables.clearCache(instance);

    if (!Tables.getTableState(instance, tableId).equals(TableState.ONLINE))

      throw new ThriftTableOperationException(tableId, null, TableOperation.MERGE, TableOperationExceptionType.OFFLINE, "table is not online");

  }

 

  public Connector getConnector() throws AccumuloException, AccumuloSecurityException {

    return instance.getConnector(SecurityConstants.SYSTEM_PRINCIPAL, SecurityConstants.getSystemToken());

  }

 

  private void waitAround(EventCoordinator.Listener listener) {

    listener.waitForEvents(ONE_SECOND);

  }

 

  // TODO: maybe move this to Property? We do this in TabletServer, Master, TableLoadBalancer, etc. - ACCUMULO-1295

  public static <T> T createInstanceFromPropertyName(AccumuloConfiguration conf, Property property, Class<T> base, T defaultInstance) {

    String clazzName = conf.get(property);

    T instance = null;

   

    try {

      Class<? extends T> clazz = AccumuloVFSClassLoader.loadClass(clazzName, base);

      instance = clazz.newInstance();

      log.info("Loaded class : " + clazzName);

    } catch (Exception e) {

      log.warn("Failed to load class ", e);

    }

   

    if (instance == null) {

      log.info("Using " + defaultInstance.getClass().getName());

      instance = defaultInstance;

    }

    return instance;

  }

 

  public Master(ServerConfiguration config, FileSystem fs, String hostname) throws IOException {

    this.serverConfig = config;

    this.instance = config.getInstance();

    this.fs = TraceFileSystem.wrap(fs);

    this.hostname = hostname;

   

    AccumuloConfiguration aconf = serverConfig.getConfiguration();

   

    log.info("Version " + Constants.VERSION);

    log.info("Instance " + instance.getInstanceID());

    ThriftTransportPool.getInstance().setIdleTime(aconf.getTimeInMillis(Property.GENERAL_RPC_TIMEOUT));

    security = AuditedSecurityOperation.getInstance();

    tserverSet = new LiveTServerSet(instance, config.getConfiguration(), this);

    this.tabletBalancer = createInstanceFromPropertyName(aconf, Property.MASTER_TABLET_BALANCER, TabletBalancer.class, new DefaultLoadBalancer());

    this.tabletBalancer.init(serverConfig);

  }

 

  public TServerConnection getConnection(TServerInstance server) {

    try {

      return tserverSet.getConnection(server);

    } catch (TException ex) {

      return null;

    }

  }

 

   

    protected String checkTableId(String tableName, TableOperation operation) throws ThriftTableOperationException {

      final String tableId = Tables.getNameToIdMap(getConfiguration().getInstance()).get(tableName);

      if (tableId == null)

        throw new ThriftTableOperationException(null, tableName, operation, TableOperationExceptionType.NOTFOUND, null);

      return tableId;

    }

   

    @Override

    public long initiateFlush(TInfo tinfo, TCredentials c, String tableId) throws ThriftSecurityException, ThriftTableOperationException, TException {

      security.canFlush(c, tableId);

     

      String zTablePath = Constants.ZROOT + "/" + getConfiguration().getInstance().getInstanceID() + Constants.ZTABLES + "/" + tableId

          + Constants.ZTABLE_FLUSH_ID;

     

      IZooReaderWriter zoo = ZooReaderWriter.getInstance();

      byte fid[];

      try {

        fid = zoo.mutate(zTablePath, null, null, new Mutator() {

          @Override

          public byte[] mutate(byte[] currentValue) throws Exception {

            long flushID = Long.parseLong(new String(currentValue, Constants.UTF8));

            flushID++;

            return Long.toString(flushID).getBytes(Constants.UTF8);

          }

        });

      } catch (NoNodeException nne) {

        throw new ThriftTableOperationException(tableId, null, TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);

      } catch (Exception e) {

        log.warn(e.getMessage(), e);

        throw new ThriftTableOperationException(tableId, null, TableOperation.FLUSH, TableOperationExceptionType.OTHER, null);

      }

      return Long.parseLong(new String(fid, Constants.UTF8));

    }

   

    @Override

    public void waitForFlush(TInfo tinfo, TCredentials c, String tableId, ByteBuffer startRow, ByteBuffer endRow, long flushID, long maxLoops)

        throws ThriftSecurityException, ThriftTableOperationException, TException {

      security.canFlush(c, tableId);

     

      if (endRow != null && startRow != null && ByteBufferUtil.toText(startRow).compareTo(ByteBufferUtil.toText(endRow)) >= 0)

        throw new ThriftTableOperationException(tableId, null, TableOperation.FLUSH, TableOperationExceptionType.BAD_RANGE,

            "start row must be less than end row");

     

      Set<TServerInstance> serversToFlush = new HashSet<TServerInstance>(tserverSet.getCurrentServers());

     

      for (long l = 0; l < maxLoops; l++) {

       

        for (TServerInstance instance : serversToFlush) {

          try {

            final TServerConnection server = tserverSet.getConnection(instance);

            if (server != null)

              server.flush(masterLock, tableId, ByteBufferUtil.toBytes(startRow), ByteBufferUtil.toBytes(endRow));

          } catch (TException ex) {

            log.error(ex.toString());

          }

        }

       

        if (l == maxLoops - 1)

          break;

       

        UtilWaitThread.sleep(50);

       

        serversToFlush.clear();

       

        try {

          Connector conn = getConnector();

          Scanner scanner = new IsolatedScanner(conn.createScanner(Constants.METADATA_TABLE_NAME, Constants.NO_AUTHS));

          Constants.METADATA_FLUSH_COLUMN.fetch(scanner);

          Constants.METADATA_DIRECTORY_COLUMN.fetch(scanner);

          scanner.fetchColumnFamily(Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY);

          scanner.fetchColumnFamily(Constants.METADATA_LOG_COLUMN_FAMILY);

          scanner.setRange(new KeyExtent(new Text(tableId), null, ByteBufferUtil.toText(startRow)).toMetadataRange());

         

          RowIterator ri = new RowIterator(scanner);

         

          int tabletsToWaitFor = 0;

          int tabletCount = 0;

         

          Text ert = ByteBufferUtil.toText(endRow);

         

          while (ri.hasNext()) {

            Iterator<Entry<Key,Value>> row = ri.next();

            long tabletFlushID = -1;

            int logs = 0;

            boolean online = false;

           

            TServerInstance server = null;

           

            Entry<Key,Value> entry = null;

            while (row.hasNext()) {

              entry = row.next();

              Key key = entry.getKey();

             

              if (Constants.METADATA_FLUSH_COLUMN.equals(key.getColumnFamily(), key.getColumnQualifier())) {

                tabletFlushID = Long.parseLong(entry.getValue().toString());

              }

             

              if (Constants.METADATA_LOG_COLUMN_FAMILY.equals(key.getColumnFamily()))

                logs++;

             

              if (Constants.METADATA_CURRENT_LOCATION_COLUMN_FAMILY.equals(key.getColumnFamily())) {

                online = true;

                server = new TServerInstance(entry.getValue(), key.getColumnQualifier());

              }

             

            }

           

            // when tablet is not online and has no logs, there is no reason to wait for it

            if ((online || logs > 0) && tabletFlushID < flushID) {

              tabletsToWaitFor++;

              if (server != null)

                serversToFlush.add(server);

            }

           

            tabletCount++;

           

            Text tabletEndRow = new KeyExtent(entry.getKey().getRow(), (Text) null).getEndRow();

            if (tabletEndRow == null || (ert != null && tabletEndRow.compareTo(ert) >= 0))

              break;

          }

         

          if (tabletsToWaitFor == 0)

            break;

         

          // TODO detect case of table offline AND tablets w/ logs? - ACCUMULO-1296

         

          if (tabletCount == 0 && !Tables.exists(instance, tableId))

            throw new ThriftTableOperationException(tableId, null, TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);

         

        } catch (AccumuloException e) {

          log.debug("Failed to scan !METADATA table to wait for flush " + tableId, e);

        } catch (TabletDeletedException tde) {

          log.debug("Failed to scan !METADATA table to wait for flush " + tableId, tde);

        } catch (AccumuloSecurityException e) {

          log.warn(e.getMessage(), e);

          throw new ThriftSecurityException();

        } catch (TableNotFoundException e) {

          log.error(e.getMessage(), e);

          throw new ThriftTableOperationException();

        }

      }

     

    }

   

    @Override

    public MasterMonitorInfo getMasterStats(TInfo info, TCredentials credentials) throws ThriftSecurityException, TException {

      final MasterMonitorInfo result = new MasterMonitorInfo();

     

      result.tServerInfo = new ArrayList<TabletServerStatus>();

      result.tableMap = new DefaultMap<String,TableInfo>(new TableInfo());

      for (Entry<TServerInstance,TabletServerStatus> serverEntry : tserverStatus.entrySet()) {

        final TabletServerStatus status = serverEntry.getValue();

        result.tServerInfo.add(status);

        for (Entry<String,TableInfo> entry : status.tableMap.entrySet()) {

          String table = entry.getKey();

          TableInfo summary = result.tableMap.get(table);

          Monitor.add(summary, entry.getValue());

        }

      }

      result.badTServers = new HashMap<String,Byte>();

      synchronized (badServers) {

        for (TServerInstance bad : badServers.keySet()) {

          result.badTServers.put(bad.hostPort(), TabletServerState.UNRESPONSIVE.getId());

        }

      }

      result.state = getMasterState();

      result.goalState = getMasterGoalState();

      result.unassignedTablets = Master.this.displayUnassigned();

      result.serversShuttingDown = new HashSet<String>();

      synchronized (serversToShutdown) {

        for (TServerInstance server : serversToShutdown)

          result.serversShuttingDown.add(server.hostPort());

      }

      DeadServerList obit = new DeadServerList(ZooUtil.getRoot(instance) + Constants.ZDEADTSERVERS);

      result.deadTabletServers = obit.getList();

      return result;

    }

   

    private void alterTableProperty(TCredentials c, String tableName, String property, String value, TableOperation op) throws ThriftSecurityException,

        ThriftTableOperationException {

      final String tableId = checkTableId(tableName, op);

      if (!security.canAlterTable(c, tableId))

        throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);

     

      try {

        if (value == null || value.isEmpty()) {

          TablePropUtil.removeTableProperty(tableId, property);

        } else if (!TablePropUtil.setTableProperty(tableId, property, value)) {

          throw new Exception("Invalid table property.");

        }

      } catch (KeeperException.NoNodeException e) {

        // race condition... table no longer exists?  This call will throw an exception if the table was deleted:

        checkTableId(tableName, op);

        log.info("Error altering table property", e);

        throw new ThriftTableOperationException(tableId, tableName, op, TableOperationExceptionType.OTHER, "Problem altering table property");

      } catch (Exception e) {

        log.error("Problem altering table property", e);

        throw new ThriftTableOperationException(tableId, tableName, op, TableOperationExceptionType.OTHER, "Problem altering table property");

      }

    }

   

    @Override

    public void removeTableProperty(TInfo info, TCredentials credentials, String tableName, String property) throws ThriftSecurityException,

        ThriftTableOperationException, TException {

      alterTableProperty(credentials, tableName, property, null, TableOperation.REMOVE_PROPERTY);

    }

   

    @Override

    public void setTableProperty(TInfo info, TCredentials credentials, String tableName, String property, String value) throws ThriftSecurityException,

        ThriftTableOperationException, TException {

      alterTableProperty(credentials, tableName, property, value, TableOperation.SET_PROPERTY);

    }

   

    @Override

    public void shutdown(TInfo info, TCredentials c, boolean stopTabletServers) throws ThriftSecurityException, TException {

      security.canPerformSystemActions(c);

      Master.this.shutdown(stopTabletServers);

    }

   

    @Override

    public void shutdownTabletServer(TInfo info, TCredentials c, String tabletServer, boolean force) throws ThriftSecurityException, TException {

      security.canPerformSystemActions(c);

     

      final InetSocketAddress addr = AddressUtil.parseAddress(tabletServer, Property.TSERV_CLIENTPORT);

      final String addrString = org.apache.accumulo.core.util.AddressUtil.toString(addr);

      final TServerInstance doomed = tserverSet.find(addrString);

      if (!force) {

        final TServerConnection server = tserverSet.getConnection(doomed);

        if (server == null) {

          log.warn("No server found for name " + tabletServer);

          return;

        }

      }

     

      long tid = fate.startTransaction();

      fate.seedTransaction(tid, new TraceRepo<Master>(new ShutdownTServer(doomed, force)), false);

      fate.waitForCompletion(tid);

      fate.delete(tid);

    }

   

    @Override

    public void reportSplitExtent(TInfo info, TCredentials credentials, String serverName, TabletSplit split) throws TException {

      KeyExtent oldTablet = new KeyExtent(split.oldTablet);

      if (migrations.remove(oldTablet) != null) {

        log.info("Canceled migration of " + split.oldTablet);

      }

      for (TServerInstance instance : tserverSet.getCurrentServers()) {

        if (serverName.equals(instance.hostPort())) {

          nextEvent.event("%s reported split %s, %s", serverName, new KeyExtent(split.newTablets.get(0)), new KeyExtent(split.newTablets.get(1)));

          return;

        }

      }

      log.warn("Got a split from a server we don't recognize: " + serverName);

    }

   

    @Override

    public void reportTabletStatus(TInfo info, TCredentials credentials, String serverName, TabletLoadState status, TKeyExtent ttablet) throws TException {

      KeyExtent tablet = new KeyExtent(ttablet);

     

      switch (status) {

        case LOAD_FAILURE:

          log.error(serverName + " reports assignment failed for tablet " + tablet);

          break;

        case LOADED:

          nextEvent.event("tablet %s was loaded on %s", tablet, serverName);