diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java index 0b012eb9234c..20ea8e5e06a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupBasedLoadBalancer.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hbase.rsgroup; +import com.google.errorprone.annotations.RestrictedApi; import edu.umd.cs.findbugs.annotations.NonNull; import java.io.IOException; import java.util.ArrayList; @@ -99,6 +100,44 @@ public class RSGroupBasedLoadBalancer implements LoadBalancer { private volatile boolean fallbackEnabled = false; + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public void setFallbackEnabledForTest(boolean enabled) { + this.fallbackEnabled = enabled; + } + + // Test-only call-tracking flags, set true whenever the corresponding assignment method below is + // invoked. Reset via resetAssignmentCallFlagsForTest() before the operation under test. + private volatile boolean roundRobinAssignmentInvoked = false; + private volatile boolean retainAssignmentInvoked = false; + private volatile boolean randomAssignmentInvoked = false; + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public boolean isRoundRobinAssignmentInvoked() { + return roundRobinAssignmentInvoked; + } + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public boolean isRetainAssignmentInvoked() { + return retainAssignmentInvoked; + } + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public boolean isRandomAssignmentInvoked() { + return randomAssignmentInvoked; + } + + @RestrictedApi(explanation = "Should only be called in tests", link = "", + allowedOnPath = ".*/src/test/.*") + public void resetAssignmentCallFlagsForTest() { + roundRobinAssignmentInvoked = false; + retainAssignmentInvoked = false; + randomAssignmentInvoked = false; + } + /** * Used by reflection in {@link org.apache.hadoop.hbase.master.balancer.LoadBalancerFactory}. */ @@ -180,6 +219,7 @@ public synchronized List balanceCluster( @NonNull public Map> roundRobinAssignment(List regions, List servers) throws IOException { + roundRobinAssignmentInvoked = true; Map> assignments = Maps.newHashMap(); List, List>> pairs = generateGroupAssignments(regions, servers); @@ -196,6 +236,7 @@ public Map> roundRobinAssignment(List r @NonNull public Map> retainAssignment(Map regions, List servers) throws HBaseIOException { + retainAssignmentInvoked = true; try { Map> assignments = new TreeMap<>(); List, List>> pairs = @@ -218,6 +259,7 @@ public Map> retainAssignment(Map servers) throws IOException { + randomAssignmentInvoked = true; List, List>> pairs = generateGroupAssignments(Lists.newArrayList(region), servers); List filteredServers = pairs.iterator().next().getSecond(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java index 783d7c7dd335..3477cae28e69 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/rsgroup/RSGroupInfoManagerImpl.java @@ -34,6 +34,8 @@ import java.util.TreeSet; import java.util.concurrent.Future; import java.util.function.Function; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; @@ -96,7 +98,6 @@ import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableMap; import org.apache.hbase.thirdparty.com.google.common.collect.Lists; import org.apache.hbase.thirdparty.com.google.common.collect.Maps; -import org.apache.hbase.thirdparty.com.google.common.collect.Sets; import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto; @@ -109,6 +110,24 @@ * This is an implementation of {@link RSGroupInfoManager} which makes use of an HBase table as the * persistence store for the group information. It also makes use of zookeeper to store group * information needed for bootstrapping during offline mode. + *

Admin-managed and auto-managed groups

An admin-managed group is a non-default + * group that is not regex-governed (see {@code hbase.rsgroup.regex.}). Its servers are set + * by an administrator and are stored in hbase:rsgroup and ZooKeeper. A group whose regex is invalid + * is admin-managed until the regex is fixed. In that time the servers that the group holds in + * memory count as admin-managed when the membership is computed + * ({@code resolveRegexBasedRSGroupMembership}). After a master restart the group holds only what + * was stored at the last flush. + *

+ * An auto-managed group is 'default' or a regex-governed group. Its servers are computed + * from the live servers and, while the regex is valid, are not stored in hbase:rsgroup and + * ZooKeeper. + *

+ * Auto-managed groups exist to make RSGroup work with auto-scaling when HBase runs in a cloud + * environment such as AWS or GCP. An admin-managed group does not give the elasticity that + * automatic upscaling and downscaling need: each new RegionServer must be moved into the group by + * hand. It fits a fixed number of RegionServers, for example to host internal system tables like + * hbase:meta and hbase:namespace. End user facing tables must be placed on an elastic group, so + * RegionServers join and leave it automatically. That is the need for auto-managed groups. *

Concurrency

RSGroup state is kept locally in Maps. There is a rsgroup name to cached * RSGroupInfo Map at {@link RSGroupInfoHolder#groupName2Group}. These Maps are persisted to the * hbase:rsgroup table (and cached in zk) on each modification. @@ -141,6 +160,10 @@ final class RSGroupInfoManagerImpl implements RSGroupInfoManager { /** Define the default number of retries */ static final int DEFAULT_MAX_RETRY_VALUE = 50; + /** Config key prefix; {@code hbase.rsgroup.regex.=} */ + static final String RS_GROUP_REGEX_PREFIX = "hbase.rsgroup.regex."; + private static final Pattern GROUP_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9_]+"); + private static final String RS_GROUP_ZNODE = "rsgroup"; static final byte[] META_FAMILY_BYTES = Bytes.toBytes("m"); @@ -198,6 +221,7 @@ private static final class RSGroupInfoHolder { private final AsyncClusterConnection conn; private final ZKWatcher watcher; private final RSGroupStartupWorker rsGroupStartupWorker; + private final ServerEventsListener serverEventsListener = new ServerEventsListener(); // contains list of groups that were last flushed to persistent store private Set prevRSGroups = new HashSet<>(); @@ -247,37 +271,31 @@ private RSGroupInfoManagerImpl(MasterServices masterServices) { this.script = new RSGroupMappingScript(masterServices.getConfiguration()); } - private synchronized void updateDefaultServers() { - LOG.info("Updating default servers."); - Map newGroupMap = Maps.newHashMap(holder.groupName2Group); - RSGroupInfo oldDefaultGroupInfo = getRSGroup(RSGroupInfo.DEFAULT_GROUP); - assert oldDefaultGroupInfo != null; - RSGroupInfo newDefaultGroupInfo = - new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, getDefaultServers()); - newDefaultGroupInfo.addAllTables(oldDefaultGroupInfo.getTables()); - newGroupMap.put(RSGroupInfo.DEFAULT_GROUP, newDefaultGroupInfo); - // do not need to persist, as we do not persist default group. - resetRSGroupMap(newGroupMap); - LOG.info("Updated default servers, {} servers", newDefaultGroupInfo.getServers().size()); - if (LOG.isDebugEnabled()) { - LOG.debug("New default servers list: {}", newDefaultGroupInfo.getServers()); + /** + * Called synchronously from {@code ServerListener#serverAdded}/{@code #serverRemoved}. Server + * membership of 'default' and of every regex-governed group is never persisted, so it is + * recomputed from the live servers and applied in memory only -- no {@code hbase:rsgroup}/ZK I/O. + */ + private synchronized void handleServerEvent() { + Map currentGroups = holder.groupName2Group; + Map> autoManagedServers = + computeAutoManagedRSGroupServers(currentGroups.values()); + Map newGroupMap = Maps.newHashMap(currentGroups); + applyAutoManagedRSGroupServers(newGroupMap, autoManagedServers); + dropRegexGovernedServersFromOtherGroups(newGroupMap, autoManagedServers); + for (Map.Entry entry : newGroupMap.entrySet()) { + if (!entry.getValue().getServers().equals(currentGroups.get(entry.getKey()).getServers())) { + resetRSGroupMap(newGroupMap); + LOG.info("Updated auto-managed RSGroup servers, {} servers", + autoManagedServers.values().stream().mapToInt(SortedSet::size).sum()); + return; + } } } private synchronized void init() throws IOException { refresh(false); - masterServices.getServerManager().registerListener(new ServerListener() { - - @Override - public void serverAdded(ServerName serverName) { - updateDefaultServers(); - } - - @Override - public void serverRemoved(ServerName serverName) { - updateDefaultServers(); - } - }); + masterServices.getServerManager().registerListener(serverEventsListener); } static RSGroupInfoManager getInstance(MasterServices masterServices) throws IOException { @@ -321,10 +339,24 @@ private Set
getOnlineServers() { .map(ServerName::getAddress).collect(Collectors.toSet()); } + /** + * Returns a copy of {@code src}, safe to mutate without affecting the live {@code + * holder.groupName2Group} entry -- callers must not mutate the object returned by + * {@link #getRSGroupInfo} or {@link #getRSGroupOfServer} in place, since a subsequent + * {@link #flushConfig(Map)} can still reject the change and the live state must stay untouched + * until the change is actually persisted. + */ + private static RSGroupInfo copyOf(RSGroupInfo src) { + RSGroupInfo copy = new RSGroupInfo(src); + // The copy constructor does not copy configuration, so do it by hand. + src.getConfiguration().forEach(copy::setConfiguration); + return copy; + } + public synchronized Set
moveServers(Set
servers, String srcGroup, String dstGroup) throws IOException { - RSGroupInfo src = getRSGroupInfo(srcGroup); - RSGroupInfo dst = getRSGroupInfo(dstGroup); + RSGroupInfo src = copyOf(getRSGroupInfo(srcGroup)); + RSGroupInfo dst = copyOf(getRSGroupInfo(dstGroup)); Set
movedServers = new HashSet<>(); // If destination is 'default' rsgroup, only add servers that are online. If not online, drop // it. If not 'default' group, add server to 'dst' rsgroup EVEN IF IT IS NOT online (could be a @@ -344,6 +376,7 @@ public synchronized Set
moveServers(Set
servers, String srcGro dst.addServer(el); movedServers.add(el); } + checkMoveAgreesWithRegex(movedServers, dst.getName()); Map newGroupMap = Maps.newHashMap(holder.groupName2Group); newGroupMap.put(src.getName(), src); newGroupMap.put(dst.getName(), dst); @@ -351,6 +384,37 @@ public synchronized Set
moveServers(Set
servers, String srcGro return movedServers; } + /** + * Regex-based membership always wins: the next server event or refresh puts a server back where + * its hostname says, so an explicit move that disagrees would only make regions flap. Such a move + * is rejected. A server whose hostname matches no regex (or an ambiguous or dangling one, which + * fall back to ordinary placement) may go anywhere except into a regex-governed group. + */ + private void checkMoveAgreesWithRegex(Set
movedServers, String dstGroup) + throws ConstraintException { + Map regexGroupMap = getRegexGroupMap(masterServices.getConfiguration()); + if (regexGroupMap.isEmpty() || movedServers.isEmpty()) { + return; + } + Map expectedGroups = + resolveServerAddrToRSGroupName(movedServers, regexGroupMap, holder.groupName2Group.keySet()); + for (Address server : movedServers) { + String expectedGroup = expectedGroups.get(server); + boolean isWrongMove = expectedGroup != null + ? !expectedGroup.equals(dstGroup) + : regexGroupMap.containsKey(dstGroup); + if (isWrongMove) { + String reason = expectedGroup != null + ? "its membership is decided by " + RS_GROUP_REGEX_PREFIX + expectedGroup + + ", under which its hostname belongs in '" + expectedGroup + "'" + : "its hostname does not match " + RS_GROUP_REGEX_PREFIX + dstGroup + + ", which decides the membership of that RSGroup"; + throw new ConstraintException("Server " + server + " cannot be moved to RSGroup '" + + dstGroup + "': " + reason + ". Change that config instead."); + } + } + } + @Override public RSGroupInfo getRSGroupOfServer(Address serverHostPort) { for (RSGroupInfo info : holder.groupName2Group.values()) { @@ -425,12 +489,10 @@ public synchronized void removeServers(Set
servers) throws IOException if (rsGroupInfo != null) { RSGroupInfo newRsGroupInfo = rsGroupInfos.get(rsGroupInfo.getName()); if (newRsGroupInfo == null) { - rsGroupInfo.removeServer(el); - rsGroupInfos.put(rsGroupInfo.getName(), rsGroupInfo); - } else { - newRsGroupInfo.removeServer(el); - rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo); + newRsGroupInfo = copyOf(rsGroupInfo); } + newRsGroupInfo.removeServer(el); + rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo); } else { LOG.warn("Server " + el + " does not belong to any rsgroup."); } @@ -585,6 +647,8 @@ public void run() { * startup of the manager. */ private synchronized void refresh(boolean forceOnline) throws IOException { + LOG.debug("Refreshing RSGroup info from source of truth: forceOnline={}, isOnline={}", + forceOnline, isOnline()); List groupList = new ArrayList<>(); // Overwrite anything read from zk, group table is source of truth @@ -599,15 +663,31 @@ private synchronized void refresh(boolean forceOnline) throws IOException { // This is added to the last of the list so it overwrites the 'default' rsgroup loaded // from region group table or zk - groupList.add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, getDefaultServers(groupList))); + groupList.add(new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP)); // populate the data HashMap newGroupMap = Maps.newHashMap(); for (RSGroupInfo group : groupList) { newGroupMap.put(group.getName(), group); } + // Server membership for 'default' and for every regex-governed group is always recomputed + // on the fly -- never persisted nor trusted from storage. + Map> autoManagedServers = + computeAutoManagedRSGroupServers(groupList); + for (Map.Entry> entry : autoManagedServers.entrySet()) { + RSGroupInfo stored = newGroupMap.get(entry.getKey()); + if (!RSGroupInfo.DEFAULT_GROUP.equals(entry.getKey()) && !stored.getServers().isEmpty()) { + LOG.info( + "Ignoring servers {} stored for regex-governed RSGroup '{}'; its membership is " + + "computed on the fly and the stored copy is cleared on the next flush", + stored.getServers(), entry.getKey()); + } + } + applyAutoManagedRSGroupServers(newGroupMap, autoManagedServers); + dropRegexGovernedServersFromOtherGroups(newGroupMap, autoManagedServers); resetRSGroupMap(newGroupMap); updateCacheOfRSGroups(newGroupMap.keySet()); + LOG.info("Refresh completed successfully"); } private void flushConfigTable(Map groupMap) throws IOException { @@ -655,13 +735,16 @@ private synchronized void flushConfig(Map newGroupMap) thro Map oldGroupMap = Maps.newHashMap(holder.groupName2Group); RSGroupInfo oldDefaultGroup = oldGroupMap.remove(RSGroupInfo.DEFAULT_GROUP); RSGroupInfo newDefaultGroup = newGroupMap.remove(RSGroupInfo.DEFAULT_GROUP); + // Servers of regex-governed groups are computed on the fly and never persisted, so their + // changes are allowed here too. if ( - !oldGroupMap.equals(newGroupMap) + !withoutRegexGovernedServers(oldGroupMap).equals(withoutRegexGovernedServers(newGroupMap)) /* compare both tables and servers in other groups */ || !oldDefaultGroup.getTables() .equals(newDefaultGroup.getTables()) /* compare tables in default group */ ) { - throw new IOException("Only servers in default group can be updated during offline mode"); + throw new IOException("Only servers in the default group and in regex-governed groups can " + + "be updated during offline mode"); } // Restore newGroupMap by putting its default group back @@ -681,15 +764,38 @@ private synchronized void flushConfig(Map newGroupMap) thro /* For online mode, persist to hbase:rsgroup and Zookeeper */ LOG.debug("Online mode, persisting to {} and ZK", RSGROUP_TABLE_NAME); - flushConfigTable(newGroupMap); + // Servers of regex-governed groups are computed on the fly and never persisted + Map persistedGroupMap = withoutRegexGovernedServers(newGroupMap); + flushConfigTable(persistedGroupMap); // Make changes visible after having been persisted to the source of truth resetRSGroupMap(newGroupMap); - saveRSGroupMapToZK(newGroupMap); + saveRSGroupMapToZK(persistedGroupMap); updateCacheOfRSGroups(newGroupMap.keySet()); LOG.info("Flush config done, new RSGroup map: {}", newGroupMap); } + /** + * Returns a copy of {@code groupMap} where every regex-governed group has an empty server set. + * Their membership is recomputed from the live servers (see {@link #refresh}), so persisting it + * would only add I/O and leave stale data behind. + */ + private Map withoutRegexGovernedServers(Map groupMap) { + Set regexGovernedGroups = getRegexGroupMap(masterServices.getConfiguration()).keySet(); + Map result = Maps.newHashMap(groupMap); + for (String groupName : regexGovernedGroups) { + RSGroupInfo info = groupMap.get(groupName); + if (info == null || info.getServers().isEmpty()) { + continue; + } + RSGroupInfo stripped = new RSGroupInfo(groupName); + stripped.addAllTables(info.getTables()); + info.getConfiguration().forEach(stripped::setConfiguration); + result.put(groupName, stripped); + } + return result; + } + private void saveRSGroupMapToZK(Map newGroupMap) throws IOException { LOG.debug("Saving RSGroup info to ZK"); try { @@ -742,30 +848,265 @@ private void updateCacheOfRSGroups(final Set currentGroups) { this.prevRSGroups.addAll(currentGroups); } - // Called by ServerEventsListenerThread. Presume it has lock on this manager when it runs. - private SortedSet
getDefaultServers() { - return getDefaultServers(listRSGroups()/* get from rsGroupMap */); + /** + * Parses {@code hbase.rsgroup.regex.=} entries out of {@code conf}. Entries + * with an invalid group name, targeting the reserved {@link RSGroupInfo#DEFAULT_GROUP}, or with + * an unparsable regex are logged and skipped. + */ + static Map getRegexGroupMap(Configuration conf) { + Map rsGroupNameToRegexMap = conf.getPropsWithPrefix(RS_GROUP_REGEX_PREFIX); + Map rsGroupNameToPatternMap = new HashMap<>(); + for (Map.Entry e : rsGroupNameToRegexMap.entrySet()) { + if (e.getValue() == null) { + // getPropsWithPrefix reads names and values separately, so an entry removed by a concurrent + // configuration reload can show up here with a null value. Treat it as unset. + continue; + } + if (!GROUP_NAME_PATTERN.matcher(e.getKey()).matches()) { + LOG.warn("Ignoring {}{} -- '{}' is not a valid RSGroup name (it must match {})", + RS_GROUP_REGEX_PREFIX, e.getKey(), e.getKey(), GROUP_NAME_PATTERN.pattern()); + continue; + } + if (RSGroupInfo.DEFAULT_GROUP.equals(e.getKey())) { + LOG.warn("Ignoring {}{} -- regex-based membership cannot target the reserved '{}' group", + RS_GROUP_REGEX_PREFIX, e.getKey(), RSGroupInfo.DEFAULT_GROUP); + continue; + } + try { + rsGroupNameToPatternMap.put(e.getKey(), Pattern.compile(e.getValue())); + } catch (PatternSyntaxException ex) { + LOG.warn("Invalid regex '{}' for RSGroup '{}' ({}{}); ignoring this entry", e.getValue(), + e.getKey(), RS_GROUP_REGEX_PREFIX, e.getKey()); + } + } + LOG.debug("Resolved regex-based RSGroup membership config: {}", rsGroupNameToPatternMap); + return rsGroupNameToPatternMap; } - // Called by ServerEventsListenerThread. Presume it has lock on this manager when it runs. - private SortedSet
getDefaultServers(List rsGroupInfoList) { - // Build a list of servers in other groups than default group, from rsGroupMap - Set
serversInOtherGroup = new HashSet<>(); - for (RSGroupInfo group : rsGroupInfoList) { - if (!RSGroupInfo.DEFAULT_GROUP.equals(group.getName())) { // not default group - serversInOtherGroup.addAll(group.getServers()); + private static List getMatchingRSGroupNames(String hostname, + Map rsGroupNameToPatternMap) { + List rsGroupNames = new ArrayList<>(); + for (Map.Entry e : rsGroupNameToPatternMap.entrySet()) { + if (e.getValue().matcher(hostname).matches()) { + rsGroupNames.add(e.getKey()); } } + return rsGroupNames; + } - // Get all online servers from Zookeeper and find out servers in default group - SortedSet
defaultServers = Sets.newTreeSet(); - for (ServerName serverName : masterServices.getServerManager().getOnlineServers().keySet()) { - Address server = Address.fromParts(serverName.getHostname(), serverName.getPort()); - if (!serversInOtherGroup.contains(server)) { // not in other groups - defaultServers.add(server); + /** + * Resolves every server in {@code onlineServers} whose hostname unambiguously matches exactly one + * entry of {@code rsGroupNameToPatternMap} naming a group in {@code existingGroupNames}. A server + * matching multiple regexes, or matching a regex for a group that does not (yet) exist, is + * omitted (warn-only) rather than resolved. + */ + static Map resolveServerAddrToRSGroupName(Set
onlineServers, + Map rsGroupNameToPatternMap, Set existingGroupNames) { + Map serverAddrToRSGroupName = new HashMap<>(); + if (rsGroupNameToPatternMap.isEmpty()) { + return serverAddrToRSGroupName; + } + Set nonExistingRSGroupNames = new HashSet<>(); + for (Address server : onlineServers) { + List matchingRSGroupNames = + getMatchingRSGroupNames(server.getHostName(), rsGroupNameToPatternMap); + if (matchingRSGroupNames.isEmpty()) { + continue; + } + if (matchingRSGroupNames.size() > 1) { + LOG.warn("Server {} hostname matches regexes for multiple RSGroups {}; treating it as " + + "unmatched by regex (falls back to default) until the overlapping " + + "hbase.rsgroup.regex.* entries are fixed", server, matchingRSGroupNames); + continue; + } + String rsGroupName = matchingRSGroupNames.get(0); + if (existingGroupNames.contains(rsGroupName)) { + serverAddrToRSGroupName.put(server, rsGroupName); + } else if (nonExistingRSGroupNames.add(rsGroupName)) { + LOG.warn( + "Config {}{} matches server {} but RSGroup '{}' does not exist -- create it with " + + "addRSGroup first; treating this server as unmatched by regex until then", + RS_GROUP_REGEX_PREFIX, rsGroupName, server, rsGroupName); + } + } + LOG.debug("Resolved server address to RSGroup name map: {}", serverAddrToRSGroupName); + return serverAddrToRSGroupName; + } + + /** Returns {@code true} if every online server is either admin-managed or regex-matched. */ + static boolean wouldEmptyDefaultGroup(Set
onlineServers, + Set
adminManagedServers, Set
regexMatchedServers) { + for (Address server : onlineServers) { + if (!adminManagedServers.contains(server) && !regexMatchedServers.contains(server)) { + return false; + } + } + return !onlineServers.isEmpty(); + } + + static final class RegexBasedRSGroupMembershipResolution { + final Set
adminManagedServers; + final Map regexMatchedServers; + + RegexBasedRSGroupMembershipResolution(Set
adminManagedServers, + Map regexMatchedServers) { + this.adminManagedServers = adminManagedServers; + this.regexMatchedServers = regexMatchedServers; + } + } + + static RegexBasedRSGroupMembershipResolution resolveRegexBasedRSGroupMembership( + Map rsGroupNameToPatternMap, Set
onlineServers, + Collection existingGroups) { + Set existingGroupNames = + existingGroups.stream().map(RSGroupInfo::getName).collect(Collectors.toSet()); + Set
adminManagedServers = new HashSet<>(); + for (RSGroupInfo existingGroup : existingGroups) { + if ( + !RSGroupInfo.DEFAULT_GROUP.equals(existingGroup.getName()) + && !rsGroupNameToPatternMap.containsKey(existingGroup.getName()) + ) { + adminManagedServers.addAll(existingGroup.getServers()); + } + } + Map regexMatchedServers = + resolveServerAddrToRSGroupName(onlineServers, rsGroupNameToPatternMap, existingGroupNames); + // A server stored in an admin-managed group that a regex now claims belongs to the regex group. + adminManagedServers.removeAll(regexMatchedServers.keySet()); + if (wouldEmptyDefaultGroup(onlineServers, adminManagedServers, regexMatchedServers.keySet())) { + // Regex placement is still applied: every server lands in its correct group. Operators are + // expected to configure regexes carefully, so this is a warning, not an error. + LOG.warn( + "Regex-based RSGroup membership leaves RSGroup '{}' with no online servers -- every " + + "online server is either admin-managed or matches a {}* entry. Tables in RSGroup '{}' " + + "(including system tables, unless they are in another RSGroup) have no server of " + + "their own to be assigned to; set {} to let them fall back to servers of other " + + "RSGroups", + RSGroupInfo.DEFAULT_GROUP, RS_GROUP_REGEX_PREFIX, RSGroupInfo.DEFAULT_GROUP, + RSGroupBasedLoadBalancer.FALLBACK_GROUP_ENABLE_KEY); + } + LOG.debug( + "Regex-based RSGroup membership resolution: onlineServers={}, " + + "adminManagedServers={}, regexMatchedServers={}", + onlineServers, adminManagedServers, regexMatchedServers); + return new RegexBasedRSGroupMembershipResolution(adminManagedServers, regexMatchedServers); + } + + private Map> + computeAutoManagedRSGroupServers(Collection existingGroups) { + Set
onlineServers = getOnlineServers(); + Map rsGroupNameToPatternMap = + getRegexGroupMap(masterServices.getConfiguration()); + RegexBasedRSGroupMembershipResolution resolution = + resolveRegexBasedRSGroupMembership(rsGroupNameToPatternMap, onlineServers, existingGroups); + + Set existingGroupNames = + existingGroups.stream().map(RSGroupInfo::getName).collect(Collectors.toSet()); + Map> result = new HashMap<>(); + for (String rsGroupName : rsGroupNameToPatternMap.keySet()) { + if (existingGroupNames.contains(rsGroupName)) { + result.put(rsGroupName, new TreeSet<>()); } } - return defaultServers; + result.put(RSGroupInfo.DEFAULT_GROUP, new TreeSet<>()); + + for (Address server : onlineServers) { + String regexGroupName = resolution.regexMatchedServers.get(server); + if (regexGroupName != null) { + result.get(regexGroupName).add(server); + continue; + } + if (resolution.adminManagedServers.contains(server)) { + continue; + } + result.get(RSGroupInfo.DEFAULT_GROUP).add(server); + } + for (Map.Entry> entry : result.entrySet()) { + if (entry.getValue().isEmpty() && !RSGroupInfo.DEFAULT_GROUP.equals(entry.getKey())) { + LOG.warn( + "Regex-governed RSGroup '{}' has no online servers; any table assigned to it has " + + "nowhere to go (check {}{} and the online servers' hostnames)", + entry.getKey(), RS_GROUP_REGEX_PREFIX, entry.getKey()); + } + } + LOG.debug("Computed auto-managed RSGroup server membership: {}", result); + return result; + } + + private static void applyAutoManagedRSGroupServers(Map groupMap, + Map> newRSGroupToServers) { + for (Map.Entry> entry : newRSGroupToServers.entrySet()) { + RSGroupInfo oldInfo = groupMap.get(entry.getKey()); + if (oldInfo == null) { + continue; + } + RSGroupInfo newInfo = new RSGroupInfo(entry.getKey(), entry.getValue()); + newInfo.addAllTables(oldInfo.getTables()); + oldInfo.getConfiguration().forEach(newInfo::setConfiguration); + groupMap.put(entry.getKey(), newInfo); + } + } + + /** + * Stored membership of admin-managed groups is not recomputed on refresh, so a server that was + * placed in one of them before a regex claimed it would end up in two groups. Unlike the runtime + * paths, refresh cannot reject this: it runs in the startup worker, which retries forever on + * failure and would keep the manager offline. Instead, resolve it the way + * {@link #computeAutoManagedRSGroupServers} already does -- the regex wins -- in memory only; + * storage is corrected by the next flush. + */ + private static void dropRegexGovernedServersFromOtherGroups(Map groupMap, + Map> autoManagedServers) { + Map regexGovernedServers = new HashMap<>(); + for (Map.Entry> entry : autoManagedServers.entrySet()) { + if (!RSGroupInfo.DEFAULT_GROUP.equals(entry.getKey())) { + entry.getValue().forEach(server -> regexGovernedServers.put(server, entry.getKey())); + } + } + if (regexGovernedServers.isEmpty()) { + return; + } + for (Map.Entry entry : groupMap.entrySet()) { + RSGroupInfo info = entry.getValue(); + if (autoManagedServers.containsKey(info.getName())) { + continue; + } + Set
overlap = new TreeSet<>(); + for (Address server : info.getServers()) { + if (regexGovernedServers.containsKey(server)) { + overlap.add(server); + } + } + if (overlap.isEmpty()) { + continue; + } + LOG.warn( + "Servers {} are stored in RSGroup '{}' but match a {}* entry; placing them in the " + + "regex-governed RSGroup instead and dropping the stored copy", + overlap, info.getName(), RS_GROUP_REGEX_PREFIX); + SortedSet
kept = new TreeSet<>(info.getServers()); + kept.removeAll(overlap); + RSGroupInfo newInfo = new RSGroupInfo(info.getName(), kept); + newInfo.addAllTables(info.getTables()); + info.getConfiguration().forEach(newInfo::setConfiguration); + entry.setValue(newInfo); + } + } + + private class ServerEventsListener implements ServerListener { + private final Logger LOG = LoggerFactory.getLogger(ServerEventsListener.class); + + @Override + public void serverAdded(ServerName serverName) { + LOG.debug("Server added: {}", serverName); + RSGroupInfoManagerImpl.this.handleServerEvent(); + } + + @Override + public void serverRemoved(ServerName serverName) { + LOG.debug("Server removed: {}", serverName); + RSGroupInfoManagerImpl.this.handleServerEvent(); + } } private class RSGroupStartupWorker extends Thread { @@ -881,8 +1222,9 @@ private void multiMutate(List mutations) throws IOException { } private void checkGroupName(String groupName) throws ConstraintException { - if (!groupName.matches("[a-zA-Z0-9_]+")) { - throw new ConstraintException("RSGroup name should only contain alphanumeric characters"); + if (!GROUP_NAME_PATTERN.matcher(groupName).matches()) { + throw new ConstraintException( + "RSGroup name '" + groupName + "' must match " + GROUP_NAME_PATTERN.pattern()); } } @@ -1309,7 +1651,10 @@ public void moveServers(Set
servers, String targetGroupName) throws IOE // MovedServers may be < passed in 'servers'. Set
movedServers = moveServers(servers, srcGrp.getName(), targetGroupName); - moveServerRegionsFromGroup(movedServers, srcGrp.getServers(), targetGroupName, + // moveServers (above) no longer mutates srcGrp in place -- re-fetch to see the post-move + // remaining server set. + RSGroupInfo refreshedSrcGrp = getRSGroupInfo(srcGrp.getName()); + moveServerRegionsFromGroup(movedServers, refreshedSrcGrp.getServers(), targetGroupName, srcGrp.getName()); LOG.info("Move servers done: moved {} servers from {} to {}", movedServers.size(), srcGrp.getName(), targetGroupName); @@ -1359,9 +1704,13 @@ public synchronized void updateRSGroupConfig(String groupName, Map rsGroupInfo.removeConfiguration(k)); + RSGroupInfo rsGroupInfo = copyOf(getRSGroupInfo(groupName)); + // Iterate over a snapshot of the keys, as removeConfiguration modifies the backing map. + new ArrayList<>(rsGroupInfo.getConfiguration().keySet()) + .forEach(rsGroupInfo::removeConfiguration); configuration.forEach((k, v) -> rsGroupInfo.setConfiguration(k, v)); - flushConfig(); + Map newGroupMap = Maps.newHashMap(holder.groupName2Group); + newGroupMap.put(groupName, rsGroupInfo); + flushConfig(newGroupMap); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java index 468b03963c4b..40ef2eb30426 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRSGroupsBase.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hbase.rsgroup; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; @@ -207,6 +209,11 @@ protected final RSGroupInfo addGroup(String groupName, int serverCount) } ADMIN.moveServersToRSGroup(set, groupName); RSGroupInfo result = ADMIN.getRSGroup(groupName); + assertEquals(set, result.getServers()); + Set
remainingDefaultServers = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers(); + for (Address server : set) { + assertFalse(remainingDefaultServers.contains(server)); + } return result; } @@ -226,8 +233,29 @@ protected final void removeGroup(String groupName) throws IOException { } } RSGroupInfo groupInfo = ADMIN.getRSGroup(groupName); - ADMIN.moveServersToRSGroup(groupInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); + Set
servers = groupInfo.getServers(); + ADMIN.moveServersToRSGroup(servers, RSGroupInfo.DEFAULT_GROUP); ADMIN.removeRSGroup(groupName); + assertFalse(ADMIN.listRSGroups().stream().anyMatch(g -> g.getName().equals(groupName))); + Set
onlineServers = new HashSet<>(); + for (ServerName sn : MASTER.getServerManager().getOnlineServersList()) { + onlineServers.add(sn.getAddress()); + } + RSGroupInfo defaultInfo = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP); + for (Address server : servers) { + // moveServersToRSGroup()-to-'default' silently drops servers that are no longer online + // (see RSGroupInfoManagerImpl#moveServers), so only online servers are guaranteed to land + // in 'default'; an offline server's former group membership is simply dropped. + if (onlineServers.contains(server)) { + assertTrue(defaultInfo.getServers().contains(server)); + } + } + for (TableName table : tables) { + // ADMIN.getRSGroup(String)/listRSGroups() do not resolve table membership from the + // TableDescriptor's RegionServerGroup attribute (only ADMIN.getRSGroup(TableName) does), + // so look up each table's group individually rather than via defaultInfo.getTables(). + assertEquals(RSGroupInfo.DEFAULT_GROUP, ADMIN.getRSGroup(table).getName()); + } } protected final void deleteTableIfNecessary() throws IOException { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegexBasedRSGroupMembership.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegexBasedRSGroupMembership.java new file mode 100644 index 000000000000..1906e06a69c3 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/TestRegexBasedRSGroupMembership.java @@ -0,0 +1,1240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.rsgroup; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.LocalHBaseCluster; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.SingleProcessHBaseCluster; +import org.apache.hadoop.hbase.client.Get; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.net.Address; +import org.apache.hadoop.hbase.testclassification.MediumTests; +import org.apache.hadoop.hbase.testclassification.RSGroupTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.DNS; +import org.apache.hadoop.hbase.util.JVMClusterUtil; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.Sets; + +import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil; +import org.apache.hadoop.hbase.shaded.protobuf.generated.RSGroupProtos; + +/** + * Black-box tests for regex-based automatic RSGroup membership + * (hbase.rsgroup.regex.<groupname>=<regex>), driven entirely through the public + * {@link org.apache.hadoop.hbase.client.Admin} RSGroup API, live RegionServer add/remove against + * the minicluster, and captured log output -- no reflection or visibility changes against + * {@link RSGroupInfoManagerImpl}. + */ +@Tag(RSGroupTests.TAG) +@Tag(MediumTests.TAG) +public class TestRegexBasedRSGroupMembership extends TestRSGroupsBase { + + private static final Logger LOG = LoggerFactory.getLogger(TestRegexBasedRSGroupMembership.class); + private static final String REGEX_PREFIX = "hbase.rsgroup.regex."; + + private final List fakeRegionServers = new ArrayList<>(); + private final Set regexConfigKeysSet = new HashSet<>(); + + @BeforeAll + public static void setUp() throws Exception { + setUpTestBeforeClass(); + } + + @AfterAll + public static void tearDown() throws Exception { + tearDownAfterClass(); + } + + @BeforeEach + public void beforeMethod(TestInfo testInfo) throws Exception { + setUpBeforeMethod(testInfo); + } + + @AfterEach + public void afterMethod() throws Exception { + // Stop any fake-hostname RS this test started before handing back to the shared teardown, + // which only ever starts servers to top back up to NUM_SLAVES_BASE, never stops extras. + for (JVMClusterUtil.RegionServerThread rst : new ArrayList<>(fakeRegionServers)) { + stopFakeRegionServer(rst); + } + // The MASTER/ADMIN Configuration objects are shared across every test method in this class; + // undo any regex config this test added to both so it can't leak into the next method. + for (String key : regexConfigKeysSet) { + MASTER.getConfiguration().unset(key); + ADMIN.getConfiguration().unset(key); + } + regexConfigKeysSet.clear(); + tearDownAfterMethod(); + RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) MASTER.getLoadBalancer(); + balancer.resetAssignmentCallFlagsForTest(); + balancer.setFallbackEnabledForTest(false); + } + + // ============================== helpers ============================== + + /** + * Starts a genuinely new RegionServer process (thread) in the shared minicluster that reports + * itself under a caller-chosen hostname, so regex membership rules can target it individually. + * The base cluster's real RS all share one real, resolvable hostname (this machine's address); + * the RPC services constructor unconditionally resolves whatever hostname is configured + * (independent of any RPC bind-address override), so the caller-chosen hostname here must itself + * be a real, resolvable, bindable literal -- "127.0.0.1" and "localhost" both work and are + * guaranteed distinct from the base cluster's real hostname, which is exactly what every test in + * this class relies on to get a controllable, non-shared identity. + */ + private JVMClusterUtil.RegionServerThread startFakeHostnameRS(String hostname) throws Exception { + Configuration rsConf = new Configuration(TEST_UTIL.getConfiguration()); + rsConf.set(DNS.UNSAFE_RS_HOSTNAME_KEY, hostname); + rsConf.set("hbase.regionserver.ipc.address", "localhost"); + // The shared TEST_UTIL configuration's regionserver port is never rewritten to an ephemeral + // value by minicluster startup (the base RS get their distinct ports through a separate, + // per-instance mechanism), so without this override every fake RS started here would try to + // bind the literal default port (16020) and collide with any other fake RS still alive. + rsConf.set(HConstants.REGIONSERVER_PORT, "0"); + // Same issue for the embedded web UI (default port 16030); these fake RS don't need one. + rsConf.set(HConstants.REGIONSERVER_INFO_PORT, "-1"); + LocalHBaseCluster localCluster = ((SingleProcessHBaseCluster) CLUSTER).hbaseCluster; + JVMClusterUtil.RegionServerThread rst = + localCluster.addRegionServer(rsConf, localCluster.getRegionServers().size()); + rst.start(); + rst.waitForServerOnline(); + fakeRegionServers.add(rst); + final ServerName sn = rst.getRegionServer().getServerName(); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> MASTER.getServerManager().getOnlineServersList().contains(sn)); + LOG.info("Started fake-hostname RS {}", sn); + return rst; + } + + private void stopFakeRegionServer(JVMClusterUtil.RegionServerThread rst) throws Exception { + fakeRegionServers.remove(rst); + final ServerName sn = rst.getRegionServer().getServerName(); + rst.getRegionServer().stop("test cleanup"); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> !MASTER.getServerManager().getOnlineServersList().contains(sn)); + LOG.info("Stopped fake-hostname RS {}", sn); + } + + private void killFakeRegionServer(JVMClusterUtil.RegionServerThread rst) throws Exception { + fakeRegionServers.remove(rst); + final ServerName sn = rst.getRegionServer().getServerName(); + ((SingleProcessHBaseCluster) CLUSTER).killRegionServer(sn); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> !MASTER.getServerManager().getOnlineServersList().contains(sn)); + LOG.info("Killed fake-hostname RS {}", sn); + } + + /** + * Server-membership recompute happens synchronously in the {@code serverAdded}/ + * {@code serverRemoved} callback -- it is only ever triggered by an actual server add/remove + * event, a bare config mutation is not enough. Forces such an event (add+remove a bystander RS) + * so a config change just made takes effect without requiring an explicit RSGroup admin RPC. Only + * used by tests that *want* the automatic reconciliation to run and correct things; the trigger + * RS's own transient membership doesn't affect assertions elsewhere since those only check for + * specific other servers' addresses. + */ + private void triggerListenerCycle() throws Exception { + JVMClusterUtil.RegionServerThread trigger = startFakeHostnameRS("127.0.0.1"); + stopFakeRegionServer(trigger); + } + + /** Reads the server set of a group straight from its row in the hbase:rsgroup table. */ + private Set
readStoredServers(String groupName) throws IOException { + try ( + Table table = TEST_UTIL.getConnection().getTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME)) { + Result result = table.get(new Get(Bytes.toBytes(groupName))); + byte[] value = result.getValue(RSGroupInfoManagerImpl.META_FAMILY_BYTES, + RSGroupInfoManagerImpl.META_QUALIFIER_BYTES); + if (value == null) { + return new HashSet<>(); + } + return new HashSet<>( + ProtobufUtil.toGroupInfo(RSGroupProtos.RSGroupInfo.parseFrom(value)).getServers()); + } + } + + /** + * Fails over to a freshly started master (the minicluster's masters are threads): a backup is + * started, the active master is stopped, and the shared MASTER/ADMIN handles are re-pointed. The + * base class keeps the master's embedded RS in a "master" group, whose stored address would go + * stale, so the master is first moved back to 'default' (whose membership is live-computed); the + * next cleanup places the new master in the "master" group again. + */ + private void restartMaster() throws Exception { + SingleProcessHBaseCluster cluster = TEST_UTIL.getMiniHBaseCluster(); + int oldIndex = -1; + List masters = cluster.getMasterThreads(); + for (int i = 0; i < masters.size(); i++) { + if (masters.get(i).getMaster() == MASTER) { + oldIndex = i; + } + } + cluster.startMaster(); + cluster.stopMaster(oldIndex); + cluster.waitOnMaster(oldIndex); + assertTrue(cluster.waitForActiveAndReadyMaster(WAIT_TIMEOUT)); + // Clients bootstrap from the configured master address, which is now the dead master's. + TEST_UTIL.getConfiguration().set(HConstants.MASTER_ADDRS_KEY, + cluster.getMaster().getServerName().getAddress().toString()); + TEST_UTIL.closeConnection(); + ADMIN.close(); + initialize(); + } + + private void setRegex(String groupName, String regex) { + String key = REGEX_PREFIX + groupName; + MASTER.getConfiguration().set(key, regex); + // ADMIN's own Configuration is a distinct object from MASTER's (the minicluster deep-copies + // configuration per daemon), so mirror the change there too -- VerifyingRSGroupAdmin#verify + // reads it to know which servers' membership is regex-driven. + ADMIN.getConfiguration().set(key, regex); + regexConfigKeysSet.add(key); + } + + /** + * Tears down a regex-governed group. Explicit moves that contradict an active regex are rejected, + * so the regex is first dropped on the master only -- ADMIN's copy still tells + * VerifyingRSGroupAdmin which servers' membership is regex-driven -- and then the group is + * emptied and removed. + */ + private void removeRegexGroup(String groupName) throws Exception { + String key = REGEX_PREFIX + groupName; + MASTER.getConfiguration().unset(key); + try { + removeGroup(groupName); + } finally { + clearRegex(groupName); + } + } + + private void clearRegex(String groupName) { + String key = REGEX_PREFIX + groupName; + MASTER.getConfiguration().unset(key); + ADMIN.getConfiguration().unset(key); + regexConfigKeysSet.remove(key); + } + + private Address addressOf(JVMClusterUtil.RegionServerThread rst) { + return rst.getRegionServer().getServerName().getAddress(); + } + + private static class LogCapturer { + private final StringWriter sw = new StringWriter(); + private final org.apache.logging.log4j.core.appender.WriterAppender appender; + private final org.apache.logging.log4j.core.Logger logger; + + LogCapturer(org.apache.logging.log4j.core.Logger logger) { + this.logger = logger; + this.appender = org.apache.logging.log4j.core.appender.WriterAppender.newBuilder() + .setName("test-regex-membership").setTarget(sw).build(); + this.appender.start(); + this.logger.addAppender(this.appender); + } + + String getOutput() { + return sw.toString(); + } + + void stopCapturing() { + this.logger.removeAppender(this.appender); + this.appender.stop(); + } + } + + private static LogCapturer captureRSGroupInfoManagerLog() { + return new LogCapturer( + (org.apache.logging.log4j.core.Logger) org.apache.logging.log4j.LogManager + .getLogger(RSGroupInfoManagerImpl.class)); + } + + // ============================== tests ============================== + + @Test + public void testNoRegexConfigBaseline() throws Exception { + // No hbase.rsgroup.regex.* configured at all: every server must remain in default, exactly + // as it would without this feature. This is also implicitly exercised by every other test in + // this module, but pin it down explicitly for this feature. + RSGroupInfo defaultGroup = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP); + assertEquals(NUM_SLAVES_BASE, defaultGroup.getServers().size()); + } + + @Test + public void testSingleRegexAutoJoinOnLiveAdd() throws Exception { + String groupName = getGroupName("auto"); + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\.1"); + + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + try { + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr)); + // No explicit moveServersToRSGroup call was made -- the server auto-joined purely from the + // ServerListener reacting to its own arrival. + assertFalse(ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testRegexWithQuantifiersMatches() throws Exception { + String groupName = getGroupName("quant"); + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\.\\d{1,3}"); + + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + try { + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr)); + // No explicit moveServersToRSGroup call was made -- the server auto-joined purely from the + // ServerListener reacting to its own arrival. + assertFalse(ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testFullStringMatchSemantics() throws Exception { + // The regex is a strict prefix of the hostname; full-string Pattern#matches semantics must + // NOT treat this as a match. + String groupName = getGroupName("prefix"); + ADMIN.addRSGroup(groupName); + // Strict prefix of "127.0.0.1" -- must NOT match under full-string Pattern#matches semantics. + setRegex(groupName, "127\\.0\\.0"); + + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + try { + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + assertFalse(ADMIN.getRSGroup(groupName).getServers().contains(addr)); + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testRegexTargetingDefaultGroupIsNoOpAndWarns() throws Exception { + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + setRegex(RSGroupInfo.DEFAULT_GROUP, "127\\.0\\.0\\..*"); + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> capturer.getOutput() + .contains("cannot target the reserved '" + RSGroupInfo.DEFAULT_GROUP + "' group")); + } finally { + capturer.stopCapturing(); + clearRegex(RSGroupInfo.DEFAULT_GROUP); + } + } + + @Test + public void testInvalidRegexSyntaxIgnoredAndWarns() throws Exception { + String groupName = getGroupName("badregex"); + ADMIN.addRSGroup(groupName); + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + setRegex(groupName, "["); + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> capturer.getOutput().contains("Invalid regex '[' for RSGroup '" + groupName + "'")); + } finally { + capturer.stopCapturing(); + removeRegexGroup(groupName); + } + } + + @Test + public void testInvalidGroupNameKeyIgnoredAndWarns() throws Exception { + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + setRegex("not a valid name", "127\\.0\\.0\\..*"); + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> capturer.getOutput().contains("is not a valid RSGroup name")); + } finally { + capturer.stopCapturing(); + clearRegex("not a valid name"); + } + } + + @Test + public void testListenerDoesNotStickToStaleServer() throws Exception { + // A server leaves, then a *different* server with a matching hostname joins later; the + // recompute must not get stuck comparing against a stale previous-assignments snapshot. + String groupName = getGroupName("stale"); + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\..*"); + try { + JVMClusterUtil.RegionServerThread first = startFakeHostnameRS("127.0.0.1"); + Address firstAddr = addressOf(first); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(firstAddr)); + + stopFakeRegionServer(first); + + // Different startcode (fresh ServerName) under the same matching hostname. + JVMClusterUtil.RegionServerThread second = startFakeHostnameRS("127.0.0.1"); + Address secondAddr = addressOf(second); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(secondAddr)); + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testDanglingGroupFallsBackThenResolvesOnceGroupCreated() throws Exception { + String groupName = getGroupName("dangling"); + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + // Group does not exist yet when the regex is configured and the matching RS joins. + setRegex(groupName, "127\\.0\\.0\\..*"); + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> capturer.getOutput() + .contains("RSGroup '" + groupName + "' does not exist -- create it with addRSGroup")); + + // Now create the group; on the next cycle the server should migrate in. + ADMIN.addRSGroup(groupName); + triggerListenerCycle(); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr)); + } finally { + capturer.stopCapturing(); + removeRegexGroup(groupName); + } + } + + @Test + public void testAmbiguousMatchFallsBackThenResolvesOnceOverlapFixed() throws Exception { + String groupOne = getGroupName("ambig1"); + String groupTwo = getGroupName("ambig2"); + ADMIN.addRSGroup(groupOne); + ADMIN.addRSGroup(groupTwo); + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + setRegex(groupOne, "127\\.0\\.0\\..*"); + setRegex(groupTwo, "127\\.0\\.0\\.1"); + + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> capturer.getOutput().contains("matches regexes for multiple RSGroups")); + assertFalse(ADMIN.getRSGroup(groupOne).getServers().contains(addr)); + assertFalse(ADMIN.getRSGroup(groupTwo).getServers().contains(addr)); + + // Fix the overlap + setRegex(groupOne, "10\\.0\\.0\\..*"); + triggerListenerCycle(); + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> ADMIN.getRSGroup(groupTwo).getServers().contains(addr)); + } finally { + capturer.stopCapturing(); + removeRegexGroup(groupOne); + removeRegexGroup(groupTwo); + } + } + + @Test + public void testRegexCoveringEveryServerEmptiesDefaultAndWarns() throws Exception { + // Regex placement is always applied: a catch-all regex matches every online server (all base + // RS share one real hostname), so they all move into the regex group and default is left + // empty. That is a warning about an unusual configuration, not an error, and fallback is what + // lets default's tables keep being assigned in that state. + // (Explicit moveServersToRSGroup can never empty default -- an unrelated, longstanding + // RSGroupInfoManagerImpl#moveServers guard keeps >=1 server there -- so the empty-default + // state is driven purely through the automatic regex-reconciliation path.) + String regexGroup = getGroupName("emptydefault"); + ADMIN.addRSGroup(regexGroup); + RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) MASTER.getLoadBalancer(); + balancer.setFallbackEnabledForTest(true); + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + setRegex(regexGroup, ".*"); + triggerListenerCycle(); + + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> capturer.getOutput().contains("leaves RSGroup 'default' with no online servers")); + assertTrue(capturer.getOutput().contains(RSGroupBasedLoadBalancer.FALLBACK_GROUP_ENABLE_KEY)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(regexGroup).getServers().size() == NUM_SLAVES_BASE); + assertTrue(ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().isEmpty()); + + // With default empty and fallback on, a table in default is still assigned -- onto servers + // of other groups. + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 5); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + assertEquals(RSGroupInfo.DEFAULT_GROUP, ADMIN.getRSGroup(tableName).getName()); + + // Narrow the regex so it only targets a new server, not the base cluster's shared + // hostname -- the base servers are unmatched again and return to default. + setRegex(regexGroup, "127\\.0\\.0\\..*"); + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> ADMIN.getRSGroup(regexGroup).getServers().contains(addr) + && ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE); + } finally { + capturer.stopCapturing(); + removeRegexGroup(regexGroup); + } + } + + @Test + public void testCatchAllRegexMovesEveryServerAndGroupTablesStayAssigned() throws Exception { + // A regex group with a live member and a bound table is broadened to a catch-all. Every + // server is placed in its regex group (default is left empty), and the group's table stays + // fully assigned on its original member. + String groupName = getGroupName("catchall"); + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\.1"); + RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) MASTER.getLoadBalancer(); + balancer.setFallbackEnabledForTest(true); + try { + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr)); + ServerName sn = getServerName(addr); + + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 5); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + ADMIN.setRSGroup(Sets.newHashSet(tableName), groupName); + assertEquals(5, getTableServerRegionMap().get(tableName).get(sn).size()); + + setRegex(groupName, ".*"); + triggerListenerCycle(); + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> { + Set
servers = ADMIN.getRSGroup(groupName).getServers(); + return servers.contains(addr) && servers.size() == NUM_SLAVES_BASE + 1 + && ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().isEmpty(); + }); + assertEquals(5, getTableServerRegionMap().get(tableName).get(sn).size()); + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testRegexGovernedGroupMembershipIsNotPersisted() throws Exception { + // Regex-governed membership is computed on the fly: it is visible even while hbase:rsgroup is + // unavailable, and neither the table row nor any later flush ever carries it. + String groupName = getGroupName("nopersist"); + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\.1"); + boolean tableDisabled = false; + try { + TEST_UTIL.getAdmin().disableTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME); + tableDisabled = true; + + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr)); + + TEST_UTIL.getAdmin().enableTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME); + tableDisabled = false; + // Any group mutation flushes every group. + String flushGroup = getGroupName("nopersistflush"); + ADMIN.addRSGroup(flushGroup); + removeGroup(flushGroup); + + assertTrue(ADMIN.getRSGroup(groupName).getServers().contains(addr)); + assertTrue(readStoredServers(groupName).isEmpty()); + } finally { + if (tableDisabled) { + TEST_UTIL.getAdmin().enableTable(RSGroupInfoManagerImpl.RSGROUP_TABLE_NAME); + } + removeRegexGroup(groupName); + } + } + + @Test + public void testMoveContradictingRegexIsRejected() throws Exception { + String groupName = getGroupName("reject"); + String otherGroup = getGroupName("rejectother"); + ADMIN.addRSGroup(groupName); + ADMIN.addRSGroup(otherGroup); + setRegex(groupName, "127\\.0\\.0\\..*"); + try { + // A server whose hostname does not match the regex cannot be moved into the regex group ... + JVMClusterUtil.RegionServerThread nonMatching = startFakeHostnameRS("localhost"); + Address nonMatchingAddr = addressOf(nonMatching); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(nonMatchingAddr)); + assertThrows(IOException.class, + () -> ADMIN.moveServersToRSGroup(Sets.newHashSet(nonMatchingAddr), groupName)); + assertTrue( + ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(nonMatchingAddr)); + assertFalse(ADMIN.getRSGroup(groupName).getServers().contains(nonMatchingAddr)); + + // ... but can still be placed in any other group. + ADMIN.moveServersToRSGroup(Sets.newHashSet(nonMatchingAddr), otherGroup); + assertTrue(ADMIN.getRSGroup(otherGroup).getServers().contains(nonMatchingAddr)); + ADMIN.moveServersToRSGroup(Sets.newHashSet(nonMatchingAddr), RSGroupInfo.DEFAULT_GROUP); + + // A server the regex claims cannot be moved out of, or to any group other than, its group. + JVMClusterUtil.RegionServerThread matching = startFakeHostnameRS("127.0.0.1"); + Address matchingAddr = addressOf(matching); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(matchingAddr)); + assertThrows(IOException.class, + () -> ADMIN.moveServersToRSGroup(Sets.newHashSet(matchingAddr), RSGroupInfo.DEFAULT_GROUP)); + assertThrows(IOException.class, + () -> ADMIN.moveServersToRSGroup(Sets.newHashSet(matchingAddr), otherGroup)); + assertTrue(ADMIN.getRSGroup(groupName).getServers().contains(matchingAddr)); + } finally { + removeGroup(otherGroup); + removeRegexGroup(groupName); + } + } + + @Test + public void testUnrelatedAdminOpsSucceedUnderActiveRegex() throws Exception { + String regexGroup = getGroupName("activeregex"); + String otherGroup = getGroupName("other"); + ADMIN.addRSGroup(regexGroup); + setRegex(regexGroup, "127\\.0\\.0\\..*"); + try { + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(regexGroup).getServers().contains(addr)); + + // Admin operations on admin-managed groups must keep working while a regex-governed group is + // active. setRSGroup requires the target group to already have at least one server, so use a + // base (non-regex-matched, admin-manageable) server rather than an empty new group. + RSGroupInfo otherGroupInfo = addGroup(otherGroup, 1); + Set
otherGroupServers = otherGroupInfo.getServers(); + TEST_UTIL.createTable(tableName, Bytes.toBytes("f")); + ADMIN.setRSGroup(Sets.newHashSet(tableName), otherGroup); + // Table-to-group membership lives on the TableDescriptor (RegionServerGroup), not in + // RSGroupInfo#getTables() -- the latter is only ever populated by flushConfig/refresh from + // persisted storage and setRSGroup never touches it. ADMIN.getRSGroup(TableName) is the + // live, authoritative direction to check this (see RSGroupUtil#getRSGroupInfo). + assertEquals(otherGroup, ADMIN.getRSGroup(tableName).getName()); + + String renamedGroup = otherGroup + "_renamed"; + ADMIN.renameRSGroup(otherGroup, renamedGroup); + RSGroupInfo renamedGroupInfo = ADMIN.getRSGroup(renamedGroup); + assertEquals(otherGroupServers, renamedGroupInfo.getServers()); + assertEquals(renamedGroup, ADMIN.getRSGroup(tableName).getName()); + assertFalse(ADMIN.listRSGroups().stream().anyMatch(g -> g.getName().equals(otherGroup))); + + ADMIN.setRSGroup(Sets.newHashSet(tableName), RSGroupInfo.DEFAULT_GROUP); + assertEquals(RSGroupInfo.DEFAULT_GROUP, ADMIN.getRSGroup(tableName).getName()); + + ADMIN.moveServersToRSGroup(ADMIN.getRSGroup(renamedGroup).getServers(), + RSGroupInfo.DEFAULT_GROUP); + Set
defaultServersAfterMoveBack = + ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers(); + for (Address server : otherGroupServers) { + assertTrue(defaultServersAfterMoveBack.contains(server)); + } + assertTrue(ADMIN.getRSGroup(renamedGroup).getServers().isEmpty()); + + ADMIN.removeRSGroup(renamedGroup); + assertFalse(ADMIN.listRSGroups().stream().anyMatch(g -> g.getName().equals(renamedGroup))); + + // Confirm the unrelated operations above never disturbed the active regex-governed group. + assertTrue(ADMIN.getRSGroup(regexGroup).getServers().contains(addr)); + } finally { + TEST_UTIL.deleteTable(tableName); + removeRegexGroup(regexGroup); + } + } + + @Test + public void testAutomaticMoveDoesNotFireCoprocessorHooks() throws Exception { + String groupName = getGroupName("hooks"); + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\..*"); + try { + assertFalse(OBSERVER.preMoveServersCalled); + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr)); + // Purely automatic, event-driven reassignment must not go through the MasterRpcServices + // moveServers RPC path, so the moveServers coprocessor hooks must not fire. + assertFalse(OBSERVER.preMoveServersCalled); + assertFalse(OBSERVER.postMoveServersCalled); + + // An explicit admin-driven move, by contrast, does fire the hooks. Use one of the + // base cluster's own (non-regex-matched) RS for this, rather than disturbing the + // regex-matched fake RS started above, which keeps running under its active regex. + String otherGroup = getGroupName("hooksother"); + ADMIN.addRSGroup(otherGroup); + Address baseServerAddr = + ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().iterator().next(); + ADMIN.moveServersToRSGroup(Sets.newHashSet(baseServerAddr), otherGroup); + assertTrue(OBSERVER.preMoveServersCalled); + assertTrue(OBSERVER.postMoveServersCalled); + removeGroup(otherGroup); + + // The unrelated explicit move above must not have disturbed the regex-governed group. + assertTrue(ADMIN.getRSGroup(groupName).getServers().contains(addr)); + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testRefreshRemovesRegexMatchedServerFromStoredAdminGroup() throws Exception { + // A server stored in an admin-managed group before a regex claimed it must not end up in two + // groups after a manager restart. refresh cannot reject (it would wedge startup), so the regex + // wins in memory and a WARN is logged. + String adminGroup = getGroupName("refreshadmin"); + String regexGroup = getGroupName("refreshregex"); + ADMIN.addRSGroup(adminGroup); + ADMIN.addRSGroup(regexGroup); + LogCapturer capturer = captureRSGroupInfoManagerLog(); + try { + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + ADMIN.moveServersToRSGroup(Sets.newHashSet(addr), adminGroup); + + // The regex arrives while the server is already stored in the admin group. A new master + // copies the minicluster configuration, so set it there too before the failover. + setRegex(regexGroup, "127\\.0\\.0\\..*"); + TEST_UTIL.getMiniHBaseCluster().getConfiguration().set(REGEX_PREFIX + regexGroup, + "127\\.0\\.0\\..*"); + restartMaster(); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(regexGroup).getServers().contains(addr)); + assertFalse(ADMIN.getRSGroup(adminGroup).getServers().contains(addr)); + assertFalse(ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + assertTrue(capturer.getOutput().contains("are stored in RSGroup '" + adminGroup + "'")); + + // The startup flush clears the stale stored copy: the admin group no longer holds the server + // and the regex-governed group stores no servers at all. + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> !readStoredServers(adminGroup).contains(addr)); + assertTrue(readStoredServers(regexGroup).isEmpty()); + } finally { + capturer.stopCapturing(); + TEST_UTIL.getMiniHBaseCluster().getConfiguration().unset(REGEX_PREFIX + regexGroup); + removeRegexGroup(regexGroup); + removeGroup(adminGroup); + } + } + + @Test + public void testRegexConfigDriftDoesNotBlockUnrelatedAdminOps() throws Exception { + // A regex that contradicts where a server was explicitly placed must not block any unrelated + // admin operation. + String groupName = getGroupName("drift"); + String bystanderGroup = getGroupName("bystander"); + ADMIN.addRSGroup(groupName); + ADMIN.addRSGroup(bystanderGroup); + String renamedBystander = bystanderGroup + "_renamed"; + try { + JVMClusterUtil.RegionServerThread rst = startFakeHostnameRS("127.0.0.1"); + Address addr = addressOf(rst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr)); + + // Legal at the time: no regex governs groupName yet. + ADMIN.moveServersToRSGroup(Sets.newHashSet(addr), groupName); + + // Now introduce a regex for groupName that does NOT match the server already placed there. + // No listener cycle runs, so the contradiction stays un-reconciled. + setRegex(groupName, "10\\.0\\.0\\..*"); + + ADMIN.renameRSGroup(bystanderGroup, renamedBystander); + assertTrue(ADMIN.listRSGroups().stream().anyMatch(g -> g.getName().equals(renamedBystander))); + } finally { + removeRegexGroup(groupName); + removeGroup(ADMIN.getRSGroup(bystanderGroup) != null ? bystanderGroup : renamedBystander); + } + } + + @Test + public void testNewlyJoinedRegexMemberReceivesRegionsOnBalance() throws Exception { + // A new RS auto-joins an already-regex-governed group while the group's sole existing + // member holds every region of a table bound to that group; running the balancer must then + // hand some of those regions to the newly-joined RS too. The first-member baseline is + // established by setRSGroup itself: RSGroupInfoManagerImpl#setRSGroup synchronously relocates + // every region of a table with no RSGroup-scoped descriptor yet onto the target group's + // servers (see moveTablesAndWait, which blocks on each ModifyTableProcedure before returning), + // so with sn1 as the sole group member at that point all regions are already there once + // setRSGroup returns -- no separate balance step is needed for that part. Only the + // second-member handoff genuinely needs an explicit balance: adding a group member does not + // by itself retrigger any region movement. + String groupName = getGroupName("balancenewmember"); + ADMIN.addRSGroup(groupName); + // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable + // identities startFakeHostnameRS supports (its RPC bind address is always "localhost" + // regardless of advertised hostname). + setRegex(groupName, "127\\.0\\.0\\.1|localhost"); + try { + JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1"); + Address addr1 = addressOf(rst1); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr1)); + ServerName sn1 = getServerName(addr1); + + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 10); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + ADMIN.setRSGroup(Sets.newHashSet(tableName), groupName); + Map> perServerAfterMove = getTableServerRegionMap().get(tableName); + assertEquals(10, perServerAfterMove.get(sn1).size()); + + ADMIN.balancerSwitch(true, true); + try { + // A second RS matching the same regex joins while the group is already active; it must + // auto-join with no explicit RPC. + JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost"); + Address addr2 = addressOf(rst2); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr2)); + ServerName sn2 = getServerName(addr2); + + ADMIN.balanceRSGroup(groupName); + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> { + Map> perServer = getTableServerRegionMap().get(tableName); + if (perServer == null) { + return false; + } + List onNewMember = perServer.get(sn2); + if (onNewMember == null || onNewMember.isEmpty()) { + return false; + } + int total = perServer.values().stream().mapToInt(List::size).sum(); + return total == 10; + }); + } finally { + ADMIN.balancerSwitch(false, true); + } + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testRestartTriggeredMembershipConfinesTableToGroupServers() throws Exception { + // RS join normally (no regex active yet), landing in default like any other RS. Only + // afterward is a regex configured targeting their hostnames; a bare config change does not + // reconcile membership by itself, so an actual lifecycle event -- restarting one of the two + // RS -- is used to force the recompute to notice and migrate both into the new group + // (the recompute triggered by one server's add/remove event considers every online server, + // not just the one that triggered it). A table is then bound to that group, and its regions + // must land only on the group's members, with those members hosting only that table's + // regions -- a bidirectional isolation check. + String groupName = getGroupName("restartisolation"); + // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable + // identities startFakeHostnameRS supports (its RPC bind address is always "localhost" + // regardless of advertised hostname -- an arbitrary loopback alias like "127.0.0.2" would + // advertise an address nothing is actually listening on, stalling any real region-open RPC). + JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1"); + JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost"); + Address addr2 = addressOf(rst2); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().contains(addr2)); + + ADMIN.addRSGroup(groupName); + setRegex(groupName, "127\\.0\\.0\\.1|localhost"); + try { + stopFakeRegionServer(rst1); + JVMClusterUtil.RegionServerThread rst1Restarted = startFakeHostnameRS("127.0.0.1"); + Address addr1Restarted = addressOf(rst1Restarted); + + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr1Restarted)); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr2)); + + ServerName sn1 = getServerName(addr1Restarted); + ServerName sn2 = getServerName(addr2); + + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + // setRSGroup synchronously relocates every region of a table with no RSGroup-scoped + // descriptor yet onto one of the target group's servers (moveTablesAndWait blocks on each + // ModifyTableProcedure before returning), so confinement to {sn1, sn2} is already guaranteed + // once this call returns -- no separate balance step is needed. + ADMIN.setRSGroup(Sets.newHashSet(tableName), groupName); + Map> perServerAfterMove = getTableServerRegionMap().get(tableName); + int totalAfterMove = 0; + for (Map.Entry> entry : perServerAfterMove.entrySet()) { + assertTrue(entry.getKey().equals(sn1) || entry.getKey().equals(sn2)); + totalAfterMove += entry.getValue().size(); + } + assertEquals(6, totalAfterMove); + + // Bidirectional isolation: the group's own members must host nothing but this table. + for (RegionInfo region : ADMIN.getRegions(sn1)) { + assertEquals(tableName, region.getTable()); + } + for (RegionInfo region : ADMIN.getRegions(sn2)) { + assertEquals(tableName, region.getTable()); + } + } finally { + removeRegexGroup(groupName); + } + } + + @Test + public void testUngracefulCrashOfRegexMemberReassignsRegionsOnlyToOtherGroupMembers() + throws Exception { + // ServerCrashProcedure recovers a dead server's regions by default with + // hbase.master.scp.retain.assignment=false, i.e. forceNewPlan=true for every ASSIGN TRSP it + // creates -- which routes through roundRobinAssignment rather than retainAssignment (see + // ServerCrashProcedure#assignRegions). This test's job is to confirm + // RSGroupBasedLoadBalancer#roundRobinAssignment still confines the + // crashed regex member's regions to the *other* online members of its own group -- never to + // default or an unrelated group -- rather than trusting the RSGroup-partitioning logic to + // "just work". It also asserts, via the test-only call-tracking flags, exactly which + // assignment methods actually ran for each step: retainAssignment during + // setRSGroup, and roundRobinAssignment (not retainAssignment) during crash recovery. + String groupName = getGroupName("crashroundrobin"); + RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) MASTER.getLoadBalancer(); + ADMIN.addRSGroup(groupName); + // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable + // identities startFakeHostnameRS supports (its RPC bind address is always "localhost" + // regardless of advertised hostname). + setRegex(groupName, "127\\.0\\.0\\.1|localhost"); + // A second, unrelated RSGroup holding one server borrowed from default -- proves crash + // recovery's per-group candidate filtering excludes every other group, not just default. + String otherGroupName = getGroupName("crashroundrobinother"); + RSGroupInfo otherGroupInfo = addGroup(otherGroupName, 1); + ServerName otherGroupServer = getServerName(otherGroupInfo.getServers().iterator().next()); + try { + JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1"); + Address addr1 = addressOf(rst1); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr1)); + ServerName sn1 = getServerName(addr1); + + JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost"); + Address addr2 = addressOf(rst2); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr2)); + ServerName sn2 = getServerName(addr2); + + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + // setRSGroup synchronously relocates every region of a table with no RSGroup-scoped + // descriptor yet onto the target group's servers (moveTablesAndWait blocks on the + // ModifyTableProcedure, whose child ReopenTableRegionsProcedure blocks on the reopened + // regions), so confinement to {sn1, sn2} is already guaranteed once this call returns. + balancer.resetAssignmentCallFlagsForTest(); + ADMIN.setRSGroup(Sets.newHashSet(tableName), groupName); + // ReopenTableRegionsProcedure closes and reassigns every region via a single batched + // retainAssignment call; since none of these regions' previous hosts are group members, + // the wrapped internal balancer's own retain logic (not this class's randomAssignment) + // places them on random candidate hosts within the group -- see BaseLoadBalancer's + // "assigned to random hosts" log path invoked from within retainAssignment. + assertFalse(balancer.isRandomAssignmentInvoked()); + assertTrue(balancer.isRetainAssignmentInvoked()); + assertFalse(balancer.isRoundRobinAssignmentInvoked()); + Map> perServerBeforeCrash = getTableServerRegionMap().get(tableName); + int totalBeforeCrash = 0; + for (Map.Entry> entry : perServerBeforeCrash.entrySet()) { + assertTrue(entry.getKey().equals(sn1) || entry.getKey().equals(sn2)); + totalBeforeCrash += entry.getValue().size(); + } + assertEquals(6, totalBeforeCrash); + + // retainAssignment's random fallback (BaseLoadBalancer#randomAssignment) places each of + // these regions independently, so it is possible -- roughly 3% of runs -- for all 6 to land + // on one of the two servers alone. Crash whichever server actually has regions, so + // ServerCrashProcedure always has at least one region to reassign and roundRobinAssignment + // is guaranteed to fire; default to the usual sn1-crashes/sn2-survives case otherwise. + boolean sn1HasNoRegions = + !perServerBeforeCrash.containsKey(sn1) || perServerBeforeCrash.get(sn1).isEmpty(); + JVMClusterUtil.RegionServerThread rstToCrash = sn1HasNoRegions ? rst2 : rst1; + ServerName survivor = sn1HasNoRegions ? sn1 : sn2; + + // Simulate an ungraceful crash + balancer.resetAssignmentCallFlagsForTest(); + killFakeRegionServer(rstToCrash); + + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> { + Map> perServer = getTableServerRegionMap().get(tableName); + if (perServer == null) { + return false; + } + int total = 0; + for (Map.Entry> entry : perServer.entrySet()) { + // Every region must have landed on the group's sole remaining member -- never on + // default or any other group, and never back on the now-dead server. + if (!entry.getKey().equals(survivor)) { + return false; + } + total += entry.getValue().size(); + } + return total == 6; + }); + // Confirms the crash-recovery ASSIGN TRSPs went through roundRobinAssignment, not + // retainAssignment -- expected since hbase.master.scp.retain.assignment defaults to false. + assertTrue(balancer.isRoundRobinAssignmentInvoked()); + assertFalse(balancer.isRetainAssignmentInvoked()); + + // The surviving group member must host nothing but this table's regions. + for (RegionInfo region : ADMIN.getRegions(survivor)) { + assertEquals(tableName, region.getTable()); + } + // The unrelated group's own member must not have received any of the crashed member's + // regions either -- recovery must stay confined to the crashed server's own group. + for (RegionInfo region : ADMIN.getRegions(otherGroupServer)) { + assertFalse(region.getTable().equals(tableName)); + } + } finally { + removeRegexGroup(groupName); + removeGroup(otherGroupName); + } + } + + @Test + public void testFallbackWhenAllRegexMembersCrash() throws Exception { + // RSGroupBasedLoadBalancer#generateGroupAssignments only ever looks at a group's currently + // online configured servers, regardless of how they became members -- regex-driven or + // admin-driven membership is indistinguishable to it. This confirms the + // hbase.rsgroup.fallback.enable=true fallback-to-default path works the same way when the + // group that loses every one of its servers is regex-governed. + String groupName = getGroupName("fallbackregex"); + ADMIN.addRSGroup(groupName); + // "127.0.0.1" and "localhost" are the only two distinct, real, independently-reachable + // identities startFakeHostnameRS supports (its RPC bind address is always "localhost" + // regardless of advertised hostname). + setRegex(groupName, "127\\.0\\.0\\.1|localhost"); + RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) MASTER.getLoadBalancer(); + balancer.setFallbackEnabledForTest(true); + try { + JVMClusterUtil.RegionServerThread rst1 = startFakeHostnameRS("127.0.0.1"); + Address addr1 = addressOf(rst1); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr1)); + ServerName sn1 = getServerName(addr1); + + JVMClusterUtil.RegionServerThread rst2 = startFakeHostnameRS("localhost"); + Address addr2 = addressOf(rst2); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(groupName).getServers().contains(addr2)); + ServerName sn2 = getServerName(addr2); + + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 6); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + // setRSGroup synchronously relocates every region of a table with no RSGroup-scoped + // descriptor yet onto the target group's servers (moveTablesAndWait blocks on each + // ModifyTableProcedure before returning), so confinement to {sn1, sn2} is already guaranteed + // once this call returns. + ADMIN.setRSGroup(Sets.newHashSet(tableName), groupName); + + // Both regex members must be hosting regions of the table before either is killed. + Map> perServerBeforeCrash = getTableServerRegionMap().get(tableName); + int totalBeforeCrash = 0; + for (ServerName member : new ServerName[] { sn1, sn2 }) { + List regionsOnMember = perServerBeforeCrash.get(member); + assertFalse(regionsOnMember == null || regionsOnMember.isEmpty()); + totalBeforeCrash += regionsOnMember.size(); + } + assertEquals(6, totalBeforeCrash); + + // Kill both regex members -- the group is left with zero online servers. Unlike the base + // cluster's servers (which remain in default throughout this test), these two fake RS were + // never part of default, so default keeps its original online servers and the fallback + // path lands there directly, with no need for a second-level fallback. + killFakeRegionServer(rst1); + killFakeRegionServer(rst2); + + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> { + Map> perServer = getTableServerRegionMap().get(tableName); + if (perServer == null) { + return false; + } + RSGroupInfo defaultGroup; + try { + defaultGroup = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP); + } catch (IOException e) { + return false; + } + int total = 0; + for (Map.Entry> entry : perServer.entrySet()) { + // Every region must have landed on a server that is currently in default -- never left + // stranded on the now-dead sn1/sn2, and never on any unrelated group. + if (!defaultGroup.getServers().contains(entry.getKey().getAddress())) { + return false; + } + total += entry.getValue().size(); + } + return total == 6; + }); + } finally { + balancer.setFallbackEnabledForTest(false); + removeRegexGroup(groupName); + } + } + + @Test + public void testExplicitMoveFromDefaultToAdminGroupTriggersRandomAssignmentBackToDefault() + throws Exception { + // Three concurrently active RSGroups -- default, a regex-governed group, and an + // admin-managed group -- each with at least one member. Explicitly moving a server out of + // default into the admin-managed group (RSGroupInfoManagerImpl#moveServers) leaves that + // server still hosting regions of a table that belongs to default; moveServerRegionsFromGroup + // detects the mismatch and relocates those orphaned regions back onto default's *remaining* + // members via LoadBalancer#randomAssignment, then actually executes each relocation through + // AssignmentManager#moveAsync -- which, because the destination is already decided, creates a + // MOVE-type TransitRegionStateProcedure whose queueAssign sets a non-null region location, so + // AssignmentManager#processAssignQueue routes it into retainMap and retainAssignment also + // fires. roundRobinAssignment never fires, since no region in this flow is ever left with a + // null location. This test confirms that exact combination of calls, plus the final, + // fully-partitioned region placement across all three groups. + RSGroupBasedLoadBalancer balancer = (RSGroupBasedLoadBalancer) MASTER.getLoadBalancer(); + String regexGroupName = getGroupName("regex3way"); + String adminGroupName = getGroupName("admin3way"); + ADMIN.addRSGroup(regexGroupName); + // "127.0.0.1" is the only fake identity needed here; it is never a member of default, so it + // cannot interfere with the base cluster's servers used below. + setRegex(regexGroupName, "127\\.0\\.0\\.1"); + try { + JVMClusterUtil.RegionServerThread regexRst = startFakeHostnameRS("127.0.0.1"); + Address regexAddr = addressOf(regexRst); + TEST_UTIL.waitFor(WAIT_TIMEOUT, + () -> ADMIN.getRSGroup(regexGroupName).getServers().contains(regexAddr)); + + // Admin-managed group starts with one server borrowed from default -- NUM_SLAVES_BASE(4) + // - 1 = 3 remain in default, enough that moving one more away below still satisfies + // moveServers' KEEP_ONE_SERVER_IN_DEFAULT_ERROR_MESSAGE constraint (default must always + // retain more servers than are being moved out of it). + RSGroupInfo adminGroupInfo = addGroup(adminGroupName, 1); + Address adminServerInitialAddr = adminGroupInfo.getServers().iterator().next(); + + // A table with no explicit RSGroup is implicitly confined to default's current members -- + // here, the 3 servers remaining in default after the admin group's initial carve-out. + TEST_UTIL.createMultiRegionTable(tableName, Bytes.toBytes("f"), 9); + TEST_UTIL.waitUntilAllRegionsAssigned(tableName); + + // Pick a default-group server that is actually hosting some of this table's regions -- the + // server we are about to move, so the orphaned-region relocation has something to do. + RSGroupInfo defaultGroupBeforeMove = ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP); + Map> perServerBeforeMove = getTableServerRegionMap().get(tableName); + int totalRegionsBeforeMove = 0; + ServerName movingServer = null; + for (Map.Entry> entry : perServerBeforeMove.entrySet()) { + assertTrue(defaultGroupBeforeMove.getServers().contains(entry.getKey().getAddress())); + totalRegionsBeforeMove += entry.getValue().size(); + if (movingServer == null && !entry.getValue().isEmpty()) { + movingServer = entry.getKey(); + } + } + assertEquals(9, totalRegionsBeforeMove); + Address movingAddr = movingServer.getAddress(); + Set
defaultServersBeforeMove = new HashSet<>(defaultGroupBeforeMove.getServers()); + + // The action under test: explicitly move that one server out of default into the + // admin-managed group. + balancer.resetAssignmentCallFlagsForTest(); + ADMIN.moveServersToRSGroup(Sets.newHashSet(movingAddr), adminGroupName); + + // See the method comment above for why both randomAssignment (picking the relocation plan) + // and retainAssignment (executing it via moveAsync) fire, while roundRobinAssignment never + // does. + assertTrue(balancer.isRandomAssignmentInvoked()); + assertTrue(balancer.isRetainAssignmentInvoked()); + assertFalse(balancer.isRoundRobinAssignmentInvoked()); + + // Membership: the moved server now belongs to the admin-managed group, not default; the + // regex group and the rest of default are untouched by this move. + Set
expectedDefaultServersAfterMove = new HashSet<>(defaultServersBeforeMove); + expectedDefaultServersAfterMove.remove(movingAddr); + assertEquals(expectedDefaultServersAfterMove, + ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers()); + assertEquals(Sets.newHashSet(adminServerInitialAddr, movingAddr), + ADMIN.getRSGroup(adminGroupName).getServers()); + assertEquals(Sets.newHashSet(regexAddr), ADMIN.getRSGroup(regexGroupName).getServers()); + + // Regions: every one of the table's 9 regions must have moved off the relocated server, + // landing only on default's remaining members -- never on the admin-managed or + // regex-governed group's servers. moveServers blocks until this relocation completes, but + // poll defensively rather than asserting immediately. + TEST_UTIL.waitFor(WAIT_TIMEOUT, () -> { + Map> perServer = getTableServerRegionMap().get(tableName); + if (perServer == null) { + return false; + } + int total = 0; + for (Map.Entry> entry : perServer.entrySet()) { + if (!expectedDefaultServersAfterMove.contains(entry.getKey().getAddress())) { + return false; + } + total += entry.getValue().size(); + } + return total == 9; + }); + + // Bidirectional isolation, now that the move has settled: neither the admin-managed group's + // servers (the original member and the one just moved in) nor the regex group's server host + // any region of this default-group table. + for (Address addr : ADMIN.getRSGroup(adminGroupName).getServers()) { + for (RegionInfo region : ADMIN.getRegions(getServerName(addr))) { + assertFalse(region.getTable().equals(tableName)); + } + } + for (RegionInfo region : ADMIN.getRegions(getServerName(regexAddr))) { + assertFalse(region.getTable().equals(tableName)); + } + } finally { + removeRegexGroup(regexGroupName); + removeGroup(adminGroupName); + } + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java index 7e69c377edbb..728c608d5c3b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/rsgroup/VerifyingRSGroupAdmin.java @@ -899,6 +899,13 @@ public List getLogEntries(Set serverNames, String logType, return admin.getLogEntries(serverNames, logType, serverType, limit, filterParams); } + private static RSGroupInfo withoutServers(RSGroupInfo group, Set
ignored) { + RSGroupInfo copy = new RSGroupInfo(group); + group.getConfiguration().forEach(copy::setConfiguration); + ignored.forEach(copy::removeServer); + return copy; + } + private void verify() throws IOException { Map groupMap = Maps.newHashMap(); Set zList = Sets.newHashSet(); @@ -938,9 +945,43 @@ private void verify() throws IOException { } } + // Servers of regex-governed groups are never persisted: storage and ZK must hold none. + Map regexGroupMap = + RSGroupInfoManagerImpl.getRegexGroupMap(admin.getConfiguration()); + for (String regexGroup : regexGroupMap.keySet()) { + RSGroupInfo persisted = groupMap.get(regexGroup); + if (persisted != null) { + assertTrue(persisted.getServers().isEmpty(), + "regex-governed group " + regexGroup + " must persist no servers"); + } + } + groupMap.put(RSGroupInfo.DEFAULT_GROUP, new RSGroupInfo(RSGroupInfo.DEFAULT_GROUP, lives, tables)); - assertEquals(Sets.newHashSet(groupMap.values()), Sets.newHashSet(admin.listRSGroups())); + // Regex-driven membership is never persisted and lags an explicit move or a config change until + // the next server event, so servers a regex matches, or that sit in a regex-governed group, are + // not compared. + List liveGroups = admin.listRSGroups(); + Set
ignored = Sets.newHashSet(); + for (RSGroupInfo group : liveGroups) { + for (Address server : group.getServers()) { + if ( + regexGroupMap.containsKey(group.getName()) || regexGroupMap.values().stream() + .anyMatch(p -> p.matcher(server.getHostName()).matches()) + ) { + ignored.add(server); + } + } + } + Set expected = Sets.newHashSet(); + for (RSGroupInfo group : groupMap.values()) { + expected.add(withoutServers(group, ignored)); + } + Set actual = Sets.newHashSet(); + for (RSGroupInfo group : liveGroups) { + actual.add(withoutServers(group, ignored)); + } + assertEquals(expected, actual); try { String groupBasePath = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode, "rsgroup"); for (String znode : ZKUtil.listChildrenNoWatch(zkw, groupBasePath)) { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java index 66838282d64b..73c97f8a4b05 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestRegionMoverFilterRSGroupServers.java @@ -45,9 +45,9 @@ * and servers are moved between groups via the real {@code moveServersToRSGroup} admin call -- no * test constructs an {@link RSGroupInfo} by hand. Every {@code RSGroupInfo} used here is read back * from the master via {@code admin.getRSGroup(...)}, which computes "default" membership as "online - * servers minus servers claimed by other groups" (see RSGroupInfoManagerImpl#getDefaultServers), so - * its membership shrinks on its own once a move happens instead of being asserted into existence by - * the test. + * servers minus servers claimed by other groups" (see + * RSGroupInfoManagerImpl#computeAutoManagedRSGroupServers), so its membership shrinks on its own + * once a move happens instead of being asserted into existence by the test. */ @Tag(MiscTests.TAG) @Tag(MediumTests.TAG) diff --git a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/configuration/dynamic.mdx b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/configuration/dynamic.mdx index 0a4a575e0897..ba2b6e086385 100644 --- a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/configuration/dynamic.mdx +++ b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/configuration/dynamic.mdx @@ -5,6 +5,13 @@ description: "How to change HBase configuration properties dynamically without s It is possible to change a subset of the configuration without requiring a server restart. In the HBase shell, the operations `update_config`, `update_all_config` and `update_rsgroup_config` will prompt a server, all servers or all servers in the RSGroup to reload configuration. +A change to `hbase.rsgroup.regex.*` properties is applied by the master in one of two ways: + +- **At HMaster restart (recommended).** Restart every master, including backup masters. The membership of all regex-governed groups is recomputed at once. +- **At the next server event, after a reload.** The master reads these properties on every regionserver join or leave. A reload therefore takes effect **only at the next server event**, not immediately. It does not recompute the in-memory membership of a group, and removing a regex by reload is not safe. + +Do not use a configuration reload to change these properties. See [Regex-Based Membership](/docs/operational-management/region-and-capacity#regex-based-membership). + Only a subset of all configurations can currently be changed in the running server. Here are those configurations: Configurations that support dynamically change diff --git a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/region-and-capacity.mdx b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/region-and-capacity.mdx index 376a471def1b..a6e18c4c83b4 100644 --- a/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/region-and-capacity.mdx +++ b/hbase-website/app/pages/_docs/docs/_mdx/(multi-page)/operational-management/region-and-capacity.mdx @@ -192,7 +192,51 @@ Servers come and go over the lifetime of a Cluster. Currently, you must manually Please use `Admin.removeServersFromRSGroup` or shell command _remove_servers_rsgroup_ to remove decommission servers from rsgroup. -The `default` group is not like other rsgroups in that it is dynamic. Its server list mirrors the current state of the cluster; i.e. if you shutdown a server that was part of the `default` rsgroup, and then do a _get_rsgroup_ `default` to list its content in the shell, the server will no longer be listed. For non-default groups, though a mode may be offline, it will persist in the non-default group's list of servers. But if you move the offline server from the non-default rsgroup to default, it will not show in the `default` list. It will just be dropped. +The `default` group, and every regex-governed group (see [Regex-Based Membership](#regex-based-membership)), are dynamic. Their server lists mirror the current state of the cluster; i.e. if you shutdown a server that was part of one of them, and then do a _get_rsgroup_ on it to list its content in the shell, the server will no longer be listed. For other non-default rsgroups, though a node may be offline, it will persist in the group's list of servers. But if you move the offline server from the non-default rsgroup to default, it will not show in the `default` list. It will just be dropped. + +### Regex-Based Membership + +Instead of moving each server into a group by hand, a non-default rsgroup can be made _regex-governed_: regionservers whose hostname matches a regular expression are placed in the group automatically as they register, and removed again when they go away. Set one property per group, where the suffix is the name of an existing rsgroup: + +```xml + + hbase.rsgroup.regex.my_group + rs-batch-[0-9]+\.example\.com + +``` + +- The regex is matched against the server's hostname only (not the port), and must match the whole hostname. +- Regexes are evaluated by the master on every regionserver join and leave, with no timeout. Like the rest of `hbase-site.xml`, they are trusted configuration, so keep them simple: a pattern prone to catastrophic backtracking (for example nested quantifiers such as `(a+)+`) can stall regionserver registration and expiry. +- The group must already exist (`add_rsgroup`); the `default` group cannot be regex-governed. +- A hostname that matches the regexes of more than one group is treated as unmatched and stays in `default` until the overlap is fixed. Invalid regexes and invalid group names are ignored with a warning in the master log. +- Membership of a regex-governed group follows the live servers and is recomputed whenever a server joins or leaves, or when the HMaster restarts, like the `default` group. It is never stored in `hbase:rsgroup` or ZooKeeper: a regex-governed group is persisted with no servers, so a server event costs no extra I/O. +- The regex always wins. A manual move that contradicts a regex is rejected, because the next server event or master restart would put the server back where its hostname says: a server matching a regex can only be moved to its regex-governed group, and a server that does not match cannot be moved into one. A regex-governed group cannot be removed while it has servers in the master's in-memory view. Remove the `hbase.rsgroup.regex.` property from the configuration of all masters, including backup masters, and restart all HMasters. The group is then an ordinary admin-managed group with no servers, and its former servers go to `default`. This holds only if you did not reload the configuration first and a group change stored the servers in the meantime; if you did, move the servers out of the group by hand. Then remove the group. Removal also fails while a table or a namespace refers to the group. +- Set the same `hbase.rsgroup.regex.*` properties on every master, including backup masters. Membership is computed from the active master's configuration, so a master that takes over with different properties would regroup the servers. +- To add, change or remove a regex, change the configuration of all masters and restart the HMasters. Do not reload the configuration; reload is not recommended for these properties. A restart recomputes the membership of every regex-governed group at once. A reload recomputes nothing: servers move only at the next server event, and the in-memory membership of a group stays as it was until then. Group operations do not recompute it either: `add_rsgroup` and a group rename do not apply a regex, so a regex for a group that does not exist yet is ignored until a later server event or restart. +- Adding or changing a regex for an existing group removes the servers that the regex does not match, but not at once. They leave the group at the next server event or master restart, and tables bound to the group then have fewer servers. Before you add or change a regex for an existing group, check the group's current servers against the regex. + + + **Removing a regex, or breaking it (typo, invalid regex), turns the group back into an ordinary + admin-managed group.** Its servers are not stored, so the group has none after a master restart; + until then, servers it already holds stay in it, and a server that joins later is placed in + `default`. Restart all masters, including backup masters, to settle it. A configuration reload is + not enough: it leaves the in-memory servers in the group, and a later change to a group stores + them as admin-managed members. Tables bound to the group may then have no regionserver of their + own (see `hbase.rsgroup.fallback.enable` below). Invalid regexes and a regex-governed group with + no online servers are both logged at WARN in the master log. + + + + Set `hbase.rsgroup.fallback.enable` to `true` whenever you use regex-based membership, unless you + have a very good reason not to. Membership follows the live servers, so a regex-governed group (or + `default`) can lose all of its servers when pods go down, when a regex is mistyped or removed, or + when the regexes cover every regionserver. Without fallback, regions of tables bound to that group + stay unassigned until a matching server returns. + + +When you enable a regex for servers that were previously placed in groups by hand, no manual cleanup is needed: after the HMaster restart, the regex places each matching server in its group at once (a reload would do so only at the next server event), any stale copy stored for it is cleared on the next flush, and a server stored in a regex-governed group that no longer matches its regex returns to `default`. The master logs a WARN naming the group the stale copy was in. + +Regex placement is always applied, even if it leaves the `default` group without any online servers (for example, when the regexes cover every regionserver). The master logs a warning in that case. Tables in `default`, including system tables unless they live in another rsgroup, then have no regionserver of their own, so set `hbase.rsgroup.fallback.enable` to `true` to let their regions fall back to the servers of other groups. Without it, those regions stay unassigned until a server that matches no regex, and is in no other group, registers. ### Best Practice @@ -208,6 +252,8 @@ Yahoo! Have found it useful at their scale to keep a special rsgroup of dead or Be careful replacing dead nodes in an rsgroup. Ensure there are enough live nodes before you start moving out the dead. Move in good live nodes first if you have to. +Servers that match a `hbase.rsgroup.regex.` entry cannot be moved into a dead-nodes group; the move is rejected because the regex decides their group. To park such a server, change the regex (for example, exclude it by hostname) and restart the HMaster to apply the change. A configuration reload is not recommended. + ### Troubleshooting [#region-and-capacity-troubleshooting] Viewing the Master log will give you insight on rsgroup operation. @@ -234,6 +280,8 @@ But if you change the 'hbase.balancer.rsgroup.enabled' to true, the old rsgroup hbase(main):005:0> move_tables_rsgroup 'default',['t1'] ``` +- Remove every `hbase.rsgroup.regex.*` property from the configuration on **all masters**, and restart all HMasters, including backup masters. Do not only reload the configuration. Moves of servers that match a regex are rejected while the property is set. After the restart, the regex-governed groups have no servers and their former servers are in `default`. + - Move all regionservers in non-default rsgroups to `default` regionserver group ```text