Elasticsearch 源码探究 001——故障探测和恢复机制

map.put(Names.SAME, ThreadPoolType.DIRECT);

static final ExecutorService DIRECT_EXECUTOR = EsExecutors.newDirectExecutorService();

executors.put(Names.SAME, new ExecutorHolder(DIRECT_EXECUTOR, new Info(Names.SAME, ThreadPoolType.DIRECT)));

@Override
public void execute(Runnable command){
    command.run(); // 直接运行，并没有实际的线程池
    rethrowErrors(command);
}

this.followersChecker = new FollowersChecker(settings, transportService, this::onFollowerCheckRequest, this::removeNode,
            nodeHealthService);

private void removeNode(DiscoveryNode discoveryNode, String reason){
        synchronized (mutex) {
            if (mode == Mode.LEADER) {
        // 提交一个node-left状态更新任务
                masterService.submitStateUpdateTask("node-left",
                    new NodeRemovalClusterStateTaskExecutor.Task(discoveryNode, reason),
                    ClusterStateTaskConfig.build(Priority.IMMEDIATE),
                    nodeRemovalExecutor,
                    nodeRemovalExecutor);
            }
        }
    }

@Override
    public ClusterTasksResult<Task> execute(final ClusterState currentState, final List<Task> tasks) throws Exception {
        final DiscoveryNodes.Builder remainingNodesBuilder = DiscoveryNodes.builder(currentState.nodes());
        boolean removed = false;
    // 如果任务指定的节点存在于当前集群状态中，则从DiscoveryNodes.Builder对象中删除该节点，
    // 并将removed标志设置为true。如果该任务指定的节点不存在于当前集群状态中，则记录一条调试级别的日志，表示忽略该任务。
        for (final Task task : tasks) {
            if (currentState.nodes().nodeExists(task.node())) {
                remainingNodesBuilder.remove(task.node());
                removed = true;
            } else {
                logger.debug("node [{}] does not exist in cluster state, ignoring", task);
            }
        }
    // 如果没有删除任何节点，则返回当前的集群状态
        if (!removed) {
            // no nodes to remove, keep the current cluster state
            return ClusterTasksResult.<Task>builder().successes(tasks).build(currentState);
        }
    // 剩余节点
        final ClusterState remainingNodesClusterState = remainingNodesClusterState(currentState, remainingNodesBuilder);

        return getTaskClusterTasksResult(currentState, tasks, remainingNodesClusterState);
    }

protected ClusterTasksResult<Task> getTaskClusterTasksResult(ClusterState currentState, List<Task> tasks,
                                                                 ClusterState remainingNodesClusterState){
        ClusterState ptasksDisassociatedState = PersistentTasksCustomMetadata.disassociateDeadNodes(remainingNodesClusterState);
        final ClusterTasksResult.Builder<Task> resultBuilder = ClusterTasksResult.<Task>builder().successes(tasks);
        return resultBuilder.build(allocationService.disassociateDeadNodes(ptasksDisassociatedState, true, describeTasks(tasks)));
    }

private void disassociateDeadNodes(RoutingAllocation allocation){
        for (Iterator<RoutingNode> it = allocation.routingNodes().mutableIterator(); it.hasNext(); ) {
            RoutingNode node = it.next();
      // 如果该节点的ID存在于RoutingAllocation对象的数据节点列表中，则表示该节点是活动节点，继续遍历下一个节点
            if (allocation.nodes().getDataNodes().containsKey(node.nodeId())) {
                // its a live node, continue
                continue;
            }
      // 否则，这个节点就离线。
      // 对于每个已离线的节点，该方法遍历该节点上的所有分片，并为每个分片创建一个UnassignedInfo对象，表示该分片未分配的原因是节点离线。
            // 然后，该方法调用failShard方法，将该分片标记为失败，并将其移动到未分配状态。该方法还会从RoutingNode列表中删除已离线的节点
            // now, go over all the shards routing on the node, and fail them
            for (ShardRouting shardRouting : node.copyShards()) {
                final IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
                boolean delayed = INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.get(indexMetadata.getSettings()).nanos() > 0;
                UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "node_left [" + node.nodeId() + "]",
                    null, 0, allocation.getCurrentNanoTime(), System.currentTimeMillis(), delayed, AllocationStatus.NO_ATTEMPT,
                    Collections.emptySet());
                allocation.routingNodes().failShard(logger, shardRouting, unassignedInfo, indexMetadata, allocation.changes());
            }
            // its a dead node, remove it, note, its important to remove it *after* we apply failed shard
            // since it relies on the fact that the RoutingNode exists in the list of nodes
            it.remove();
        }
    }

    // fail actual shard
        if (failedShard.initializing()) {
            if (failedShard.relocatingNodeId() == null) {
                if (failedShard.primary()) {
                    // promote active replica to primary if active replica exists (only the case for shadow replicas)
                    unassignPrimaryAndPromoteActiveReplicaIfExists(failedShard, unassignedInfo, routingChangesObserver);
                } else {
                    // initializing shard that is not relocation target, just move to unassigned
                    moveToUnassigned(failedShard, unassignedInfo);
                }
            } 

private void unassignPrimaryAndPromoteActiveReplicaIfExists(ShardRouting failedShard, UnassignedInfo unassignedInfo,
                                                                RoutingChangesObserver routingChangesObserver){
        assert failedShard.primary();
    // 这里会从所有副本分片中找最高版本的副本
        ShardRouting activeReplica = activeReplicaWithHighestVersion(failedShard.shardId());
        if (activeReplica == null) {
            moveToUnassigned(failedShard, unassignedInfo);
        } else {
      // 将主分片移动到未分配状态，并将其降级为副本， 这里还会将主分片加入到unassignedShards列表中， 根据索引的配置会做分片恢复
            movePrimaryToUnassignedAndDemoteToReplica(failedShard, unassignedInfo);
      // 将活动副本升级为主分片，并通知路由变更观察器进行相应的处理
            promoteReplicaToPrimary(activeReplica, routingChangesObserver);
        }
    }

private void onFailure(Node node){
    while(true) {
        DeadHostState previousDeadHostState =
            blacklist.putIfAbsent(node.getHost(), new DeadHostState(DeadHostState.DEFAULT_TIME_SUPPLIER));
        if (previousDeadHostState == null) {
            if (logger.isDebugEnabled()) {
                logger.debug("added [" + node + "] to blacklist");
            }
            break;
        }
        if (blacklist.replace(node.getHost(), previousDeadHostState,
                new DeadHostState(previousDeadHostState))) {
            if (logger.isDebugEnabled()) {
                logger.debug("updated [" + node + "] already in blacklist");
            }
            break;
        }
    }
    failureListener.onFailure(node);
}

static Iterable<Node> selectNodes(NodeTuple<List<Node>> nodeTuple, Map<HttpHost, DeadHostState> blacklist,
                                      AtomicInteger lastNodeIndex, NodeSelector nodeSelector) throws IOException {
        /*
         * Sort the nodes into living and dead lists.
         */
        List<Node> livingNodes = new ArrayList<>(Math.max(0, nodeTuple.nodes.size() - blacklist.size()));
        List<DeadNode> deadNodes = new ArrayList<>(blacklist.size());
        for (Node node : nodeTuple.nodes) {
            DeadHostState deadness = blacklist.get(node.getHost());
      // 没有被列入黑名单或已经到了重试时间的节点 , 默认是1min重试间隔， 只有1min之后才会再次访问这个节点， 无法修改，写死的
            if (deadness == null || deadness.shallBeRetried()) {
                livingNodes.add(node);
            } else {
                deadNodes.add(new DeadNode(node, deadness));
            }
        }

<dependency>
  <groupId>org.elasticsearch.client</groupId>
  <artifactId>elasticsearch-rest-high-level-client</artifactId>
  <version>7.10.2</version>
</dependency>