图解 Kafka 生产者元数据拉取管理全流程（下）

public final class Cluster {
    private final boolean isBootstrapConfigured;
    // kafka集群中的broker节点
    private final List<Node> nodes;
    // 未授权的主题集合
    private final Set<String> unauthorizedTopics;
    // 无效的主题集合
    private final Set<String> invalidTopics;
    // 内部主题集合
    private final Set<String> internalTopics;
    // controller 节点
    private final Node controller;
    // topic对应的 partition 信息字典,存放的 partition 不一定有 Leader 副本, 键为topic,值为 partition 信息集合。
    private final Map<TopicPartition, PartitionInfo> partitionsByTopicPartition;
    // 键为topic,值为可用 partition 信息集合,存放的 partition 一定有 Leader 副本
    private final Map<String, List<PartitionInfo>> partitionsByTopic;
    // 键为broker的id,值为partition 信息集合
    private final Map<String, List<PartitionInfo>> availablePartitionsByTopic;
    // 键为broker的id,值为表示该节点的node实例
    private final Map<Integer, List<PartitionInfo>> partitionsByNode;
    private final Map<Integer, Node> nodesById;
    private final ClusterResource clusterResource;
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.

 public class Node {
    // broker 节点id
    private final int id;
    // broker 节点id的字符串
    private final String idString;
    // broker 节点的地址用于socket连接
    private final String host;
    // 端口
    private final int port;
    // broker 节点的机架
    private final String rack;
    // 初始化节点属性
    public Node(int id, String host, int port, String rack) {
        this.id = id;
        this.idString = Integer.toString(id);
        this.host = host;
        this.port = port;
        this.rack = rack;
    }
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.

public class PartitionInfo {
    // 主题
    private final String topic;
    // 分区编号id
    private final int partition;
    // 分区 Leader 副本信息,唯一进行通信的节点
    private final Node leader;
    // 全部副本信息
    private final Node[] replicas;
    // ISR 副本信息, follower角色
    private final Node[] inSyncReplicas;
    // 离线副本信息
    private final Node[] offlineReplicas;
    // 初始化分区信息
    public PartitionInfo(String topic,
                         int partition,
                         Node leader,
                         Node[] replicas,
                         Node[] inSyncReplicas,
                         Node[] offlineReplicas) {
        this.topic = topic;
        this.partition = partition;
        this.leader = leader;
        this.replicas = replicas;
        this.inSyncReplicas = inSyncReplicas;
        this.offlineReplicas = offlineReplicas;
    }
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.

public final class TopicPartition implements Serializable {
    // hash值，用来hashCode方法缓存
    private int hash = 0;
    // 分区编号
    private final int partition;
    // 主题名称
    private final String topic;
    // 初始化主题分区索引
    public TopicPartition(String topic, int partition) {
        this.partition = partition;
        this.topic = topic;
    }
    // 计算hash值
    public int hashCode() {
        if (hash != 0)
            return hash;
        final int prime = 31;
        int result = 1;
        result = prime * result + partition;
        result = prime * result + Objects.hashCode(topic);
        this.hash = result;
        return result;
    }
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.

// 向 topic 异步地发送数据，当发送确认后唤起回调函数
public Future<RecordMetadata> send(ProducerRecord<K, V> record, Callback callback) {
   ProducerRecord<K, V> interceptedRecord = this.interceptors.onSend(record);
   // 调用 doSend 发送，支持回调函数
   return doSend(interceptedRecord, callback);
}

private Future<RecordMetadata> doSend(ProducerRecord<K, V> record, Callback callback) {
        TopicPartition tp = null;
        try {
            // 假如 sender 线程为空或者不运行，报错
            throwIfProducerClosed();
            // 当前时间【毫秒】 
            long nowMs = time.milliseconds();
            ClusterAndWaitTime clusterAndWaitTime;
            try {
                // 等待元数据更新
                clusterAndWaitTime = waitOnMetadata(record.topic(), record.partition(), nowMs, maxBlockTimeMs);
            } catch (KafkaException e) {
                if (metadata.isClosed())
                    throw new KafkaException("Producer closed while send in progress", e);
                throw e;
            }
            nowMs += clusterAndWaitTime.waitedOnMetadataMs;
            long remainingWaitMs = Math.max(0, maxBlockTimeMs - clusterAndWaitTime.waitedOnMetadataMs);
            Cluster cluster = clusterAndWaitTime.cluster;
            ....
            // 将消息记录追加到消息缓冲区
            RecordAccumulator.RecordAppendResult result = accumulator.append(tp, timestamp, serializedKey,
                    serializedValue, headers, interceptCallback, remainingWaitMs, true, nowMs);
                    
            // 当批次满了或者新批次被创建了，开始唤醒sender线程数据发送
            if (result.batchIsFull || result.newBatchCreated) {
                this.sender.wakeup();
            }
            return result.future;
        } catch (ApiException e) {
            ....
            return new FutureFailure(e);
        } catch (InterruptedException e) {
            ....
            throw new InterruptException(e);
        } catch (KafkaException e) {
            ....
            throw e;
        } catch (Exception e) {
            ....
            throw e;
        }
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.

private ClusterAndWaitTime waitOnMetadata(String topic, Integer partition, long nowMs, long maxWaitMs) throws InterruptedException {
        // 1. 从元数据缓存中获取元数据
        Cluster cluster = metadata.fetch();
        // 2. 判断该主题是否是无效的主题
        if (cluster.invalidTopics().contains(topic))
            throw new InvalidTopicException(topic);
        // 3. 将该主题放入元数据主题列表中
        metadata.add(topic, nowMs);
        // 4. 从元数据缓存中获取主题对应的分区数
        Integer partitionsCount = cluster.partitionCountForTopic(topic);
        // 5. 满足这些条件后就不用拉取元数据，直接从元数据缓存中返回
        if (partitionsCount != null && (partition == null || partition < partitionsCount))
            // 返回元数据信息
            return new ClusterAndWaitTime(cluster, 0);

        long remainingWaitMs = maxWaitMs;
        long elapsed = 0;
        // 6. 不停的轮询唤醒 sender 线程更新元数据
        do {
            // 7. 将主题 topic及过期时间添加到元数据主题列表中
            metadata.add(topic, nowMs + elapsed);
            // 8. 标记元数据更新标识，获取元数据版本号
            int version = metadata.requestUpdateForTopic(topic);
            // 9. 唤醒 sender 子线程
            sender.wakeup();
            try {
                // 10. 阻塞线程等待元数据更新成功
                metadata.awaitUpdate(version, remainingWaitMs);
            } catch (TimeoutException ex) {
                ...
            }
            // 11. 到这里，应该是当前cluster更新成功了，再次获取元数据
            cluster = metadata.fetch();
            // 12. 计算等待更新完成元数据消耗时间
            elapsed = time.milliseconds() - nowMs;
            // 13. 如果超时，抛超时异常
            if (elapsed >= maxWaitMs) {
               ...
            }
            metadata.maybeThrowExceptionForTopic(topic);
            remainingWaitMs = maxWaitMs - elapsed;
            // 14. 获取元数据分区数
            partitionsCount = cluster.partitionCountForTopic(topic);
        } while (partitionsCount == null || (partition != null && partition >= partitionsCount));
        // 15. 返回元数据以及消耗时间
        return new ClusterAndWaitTime(cluster, elapsed);
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.

 public class Sender implements Runnable {
     public void run() {
        // running 字段用来标识当前 Sender 线程是否正常执行
        while (running) {
            try {
                // 如果正常运行，则执行运行周期
                runOnce();
            } catch (Exception e) {
                log.error("Uncaught error in kafka producer I/O thread: ", e);
            }
        }
    }
    
    void runOnce() {
        .....  // 此处省略事务消息相关的处理逻辑
        long currentTimeMs = time.milliseconds();
        // 创建发送到 kafka 集群的请求
        long pollTimeout = sendProducerData(currentTimeMs);
        // 真正执行网络IO的地方，会将请求发送出去，并处理收到的响应
        client.poll(pollTimeout, currentTimeMs);
    }
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.

 private long sendProducerData(long now) {
        // 从元数据缓存中获取元数据
        Cluster cluster = metadata.fetch();
        // 通过元数据cluster获取要发送的节点 Leader 分区信息
        RecordAccumulator.ReadyCheckResult result = this.accumulator.ready(cluster, now);
        // 如果主题的 Leader 分区对应的节点不存在就强制更新元数据
        if (!result.unknownLeaderTopics.isEmpty()) {
            for (String topic : result.unknownLeaderTopics)
                // 将主题加入元数据主题列表
                this.metadata.add(topic, now);
            // 强制标记元数据更新标识
            this.metadata.requestUpdate();
        }
      ..... // 忽略能发送的节点数据
  }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.

public class NetworkClient implements KafkaClient {
   public List<ClientResponse> poll(long timeout, long now) {
        ....
        // 尝试更新元数据
        long metadataTimeout = metadataUpdater.maybeUpdate(now);
        try {
            // 执行IO操作
            this.selector.poll(Utils.min(timeout, metadataTimeout, defaultRequestTimeoutMs));
        } catch (IOException e) {
            log.error("Unexpected error during I/O", e);
        }
        ....
        List<ClientResponse> responses = new ArrayList<>();
        // 处理更新命令的返回
        handleCompletedReceives(responses, updatedNow);
        return responses;
   }
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.

 public class NetworkClient implements KafkaClient {
    private NetworkClient(MetadataUpdater metadataUpdater,
                          Metadata metadata,
                          Selectable selector,
                          String clientId,
                          int maxInFlightRequestsPerConnection,
                          long reconnectBackoffMs,
                          long reconnectBackoffMax,
                          int socketSendBuffer,
                          int socketReceiveBuffer,
                          int defaultRequestTimeoutMs,
                          long connectionSetupTimeoutMs,
                          long connectionSetupTimeoutMaxMs,
                          ClientDnsLookup clientDnsLookup,
                          Time time,
                          boolean discoverBrokerVersions,
                          ApiVersions apiVersions,
                          Sensor throttleTimeSensor,
                          LogContext logContext) {
       
        if (metadataUpdater == null) {
            // 通过这里可以看出 metadataUpdate 实例化对象
            this.metadataUpdater = new DefaultMetadataUpdater(metadata);
        } else {
            this.metadataUpdater = metadataUpdater;
        }
        ...
    } 
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.

 public long maybeUpdate(long now) {
      // 计算下次要更新元数据的时间，其中会检测needUpdate的值、退避时间、是否长时间未更新
      long timeToNextMetadataUpdate = metadata.timeToNextUpdate(now);
      // 检测是否已经发送了元数据的请求，如果一个元数据的请求，还未从服务端返回，那么时间设置为 waitForMetadataFetch（默认30s）
      long waitForMetadataFetch = hasFetchInProgress() ? defaultRequestTimeoutMs : 0;
      // 计算元数据超时时间
      long metadataTimeout = Math.max(timeToNextMetadataUpdate, waitForMetadataFetch);
      if (metadataTimeout > 0) {
          return metadataTimeout;
      }
      // 表示需要立即更新，取最空闲的节点node
      Node node = leastLoadedNode(now);
      // 发送更新元数据的请求     
      return maybeUpdate(now, node);
  }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.

public class Metadata implements Closeable {
  // 计算下次更新元数据的时间
  public synchronized long timeToNextUpdate(long nowMs) {
      long timeToExpire = updateRequested() ? 0 : Math.max(this.lastSuccessfulRefreshMs + this.metadataExpireMs - nowMs, 0);
      return Math.max(timeToExpire, timeToAllowUpdate(nowMs));
  }
  
  // 判断这2个标识是否为true
  public synchronized boolean updateRequested() {
      return this.needFullUpdate || this.needPartialUpdate;
  }
  // 计算允许更新元数据的时机
  public synchronized long timeToAllowUpdate(long nowMs) {
      return Math.max(this.lastRefreshMs + this.refreshBackoffMs - nowMs, 0);
  }
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.

private long maybeUpdate(long now, Node node) {
      String nodeConnectionId = node.idString();
      // 判断当前node的状态是否可以发送Request请求
      if (canSendRequest(nodeConnectionId, now)) {
          // 构建元数据请求
          Metadata.MetadataRequestAndVersion requestAndVersion = metadata.newMetadataRequestAndVersion(now);
          MetadataRequest.Builder metadataRequest = requestAndVersion.requestBuilder;
          // 向 nodeConnectionId 发送元数据请求
          sendInternalMetadataRequest(metadataRequest, nodeConnectionId, now);
          inProgress = new InProgressData(requestAndVersion.requestVersion, requestAndVersion.isPartialUpdate);
          return defaultRequestTimeoutMs;
      }
      // 判断Node是否正在连接
      if (isAnyNodeConnecting()) {
          return reconnectBackoffMs;
      }
      // 如果存在可用的Node，则尝试初始化连接
      if (connectionStates.canConnect(nodeConnectionId, now)) {
          // 初始化与node的连接
          initiateConnect(node, now);
          return reconnectBackoffMs;
      }
      // 阻塞等待有新的节点可用
      return Long.MAX_VALUE;
  }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.

  private void handleCompletedReceives(List<ClientResponse> responses, long now) {
    for (NetworkReceive receive : this.selector.completedReceives()) {
        ....
        // 如果是MetadataResponse类的响应，由metadataUpdater来处理
        if (req.isInternalRequest && response instanceof MetadataResponse)
          // 处理成功返回的响应信息
          metadataUpdater.handleSuccessfulResponse(req.header, now, (MetadataResponse) response);
        ....
   }
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.

 public void handleSuccessfulResponse(RequestHeader requestHeader, long now, MetadataResponse response) {
     ....
    // Check if any topic's metadata failed to get updated
    Map<String, Errors> errors = response.errors();
    // 如果返回错误，直接报错
    if (!errors.isEmpty())
        log.warn("Error while fetching metadata with correlation id {} : {}", requestHeader.correlationId(), errors);
    // 判断broker信息是否为空，如果为空表示没有获得元数据
    if (response.brokers().isEmpty()) {
        // 更新失败
        this.metadata.failedUpdate(now);
    } else {
        // 如果成功 则开始更新元数据
        this.metadata.update(inProgress.requestVersion, response, inProgress.isPartialUpdate, now);
    }
    inProgress = null;
 }
 
 // 元数据更新失败
 public synchronized void failedUpdate(long now) {
     // 最后一次更新元数据的时间为当前时间
     this.lastRefreshMs = now;
 }
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.