图解 Kafka 源码之 NetworkClient 网络通信组件架构设计（上篇）

 public class NetworkClient implements KafkaClient {
    // 状态枚举值
    private enum State {
        ACTIVE,
        CLOSING,
        CLOSED
    }
    /* the selector used to perform network i/o */
    // 用于执行网络 I/O 的选择器
    private final Selectable selector;
    // Metadata元信息的更新器, 它可以尝试更新元信息
    private final MetadataUpdater metadataUpdater;
    /* the state of each node's connection */
    // 管理集群所有节点连接的状态
    private final ClusterConnectionStates connectionStates;
    /* the set of requests currently being sent or awaiting a response */
    // 当前正在发送或等待响应的请求集合
    private final InFlightRequests inFlightRequests;
    /* the socket send buffer size in bytes */
    // 套接字发送数据的缓冲区的大小（以字节为单位）
    private final int socketSendBuffer;
    /* the socket receive size buffer in bytes */
    // 套接字接收数据的缓冲区的大小（以字节为单位）
    private final int socketReceiveBuffer;
    /* the client id used to identify this client in requests to the server */
    // 表示客户端id，标识客户端身份
    private final String clientId;
    /* the current correlation id to use when sending requests to servers */
    // 向服务器发送请求时使用的当前关联 ID
    private int correlation;
    /* default timeout for individual requests to await acknowledgement from servers */
    // 单个请求等待服务器确认的默认超时
    private final int defaultRequestTimeoutMs;
    /* time in ms to wait before retrying to create connection to a server */
    // 重连的退避时间
    private final long reconnectBackoffMs;
    /**
     * True if we should send an ApiVersionRequest when first connecting to a broker.
     * 是否需要与 Broker 端的版本协调，默认为 true
     * 如果为 true 当第一次连接到一个 broker 时，应当发送一个 version 的请求，用来得知 broker 的版本， 如果为 false 则不需要发送 version 的请求。
     */
    private final boolean discoverBrokerVersions;
    // broker 端版本
    private final ApiVersions apiVersions;
    // 存储着要发送的版本请求，key 为 nodeId，value 为构建请求的 Builder
    private final Map<String, ApiVersionsRequest.Builder> nodesNeedingApiVersionsFetch = new HashMap<>();
    // 取消的请求集合
    private final List<ClientResponse> abortedSends = new LinkedList<>();
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.

/**
 * Begin connecting to the given node, return true if we are already connected and ready to send to that node.
 *
 * @param node The node to check
 * @param now The current timestamp
 * @return True if we are ready to send to the given node
 */
 @Override
 public boolean ready(Node node, long now){
     // 空节点
     if (node.isEmpty())
        throw new IllegalArgumentException("Cannot connect to empty node " + node);
     // 1、判断节点是否准备好发送请求
     if (isReady(node, now))
        return true;
     // 2、判断节点连接状态
     if (connectionStates.canConnect(node.idString(), now))
        // if we are interested in sending to a node and we don't have a connection to it, initiate one
        // 3、初始化连接，但此时不一定连接成功了
        initiateConnect(node, now);

     return false;
}

/**
 * Check if the node with the given id is ready to send more requests.
 * @param node The node
 * @param now The current time in ms
 * @return true if the node is ready
*/
@Override
public boolean isReady(Node node, long now){
    // if we need to update our metadata now declare all requests unready to make metadata requests first priority
    // 当发现正在更新元数据时，会禁止发送请求 && 当连接没有创建完毕或者当前发送的请求过多时，也会禁止发送请求
    return !metadataUpdater.isUpdateDue(now) && canSendRequest(node.idString(), now);
}

/**
 * Are we connected and ready and able to send more requests to the given connection?
 * 检测连接状态、发送请求是否过多
 * @param node The node
 * @param now the current timestamp
 */
 private boolean canSendRequest(String node, long now){
    // 三个条件必须都满足
    return connectionStates.isReady(node, now) && selector.isChannelReady(node) &&
            inFlightRequests.canSendMore(node);
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.

  /**
  * 创建连接 
  * Initiate a connection to the given node
  * @param node the node to connect to
  * @param now current time in epoch milliseconds
  */
 private void initiateConnect(Node node, long now){
    String nodeConnectionId = node.idString();
    try {
        // 1、更新连接状态为正在连接
        connectionStates.connecting(nodeConnectionId, now, node.host(), clientDnsLookup);
        // 获取连接地址
        InetAddress address = connectionStates.currentAddress(nodeConnectionId);
        log.debug("Initiating connection to node {} using address {}", node, address);
        // 2、调用 selector 尝试异步进行连接，后续通过selector.poll进行监听事件就绪 
        selector.connect(nodeConnectionId,
                    new InetSocketAddress(address, node.port()),
                    this.socketSendBuffer,
                    this.socketReceiveBuffer);
    } catch (IOException e) {
        log.warn("Error connecting to node {}", node, e);
        // Attempt failed, we'll try again after the backoff
        connectionStates.disconnected(nodeConnectionId, now);
        // Notify metadata updater of the connection failure
        metadataUpdater.handleServerDisconnect(now, nodeConnectionId, Optional.empty());
    }
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.

 /**
 * ClientRequest 是客户端的请求，封装了 requestBuilder 
 */
public final class ClientRequest {
    // 节点地址
    private final String destination;
    // ClientRequest 中通过 requestBuilder 给不同类型的请求设置不同的请求内容
    private final AbstractRequest.Builder<?> requestBuilder;
    // 请求头的 correlationId
    private final int correlationId;
    // 请求头的 clientid
    private final String clientId;
    // 创建时间
    private final long createdTimeMs;
    // 是否需要进行响应
    private final boolean expectResponse;
    // 请求的超时时间
    private final int requestTimeoutMs;
    // 回调函数 用来处理响应
    private final RequestCompletionHandler callback;
    ......
}

/**
 * Queue up the given request for sending. Requests can only be sent out to ready nodes.
 * @param request The request
 * @param now The current timestamp
 * 发送请求，这个方法 生产者和消费者都会调用，其中 ClientRequest 表示客户端的请求。
 */
 @Override
 public void send(ClientRequest request, long now){
     doSend(request, false, now);
 }

 // 检测请求版本是否支持，如果支持则发送请求
 private void doSend(ClientRequest clientRequest, boolean isInternalRequest, long now){
        // 确认是否活跃
        ensureActive();
        // 目标节点id
        String nodeId = clientRequest.destination();
        // 是否是 NetworkClient 内部请求 这里为 false
        if (!isInternalRequest) {
             // 检测是否可以向指定 Node 发送请求，如果还不能发送请求则抛异常
             if (!canSendRequest(nodeId, now))
                throw new IllegalStateException("Attempt to send a request to node " + nodeId + " which is not ready.");
        }
        AbstractRequest.Builder<?> builder = clientRequest.requestBuilder();
        try {
            // 检测版本
            NodeApiVersions versionInfo = apiVersions.get(nodeId);
            // ... 忽略
            // builder.build()是 ProduceRequest.Builder，结果是ProduceRequest
            // 调用 doSend 方法
            doSend(clientRequest, isInternalRequest, now, builder.build(version));
        } catch (UnsupportedVersionException unsupportedVersionException) {            log.debug("Version mismatch when attempting to send {} with correlation id {} to {}", builder, clientRequest.correlationId(), clientRequest.destination(), unsupportedVersionException);
           // 请求的版本不协调，那么生成 clientResponse
           ClientResponse clientResponse = new ClientResponse(clientRequest.makeHeader(builder.latestAllowedVersion()),
                    clientRequest.callback(), clientRequest.destination(), now, now,
                    false, unsupportedVersionException, null, null);
            // 添加到 abortedSends 集合里
            abortedSends.add(clientResponse);
        }
  }

  /**
   * isInternalRequest 表示发送前是否需要验证连接状态，如果为 true 则表示客户端已经确定连接是好的
   * request表示请求体
   */
  private void doSend(ClientRequest clientRequest, boolean isInternalRequest, long now, AbstractRequest request){
        // 目标节点地址
        String destination = clientRequest.destination();
        // 生成请求头
        RequestHeader header = clientRequest.makeHeader(request.version());
        if (log.isDebugEnabled()) {
            log.debug("Sending {} request with header {} and timeout {} to node {}: {}",
                clientRequest.apiKey(), header, clientRequest.requestTimeoutMs(), destination, request);
        }
        // 1、构建 NetworkSend 对象 结合请求头和请求体，序列化数据，保存到 NetworkSend 
        Send send = request.toSend(destination, header);
        // 2、构建 inFlightRequest 对象 保存了发送前的所有信息
        InFlightRequest inFlightRequest = new InFlightRequest(
                clientRequest,
                header,
                isInternalRequest,
                request,
                send,
                now);
        // 3、把 inFlightRequest 加入 inFlightRequests 集合里
        this.inFlightRequests.add(inFlightRequest);
        // 4、调用 Selector 异步发送数据，并将 send 和对应 kafkaChannel 绑定起来，并开启该 kafkaChannel 底层 socket 的写事件，等待下一步真正的网络发送
        selector.send(send);
}

@Override
public boolean active(){
    // 判断状态是否是活跃的
    return state.get() == State.ACTIVE;
}

// 确认是否活跃
private void ensureActive(){
   if (!active())
      throw new DisconnectException("NetworkClient is no longer active, state is " + state);
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.
72.
73.
74.
75.
76.
77.
78.
79.
80.
81.
82.
83.
84.
85.
86.
87.
88.
89.
90.
91.
92.
93.
94.
95.
96.
97.
98.
99.
100.
101.
102.
103.
104.

/**
 * Do actual reads and writes to sockets.
 * @param timeout The maximum amount of time to wait (in ms) for responses if there are none immediately,
 * must be non-negative. The actual timeout will be the minimum of timeout, request timeout and
 * metadata timeout
 * @param now The current time in milliseconds
 * @return The list of responses received
*/
@Override
public List<ClientResponse> poll(long timeout, long now){
   // 确认是否活跃
   ensureActive();
   // 取消发送是否为空
   if (!abortedSends.isEmpty()) {
      // If there are aborted sends because of unsupported version exceptions or disconnects,
      // handle them immediately without waiting for Selector#poll.
      List<ClientResponse> responses = new ArrayList<>();
      handleAbortedSends(responses);
      completeResponses(responses);
      return responses;
   }
   // 1、尝试更新元数据
   long metadataTimeout = metadataUpdater.maybeUpdate(now);
   try {
      // 2、执行网络 I/O 操作，真正读写发送的地方，如果客户端的请求被完整的处理过了，会加入到completeSends 或 complteReceives 集合中
      this.selector.poll(Utils.min(timeout, metadataTimeout, defaultRequestTimeoutMs));
   } catch (IOException e) {
      log.error("Unexpected error during I/O", e);
   }

   // process completed actions
   long updatedNow = this.time.milliseconds();
   // 响应结果集合：真正的读写操作, 会生成responses
   List<ClientResponse> responses = new ArrayList<>();
   // 3、完成发送的handler，处理 completedSends 集合
   handleCompletedSends(responses, updatedNow);
   // 4、完成接收的handler，处理 completedReceives 队列
   handleCompletedReceives(responses, updatedNow);
   // 5、断开连接的handler，处理 disconnected 列表
   handleDisconnections(responses, updatedNow);
   // 6、处理连接的handler，处理 connected 列表
   handleConnections();
   // 7、处理版本协调请求（获取api版本号） handler
   handleInitiateApiVersionRequests(updatedNow);
   // 8、超时连接的handler，处理超时连接集合
   handleTimedOutConnections(responses, updatedNow);
   // 9、超时请求的handler，处理超时请求集合
   handleTimedOutRequests(responses, updatedNow);
   // 10、完成响应回调
   completeResponses(responses);

   return responses;
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.

/**
 * Handle any completed request send. In particular if no response is expected consider the request complete.
 * @param responses The list of responses to update
 * @param now The current time
*/
private void handleCompletedSends(List<ClientResponse> responses, long now){
   // if no response is expected then when the send is completed, return it
   // 1、遍历 completedSends 发送完成的请求集合，通过调用 Selector 获取从上一次 poll 开始的请求
   for (Send send : this.selector.completedSends()) {
       // 2、从 inFlightRequests 集合获取该 Send 关联对应 Node 的队列取出最新的请求，但并没有从队列中删除，取出后判断这个请求是否期望得到响应
       InFlightRequest request = this.inFlightRequests.lastSent(send.destination());
       // 3、是否需要响应, 如果不需要响应,当Send请求完成时,就直接返回.还是有request.completed生成的ClientResponse对象
       if (!request.expectResponse) {
           // 4、如果不需要响应就取出 inFlightRequests 中该 Sender 关联对应 Node 的 inFlightRequest，即提取最新的请求
           this.inFlightRequests.completeLastSent(send.destination());
           // 5、调用 completed() 生成 ClientResponse,第一个参数为null,表示没有响应内容,把请求添加到 Responses 集合
           responses.add(request.completed(null, now));
       }
   }
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.

/**
 * Handle any completed receives and update the response list with the responses received.
 * @param responses The list of responses to update
 * @param now The current time
 * 处理 CompletedReceives 队列,根据返回的响应信息实例化 ClientResponse ,并加到响应集合里
*/
private void handleCompletedReceives(List<ClientResponse> responses, long now){
   // 1、遍历 CompletedReceives 响应集合，通过 Selector 返回未处理的响应
   for (NetworkReceive receive : this.selector.completedReceives()) {
       // 2、获取发送请求的 Node id
       String source = receive.source();
       // 3、从 inFlightRequests 集合队列获取已发送请求「最老的请求」并删除（从 inFlightRequests 删除，因为inFlightRequests 存储的是未收到请求响应的 ClientRequest，现在请求已经有响应了，就不需要保存了）
       InFlightRequest req = inFlightRequests.completeNext(source);
       // 4、解析响应，并且验证响应头，生成 responseStruct 实例
       Struct responseStruct = parseStructMaybeUpdateThrottleTimeMetrics(receive.payload(), req.header,throttleTimeSensor, now);
       // 生成响应体
       AbstractResponse response = AbstractResponse.parseResponse(req.header.apiKey(), responseStruct, req.header.apiVersion());       
      ....
      // If the received response includes a throttle delay, throttle the connection.
      // 流控处理
      maybeThrottle(response, req.header.apiVersion(), req.destination, now);
      // 5、判断返回类型
      if (req.isInternalRequest && response instanceof MetadataResponse)
          // 处理元数据请求响应
          metadataUpdater.handleSuccessfulResponse(req.header, now, (MetadataResponse) response);
      else if (req.isInternalRequest && response instanceof ApiVersionsResponse)
          // 处理版本协调响应
          handleApiVersionsResponse(responses, req, now, (ApiVersionsResponse) response);
      else
          // 普通发送消息的响应，通过 InFlightRequest.completed()，生成 ClientResponse，将响应添加到 responses 集合中
          responses.add(req.completed(response, now));
    }
}

// 解析响应，并且验证响应头，生成 responseStruct 实例
private static Struct parseStructMaybeUpdateThrottleTimeMetrics(ByteBuffer responseBuffer, RequestHeader requestHeader, Sensor throttleTimeSensor, long now){
    // 解析响应头
    ResponseHeader responseHeader = ResponseHeader.parse(responseBuffer,
            requestHeader.apiKey().responseHeaderVersion(requestHeader.apiVersion()));
    // 解析响应体
    Struct responseBody = requestHeader.apiKey().parseResponse(requestHeader.apiVersion(), responseBuffer);
    // 验证请求头与响应头的 correlation id 必须相等
    correlate(requestHeader, responseHeader);
    if (throttleTimeSensor != null && responseBody.hasField(CommonFields.THROTTLE_TIME_MS))
            throttleTimeSensor.record(responseBody.get(CommonFields.THROTTLE_TIME_MS), now);
    return responseBody;
}
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.