Elasticsearch 删除重复文档实现方式，你知道几个?

#!/usr/local/bin/python3
from elasticsearch import Elasticsearch, helpers
import hashlib
import ssl

# 全局变量
ES_HOST = 'https://192.168.1.10:9200'
ES_USER = 'elastic'
ES_PASSWORD = '9Z=T2wOWIXXXXXXXX'
CERT_FINGERPRINT = "a4d0fe1eb7e1fd9874XXXXXXXXXX"

# 全局词典
dict_of_duplicate_docs = {}

# https://www.elastic.co/guide/en/elasticsearch/client/python-api/current/config.html
# 要求python 版本3.10及以上
# fingerprint 生成方式，方式一：Elasticsearch 首次启动的时候自动生成。
# 方式二：借助命令行再次生成（结果同方式一）
# bash-4.2$ openssl x509 -fingerprint -sha256 -in config/certs/http_ca.crt
# SHA256 Fingerprint=A4:D0:FE:1E:B7:E1:FD:98:74:A4:10:6F:E1:XXXXXXXX
def es_init():
    es = Elasticsearch( ES_HOST,
    ssl_assert_fingerprint=CERT_FINGERPRINT,
    basic_auth=(ES_USER, ES_PASSWORD),
    verify_certs=False )
    return es
    
# 对每个文档生成唯一id（自定义生成）
def populate_dict_of_duplicate_docs(hit):
    combined_key = ""
    
    # 三个字段决定索引是否重复，自动是根据业务指定的
    keys_to_include_in_hash = ["CAC", "FTSE", "SMI"]
    for mykey in keys_to_include_in_hash:
      combined_key += str(hit['_source'][mykey])
      
      _id = hit["_id"]
      
      # 基于三个字段的组合，生成md5值，便于后续去重用。
      hashval = hashlib.md5(combined_key.encode('utf-8')).digest()
      
      # 生成键值对词典，key使用md5值，value 为数组类型。
      # 相同的 key 值也就是相同的文档，value 是文档id列表
      dict_of_duplicate_docs.setdefault(hashval, []).append(_id)
      print(dict_of_duplicate_docs)
      
  
  # 待去重索引遍历
  def scroll_over_all_docs():
      es = es_init()
      for hit in helpers.scan(es, index='stocks'):
        populate_dict_of_duplicate_docs(hit)
        
    # 去重处理函数
    def loop_over_hashes_and_remove_duplicates():
      es = es_init()
      for hashval, array_of_ids in dict_of_duplicate_docs.items():
      # 对id列表长度大于1的做去重处理操作
      if len(array_of_ids) > 1:
      print(" Duplicate docs hash=%s ****" % hashval)
      # 获取相同的文档
      matching_docs = es.mget(index="stocks", ids= array_of_ids[0:len(array_of_ids)-1])
      for doc in matching_docs['docs']:
        print("doc=%s\n" % doc)
        es.delete(index="stocks", id = doc['_id'])
        
def main():
    scroll_over_all_docs()
    loop_over_hashes_and_remove_duplicates()
    
main()
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.
33.
34.
35.
36.
37.
38.
39.
40.
41.
42.
43.
44.
45.
46.
47.
48.
49.
50.
51.
52.
53.
54.
55.
56.
57.
58.
59.
60.
61.
62.
63.
64.
65.
66.
67.
68.
69.
70.
71.

openssl x509 -fingerprint -sha256 -in config/certs/http_ca.crt
1.