readDir = ‘去重文本.txt‘
writeDir = ‘新文本.txt‘ # 结果文件的路径
lines_seen = set() #空集合,集合的特性,去重
outfile=open(writeDir,"w+",encoding=‘utf-8-sig‘)
f = open(readDir,"r",encoding=‘utf-8-sig‘)
for line in f:
if line not in lines_seen:
outfile.write(line)
lines_seen.add(line)
outfile.close()
f.close()
原文：https://www.cnblogs.com/JIM-FAN/p/12904903.html