Spaces:
Sleeping
Sleeping
File size: 1,459 Bytes
ab95a25 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import argparse
def deup(src_file, tgt_file, src_file_out, tgt_file_out):
seen = set()
dup_count = 0
with open(src_file, encoding='utf-8') as fsrc, \
open(tgt_file, encoding='utf-8') as ftgt, \
open(src_file_out, 'w', encoding='utf-8') as fsrc_out, \
open(tgt_file_out, 'w', encoding='utf-8') as ftgt_out:
for s, t in zip(fsrc, ftgt):
if (s, t) not in seen:
fsrc_out.write(s)
ftgt_out.write(t)
seen.add((s, t))
else:
dup_count += 1
print(f'number of duplication: {dup_count}')
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--src-file", type=str, required=True,
help="src file")
parser.add_argument("--tgt-file", type=str, required=True,
help="tgt file")
parser.add_argument("--src-file-out", type=str, required=True,
help="src ouptut file")
parser.add_argument("--tgt-file-out", type=str, required=True,
help="tgt ouput file")
args = parser.parse_args()
deup(args.src_file, args.tgt_file, args.src_file_out, args.tgt_file_out)
if __name__ == "__main__":
main()
|