filters/dedup.py

20 lines
No EOL
818 B
Python

def remove_duplicates(input_filename, output_filename):
seen_lines = set()
try:
with open(input_filename, 'r') as infile, \
open(output_filename, 'w') as outfile:
for line in infile:
if line not in seen_lines:
outfile.write(line)
seen_lines.add(line)
print(f"Successfully processed {input_filename}")
except FileNotFoundError:
print(f"Error: Could not find input file '{input_filename}'")
except PermissionError:
print(f"Error: Permission denied for accessing files")
except Exception as e:
print(f"An error occurred: {e}")
if __name__ == "__main__":
input_file = input("Input file: ")
output_file = input("Output file: ")
remove_duplicates(input_file, output_file)