no need to de-duplicate as we are collapsing the results by urlkey

Same urls aren't recieved
This commit is contained in:
Akash Mahanty 2021-01-10 11:36:46 +05:30 committed by GitHub
parent dafba5d0cb
commit d6b7df6837
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -308,8 +308,6 @@ class Url:
for snapshot in snapshots: for snapshot in snapshots:
url_list.append(snapshot.original) url_list.append(snapshot.original)
url_list = list(set(url_list)) # remove duplicates
# Remove all deadURLs from url_list if alive=True # Remove all deadURLs from url_list if alive=True
if alive: if alive:
with concurrent.futures.ThreadPoolExecutor() as executor: with concurrent.futures.ThreadPoolExecutor() as executor: