no need to de-duplicate as we are collapsing the results by urlkey
Same urls aren't recieved
This commit is contained in:
parent
dafba5d0cb
commit
d6b7df6837
@ -308,8 +308,6 @@ class Url:
|
||||
for snapshot in snapshots:
|
||||
url_list.append(snapshot.original)
|
||||
|
||||
url_list = list(set(url_list)) # remove duplicates
|
||||
|
||||
# Remove all deadURLs from url_list if alive=True
|
||||
if alive:
|
||||
with concurrent.futures.ThreadPoolExecutor() as executor:
|
||||
|
Loading…
Reference in New Issue
Block a user