From d6b7df6837501e18c4a365765964354abf54334b Mon Sep 17 00:00:00 2001
From: Akash Mahanty <akamhy@yahoo.com>
Date: Sun, 10 Jan 2021 11:36:46 +0530
Subject: [PATCH] no need to de-duplicate as we are collapsing the results by
 urlkey

Same urls aren't recieved
---
 waybackpy/wrapper.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/waybackpy/wrapper.py b/waybackpy/wrapper.py
index e2c33b6..ed9dd8f 100644
--- a/waybackpy/wrapper.py
+++ b/waybackpy/wrapper.py
@@ -308,8 +308,6 @@ class Url:
         for snapshot in snapshots:
             url_list.append(snapshot.original)
 
-        url_list = list(set(url_list))  # remove duplicates
-
         # Remove all deadURLs from url_list if alive=True
         if alive:
             with concurrent.futures.ThreadPoolExecutor() as executor: