bookys.py

@@ -1,26 +1,24 @@
 import requests
-from lxml import html
-from xml.etree.ElementTree import fromstring
 
 def getContentFromURL(url):
     return requests.get(url).text
 
 c = 0
 
-# 36 is the first `i` value containing `livres`.
+for i in range(36, 203):#1, 203):
-for i in range(1, 203):
     print(i)
     url = f'https://ww9.bookys-ebooks.com/sitemaps/items{i}.xml'
     sitemapContent = getContentFromURL(url)
-    sitemapXML = fromstring(sitemapContent)
+    jsonData = xmltodict.parse(sitemapContent)['urlset']['url'][1:]
-    for entry in sitemapXML:
+    for entry in jsonData:
-        loc = entry.findtext('{*}loc')
+        #print(entry)
+        loc = entry['loc']
         if loc.startswith('https://ww9.bookys-ebooks.com/livres/'):
+            #print(entry)
             print(i, c)
             print(loc)
             content = getContentFromURL(loc)
-            tree = html.fromstring(content)
+            title = content.split('<title>')[1].split('\n')[0]
-            title = tree.xpath('/html/body/div/div/div[2]/div[1]/div/div[2]/div[2]/h1')[0].text.strip()
             print(title)
             print()
             c += 1

scanlibs.py

@@ -1,6 +1,6 @@
-import requests
+import requests, re
 from lxml import html
-from xml.etree.ElementTree import fromstring
+from xml.etree.ElementTree import XML, fromstring
 
 def getContentFromURL(url):
     return requests.get(url).text
@@ -8,25 +8,38 @@ def getContentFromURL(url):
 sitemapUrl = 'https://scanlibs.com/sitemap.xml'
 
 sitemapContent = getContentFromURL(sitemapUrl)
+"""
 sitemapXML = fromstring(sitemapContent)
 
-for entry in sitemapXML.findall('{*}url')[1:]:
+for elem in sitemapXML.iter():
-    loc = entry.findtext('{*}loc')
+    for el in elem.iter():
+        print(el.tag, ':', el.text)
+    #print('!', elem.text, '!')
+
+import xmltodict, json
+"""
+
+jsonData = xmltodict.parse(sitemapContent)['urlset']['url'][1:]
+
+for entry in jsonData:
+    loc = entry['loc']
     print(loc)
-    print(entry.findtext('{*}lastmod'))
+    print(entry['lastmod'])
-    print(entry.find('{*}image').findtext('{*}loc'))
+    print(entry['image:image']['image:loc'])
     while True:
         content = getContentFromURL(loc)
         tree = html.fromstring(content)
-        title = tree.xpath('/html/head/title')[0].text
+        title = content.split('<title>')[1].split('</title>')[0]
         if not title.startswith('503 Service Temporarily Unavailable'):
             break
         else:
             print('MISS')
     print(title)
-    meta = tree.xpath('/html/body/div[1]/div/article/div/div/div/div/div[1]/p[1]')[0].text
+    print()
+    continue
+    metaPattern = 'border-bottom:1px solid #f0f0f0">'
+    meta = content.split(metaPattern)[1].split('</p>')[0]
     print(meta)
-    description = tree.xpath('/html/body/div[1]/div/article/div/div/div/div/div[1]/p[2]')[0].text
+    description = content.split('"></span>')[1].split('<a class="rewlink"')[0]
     print(description)
     print()
-