Compare commits

..

2 Commits

2 changed files with 20 additions and 31 deletions

View File

@ -1,24 +1,26 @@
import requests
from lxml import html
from xml.etree.ElementTree import fromstring
def getContentFromURL(url):
return requests.get(url).text
c = 0
for i in range(36, 203):#1, 203):
# 36 is the first `i` value containing `livres`.
for i in range(1, 203):
print(i)
url = f'https://ww9.bookys-ebooks.com/sitemaps/items{i}.xml'
sitemapContent = getContentFromURL(url)
jsonData = xmltodict.parse(sitemapContent)['urlset']['url'][1:]
for entry in jsonData:
#print(entry)
loc = entry['loc']
sitemapXML = fromstring(sitemapContent)
for entry in sitemapXML:
loc = entry.findtext('{*}loc')
if loc.startswith('https://ww9.bookys-ebooks.com/livres/'):
#print(entry)
print(i, c)
print(loc)
content = getContentFromURL(loc)
title = content.split('<title>')[1].split('\n')[0]
tree = html.fromstring(content)
title = tree.xpath('/html/body/div/div/div[2]/div[1]/div/div[2]/div[2]/h1')[0].text.strip()
print(title)
print()
c += 1

View File

@ -1,6 +1,6 @@
import requests, re
import requests
from lxml import html
from xml.etree.ElementTree import XML, fromstring
from xml.etree.ElementTree import fromstring
def getContentFromURL(url):
return requests.get(url).text
@ -8,38 +8,25 @@ def getContentFromURL(url):
sitemapUrl = 'https://scanlibs.com/sitemap.xml'
sitemapContent = getContentFromURL(sitemapUrl)
"""
sitemapXML = fromstring(sitemapContent)
for elem in sitemapXML.iter():
for el in elem.iter():
print(el.tag, ':', el.text)
#print('!', elem.text, '!')
import xmltodict, json
"""
jsonData = xmltodict.parse(sitemapContent)['urlset']['url'][1:]
for entry in jsonData:
loc = entry['loc']
for entry in sitemapXML.findall('{*}url')[1:]:
loc = entry.findtext('{*}loc')
print(loc)
print(entry['lastmod'])
print(entry['image:image']['image:loc'])
print(entry.findtext('{*}lastmod'))
print(entry.find('{*}image').findtext('{*}loc'))
while True:
content = getContentFromURL(loc)
tree = html.fromstring(content)
title = content.split('<title>')[1].split('</title>')[0]
title = tree.xpath('/html/head/title')[0].text
if not title.startswith('503 Service Temporarily Unavailable'):
break
else:
print('MISS')
print(title)
print()
continue
metaPattern = 'border-bottom:1px solid #f0f0f0">'
meta = content.split(metaPattern)[1].split('</p>')[0]
meta = tree.xpath('/html/body/div[1]/div/article/div/div/div/div/div[1]/p[1]')[0].text
print(meta)
description = content.split('"></span>')[1].split('<a class="rewlink"')[0]
description = tree.xpath('/html/body/div[1]/div/article/div/div/div/div/div[1]/p[2]')[0].text
print(description)
print()