Linguee_webscraper/main.py

import requests
from lxml import html
import string

charset = string.ascii_lowercase
url = 'https://www.linguee.fr/francais-anglais/search'
params = {
    'ch': 0
}

MAXIMUM_SUGGESTIONS = 4

entries = set()

def treatSuffixes(prefix):
    print(prefix)
    for char in charset:
        base = prefix + char
        params['qe'] = base
        text = requests.get(url, params = params).text
        tree = html.fromstring(text)
        items = tree.xpath('//div[@class="main_item"]')
        itemsLen = len(items)
        assert itemsLen <= MAXIMUM_SUGGESTIONS, f'More than {MAXIMUM_SUGGESTIONS} items!'
        for item in items:
            entry = item.text_content()
            if not entry in entries:
                print(len(entries), entry)
            entries.add(entry)
        if itemsLen == MAXIMUM_SUGGESTIONS:
            treatSuffixes(base)

treatSuffixes('')