import requests from lxml import html import string charset = string.ascii_lowercase url = 'https://www.linguee.fr/francais-anglais/search' params = { 'ch': 0 } MAXIMUM_SUGGESTIONS = 4 entries = set() def treatSuffixes(prefix): #print(prefix) for char in charset: base = prefix + char print(base) params['qe'] = base text = requests.get(url, params = params).text tree = html.fromstring(text) rows = tree.xpath('//div[@class="main_row"]') rowsLen = len(rows) assert rowsLen <= MAXIMUM_SUGGESTIONS, f'More than {MAXIMUM_SUGGESTIONS} rows!' for row in rows: item = row.xpath('div[@class="main_item"]')[0] entry = item.text_content() wordType = row.xpath('div[@class="main_wordtype"]') if wordType != [] and item.attrib['lc'] == 'FR' and not entry in entries: print(len(entries), entry, wordType[0].text_content()) entries.add(entry) if rowsLen == MAXIMUM_SUGGESTIONS: treatSuffixes(base) treatSuffixes('')