dece/TLFi
1
0
Fork 0
Code Issues Pull requests Projects Releases 3 Activity

get-definitions: slow down big boy

Browse source
This commit is contained in:
dece 2021-03-15 20:19:30 +01:00
parent fb43e28c6e
commit da64b59cfb
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
get_definitions.py → get-definitions.py
View file

@ -1,5 +1,6 @@
import os import os
import sys import sys
import time
import urllib.parse import urllib.parse
from pathlib import Path from pathlib import Path
@ -26,6 +27,7 @@ def main():
else: else:
start_from = "" start_from = ""
get_definitions(form) get_definitions(form)
time.sleep(5000)
except OSError: except OSError:
exit("Could not open forms list file.") exit("Could not open forms list file.")
@ -39,8 +41,7 @@ def get_definitions(form):
# Check other definitions. # Check other definitions.
tabs_bar = soup.find("div", id="vtoolbar") tabs_bar = soup.find("div", id="vtoolbar")
if not tabs_bar: if not tabs_bar:
sys.stderr.write("No tabs bar!\n") exit("No tabs bar!\n")
return
num_defs = len(tabs_bar("li")) num_defs = len(tabs_bar("li"))
for variant in range(1, num_defs): # skip the first, we got it above. for variant in range(1, num_defs): # skip the first, we got it above.
variant_url = url + f"/{variant}" variant_url = url + f"/{variant}"
@ -60,8 +61,7 @@ def get_page(url):
def process_definition_page(soup, form, variant): def process_definition_page(soup, form, variant):
definition = soup.find("div", id="lexicontent") definition = soup.find("div", id="lexicontent")
if not definition: if not definition:
sys.stderr.write("No definition!\n") exit("No definition!\n")
return
save_definition(form, variant, definition.prettify()) save_definition(form, variant, definition.prettify())