You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
38 lines
1005 B
38 lines
1005 B
import random
|
|
from typing import Callable, Optional
|
|
|
|
import requests
|
|
|
|
|
|
def http_get(url: str, log_e: Optional[Callable] = None) -> Optional[str]:
|
|
"""Get the HTML as text from this URL.
|
|
log_e is an optional error logging function."""
|
|
try:
|
|
response = requests.get(url)
|
|
response.raise_for_status()
|
|
except OSError as exc:
|
|
if log_e:
|
|
log_e(f"http_get error: {exc}")
|
|
return None
|
|
return response.text
|
|
|
|
|
|
def proc(proba_percentage: int) -> bool:
|
|
return random.random() < (proba_percentage / 100.0)
|
|
|
|
|
|
def limit_text_length(text, max_length=400):
|
|
"""Limit text size to 400 characters max."""
|
|
words = text.split(" ")
|
|
cut_text = ""
|
|
while words:
|
|
next_word = words.pop(0)
|
|
if len(cut_text) + len(next_word) + 1 >= max_length:
|
|
break
|
|
cut_text += next_word + " "
|
|
if len(cut_text) < len(text):
|
|
cut_text = cut_text[:-1] + "…"
|
|
else:
|
|
cut_text = cut_text.rstrip()
|
|
return cut_text
|