Spaces:
No application file
No application file
File size: 466 Bytes
7645142 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
import time, urllib.parse, logging, random
from dateutil import parser as dateparser
logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)
def canonicalize_url(url, base=None):
if base:
return urllib.parse.urljoin(base, url)
return url
def parse_date(s):
try:
return dateparser.parse(s)
except Exception:
return None
def backoff_sleep(attempt):
time.sleep(min(2**attempt + random.random(), 10))
|