utils.py

import time, urllib.parse, logging, random
from dateutil import parser as dateparser

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

def canonicalize_url(url, base=None):
    if base:
        return urllib.parse.urljoin(base, url)
    return url

def parse_date(s):
    try:
        return dateparser.parse(s)
    except Exception:
        return None

def backoff_sleep(attempt):
    time.sleep(min(2**attempt + random.random(), 10))