simono41/FediFetcher
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Be timezone aware in processing ratelimits

Browse source
This commit is contained in:
Michael Thomas 2023-03-15 07:20:49 +00:00
parent ce69c2a033
commit ffc80a5081
2 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
get_context.py
View file

@ -1,6 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
from datetime import datetime, timedelta from datetime import datetime, timedelta
from dateutil import parser
import itertools import itertools
import json import json
import os import os
@ -584,8 +585,9 @@ def get(url, headers = {}, timeout = 5, max_tries = 5):
response = requests.get( url, headers= h, timeout=timeout) response = requests.get( url, headers= h, timeout=timeout)
if response.status_code == 429: if response.status_code == 429:
if max_tries > 0: if max_tries > 0:
reset = datetime.strptime(response.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ') reset = parser.parse(response.headers['x-ratelimit-reset'])
wait = (reset - datetime.now()).total_seconds() + 1 now = datetime.now(datetime.now().astimezone().tzinfo)
wait = (reset - now).total_seconds() + 1
log(f"Rate Limit hit requesting {url}. Waiting {wait} sec to retry at {response.headers['x-ratelimit-reset']}") log(f"Rate Limit hit requesting {url}. Waiting {wait} sec to retry at {response.headers['x-ratelimit-reset']}")
time.sleep(wait) time.sleep(wait)
return get(url, headers, timeout, max_tries - 1) return get(url, headers, timeout, max_tries - 1)

1
requirements.txt
View file

@ -5,3 +5,4 @@ idna==3.4
requests==2.28.2 requests==2.28.2
six==1.16.0 six==1.16.0
urllib3==1.26.14 urllib3==1.26.14
python-dateutil==2.8.2