simono41/FediFetcher
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

remove redundant code for retrying on HTTP 429

Browse source
This commit is contained in:
Toad King 2023-08-05 11:44:48 -05:00
parent 712d88cf0d
commit db2dcce2ff
1 changed files with 0 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
find_posts.py
View file

@ -734,11 +734,6 @@ def get_mastodon_urls(webserver, toot_id, toot_url):
except Exception as ex: except Exception as ex:
log(f"Error parsing context for toot {toot_url}. Exception: {ex}") log(f"Error parsing context for toot {toot_url}. Exception: {ex}")
return [] return []
elif resp.status_code == 429:
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1)
return get_mastodon_urls(webserver, toot_id, toot_url)
log( log(
f"Error getting context for toot {toot_url}. Status code: {resp.status_code}" f"Error getting context for toot {toot_url}. Status code: {resp.status_code}"
@ -771,11 +766,6 @@ def get_lemmy_comment_context(webserver, toot_id, toot_url):
except Exception as ex: except Exception as ex:
log(f"Error parsing context for comment {toot_url}. Exception: {ex}") log(f"Error parsing context for comment {toot_url}. Exception: {ex}")
return [] return []
elif resp.status_code == 429:
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1)
return get_lemmy_comment_context(webserver, toot_id, toot_url)
def get_lemmy_comments_urls(webserver, post_id, toot_url): def get_lemmy_comments_urls(webserver, post_id, toot_url):
"""get the URLs of the comments of the given post""" """get the URLs of the comments of the given post"""
@ -812,11 +802,6 @@ def get_lemmy_comments_urls(webserver, post_id, toot_url):
return urls return urls
except Exception as ex: except Exception as ex:
log(f"Error parsing comments for post {toot_url}. Exception: {ex}") log(f"Error parsing comments for post {toot_url}. Exception: {ex}")
elif resp.status_code == 429:
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when getting comments for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1)
return get_lemmy_comments_urls(webserver, post_id, toot_url)
log(f"Error getting comments for post {toot_url}. Status code: {resp.status_code}") log(f"Error getting comments for post {toot_url}. Status code: {resp.status_code}")
return [] return []
@ -902,11 +887,6 @@ def add_context_url(url, server, access_token):
"Make sure you have the read:search scope enabled for your access token." "Make sure you have the read:search scope enabled for your access token."
) )
return False return False
elif resp.status_code == 429:
reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
log(f"Rate Limit hit when adding url {search_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
time.sleep((reset - datetime.now()).total_seconds() + 1)
return add_context_url(url, server, access_token)
else: else:
log( log(
f"Error adding url {search_url} to server {server}. Status code: {resp.status_code}" f"Error adding url {search_url} to server {server}. Status code: {resp.status_code}"