simono41/FediFetcher
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Save seen_hosts on disk

Browse source
This commit is contained in:
nanos 2023-08-03 07:51:46 +01:00
parent a7611c6e6f
commit 5a2b2c2311
1 changed files with 9 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
find_posts.py
View file

@ -1291,6 +1291,7 @@ if __name__ == "__main__":
REPLIED_TOOT_SERVER_IDS_FILE = os.path.join(arguments.state_dir, "replied_toot_server_ids") REPLIED_TOOT_SERVER_IDS_FILE = os.path.join(arguments.state_dir, "replied_toot_server_ids")
KNOWN_FOLLOWINGS_FILE = os.path.join(arguments.state_dir, "known_followings") KNOWN_FOLLOWINGS_FILE = os.path.join(arguments.state_dir, "known_followings")
RECENTLY_CHECKED_USERS_FILE = os.path.join(arguments.state_dir, "recently_checked_users") RECENTLY_CHECKED_USERS_FILE = os.path.join(arguments.state_dir, "recently_checked_users")
SEEN_HOSTS_FILE = os.path.join(arguments.state_dir, "seen_hosts")
seen_urls = OrderedSet([]) seen_urls = OrderedSet([])
@ -1325,6 +1326,10 @@ if __name__ == "__main__":
all_known_users = OrderedSet(list(known_followings) + list(recently_checked_users)) all_known_users = OrderedSet(list(known_followings) + list(recently_checked_users))
# NOTE: explicitly not cached in a file so we get server version upgrades or migrations to new software # NOTE: explicitly not cached in a file so we get server version upgrades or migrations to new software
if os.path.exists(SEEN_HOSTS_FILE):
with open(SEEN_HOSTS_FILE, "r", encoding="utf-8") as f:
seen_hosts = json.load(f)
else:
seen_hosts = {} seen_hosts = {}
if(isinstance(arguments.access_token, str)): if(isinstance(arguments.access_token, str)):
@ -1421,6 +1426,9 @@ if __name__ == "__main__":
with open(RECENTLY_CHECKED_USERS_FILE, "w", encoding="utf-8") as f: with open(RECENTLY_CHECKED_USERS_FILE, "w", encoding="utf-8") as f:
recently_checked_users.toJSON() recently_checked_users.toJSON()
with open(SEEN_HOSTS_FILE, "w", encoding="utf-8") as f:
json.dump(seen_hosts, f)
os.remove(LOCK_FILE) os.remove(LOCK_FILE)
if(arguments.on_done != None and arguments.on_done != ''): if(arguments.on_done != None and arguments.on_done != ''):