simono41/FediFetcher
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

make the state file configurable

Browse source
This commit is contained in:
Robert Gerus 2023-03-28 20:09:04 +02:00
parent 44f15de367
commit 73a296c310
2 changed files with 11 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -112,6 +112,8 @@ Please find the list of all configuration options, including descriptions, below
| `REMEMBER_USERS_FOR_HOURS` | `--remember-users-for-hours` | No | How long between back-filling attempts for non-followed accounts? Defaults to `168`, i.e. one week. | `REMEMBER_USERS_FOR_HOURS` | `--remember-users-for-hours` | No | How long between back-filling attempts for non-followed accounts? Defaults to `168`, i.e. one week.
| `HTTP_TIMEOUT` | `--http-timeout` | No | The timeout for any HTTP requests to the Mastodon API in seconds. Defaults to `5`. | `HTTP_TIMEOUT` | `--http-timeout` | No | The timeout for any HTTP requests to the Mastodon API in seconds. Defaults to `5`.
| -- | `--lock-hours` | No | Determines after how many hours a lock file should be discarded. Not relevant when running the script as GitHub Action, as concurrency is prevented using a different mechanism. Recommended value: `24`. | -- | `--lock-hours` | No | Determines after how many hours a lock file should be discarded. Not relevant when running the script as GitHub Action, as concurrency is prevented using a different mechanism. Recommended value: `24`.
| -- | `--lock-file` | No | Location for the lock file. If not specified, will use `lock.lock` under the state directory. Not relevant when running the script as GitHub Action.
| -- | `--state-dir` | No | Directory storing persistent files, and the default location for lock file. Not relevant when running the script as GitHub Action.
| `ON_START` | `--on-start` | No | Optionally provide a callback URL that will be pinged when processing is starting. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io. | `ON_START` | `--on-start` | No | Optionally provide a callback URL that will be pinged when processing is starting. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io.
| `ON_DONE` | `--on-done` | No | Optionally provide a callback URL that will be called when processing is finished. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io. | `ON_DONE` | `--on-done` | No | Optionally provide a callback URL that will be called when processing is finished. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io.
| `ON_FAIL` | `--on-fail` | No | Optionally provide a callback URL that will be called when processing has failed. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io. | `ON_FAIL` | `--on-fail` | No | Optionally provide a callback URL that will be called when processing has failed. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io.

14
find_posts.py
View file

@ -27,6 +27,8 @@ argparser.add_argument('--from-notifications', required = False, type=int, defau
argparser.add_argument('--remember-users-for-hours', required=False, type=int, default=24*7, help="How long to remember users that you aren't following for, before trying to backfill them again.") argparser.add_argument('--remember-users-for-hours', required=False, type=int, default=24*7, help="How long to remember users that you aren't following for, before trying to backfill them again.")
argparser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.") argparser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.")
argparser.add_argument('--lock-hours', required = False, type=int, default=24, help="The lock timeout in hours.") argparser.add_argument('--lock-hours', required = False, type=int, default=24, help="The lock timeout in hours.")
argparser.add_argument('--lock-file', required = False, default=None, help="Location of the lock file")
argparser.add_argument('--state-dir', required = False, default="artifacts", help="Directory to store persistent files and possibly lock file")
argparser.add_argument('--on-done', required = False, default=None, help="Provide a url that will be pinged when processing has completed. You can use this for 'dead man switch' monitoring of your task") argparser.add_argument('--on-done', required = False, default=None, help="Provide a url that will be pinged when processing has completed. You can use this for 'dead man switch' monitoring of your task")
argparser.add_argument('--on-start', required = False, default=None, help="Provide a url that will be pinged when processing is starting. You can use this for 'dead man switch' monitoring of your task") argparser.add_argument('--on-start', required = False, default=None, help="Provide a url that will be pinged when processing is starting. You can use this for 'dead man switch' monitoring of your task")
argparser.add_argument('--on-fail', required = False, default=None, help="Provide a url that will be pinged when processing has failed. You can use this for 'dead man switch' monitoring of your task") argparser.add_argument('--on-fail', required = False, default=None, help="Provide a url that will be pinged when processing has failed. You can use this for 'dead man switch' monitoring of your task")
@ -778,7 +780,9 @@ if __name__ == "__main__":
except Exception as ex: except Exception as ex:
log(f"Error getting callback url: {ex}") log(f"Error getting callback url: {ex}")
LOCK_FILE = 'artifacts/lock.lock' if arguments.lock_file is None:
arguments.lock_file = os.path.join(arguments.state, 'lock.lock')
LOCK_FILE = arguments.lock_file
if( os.path.exists(LOCK_FILE)): if( os.path.exists(LOCK_FILE)):
log(f"Lock file exists at {LOCK_FILE}") log(f"Lock file exists at {LOCK_FILE}")
@ -813,10 +817,10 @@ if __name__ == "__main__":
try: try:
SEEN_URLS_FILE = "artifacts/seen_urls" SEEN_URLS_FILE = os.path.join(arguments.state, "seen_urls")
REPLIED_TOOT_SERVER_IDS_FILE = "artifacts/replied_toot_server_ids" REPLIED_TOOT_SERVER_IDS_FILE = os.path.join(arguments.state, "replied_toot_server_ids")
KNOWN_FOLLOWINGS_FILE = "artifacts/known_followings" KNOWN_FOLLOWINGS_FILE = os.path.join(arguments.state, "known_followings")
RECENTLY_CHECKED_USERS_FILE = "artifacts/recently_checked_users" RECENTLY_CHECKED_USERS_FILE = os.path.join(arguments.state, "recently_checked_users")
SEEN_URLS = OrderedSet([]) SEEN_URLS = OrderedSet([])