From afc981930c1ef0a75e16f8d786d71ccf85357ff3 Mon Sep 17 00:00:00 2001 From: nanos Date: Tue, 21 Mar 2023 19:23:14 +0000 Subject: [PATCH] Implement option to get replies to bookmarks (#19) --- .github/workflows/get_context.yml | 2 +- README.md | 6 +++++- find_posts.py | 19 ++++++++++++++++--- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/.github/workflows/get_context.yml b/.github/workflows/get_context.yml index 3b3bbd7..e0b8f3f 100644 --- a/.github/workflows/get_context.yml +++ b/.github/workflows/get_context.yml @@ -32,7 +32,7 @@ jobs: path: artifacts - name: Get Directory structure run: ls -lR - - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }} --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} + - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }} --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} --max-bookmarks=${{ vars.MAX_BOOKMARKS || 0 }} - name: Upload artifacts uses: actions/upload-artifact@v3 with: diff --git a/README.md b/README.md index 0c3701e..7889f60 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ This GitHub repository provides a GitHub action that runs every 10 mins, doing t 1. It can [pull remote replies into your instance](https://blog.thms.uk/2023/03/pull-missing-responses-into-mastodon?utm_source=github), using the Mastodon API. That part itself has two parts: 1. It gets remote replies to posts that users on your instance have already replied to during the last `REPLY_INTERVAL_IN_HOURS` hours, and adds them to your own server. 2. It gets remote replies to the last `HOME_TIMELINE_LENGTH` posts from your home timeline, and adds them to your own server. + 3. It gets remote replies to the last `MAX_BOOKMARKS` of your bookmarks, and adds them to your own server. 2. It can also [backfill posts](https://blog.thms.uk/2023/03/backfill-recently-followed-accounts?utm_source=github): 1. from the last `MAX_FOLLOWINGS` users that you have followed. 2. form the last `MAX_FOLLOWERS` users that have followed you. @@ -28,7 +29,7 @@ Regardless of how you want to run this script, you must first get an access toke 1. In Mastodon go to Preferences > Development > New Application 1. give it a nice name - 2. enable `read:search`, `read:statuses`, `read:follows`, and `admin:read:accounts` + 2. Enable the required scopes for your options. See below for details, but if you want to use all parts of this script, you'll need these scopes: `read:search`, `read:statuses`, `read:follows`, `read:bookmarks`, and `admin:read:accounts` 3. Save 4. Copy the value of `Your access token` @@ -87,6 +88,7 @@ Please see below for a list of configuration options. | `MAX_FOLLOWINGS` | `--max-followings` | No | Provide to backfill profiles for your most recent followings. Determines how many of your last followings you want to backfill. (An integer number, e.g. `80`. Ensure you also provide `USER`). | `MAX_FOLLOWERS` | `--max-followers` | No | Provide to backfill profiles for your most recent followers. Determines how many of your last followers you want to backfill. (An integer number, e.g. `80`. Ensure you also provide `USER`). | `MAX_FOLLOW_REQUESTS` | `--max-follow-requests` | No | Provide to backfill profiles for the API key owner's most recent pending follow requests. Determines how many of your last follow requests you want to backfill. (An integer number, e.g. `80`.). Requires an access token with `read:follows` scope. +| `MAX_BOOKMARKS` | `--max-bookmarks` | No | Provide to fetch remote replies to any posts you have bookmarked. Determines how many of your bookmarks you want to get replies to. (An integer number, e.g. `80`.). Requires an access token with `read:bookmarks` scope. | `HTTP_TIMEOUT` | `--http-timeout` | No | The timeout for any HTTP requests to the Mastodon API in seconds. Defaults to `5`. | -- | `--lock-hours` | No | Determines after how many hours a lock file should be discarded. Not relevant when running the script as GitHub Action, as concurrency is prevented using a different mechanism. | `ON_START` | `--on-start` | No | Optionally provide a callback URL that will be pinged when processing is starting. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io. @@ -102,6 +104,8 @@ Please see below for a list of configuration options. - `admin:read:accounts` - If you are supplying `MAX_FOLLOW_REQUESTS` / `--max-follow-requests` you must additionally enable this scope: - `read:follows` + - If you are supplying `MAX_BOOKMARKS` / `--max-bookmarks` you must additionally enable this scope: + - `read:bookmarks` ## Acknowledgments diff --git a/find_posts.py b/find_posts.py index 03c96a0..90b0997 100644 --- a/find_posts.py +++ b/find_posts.py @@ -22,6 +22,7 @@ argparser.add_argument('--user', required = False, default='', help="Use togethe argparser.add_argument('--max-followings', required = False, type=int, default=0, help="Backfill posts for new accounts followed by --user. We'll backfill at most this many followings' posts") argparser.add_argument('--max-followers', required = False, type=int, default=0, help="Backfill posts for new accounts following --user. We'll backfill at most this many followers' posts") argparser.add_argument('--max-follow-requests', required = False, type=int, default=0, help="Backfill posts of the API key owners pending follow requests. We'll backfill at most this many requester's posts") +argparser.add_argument('--max-bookmarks', required = False, type=int, default=0, help="Fetch remote replies to the API key owners Bookmarks. We'll fetch replies to at most this many bookmarks") argparser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.") argparser.add_argument('--lock-hours', required = False, type=int, default=24, help="The lock timeout in hours.") argparser.add_argument('--on-done', required = False, default=None, help="Provide a url that will be pinged when processing has completed. You can use this for 'dead man switch' monitoring of your task") @@ -39,7 +40,8 @@ def pull_context( backfill_followings_for_user, known_followings, max_followers, - max_follow_requests + max_follow_requests, + max_bookmarks ): parsed_urls = {} @@ -80,10 +82,20 @@ def pull_context( if max_follow_requests > 0: log(f"Getting posts from last {max_follow_requests} follow requests") - user_id = get_user_id(server, backfill_followings_for_user) follow_requests = get_new_follow_requests(server, access_token, max_follow_requests, known_followings) add_following_posts(server, access_token, follow_requests, known_followings, seen_urls) + if max_bookmarks > 0: + log(f"Pulling replies to the last {max_bookmarks} bookmarks") + bookmarks = get_bookmarks(server, access_token, max_bookmarks) + known_context_urls = get_all_known_context_urls(server, bookmarks,parsed_urls) + add_context_urls(server, access_token, known_context_urls, seen_urls) + +def get_bookmarks(server, access_token, max): + return get_paginated_mastodon(f"https://{server}/api/v1/bookmarks", max, { + "Authorization": f"Bearer {access_token}", + }) + def add_following_posts(server, access_token, followings, know_followings, seen_urls): for user in followings: posts = get_user_posts(user, know_followings, server) @@ -767,7 +779,8 @@ if __name__ == "__main__": arguments.user, KNOWN_FOLLOWINGS, arguments.max_followers, - arguments.max_follow_requests + arguments.max_follow_requests, + arguments.max_bookmarks ) with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f: