From 1fffddcb231beb427ca190e6e6ef9ddc344ea30a Mon Sep 17 00:00:00 2001 From: nanos Date: Fri, 28 Apr 2023 20:40:24 +0100 Subject: [PATCH] support for fetching context to favourites (#42) --- .github/workflows/get_context.yml | 2 +- README.md | 3 +++ find_posts.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/get_context.yml b/.github/workflows/get_context.yml index 7a98539..7fc48bd 100644 --- a/.github/workflows/get_context.yml +++ b/.github/workflows/get_context.yml @@ -33,7 +33,7 @@ jobs: path: artifacts - name: Get Directory structure run: ls -lR - - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }} --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} --max-bookmarks=${{ vars.MAX_BOOKMARKS || 0 }} --remember-users-for-hours=${{ vars.REMEMBER_USERS_FOR_HOURS || 168 }} --from-notifications=${{ vars.FROM_NOTIFICATIONS || 0 }} --backfill-with-context=${{ vars.BACKFILL_WITH_CONTEXT || 1 }} --backfill-mentioned-users=${{ vars.BACKFILL_MENTIONED_USERS || 1 }} + - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }} --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} --max-bookmarks=${{ vars.MAX_BOOKMARKS || 0 }} --remember-users-for-hours=${{ vars.REMEMBER_USERS_FOR_HOURS || 168 }} --from-notifications=${{ vars.FROM_NOTIFICATIONS || 0 }} --backfill-with-context=${{ vars.BACKFILL_WITH_CONTEXT || 1 }} --backfill-mentioned-users=${{ vars.BACKFILL_MENTIONED_USERS || 1 }} --max-favourites=${{ vars.MAX_FAVOURITES || 0}} - name: Upload artifacts uses: actions/upload-artifact@v3 with: diff --git a/README.md b/README.md index 3403c4a..7d3f1ac 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ Please find the list of all configuration options, including descriptions, below | `HOME_TIMELINE_LENGTH` | `--home-timeline-length` | No | Provide to fetch remote replies to posts in the API-Key owner's home timeline. Determines how many posts we'll fetch replies for. Recommended value: `200`. | `REPLY_INTERVAL_IN_HOURS` | `--reply-interval-in-hours` | No | Provide to fetch remote replies to posts that have received replies from users on your own instance. Determines how far back in time we'll go to find posts that have received replies. Recommend value: `0` (disabled). Requires an access token with `admin:read:accounts`. | `MAX_BOOKMARKS` | `--max-bookmarks` | No | Provide to fetch remote replies to any posts you have bookmarked. Determines how many of your bookmarks you want to get replies to. Recommended value: `80`. Requires an access token with `read:bookmarks` scope. +| `MAX_FAVOURITES` | `--max-favourites` | No | Provide to fetch remote replies to any posts you have favourited. Determines how many of your favourites you want to get replies to. Recommended value: `40`. Requires an access token with `read:favourites` scope. | `MAX_FOLLOWINGS` | `--max-followings` | No | Provide to backfill profiles for your most recent followings. Determines how many of your last followings you want to backfill. Recommended value: `80`. | `MAX_FOLLOWERS` | `--max-followers` | No | Provide to backfill profiles for your most recent followers. Determines how many of your last followers you want to backfill. Recommended value: `80`. | `MAX_FOLLOW_REQUESTS` | `--max-follow-requests` | No | Provide to backfill profiles for the API key owner's most recent pending follow requests. Determines how many of your last follow requests you want to backfill. Recommended value: `80`. @@ -147,6 +148,8 @@ This is only supported when running FediFetcher as cron job, or container. Multi - `read:follows` - If you are supplying `MAX_BOOKMARKS` / `--max-bookmarks` you must additionally enable this scope: - `read:bookmarks` + - If you are supplying `MAX_FAVOURITES` / `--max-favourites` you must additionally enable this scope: + - `read:favourites` - If you are supplying `FROM_NOTIFICATIONS` / `--from-notifications` you must additionally enable this scope: - `read:notifications` diff --git a/find_posts.py b/find_posts.py index 4485895..2045720 100644 --- a/find_posts.py +++ b/find_posts.py @@ -23,6 +23,7 @@ argparser.add_argument('--max-followings', required = False, type=int, default=0 argparser.add_argument('--max-followers', required = False, type=int, default=0, help="Backfill posts for new accounts following --user. We'll backfill at most this many followers' posts") argparser.add_argument('--max-follow-requests', required = False, type=int, default=0, help="Backfill posts of the API key owners pending follow requests. We'll backfill at most this many requester's posts") argparser.add_argument('--max-bookmarks', required = False, type=int, default=0, help="Fetch remote replies to the API key owners Bookmarks. We'll fetch replies to at most this many bookmarks") +argparser.add_argument('--max-favourites', required = False, type=int, default=0, help="Fetch remote replies to the API key owners Favourites. We'll fetch replies to at most this many favourites") argparser.add_argument('--from-notifications', required = False, type=int, default=0, help="Backfill accounts of anyone appearing in your notifications, during the last hours") argparser.add_argument('--remember-users-for-hours', required=False, type=int, default=24*7, help="How long to remember users that you aren't following for, before trying to backfill them again.") argparser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.") @@ -57,6 +58,11 @@ def get_bookmarks(server, access_token, max): "Authorization": f"Bearer {access_token}", }) +def get_favourites(server, access_token, max): + return get_paginated_mastodon(f"https://{server}/api/v1/favourites", max, { + "Authorization": f"Bearer {access_token}", + }) + def add_user_posts(server, access_token, followings, know_followings, all_known_users, seen_urls): for user in followings: if user['acct'] not in all_known_users and not user['url'].startswith(f"https://{server}/"): @@ -899,6 +905,12 @@ if __name__ == "__main__": known_context_urls = get_all_known_context_urls(arguments.server, bookmarks,parsed_urls) add_context_urls(arguments.server, token, known_context_urls, seen_urls) + if arguments.max_favourites > 0: + log(f"Pulling replies to the last {arguments.max_favourites} favourites") + favourites = get_favourites(arguments.server, token, arguments.max_favourites) + known_context_urls = get_all_known_context_urls(arguments.server, favourites,parsed_urls) + add_context_urls(arguments.server, token, known_context_urls, seen_urls) + with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f: f.write("\n".join(list(known_followings)[-10000:]))