Merge pull request #43 from nanos/dev

support for fetching context to favourites (#42)

.github/workflows/get_context.yml

@@ -33,7 +33,7 @@ jobs:
           path: artifacts
       - name: Get Directory structure
         run: ls -lR
-      - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }}  --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} --max-bookmarks=${{ vars.MAX_BOOKMARKS || 0 }} --remember-users-for-hours=${{ vars.REMEMBER_USERS_FOR_HOURS || 168 }} --from-notifications=${{ vars.FROM_NOTIFICATIONS || 0 }} --backfill-with-context=${{ vars.BACKFILL_WITH_CONTEXT || 1 }}  --backfill-mentioned-users=${{ vars.BACKFILL_MENTIONED_USERS || 1 }} 
+      - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }}  --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} --max-bookmarks=${{ vars.MAX_BOOKMARKS || 0 }} --remember-users-for-hours=${{ vars.REMEMBER_USERS_FOR_HOURS || 168 }} --from-notifications=${{ vars.FROM_NOTIFICATIONS || 0 }} --backfill-with-context=${{ vars.BACKFILL_WITH_CONTEXT || 1 }}  --backfill-mentioned-users=${{ vars.BACKFILL_MENTIONED_USERS || 1 }}  --max-favourites=${{ vars.MAX_FAVOURITES || 0}}
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:

README.md

@@ -114,6 +114,7 @@ Please find the list of all configuration options, including descriptions, below
 | `HOME_TIMELINE_LENGTH` | `--home-timeline-length` | No | Provide to fetch remote replies to posts in the API-Key owner's home timeline. Determines how many posts we'll fetch replies for. Recommended value: `200`.
 | `REPLY_INTERVAL_IN_HOURS` | `--reply-interval-in-hours` | No | Provide to fetch remote replies to posts that have received replies from users on your own instance. Determines how far back in time we'll go to find posts that have received replies. Recommend value: `0` (disabled). Requires an access token with `admin:read:accounts`.
 | `MAX_BOOKMARKS` | `--max-bookmarks` | No | Provide to fetch remote replies to any posts you have bookmarked. Determines how many of your bookmarks you want to get replies to. Recommended value: `80`. Requires an access token with `read:bookmarks` scope.
+| `MAX_FAVOURITES` | `--max-favourites` | No | Provide to fetch remote replies to any posts you have favourited. Determines how many of your favourites you want to get replies to. Recommended value: `40`. Requires an access token with `read:favourites` scope.
 | `MAX_FOLLOWINGS` | `--max-followings` | No | Provide to backfill profiles for your most recent followings. Determines how many of your last followings you want to backfill. Recommended value: `80`.
 | `MAX_FOLLOWERS` | `--max-followers` | No | Provide to backfill profiles for your most recent followers. Determines how many of your last followers you want to backfill. Recommended value: `80`.
 | `MAX_FOLLOW_REQUESTS` | `--max-follow-requests` | No | Provide to backfill profiles for the API key owner's most recent pending follow requests. Determines how many of your last follow requests you want to backfill. Recommended value: `80`.
@@ -147,6 +148,8 @@ This is only supported when running FediFetcher as cron job, or container. Multi
    - `read:follows`
  - If you are supplying `MAX_BOOKMARKS` / `--max-bookmarks` you must additionally enable this scope:
    - `read:bookmarks`
+ - If you are supplying `MAX_FAVOURITES` / `--max-favourites` you must additionally enable this scope:
+   - `read:favourites`
  - If you are supplying `FROM_NOTIFICATIONS` / `--from-notifications` you must additionally enable this scope:
    - `read:notifications`
 

find_posts.py

@@ -23,6 +23,7 @@ argparser.add_argument('--max-followings', required = False, type=int, default=0
 argparser.add_argument('--max-followers', required = False, type=int, default=0, help="Backfill posts for new accounts following --user. We'll backfill at most this many followers' posts")
 argparser.add_argument('--max-follow-requests', required = False, type=int, default=0, help="Backfill posts of the API key owners pending follow requests. We'll backfill at most this many requester's posts")
 argparser.add_argument('--max-bookmarks', required = False, type=int, default=0, help="Fetch remote replies to the API key owners Bookmarks. We'll fetch replies to at most this many bookmarks")
+argparser.add_argument('--max-favourites', required = False, type=int, default=0, help="Fetch remote replies to the API key owners Favourites. We'll fetch replies to at most this many favourites")
 argparser.add_argument('--from-notifications', required = False, type=int, default=0, help="Backfill accounts of anyone appearing in your notifications, during the last hours")
 argparser.add_argument('--remember-users-for-hours', required=False, type=int, default=24*7, help="How long to remember users that you aren't following for, before trying to backfill them again.")
 argparser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.")
@@ -57,6 +58,11 @@ def get_bookmarks(server, access_token, max):
         "Authorization": f"Bearer {access_token}",
     })
 
+def get_favourites(server, access_token, max):
+    return get_paginated_mastodon(f"https://{server}/api/v1/favourites", max, {
+        "Authorization": f"Bearer {access_token}",
+    })
+
 def add_user_posts(server, access_token, followings, know_followings, all_known_users, seen_urls):
     for user in followings:
         if user['acct'] not in all_known_users and not user['url'].startswith(f"https://{server}/"):
@@ -899,6 +905,12 @@ if __name__ == "__main__":
                 known_context_urls = get_all_known_context_urls(arguments.server, bookmarks,parsed_urls)
                 add_context_urls(arguments.server, token, known_context_urls, seen_urls)
 
+            if arguments.max_favourites > 0:
+                log(f"Pulling replies to the last {arguments.max_favourites} favourites")
+                favourites = get_favourites(arguments.server, token, arguments.max_favourites)
+                known_context_urls = get_all_known_context_urls(arguments.server, favourites,parsed_urls)
+                add_context_urls(arguments.server, token, known_context_urls, seen_urls)
+
         with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f:
             f.write("\n".join(list(known_followings)[-10000:]))