Implement option to get replies to bookmarks (#19)

.github/workflows/get_context.yml

@@ -32,7 +32,7 @@ jobs:
           path: artifacts
       - name: Get Directory structure
         run: ls -lR
-      - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }}  --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }}
+      - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} --server=${{ vars.MASTODON_SERVER }} --reply-interval-in-hours=${{ vars.REPLY_INTERVAL_IN_HOURS || 0 }} --home-timeline-length=${{ vars.HOME_TIMELINE_LENGTH || 0 }} --max-followings=${{ vars.MAX_FOLLOWINGS || 0 }} --user=${{ vars.USER }} --max-followers=${{ vars.MAX_FOLLOWERS || 0 }}  --http-timeout=${{ vars.HTTP_TIMEOUT || 5 }} --max-follow-requests=${{ vars.MAX_FOLLOW_REQUESTS || 0 }} --on-fail=${{ vars.ON_FAIL }} --on-start=${{ vars.ON_START }} --on-done=${{ vars.ON_DONE }} --max-bookmarks=${{ vars.MAX_BOOKMARKS || 0 }}
       - name: Upload artifacts
         uses: actions/upload-artifact@v3
         with:

README.md

@@ -5,6 +5,7 @@ This GitHub repository provides a GitHub action that runs every 10 mins, doing t
 1. It can [pull remote replies into your instance](https://blog.thms.uk/2023/03/pull-missing-responses-into-mastodon?utm_source=github), using the Mastodon API. That part itself has two parts:
    1. It gets remote replies to posts that users on your instance have already replied to during the last `REPLY_INTERVAL_IN_HOURS` hours, and adds them to your own server.
    2. It gets remote replies to the last `HOME_TIMELINE_LENGTH` posts from your home timeline, and adds them to your own server.
+   3. It gets remote replies to the last `MAX_BOOKMARKS` of your bookmarks, and adds them to your own server.
 2. It can also [backfill posts](https://blog.thms.uk/2023/03/backfill-recently-followed-accounts?utm_source=github):
    1. from the last `MAX_FOLLOWINGS` users that you have followed.
    2. form the last `MAX_FOLLOWERS` users that have followed you.
@@ -28,7 +29,7 @@ Regardless of how you want to run this script, you must first get an access toke
 
 1. In Mastodon go to Preferences > Development > New Application
    1. give it a nice name
-   2. enable `read:search`, `read:statuses`, `read:follows`, and `admin:read:accounts`
+   2. Enable the required scopes for your options. See below for details, but if you want to use all parts of this script, you'll need these scopes: `read:search`, `read:statuses`, `read:follows`, `read:bookmarks`, and `admin:read:accounts`
    3. Save
    4. Copy the value of `Your access token`
 
@@ -87,6 +88,7 @@ Please see below for a list of configuration options.
 | `MAX_FOLLOWINGS` | `--max-followings` | No | Provide to backfill profiles for your most recent followings. Determines how many of your last followings you want to backfill. (An integer number, e.g. `80`. Ensure you also provide `USER`).
 | `MAX_FOLLOWERS` | `--max-followers` | No | Provide to backfill profiles for your most recent followers. Determines how many of your last followers you want to backfill. (An integer number, e.g. `80`. Ensure you also provide `USER`).
 | `MAX_FOLLOW_REQUESTS` | `--max-follow-requests` | No | Provide to backfill profiles for the API key owner's most recent pending follow requests. Determines how many of your last follow requests you want to backfill. (An integer number, e.g. `80`.). Requires an access token with `read:follows` scope.
+| `MAX_BOOKMARKS` | `--max-bookmarks` | No | Provide to fetch remote replies to any posts you have bookmarked. Determines how many of your bookmarks you want to get replies to. (An integer number, e.g. `80`.). Requires an access token with `read:bookmarks` scope.
 | `HTTP_TIMEOUT` | `--http-timeout` | No | The timeout for any HTTP requests to the Mastodon API in seconds. Defaults to `5`.
 | -- | `--lock-hours` | No | Determines after how many hours a lock file should be discarded. Not relevant when running the script as GitHub Action, as concurrency is prevented using a different mechanism.
 | `ON_START` | `--on-start` | No | Optionally provide a callback URL that will be pinged when processing is starting. A query parameter `rid={uuid}` will automatically be appended to uniquely identify each execution. This can be used to monitor your script using a service such as healthchecks.io.
@@ -102,6 +104,8 @@ Please see below for a list of configuration options.
    - `admin:read:accounts`
  - If you are supplying `MAX_FOLLOW_REQUESTS` / `--max-follow-requests` you must additionally enable this scope:
    - `read:follows`
+ - If you are supplying `MAX_BOOKMARKS` / `--max-bookmarks` you must additionally enable this scope:
+   - `read:bookmarks`
 
 ## Acknowledgments
 

find_posts.py

@@ -22,6 +22,7 @@ argparser.add_argument('--user', required = False, default='', help="Use togethe
 argparser.add_argument('--max-followings', required = False, type=int, default=0, help="Backfill posts for new accounts followed by --user. We'll backfill at most this many followings' posts")
 argparser.add_argument('--max-followers', required = False, type=int, default=0, help="Backfill posts for new accounts following --user. We'll backfill at most this many followers' posts")
 argparser.add_argument('--max-follow-requests', required = False, type=int, default=0, help="Backfill posts of the API key owners pending follow requests. We'll backfill at most this many requester's posts")
+argparser.add_argument('--max-bookmarks', required = False, type=int, default=0, help="Fetch remote replies to the API key owners Bookmarks. We'll fetch replies to at most this many bookmarks")
 argparser.add_argument('--http-timeout', required = False, type=int, default=5, help="The timeout for any HTTP requests to your own, or other instances.")
 argparser.add_argument('--lock-hours', required = False, type=int, default=24, help="The lock timeout in hours.")
 argparser.add_argument('--on-done', required = False, default=None, help="Provide a url that will be pinged when processing has completed. You can use this for 'dead man switch' monitoring of your task")
@@ -39,7 +40,8 @@ def pull_context(
     backfill_followings_for_user,
     known_followings,
     max_followers,
-    max_follow_requests
+    max_follow_requests,
+    max_bookmarks
 ):
     
     parsed_urls = {}
@@ -80,10 +82,20 @@ def pull_context(
 
     if max_follow_requests > 0:
         log(f"Getting posts from last {max_follow_requests} follow requests")
-        user_id = get_user_id(server, backfill_followings_for_user)
         follow_requests = get_new_follow_requests(server, access_token, max_follow_requests, known_followings)
         add_following_posts(server, access_token, follow_requests, known_followings, seen_urls)
 
+    if max_bookmarks > 0:
+        log(f"Pulling replies to the last {max_bookmarks} bookmarks")
+        bookmarks = get_bookmarks(server, access_token, max_bookmarks)
+        known_context_urls = get_all_known_context_urls(server, bookmarks,parsed_urls)
+        add_context_urls(server, access_token, known_context_urls, seen_urls)
+
+def get_bookmarks(server, access_token, max):
+    return get_paginated_mastodon(f"https://{server}/api/v1/bookmarks", max, {
+        "Authorization": f"Bearer {access_token}",
+    })
+
 def add_following_posts(server, access_token, followings, know_followings, seen_urls):
     for user in followings:
         posts = get_user_posts(user, know_followings, server)
@@ -767,7 +779,8 @@ if __name__ == "__main__":
             arguments.user,
             KNOWN_FOLLOWINGS,
             arguments.max_followers,
-            arguments.max_follow_requests
+            arguments.max_follow_requests,
+            arguments.max_bookmarks
         )
 
         with open(KNOWN_FOLLOWINGS_FILE, "w", encoding="utf-8") as f: