Merge pull request #103 from likeazir/patch-1

Add sharkey as misskey-api-capable

.github/workflows/build-container.yaml

@@ -9,11 +9,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+      
+        uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       - name: Login to GHCR
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         if: github.event_name != 'pull_request'
         with:
           registry: ghcr.io
@@ -21,9 +22,10 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
+          platforms: linux/amd64,linux/arm64
           tags: |
             ghcr.io/${{ github.repository_owner }}/fedifetcher:${{ github.ref_name }}
             ghcr.io/${{ github.repository_owner }}/fedifetcher:latest

.github/workflows/get_context.yml

@@ -12,17 +12,17 @@ jobs:
     environment: mastodon
     steps:
       - name: Checkout original repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
          python-version: '3.10'
          cache: 'pip' # caching pip dependencies
       - run: pip install -r requirements.txt
       - name: Download all workflow run artifacts
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@v3
         with:
           name: artifacts
           workflow: get_context.yml
@@ -32,12 +32,12 @@ jobs:
         run: ls -lR
       - run: python find_posts.py --lock-hours=0 --access-token=${{ secrets.ACCESS_TOKEN }} -c="./config.json"
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: artifacts
           path: |
             artifacts
       - name: Checkout user's forked repository for keeping workflow alive
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Keep workflow alive
-        uses: gautamkrishnar/keepalive-workflow@v1
+        uses: gautamkrishnar/keepalive-workflow@v1

README.md

@@ -97,6 +97,8 @@ Persistent files are stored in `/app/artifacts` within the container, so you may
 
 An [example Kubernetes CronJob](./examples/k8s-cronjob.yaml) for running the container is included in the `examples` folder.
 
+An [example Docker Compose Script](./examples/docker-compose.yaml) for running the container periodically is included in the `examples` folder.
+
 ### Configuration options
 
 FediFetcher has quite a few configuration options, so here is my quick configuration advice, that should probably work for most people:

examples/docker-compose.yaml

@@ -0,0 +1,19 @@
+name: fedifetcher
+services:
+  fedifetcher:
+    stdin_open: true
+    tty: true
+    image: ghcr.io/nanos/fedifetcher:latest
+    command: "--access-token=<TOKEN> --server=<SERVER>"
+    # Persist our data
+    volumes:
+      - ./data:/app/artifacts
+    # Use the `deploy` option to enable `restart_policy`
+    deploy:
+      # Don't go above 1 replica to avoid multiple overlapping executions of the script
+      replicas: 1
+      restart_policy:
+        # The `any` condition means even after successful runs, we'll restart the script
+        condition: any
+        # Specify how often the script should run - for example; after 1 hour.
+        delay: 1h

examples/k8s-cronjob.yaml

@@ -14,7 +14,7 @@ spec:
 apiVersion: batch/v1
 kind: CronJob
 metadata:
-  name: FediFetcher
+  name: fedifetcher
 spec:
   # Run every 2 hours
   schedule: "0 */2 * * *"
@@ -30,7 +30,7 @@ spec:
               persistentVolumeClaim:
                 claimName: fedifetcher-pvc
           containers:
-            - name: FediFetcher
+            - name: fedifetcher
               image: ghcr.io/nanos/fedifetcher:latest
               args:
                 - --server=your.server.social

find_posts.py

@@ -12,7 +12,6 @@ import requests
 import time
 import argparse
 import uuid
-import git
 import defusedxml.ElementTree as ET
 
 argparser=argparse.ArgumentParser()
@@ -138,7 +137,7 @@ def get_user_posts_mastodon(userName, webserver):
     try:
         user_id = get_user_id(webserver, userName)
     except Exception as ex:
-        log(f"Error getting user ID for user {user['acct']}: {ex}")
+        log(f"Error getting user ID for user {userName}: {ex}")
         return None
 
     try:
@@ -149,14 +148,14 @@ def get_user_posts_mastodon(userName, webserver):
             return response.json()
         elif response.status_code == 404:
             raise Exception(
-                f"User {user['acct']} was not found on server {webserver}"
+                f"User {userName} was not found on server {webserver}"
             )
         else:
             raise Exception(
                 f"Error getting URL {url}. Status code: {response.status_code}"
             )
     except Exception as ex:
-        log(f"Error getting posts for user {user['acct']}: {ex}")
+        log(f"Error getting posts for user {userName}: {ex}")
         return None
 
 def get_user_posts_lemmy(userName, userUrl, webserver):
@@ -557,6 +556,11 @@ def parse_url(url, parsed_urls):
         match = parse_mastodon_url(url)
         if match is not None:
             parsed_urls[url] = match
+
+    if url not in parsed_urls:
+        match = parse_mastodon_uri(url)
+        if match is not None:
+            parsed_urls[url] = match
     
     if url not in parsed_urls:
         match = parse_pleroma_url(url)
@@ -602,6 +606,14 @@ def parse_mastodon_url(url):
         return (match.group("server"), match.group("toot_id"))
     return None
 
+def parse_mastodon_uri(uri):
+    """parse a Mastodon URI and return the server and ID"""
+    match = re.match(
+        r"https://(?P<server>[^/]+)/users/(?P<username>[^/]+)/statuses/(?P<toot_id>[^/]+)", uri
+    )
+    if match is not None:
+        return (match.group("server"), match.group("toot_id"))
+    return None
 
 def parse_pleroma_url(url):
     """parse a Pleroma URL and return the server and ID"""
@@ -734,11 +746,6 @@ def get_mastodon_urls(webserver, toot_id, toot_url):
         except Exception as ex:
             log(f"Error parsing context for toot {toot_url}. Exception: {ex}")
         return []
-    elif resp.status_code == 429:
-        reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
-        log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
-        time.sleep((reset - datetime.now()).total_seconds() + 1)
-        return get_mastodon_urls(webserver, toot_id, toot_url)
 
     log(
         f"Error getting context for toot {toot_url}. Status code: {resp.status_code}"
@@ -771,11 +778,6 @@ def get_lemmy_comment_context(webserver, toot_id, toot_url):
         except Exception as ex:
             log(f"Error parsing context for comment {toot_url}. Exception: {ex}")
         return []
-    elif resp.status_code == 429:
-        reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
-        log(f"Rate Limit hit when getting context for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
-        time.sleep((reset - datetime.now()).total_seconds() + 1)
-        return get_lemmy_comment_context(webserver, toot_id, toot_url)
 
 def get_lemmy_comments_urls(webserver, post_id, toot_url):
     """get the URLs of the comments of the given post"""
@@ -812,11 +814,6 @@ def get_lemmy_comments_urls(webserver, post_id, toot_url):
             return urls
         except Exception as ex:
             log(f"Error parsing comments for post {toot_url}. Exception: {ex}")
-    elif resp.status_code == 429:
-        reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
-        log(f"Rate Limit hit when getting comments for {toot_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
-        time.sleep((reset - datetime.now()).total_seconds() + 1)
-        return get_lemmy_comments_urls(webserver, post_id, toot_url)
 
     log(f"Error getting comments for post {toot_url}. Status code: {resp.status_code}")
     return []
@@ -902,11 +899,6 @@ def add_context_url(url, server, access_token):
             "Make sure you have the read:search scope enabled for your access token."
         )
         return False
-    elif resp.status_code == 429:
-        reset = datetime.strptime(resp.headers['x-ratelimit-reset'], '%Y-%m-%dT%H:%M:%S.%fZ')
-        log(f"Rate Limit hit when adding url {search_url}. Waiting to retry at {resp.headers['x-ratelimit-reset']}")
-        time.sleep((reset - datetime.now()).total_seconds() + 1)
-        return add_context_url(url, server, access_token)
     else:
         log(
             f"Error adding url {search_url} to server {server}. Status code: {resp.status_code}"
@@ -1143,6 +1135,7 @@ def get_nodeinfo(server, seen_hosts, host_meta_fallback = False):
             return None
 
     if resp.status_code == 200:
+        nodeLoc = None
         try:
             nodeInfo = resp.json()
             for link in nodeInfo['links']:
@@ -1175,7 +1168,7 @@ def get_nodeinfo(server, seen_hosts, host_meta_fallback = False):
 
     # return early if the web domain has been seen previously (in cases with host-meta lookups)
     if server in seen_hosts:
-        return seen_hosts[server]
+        return seen_hosts.get(server)
 
     try:
         resp = get(nodeLoc, timeout = 30)
@@ -1225,8 +1218,8 @@ def get_server_info(server, seen_hosts):
 def set_server_apis(server):
     # support for new server software should be added here
     software_apis = {
-        'mastodonApiSupport': ['mastodon', 'pleroma', 'akkoma', 'pixelfed', 'hometown'],
-        'misskeyApiSupport': ['misskey', 'calckey', 'firefish', 'foundkey'],
+        'mastodonApiSupport': ['mastodon', 'pleroma', 'akkoma', 'pixelfed', 'hometown', 'iceshrimp'],
+        'misskeyApiSupport': ['misskey', 'calckey', 'firefish', 'foundkey', 'sharkey'],
         'lemmyApiSupport': ['lemmy']
     }
 
@@ -1247,16 +1240,7 @@ def set_server_apis(server):
 if __name__ == "__main__":
     start = datetime.now()
 
-    repo = git.Repo(os.getcwd())
-
-    tag = next((tag for tag in repo.tags if tag.commit == repo.head.commit), None)
-
-    if(isinstance(tag, git.TagReference)) :
-        version = tag.name
-    else:
-        version = f"on commit {repo.head.commit.name_rev}"
-
-    log(f"Starting FediFetcher {version}")
+    log(f"Starting FediFetcher")
 
     arguments = argparser.parse_args()
 

requirements.txt

@@ -1,8 +1,6 @@
 certifi==2022.12.7
 charset-normalizer==3.0.1
 docutils==0.19
-gitdb==4.0.10
-GitPython==3.1.31
 idna==3.4
 python-dateutil==2.8.2
 requests==2.28.2