add mastodon fetcher

src/mastodon.ts

@@ -0,0 +1,108 @@
+import pino from "pino";
+import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda";
+import { MicroBlogBackend } from "./pocketbase";
+
+// custom destination formatter
+const destination = pinoLambdaDestination();
+const logger = pino({}, destination);
+const withRequest = lambdaRequestTracker();
+
+const pb = new MicroBlogBackend(logger);
+
+const baseURL = "https://dice.camp";
+const accountId = "112364858295724350";
+const username = "travisshears";
+
+type MastodonPost = {
+  media_attachments: {
+    type: string; //'image',
+    url: string;
+    description: string; // 'Blurry gate',
+  }[];
+  id: string;
+  content: string;
+  account: {
+    id: string;
+  };
+  created_at: string;
+  tags: { name: string }[];
+};
+
+const getPostUntilId = async ({
+  lastSavedId,
+  maxId,
+  carryPosts = [],
+}: {
+  lastSavedId?: string;
+  maxId?: string;
+  carryPosts?: MastodonPost[];
+}): Promise<MastodonPost[]> => {
+  const params = new URLSearchParams();
+  params.append("limit", "10");
+  const urlWithParams = new URL(
+    `${baseURL}/api/v1/accounts/${accountId}/statuses`
+  );
+  urlWithParams.search = params.toString();
+
+  const res = await fetch(urlWithParams);
+  const posts = (await res.json()) as MastodonPost[];
+  const containsId = posts.some((post) => post.id === lastSavedId);
+
+  if (!containsId && posts.length >= 5) {
+    return getPostUntilId({
+      lastSavedId,
+      carryPosts: carryPosts?.concat(posts),
+      maxId: posts[posts.length - 1]?.id,
+    });
+  }
+
+  const allPosts = carryPosts?.concat(posts).reverse();
+  if (lastSavedId) {
+    const index = allPosts.findIndex((post) => post.id === lastSavedId);
+    return allPosts.slice(index + 1);
+  }
+  return allPosts;
+};
+
+const savePost = async (post: MastodonPost) => {
+  const postData = {
+    remoteId: post.id,
+    authorId: post.account.id,
+    posted: post.created_at,
+    source: "mastodon" as const,
+    fullPost: post,
+  };
+  return await pb.savePost(postData);
+};
+
+const saveTags = async (post: MastodonPost, postId: string) => {
+  logger.info({ tags: post.tags }, "saving tags");
+  for (const tag of post.tags) {
+    await pb.setTag(tag.name, postId);
+  }
+};
+
+const saveImages = async (post: MastodonPost, postId: string) => {
+  logger.info({ images: post.media_attachments }, "saving images");
+  for (const image of post.media_attachments) {
+    await pb.saveAndSetImage(
+      { remoteURL: image.url, alt: image.description },
+      postId
+    );
+  }
+};
+
+exports.run = async (event: any, context: any) => {
+  withRequest(event, context);
+  const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("mastodon");
+  console.log({ lastSavedPostId });
+  const posts = await getPostUntilId({ lastSavedId: lastSavedPostId });
+  console.log({ posts });
+
+  for (const post of posts) {
+    logger.info({ post }, "saving post");
+    const savedNewPost = await savePost(post);
+    await saveTags(post, savedNewPost.id);
+    await saveImages(post, savedNewPost.id);
+  }
+};