travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

convert mastodon scraper to docker

Browse source
This commit is contained in:
Travis Shears 2025-05-22 19:04:40 +02:00
parent 5788151840
commit adbc1f5133
4 changed files with 27 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/mastodon.ts
View file

@ -1,11 +1,8 @@
import pino from "pino";
import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda";
import { MicroBlogBackend } from "./pocketbase";
// custom destination formatter
const destination = pinoLambdaDestination();
const logger = pino({}, destination);
const withRequest = lambdaRequestTracker();
const logger = pino();
const pb = new MicroBlogBackend(logger);
@ -39,7 +36,7 @@ const getPostUntilId = async ({
const params = new URLSearchParams();
params.append("limit", "10");
const urlWithParams = new URL(
`${baseURL}/api/v1/accounts/${accountId}/statuses`
`${baseURL}/api/v1/accounts/${accountId}/statuses`,
);
urlWithParams.search = params.toString();
@ -86,13 +83,12 @@ const saveImages = async (post: MastodonPost, postId: string) => {
for (const image of post.media_attachments) {
await pb.saveAndSetImage(
{ remoteURL: image.url, alt: image.description },
postId
postId,
);
}
};
exports.run = async (event: any, context: any) => {
withRequest(event, context);
(async () => {
const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("mastodon");
console.log({ lastSavedPostId });
const posts = await getPostUntilId({ lastSavedId: lastSavedPostId });
@ -104,4 +100,4 @@ exports.run = async (event: any, context: any) => {
await saveTags(post, savedNewPost.id);
await saveImages(post, savedNewPost.id);
}
};
})();