travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

convert mastodon scraper to docker

Browse source
This commit is contained in:
Travis Shears 2025-05-22 19:04:40 +02:00
parent 5788151840
commit adbc1f5133
4 changed files with 27 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

6
build.mjs
View file

@ -17,3 +17,9 @@ await esbuild.build({
entryPoints: ["src/bluesky.ts"], entryPoints: ["src/bluesky.ts"],
outfile: "dist/bluesky.js", outfile: "dist/bluesky.js",
}); });
await esbuild.build({
...commonProps,
entryPoints: ["src/mastodon.ts"],
outfile: "dist/mastodon.js",
});

11
create_docker_image.sh Executable file
View file

@ -0,0 +1,11 @@
#!/bin/sh
set -e
VERSION=$(jq '.version' < package.json | sed 's/"//g')
IMAGE="853019563312.dkr.ecr.eu-central-1.amazonaws.com/micro-blog-fetchers:${VERSION}"
echo Building and deploying micro blog fetchers version: $VERSION;
# # Build the Docker image
docker buildx build --platform linux/amd64,linux/arm64 -t $IMAGE .
docker push $IMAGE

7
package.json
View file

@ -1,8 +1,11 @@
{ {
"name": "micro_blog_fetchers", "name": "micro_blog_fetchers",
"version": "1.1.0", "version": "1.2.0",
"scripts": { "scripts": {
"dev:nostr": "node build.mjs && node ./dist/nostr.js | pino-pretty" "dev:nostr": "node build.mjs && node ./dist/nostr.js | pino-pretty",
"dev:bluesky": "node build.mjs && node ./dist/bluesky.js | pino-pretty",
"dev:mastodon": "node build.mjs && node ./dist/mastodon.js | pino-pretty",
"build": "node build.mjs"
}, },
"devDependencies": { "devDependencies": {
"@types/node": "^20.14.8", "@types/node": "^20.14.8",

14
src/mastodon.ts
View file

@ -1,11 +1,8 @@
import pino from "pino"; import pino from "pino";
import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda";
import { MicroBlogBackend } from "./pocketbase"; import { MicroBlogBackend } from "./pocketbase";
// custom destination formatter // custom destination formatter
const destination = pinoLambdaDestination(); const logger = pino();
const logger = pino({}, destination);
const withRequest = lambdaRequestTracker();
const pb = new MicroBlogBackend(logger); const pb = new MicroBlogBackend(logger);
@ -39,7 +36,7 @@ const getPostUntilId = async ({
const params = new URLSearchParams(); const params = new URLSearchParams();
params.append("limit", "10"); params.append("limit", "10");
const urlWithParams = new URL( const urlWithParams = new URL(
`${baseURL}/api/v1/accounts/${accountId}/statuses` `${baseURL}/api/v1/accounts/${accountId}/statuses`,
); );
urlWithParams.search = params.toString(); urlWithParams.search = params.toString();
@ -86,13 +83,12 @@ const saveImages = async (post: MastodonPost, postId: string) => {
for (const image of post.media_attachments) { for (const image of post.media_attachments) {
await pb.saveAndSetImage( await pb.saveAndSetImage(
{ remoteURL: image.url, alt: image.description }, { remoteURL: image.url, alt: image.description },
postId postId,
); );
} }
}; };
exports.run = async (event: any, context: any) => { (async () => {
withRequest(event, context);
const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("mastodon"); const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("mastodon");
console.log({ lastSavedPostId }); console.log({ lastSavedPostId });
const posts = await getPostUntilId({ lastSavedId: lastSavedPostId }); const posts = await getPostUntilId({ lastSavedId: lastSavedPostId });
@ -104,4 +100,4 @@ exports.run = async (event: any, context: any) => {
await saveTags(post, savedNewPost.id); await saveTags(post, savedNewPost.id);
await saveImages(post, savedNewPost.id); await saveImages(post, savedNewPost.id);
} }
}; })();