travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

convert bluesky scraper to docker

Browse source
This commit is contained in:
Travis Shears 2025-05-22 14:45:33 +02:00
parent 28470e0eba
commit 5788151840
4 changed files with 35 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/bluesky.ts
View file

@ -1,11 +1,8 @@
import pino from "pino";
import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda";
import { MicroBlogBackend } from "./pocketbase";
// logger
const destination = pinoLambdaDestination();
const logger = pino({}, destination);
const withRequest = lambdaRequestTracker();
const logger = pino();
// pocketbase
const pb = new MicroBlogBackend(logger);
@ -59,7 +56,7 @@ const getPostsUntilID = async (
session: Session,
id: string,
cursor: string | null = null,
oldFeed: BlueSkyPost[] = []
oldFeed: BlueSkyPost[] = [],
): Promise<BlueSkyPost[]> => {
const params = new URLSearchParams();
params.append("actor", session.did);
@ -69,7 +66,7 @@ const getPostsUntilID = async (
}
const urlWithParams = new URL(
"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed"
"https://bsky.social/xrpc/app.bsky.feed.getAuthorFeed",
);
urlWithParams.search = params.toString();
@ -136,13 +133,12 @@ const saveImages = async (post: BlueSkyPost, postId: string) => {
for (const image of images) {
await pb.saveAndSetImage(
{ remoteURL: image.fullsize, alt: image.alt },
postId
postId,
);
}
};
exports.run = async (event: any, context: any) => {
withRequest(event, context);
(async () => {
const session = await createSession();
const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("blue_sky");
const posts = await getPostsUntilID(session, lastSavedPostId ?? "");
@ -156,4 +152,4 @@ exports.run = async (event: any, context: any) => {
await saveTags(post, savedNewPost.id);
await saveImages(post, savedNewPost.id);
}
};
})();