init project with pixelfed fetcher
This commit is contained in:
commit
cf01dea155
12 changed files with 929 additions and 0 deletions
99
src/pixelfed.ts
Normal file
99
src/pixelfed.ts
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
const baseURL = `https://gram.social/api/pixelfed/v1/accounts/`;
|
||||
const accountID = `703621281309160235`;
|
||||
import { microBlogBackend as pb } from "./pocketbase";
|
||||
|
||||
type PixelFedPost = {
|
||||
media_attachments: {
|
||||
type: string; //'image',
|
||||
url: string; // 'https://gram.social/storage/m/_v2/703621281309160235/530d83cd3-f15549/FR41GdSiUQY0/bNHMrzuQkuhXKKfR1zG4HHcjFTe6G2YF02SOr2zi.jpg',
|
||||
description: string; // 'Blurry gate',
|
||||
}[];
|
||||
id: string;
|
||||
content: string;
|
||||
account: {
|
||||
id: string;
|
||||
};
|
||||
created_at: string;
|
||||
tags: { name: string }[];
|
||||
};
|
||||
|
||||
const getPostUntilId = async ({
|
||||
lastSavedId,
|
||||
maxId,
|
||||
carryPosts = [],
|
||||
}: {
|
||||
lastSavedId?: string;
|
||||
maxId?: string;
|
||||
carryPosts?: PixelFedPost[];
|
||||
}): Promise<PixelFedPost[]> => {
|
||||
const params = new URLSearchParams();
|
||||
params.append("limit", "5");
|
||||
params.append("only_media", "true");
|
||||
if (maxId) {
|
||||
params.append("max_id", maxId);
|
||||
}
|
||||
|
||||
const urlWithParams = new URL(`${baseURL}${accountID}/statuses`);
|
||||
urlWithParams.search = params.toString();
|
||||
|
||||
const res = await fetch(urlWithParams.toString());
|
||||
const posts = (await res.json()) as PixelFedPost[];
|
||||
const containsId = posts.some((post) => post.id === lastSavedId);
|
||||
|
||||
if (!containsId && posts.length >= 5) {
|
||||
return getPostUntilId({
|
||||
lastSavedId,
|
||||
carryPosts: carryPosts?.concat(posts),
|
||||
maxId: posts[posts.length - 1]?.id,
|
||||
});
|
||||
}
|
||||
|
||||
const allPosts = carryPosts?.concat(posts).reverse();
|
||||
if (lastSavedId) {
|
||||
const index = allPosts.findIndex((post) => post.id === lastSavedId);
|
||||
return allPosts.slice(index + 1);
|
||||
}
|
||||
return allPosts;
|
||||
};
|
||||
|
||||
const savePost = async (post: PixelFedPost) => {
|
||||
const postData = {
|
||||
remoteId: post.id,
|
||||
authorId: post.account.id,
|
||||
posted: post.created_at,
|
||||
source: "pixelfed" as const,
|
||||
fullPost: post,
|
||||
};
|
||||
return await pb.savePost(postData);
|
||||
};
|
||||
|
||||
const saveTags = async (post: PixelFedPost, postId: string) => {
|
||||
console.log({ tags: post.tags }, "saving tags");
|
||||
for (const tag of post.tags) {
|
||||
await pb.setTag(tag.name, postId);
|
||||
}
|
||||
};
|
||||
|
||||
const saveImages = async (post: PixelFedPost, postId: string) => {
|
||||
console.log({ images: post.media_attachments }, "saving images");
|
||||
for (const image of post.media_attachments) {
|
||||
await pb.saveAndSetImage(
|
||||
{ remoteURL: image.url, alt: image.description },
|
||||
postId
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
exports.run = async () => {
|
||||
const lastSavedPostId = await pb.getLatestPostId("pixelfed");
|
||||
const posts = await getPostUntilId({ lastSavedId: lastSavedPostId });
|
||||
const post = posts[0];
|
||||
if (post) {
|
||||
console.log({ post }, "saving post");
|
||||
const savedNewPost = await savePost(post);
|
||||
if (savedNewPost) {
|
||||
await saveTags(post, savedNewPost.id);
|
||||
await saveImages(post, savedNewPost.id);
|
||||
}
|
||||
}
|
||||
};
|
||||
202
src/pocketbase.ts
Normal file
202
src/pocketbase.ts
Normal file
|
|
@ -0,0 +1,202 @@
|
|||
import PocketBase from "pocketbase";
|
||||
|
||||
export type MicroBlogPostImage = {
|
||||
id: string;
|
||||
collectionId: string;
|
||||
image: string;
|
||||
alt?: string;
|
||||
remoteURL: string;
|
||||
};
|
||||
|
||||
export type MicroBlogPostTag = {
|
||||
id: string;
|
||||
tag: string;
|
||||
};
|
||||
|
||||
export type MicroBlogPostSource =
|
||||
| "blue_sky"
|
||||
| "mastodon"
|
||||
| "pleroma"
|
||||
| "pixelfed";
|
||||
|
||||
export type MicroBlogPost = {
|
||||
source: MicroBlogPostSource;
|
||||
fullPost: any;
|
||||
remoteId: string;
|
||||
authorId: string;
|
||||
id: string;
|
||||
expand: {
|
||||
images?: MicroBlogPostImage[];
|
||||
tags?: {
|
||||
id: string;
|
||||
}[];
|
||||
};
|
||||
};
|
||||
|
||||
class MicroBlogBackend {
|
||||
private pb: PocketBase;
|
||||
private clientSetTime?: Date;
|
||||
constructor() {
|
||||
this.pb = new PocketBase("https://personal-pocket-base.fly.dev");
|
||||
}
|
||||
|
||||
private async login() {
|
||||
const pw = process.env.POCKET_BASE_PW!;
|
||||
const userName = process.env.POCKET_BASE_USER!;
|
||||
console.log({ userName }, "Logging in to pocketbase");
|
||||
await this.pb.collection("users").authWithPassword(userName, pw);
|
||||
this.clientSetTime = new Date();
|
||||
}
|
||||
|
||||
private async checkLogin() {
|
||||
if (!this.clientSetTime) {
|
||||
await this.login();
|
||||
return;
|
||||
}
|
||||
const now = new Date();
|
||||
const diff = now.getTime() - this.clientSetTime.getTime();
|
||||
const day = 86_400_000;
|
||||
|
||||
if (diff > day) {
|
||||
await this.login();
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
public async getLatestPostId(
|
||||
postSource: MicroBlogPostSource
|
||||
): Promise<string | undefined> {
|
||||
await this.checkLogin();
|
||||
try {
|
||||
const post = await this.pb
|
||||
.collection<MicroBlogPost>("micro_blog_posts")
|
||||
.getFirstListItem(`source = '${postSource}'`, { sort: "-posted" });
|
||||
return post.id;
|
||||
} catch (error: any) {
|
||||
if (error.status === 404) {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async getTag(tag: string): Promise<MicroBlogPostTag | undefined> {
|
||||
await this.checkLogin();
|
||||
try {
|
||||
const remoteTag = await this.pb
|
||||
.collection<MicroBlogPostTag>("micro_blog_tags")
|
||||
.getFirstListItem(`tag = '${tag}'`);
|
||||
return remoteTag;
|
||||
} catch (e: any) {
|
||||
if (e.status === 404) {
|
||||
return undefined;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async getImageByRemoteURL(
|
||||
remoteURL: string
|
||||
): Promise<MicroBlogPostImage | undefined> {
|
||||
await this.checkLogin();
|
||||
try {
|
||||
const remoteImage = await this.pb
|
||||
.collection<MicroBlogPostImage>("micro_blog_images")
|
||||
.getFirstListItem(`remoteURL = '${remoteURL}'`);
|
||||
return remoteImage;
|
||||
} catch (e: any) {
|
||||
if (e.status === 404) {
|
||||
return undefined;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
private async checkForPost(
|
||||
remoteId: string
|
||||
): Promise<MicroBlogPost | undefined> {
|
||||
await this.checkLogin();
|
||||
try {
|
||||
return await this.pb
|
||||
.collection<MicroBlogPost>("micro_blog_posts")
|
||||
.getFirstListItem(`remoteId = '${remoteId}'`);
|
||||
} catch (e: any) {
|
||||
if (e.status === 404) {
|
||||
return undefined;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
public async savePost(
|
||||
post: Omit<MicroBlogPost, "id" | "expand">
|
||||
): Promise<MicroBlogPost> {
|
||||
const existingPost = await this.checkForPost(post.remoteId);
|
||||
if (!existingPost) {
|
||||
return await this.pb
|
||||
.collection<MicroBlogPost>("micro_blog_posts")
|
||||
.create(post);
|
||||
}
|
||||
return existingPost;
|
||||
}
|
||||
|
||||
public async setTag(rawTag: string, postId: string) {
|
||||
let tag = await this.getTag(rawTag);
|
||||
if (!tag) {
|
||||
tag = await this.pb
|
||||
.collection<MicroBlogPostTag>("micro_blog_tags")
|
||||
.create({ tag: rawTag });
|
||||
}
|
||||
if (!tag) {
|
||||
throw new Error("Failed to create tag");
|
||||
}
|
||||
await this.pb.collection("micro_blog_posts").update(postId, {
|
||||
"tags+": tag.id,
|
||||
});
|
||||
}
|
||||
|
||||
public async saveAndSetImage(
|
||||
imageToSave: Omit<MicroBlogPostImage, "id" | "image" | "collectionId">,
|
||||
postId: string
|
||||
) {
|
||||
let image = await this.getImageByRemoteURL(imageToSave.remoteURL);
|
||||
if (!image) {
|
||||
const imageResponse = await fetch(imageToSave.remoteURL);
|
||||
if (!imageResponse.ok) {
|
||||
throw new Error("Failed to download image");
|
||||
}
|
||||
const imageBlob = await imageResponse.blob();
|
||||
const imageFile = new File([imageBlob], "image.jpg", {
|
||||
type: imageBlob.type,
|
||||
});
|
||||
const data = {
|
||||
...imageToSave,
|
||||
image: imageFile,
|
||||
};
|
||||
image = await this.pb
|
||||
.collection<MicroBlogPostTag>("micro_blog_images")
|
||||
.create(data);
|
||||
}
|
||||
if (!image) {
|
||||
throw new Error("Failed to create image");
|
||||
}
|
||||
await this.pb.collection("micro_blog_posts").update(postId, {
|
||||
"images+": image.id,
|
||||
});
|
||||
}
|
||||
|
||||
public async getPosts(page: number, limit = 20) {
|
||||
await this.checkLogin();
|
||||
const resultList = await this.pb
|
||||
.collection<MicroBlogPost>("micro_blog_posts")
|
||||
.getList(page, limit, {
|
||||
sort: "-posted",
|
||||
expand: "images,tags",
|
||||
filter: `(source = "blue_sky" || source = "pleroma")`,
|
||||
// filter: 'source = "pleroma"',
|
||||
});
|
||||
return resultList;
|
||||
}
|
||||
}
|
||||
|
||||
export const microBlogBackend = new MicroBlogBackend();
|
||||
Loading…
Add table
Add a link
Reference in a new issue