travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

add logging

Browse source
This commit is contained in:
Travis Shears 2024-06-24 15:44:17 +02:00
parent cf01dea155
commit e8234e41e8
4 changed files with 230 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -9,6 +9,8 @@
"serverless": "4"
},
"dependencies": {
"pino": "^9.2.0",
"pino-lambda": "^4.4.0",
"pocketbase": "^0.21.3"
}
}

24
src/pixelfed.ts
View file

@ -1,6 +1,16 @@
import pino from "pino";
import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda";
import { MicroBlogBackend } from "./pocketbase";
// custom destination formatter
const destination = pinoLambdaDestination();
const logger = pino({}, destination);
const withRequest = lambdaRequestTracker();
const baseURL = `https://gram.social/api/pixelfed/v1/accounts/`;
const accountID = `703621281309160235`;
import { microBlogBackend as pb } from "./pocketbase";
const pb = new MicroBlogBackend(logger);
type PixelFedPost = {
media_attachments: {
@ -68,14 +78,14 @@ const savePost = async (post: PixelFedPost) => {
};
const saveTags = async (post: PixelFedPost, postId: string) => {
console.log({ tags: post.tags }, "saving tags");
logger.info({ tags: post.tags }, "saving tags");
for (const tag of post.tags) {
await pb.setTag(tag.name, postId);
}
};
const saveImages = async (post: PixelFedPost, postId: string) => {
console.log({ images: post.media_attachments }, "saving images");
logger.info({ images: post.media_attachments }, "saving images");
for (const image of post.media_attachments) {
await pb.saveAndSetImage(
{ remoteURL: image.url, alt: image.description },
@ -84,12 +94,12 @@ const saveImages = async (post: PixelFedPost, postId: string) => {
}
};
exports.run = async () => {
exports.run = async (event: any, context: any) => {
withRequest(event, context);
const lastSavedPostId = await pb.getLatestPostId("pixelfed");
const posts = await getPostUntilId({ lastSavedId: lastSavedPostId });
const post = posts[0];
if (post) {
console.log({ post }, "saving post");
for (const post of posts) {
logger.info({ post }, "saving post");
const savedNewPost = await savePost(post);
if (savedNewPost) {
await saveTags(post, savedNewPost.id);

10
src/pocketbase.ts
View file

@ -1,3 +1,4 @@
import type { Logger } from "pino";
import PocketBase from "pocketbase";
export type MicroBlogPostImage = {
@ -33,17 +34,17 @@ export type MicroBlogPost = {
};
};
class MicroBlogBackend {
export class MicroBlogBackend {
private pb: PocketBase;
private clientSetTime?: Date;
constructor() {
constructor(private logger: Logger) {
this.pb = new PocketBase("https://personal-pocket-base.fly.dev");
}
private async login() {
const pw = process.env.POCKET_BASE_PW!;
const userName = process.env.POCKET_BASE_USER!;
console.log({ userName }, "Logging in to pocketbase");
this.logger.info({ userName }, "Logging in to pocketbase");
await this.pb.collection("users").authWithPassword(userName, pw);
this.clientSetTime = new Date();
}
@ -137,6 +138,7 @@ class MicroBlogBackend {
.collection<MicroBlogPost>("micro_blog_posts")
.create(post);
}
this.logger.info({ existingPost }, "Found existing post");
return existingPost;
}
@ -198,5 +200,3 @@ class MicroBlogBackend {
return resultList;
}
}
export const microBlogBackend = new MicroBlogBackend();

206
yarn.lock
View file

@ -35,6 +35,15 @@ __metadata:
languageName: node
linkType: hard
"abort-controller@npm:^3.0.0":
version: 3.0.0
resolution: "abort-controller@npm:3.0.0"
dependencies:
event-target-shim: "npm:^5.0.0"
checksum: 10c0/90ccc50f010250152509a344eb2e71977fbf8db0ab8f1061197e3275ddf6c61a41a6edfd7b9409c664513131dd96e962065415325ef23efa5db931b382d24ca5
languageName: node
linkType: hard
"ansi-regex@npm:^5.0.1":
version: 5.0.1
resolution: "ansi-regex@npm:5.0.1"
@ -72,6 +81,13 @@ __metadata:
languageName: node
linkType: hard
"atomic-sleep@npm:^1.0.0":
version: 1.0.0
resolution: "atomic-sleep@npm:1.0.0"
checksum: 10c0/e329a6665512736a9bbb073e1761b4ec102f7926cce35037753146a9db9c8104f5044c1662e4a863576ce544fb8be27cd2be6bc8c1a40147d03f31eb1cfb6e8a
languageName: node
linkType: hard
"axios-proxy-builder@npm:^0.1.2":
version: 0.1.2
resolution: "axios-proxy-builder@npm:0.1.2"
@ -99,6 +115,13 @@ __metadata:
languageName: node
linkType: hard
"base64-js@npm:^1.3.1":
version: 1.5.1
resolution: "base64-js@npm:1.5.1"
checksum: 10c0/f23823513b63173a001030fae4f2dabe283b99a9d324ade3ad3d148e218134676f1ee8568c877cd79ec1c53158dcf2d2ba527a97c606618928ba99dd930102bf
languageName: node
linkType: hard
"brace-expansion@npm:^2.0.1":
version: 2.0.1
resolution: "brace-expansion@npm:2.0.1"
@ -108,6 +131,16 @@ __metadata:
languageName: node
linkType: hard
"buffer@npm:^6.0.3":
version: 6.0.3
resolution: "buffer@npm:6.0.3"
dependencies:
base64-js: "npm:^1.3.1"
ieee754: "npm:^1.2.1"
checksum: 10c0/2a905fbbcde73cc5d8bd18d1caa23715d5f83a5935867c2329f0ac06104204ba7947be098fe1317fbd8830e26090ff8e764f08cd14fefc977bb248c3487bcbd0
languageName: node
linkType: hard
"color-convert@npm:^2.0.1":
version: 2.0.1
resolution: "color-convert@npm:2.0.1"
@ -172,6 +205,27 @@ __metadata:
languageName: node
linkType: hard
"event-target-shim@npm:^5.0.0":
version: 5.0.1
resolution: "event-target-shim@npm:5.0.1"
checksum: 10c0/0255d9f936215fd206156fd4caa9e8d35e62075d720dc7d847e89b417e5e62cf1ce6c9b4e0a1633a9256de0efefaf9f8d26924b1f3c8620cffb9db78e7d3076b
languageName: node
linkType: hard
"events@npm:^3.3.0":
version: 3.3.0
resolution: "events@npm:3.3.0"
checksum: 10c0/d6b6f2adbccbcda74ddbab52ed07db727ef52e31a61ed26db9feb7dc62af7fc8e060defa65e5f8af9449b86b52cc1a1f6a79f2eafcf4e62add2b7a1fa4a432f6
languageName: node
linkType: hard
"fast-redact@npm:^3.1.1":
version: 3.5.0
resolution: "fast-redact@npm:3.5.0"
checksum: 10c0/7e2ce4aad6e7535e0775bf12bd3e4f2e53d8051d8b630e0fa9e67f68cb0b0e6070d2f7a94b1d0522ef07e32f7c7cda5755e2b677a6538f1e9070ca053c42343a
languageName: node
linkType: hard
"follow-redirects@npm:^1.15.6":
version: 1.15.6
resolution: "follow-redirects@npm:1.15.6"
@ -219,6 +273,13 @@ __metadata:
languageName: node
linkType: hard
"ieee754@npm:^1.2.1":
version: 1.2.1
resolution: "ieee754@npm:1.2.1"
checksum: 10c0/b0782ef5e0935b9f12883a2e2aa37baa75da6e66ce6515c168697b42160807d9330de9a32ec1ed73149aea02e0d822e572bca6f1e22bdcbd2149e13b050b17bb
languageName: node
linkType: hard
"is-fullwidth-code-point@npm:^3.0.0":
version: 3.0.0
resolution: "is-fullwidth-code-point@npm:3.0.0"
@ -285,6 +346,13 @@ __metadata:
languageName: node
linkType: hard
"on-exit-leak-free@npm:^2.1.0":
version: 2.1.2
resolution: "on-exit-leak-free@npm:2.1.2"
checksum: 10c0/faea2e1c9d696ecee919026c32be8d6a633a7ac1240b3b87e944a380e8a11dc9c95c4a1f8fb0568de7ab8db3823e790f12bda45296b1d111e341aad3922a0570
languageName: node
linkType: hard
"package-json-from-dist@npm:^1.0.0":
version: 1.0.0
resolution: "package-json-from-dist@npm:1.0.0"
@ -309,6 +377,53 @@ __metadata:
languageName: node
linkType: hard
"pino-abstract-transport@npm:^1.2.0":
version: 1.2.0
resolution: "pino-abstract-transport@npm:1.2.0"
dependencies:
readable-stream: "npm:^4.0.0"
split2: "npm:^4.0.0"
checksum: 10c0/b4ab59529b7a91f488440147fc58ee0827a6c1c5ca3627292339354b1381072c1a6bfa9b46d03ad27872589e8477ecf74da12cf286e1e6b665ac64a3b806bf07
languageName: node
linkType: hard
"pino-lambda@npm:^4.4.0":
version: 4.4.0
resolution: "pino-lambda@npm:4.4.0"
peerDependencies:
pino: ">=6.0.0"
checksum: 10c0/5166f439915c322cadec5bff6b991f29b5880c88d6a66dfe12bce9ac7537318c72041898f5bf2218303c3510350d935727902d0e3f901561a35cc7a5c115cb76
languageName: node
linkType: hard
"pino-std-serializers@npm:^7.0.0":
version: 7.0.0
resolution: "pino-std-serializers@npm:7.0.0"
checksum: 10c0/73e694d542e8de94445a03a98396cf383306de41fd75ecc07085d57ed7a57896198508a0dec6eefad8d701044af21eb27253ccc352586a03cf0d4a0bd25b4133
languageName: node
linkType: hard
"pino@npm:^9.2.0":
version: 9.2.0
resolution: "pino@npm:9.2.0"
dependencies:
atomic-sleep: "npm:^1.0.0"
fast-redact: "npm:^3.1.1"
on-exit-leak-free: "npm:^2.1.0"
pino-abstract-transport: "npm:^1.2.0"
pino-std-serializers: "npm:^7.0.0"
process-warning: "npm:^3.0.0"
quick-format-unescaped: "npm:^4.0.3"
real-require: "npm:^0.2.0"
safe-stable-stringify: "npm:^2.3.1"
sonic-boom: "npm:^4.0.1"
thread-stream: "npm:^3.0.0"
bin:
pino: bin.js
checksum: 10c0/5fbd226ff7dab0961232b5aa5eca0530cdc5bb29f6bf17d929e42239293b1a587a26cc311db6abc1090c9dd57e8f7b031eae341b41d00d4a642b4f1736474c80
languageName: node
linkType: hard
"pocketbase@npm:^0.21.3":
version: 0.21.3
resolution: "pocketbase@npm:0.21.3"
@ -325,6 +440,20 @@ __metadata:
languageName: node
linkType: hard
"process-warning@npm:^3.0.0":
version: 3.0.0
resolution: "process-warning@npm:3.0.0"
checksum: 10c0/60f3c8ddee586f0706c1e6cb5aa9c86df05774b9330d792d7c8851cf0031afd759d665404d07037e0b4901b55c44a423f07bdc465c63de07d8d23196bb403622
languageName: node
linkType: hard
"process@npm:^0.11.10":
version: 0.11.10
resolution: "process@npm:0.11.10"
checksum: 10c0/40c3ce4b7e6d4b8c3355479df77aeed46f81b279818ccdc500124e6a5ab882c0cc81ff7ea16384873a95a74c4570b01b120f287abbdd4c877931460eca6084b3
languageName: node
linkType: hard
"proxy-from-env@npm:^1.1.0":
version: 1.1.0
resolution: "proxy-from-env@npm:1.1.0"
@ -332,6 +461,33 @@ __metadata:
languageName: node
linkType: hard
"quick-format-unescaped@npm:^4.0.3":
version: 4.0.4
resolution: "quick-format-unescaped@npm:4.0.4"
checksum: 10c0/fe5acc6f775b172ca5b4373df26f7e4fd347975578199e7d74b2ae4077f0af05baa27d231de1e80e8f72d88275ccc6028568a7a8c9ee5e7368ace0e18eff93a4
languageName: node
linkType: hard
"readable-stream@npm:^4.0.0":
version: 4.5.2
resolution: "readable-stream@npm:4.5.2"
dependencies:
abort-controller: "npm:^3.0.0"
buffer: "npm:^6.0.3"
events: "npm:^3.3.0"
process: "npm:^0.11.10"
string_decoder: "npm:^1.3.0"
checksum: 10c0/a2c80e0e53aabd91d7df0330929e32d0a73219f9477dbbb18472f6fdd6a11a699fc5d172a1beff98d50eae4f1496c950ffa85b7cc2c4c196963f289a5f39275d
languageName: node
linkType: hard
"real-require@npm:^0.2.0":
version: 0.2.0
resolution: "real-require@npm:0.2.0"
checksum: 10c0/23eea5623642f0477412ef8b91acd3969015a1501ed34992ada0e3af521d3c865bb2fe4cdbfec5fe4b505f6d1ef6a03e5c3652520837a8c3b53decff7e74b6a0
languageName: node
linkType: hard
"rimraf@npm:^5.0.5":
version: 5.0.7
resolution: "rimraf@npm:5.0.7"
@ -348,12 +504,28 @@ __metadata:
resolution: "root-workspace-0b6124@workspace:."
dependencies:
"@types/node": "npm:^20.14.8"
pino: "npm:^9.2.0"
pino-lambda: "npm:^4.4.0"
pocketbase: "npm:^0.21.3"
prettier: "npm:^3.3.2"
serverless: "npm:4"
languageName: unknown
linkType: soft
"safe-buffer@npm:~5.2.0":
version: 5.2.1
resolution: "safe-buffer@npm:5.2.1"
checksum: 10c0/6501914237c0a86e9675d4e51d89ca3c21ffd6a31642efeba25ad65720bce6921c9e7e974e5be91a786b25aa058b5303285d3c15dbabf983a919f5f630d349f3
languageName: node
linkType: hard
"safe-stable-stringify@npm:^2.3.1":
version: 2.4.3
resolution: "safe-stable-stringify@npm:2.4.3"
checksum: 10c0/81dede06b8f2ae794efd868b1e281e3c9000e57b39801c6c162267eb9efda17bd7a9eafa7379e1f1cacd528d4ced7c80d7460ad26f62ada7c9e01dec61b2e768
languageName: node
linkType: hard
"serverless@npm:4":
version: 4.1.5
resolution: "serverless@npm:4.1.5"
@ -391,6 +563,22 @@ __metadata:
languageName: node
linkType: hard
"sonic-boom@npm:^4.0.1":
version: 4.0.1
resolution: "sonic-boom@npm:4.0.1"
dependencies:
atomic-sleep: "npm:^1.0.0"
checksum: 10c0/7b467f2bc8af7ff60bf210382f21c59728cc4b769af9b62c31dd88723f5cc472752d2320736cc366acc7c765ddd5bec3072c033b0faf249923f576a7453ba9d3
languageName: node
linkType: hard
"split2@npm:^4.0.0":
version: 4.2.0
resolution: "split2@npm:4.2.0"
checksum: 10c0/b292beb8ce9215f8c642bb68be6249c5a4c7f332fc8ecadae7be5cbdf1ea95addc95f0459ef2e7ad9d45fd1064698a097e4eb211c83e772b49bc0ee423e91534
languageName: node
linkType: hard
"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0":
version: 4.2.3
resolution: "string-width@npm:4.2.3"
@ -413,6 +601,15 @@ __metadata:
languageName: node
linkType: hard
"string_decoder@npm:^1.3.0":
version: 1.3.0
resolution: "string_decoder@npm:1.3.0"
dependencies:
safe-buffer: "npm:~5.2.0"
checksum: 10c0/810614ddb030e271cd591935dcd5956b2410dd079d64ff92a1844d6b7588bf992b3e1b69b0f4d34a3e06e0bd73046ac646b5264c1987b20d0601f81ef35d731d
languageName: node
linkType: hard
"strip-ansi-cjs@npm:strip-ansi@^6.0.1, strip-ansi@npm:^6.0.0, strip-ansi@npm:^6.0.1":
version: 6.0.1
resolution: "strip-ansi@npm:6.0.1"
@ -431,6 +628,15 @@ __metadata:
languageName: node
linkType: hard
"thread-stream@npm:^3.0.0":
version: 3.1.0
resolution: "thread-stream@npm:3.1.0"
dependencies:
real-require: "npm:^0.2.0"
checksum: 10c0/c36118379940b77a6ef3e6f4d5dd31e97b8210c3f7b9a54eb8fe6358ab173f6d0acfaf69b9c3db024b948c0c5fd2a7df93e2e49151af02076b35ada3205ec9a6
languageName: node
linkType: hard
"tunnel@npm:^0.0.6":
version: 0.0.6
resolution: "tunnel@npm:0.0.6"