From a481626e472eeaefa6526522b7d643de51f9c375 Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Fri, 5 Jul 2024 17:27:41 +0200 Subject: [PATCH] switch to simple ws backed nostr fetcher --- .gitignore | 1 + README.md | 2 +- package.json | 3 - src/nostr.ts | 215 ++++++++++++++++-------------------------- src/pocketbase.ts | 4 +- yarn.lock | 234 +--------------------------------------------- 6 files changed, 84 insertions(+), 375 deletions(-) diff --git a/.gitignore b/.gitignore index bad62d7..bfc2cb2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ node_modules .serverless .env .yarn/ +nostr/* diff --git a/README.md b/README.md index 5559ecf..a1108b6 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ The Micro Blog Repo is made up of a PocketBase db and these lambda functions. To - [X] Post with image support - [X] [Pixelfed via gram.social](https://gram.social/i/web/profile/703621281309160235) - [X] [Mastodon via dice.camp](https://dice.camp/@travisshears) -- [ ] [Nostr profile1qqs9....](https://snort.social/nprofile1qqs9udcv9uhqggjz87js9rtaph4lajlxnxsvwvm7zwdjt6etzyk52rgpypmhxue69uhkummnw3ezuetfde6kuer6wasku7nfvuh8xurpvdjj7qgkwaehxw309ajkgetw9ehx7um5wghxcctwvshszrnhwden5te0dehhxtnvdakz7z94haj) +- [X] [Nostr profile1qqs9....](https://snort.social/nprofile1qqs9udcv9uhqggjz87js9rtaph4lajlxnxsvwvm7zwdjt6etzyk52rgpypmhxue69uhkummnw3ezuetfde6kuer6wasku7nfvuh8xurpvdjj7qgkwaehxw309ajkgetw9ehx7um5wghxcctwvshszrnhwden5te0dehhxtnvdakz7z94haj) ## Deployment diff --git a/package.json b/package.json index 0854f87..3a22a54 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,6 @@ "serverless": "4" }, "dependencies": { - "@nostr-ts/common": "^0.0.5", - "@nostr-ts/node": "^0.0.5", - "nostr-tools": "^2.7.0", "pino": "^9.2.0", "pino-lambda": "^4.4.0", "pocketbase": "^0.21.3", diff --git a/src/nostr.ts b/src/nostr.ts index c959786..8fcee7c 100644 --- a/src/nostr.ts +++ b/src/nostr.ts @@ -1,9 +1,7 @@ import pino from "pino"; +import WebSocket from "ws"; import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda"; import { MicroBlogBackend } from "./pocketbase"; -import { RelayClient } from "@nostr-ts/node"; -import { NFilters, EventsRequest, CLIENT_MESSAGE_TYPE } from "@nostr-ts/common"; -import Client from "pocketbase"; // custom destination formatter const destination = pinoLambdaDestination(); @@ -12,147 +10,90 @@ const withRequest = lambdaRequestTracker(); const pb = new MicroBlogBackend(logger); -// const feedEvents = await npubA.feed(); +const fetcherNpub = + "npub1qqqqqqp350slvdds7028l4yre5cuh8v38zseert25mxf7lkr2trsy0j2m8"; +const myNpub = + "5e370c2f2e0422423fa5028d7d0debfecbe699a0c7337e139b25eb2b112d450d"; -// type MastodonPost = { -// media_attachments: { -// type: string; //'image', -// url: string; -// description: string; // 'Blurry gate', -// }[]; -// id: string; -// content: string; -// account: { -// id: string; -// }; -// created_at: string; -// tags: { name: string }[]; -// }; - -// const getPostUntilId = async ({ -// lastSavedId, -// maxId, -// carryPosts = [], -// }: { -// lastSavedId?: string; -// maxId?: string; -// carryPosts?: MastodonPost[]; -// }): Promise => { -// const params = new URLSearchParams(); -// params.append("limit", "10"); -// const urlWithParams = new URL( -// `${baseURL}/api/v1/accounts/${accountId}/statuses` -// ); -// urlWithParams.search = params.toString(); - -// const res = await fetch(urlWithParams); -// const posts = (await res.json()) as MastodonPost[]; -// const containsId = posts.some((post) => post.id === lastSavedId); - -// if (!containsId && posts.length >= 5) { -// return getPostUntilId({ -// lastSavedId, -// carryPosts: carryPosts?.concat(posts), -// maxId: posts[posts.length - 1]?.id, -// }); -// } - -// const allPosts = carryPosts?.concat(posts).reverse(); -// if (lastSavedId) { -// const index = allPosts.findIndex((post) => post.id === lastSavedId); -// return allPosts.slice(index + 1); -// } -// return allPosts; -// }; - -// const savePost = async (post: MastodonPost) => { -// const postData = { -// remoteId: post.id, -// authorId: post.account.id, -// posted: post.created_at, -// source: "mastodon" as const, -// fullPost: post, -// }; -// return await pb.savePost(postData); -// }; - -// const saveTags = async (post: MastodonPost, postId: string) => { -// logger.info({ tags: post.tags }, "saving tags"); -// for (const tag of post.tags) { -// await pb.setTag(tag.name, postId); -// } -// }; - -// const saveImages = async (post: MastodonPost, postId: string) => { -// logger.info({ images: post.media_attachments }, "saving images"); -// for (const image of post.media_attachments) { -// await pb.saveAndSetImage( -// { remoteURL: image.url, alt: image.description }, -// postId -// ); -// } -// }; +type NostrTag = ["t" | "r" | "imeta", string]; +type NostrEvent = [ + "EVENT" | "EOSE", + string, + { + id: string; + pubkey: string; + created_at: number; + kind: number; // 1 + tags: NostrTag[]; + content: string; + sig: string; + }, +]; exports.run = async (event: any, context: any) => { withRequest(event, context); - logger.info("connecting to nostr relays"); - const npub = - "npub1tcmsctewqs3yy0a9q2xh6r0tlm97dxdqcuehuyumyh4jkyfdg5xsmyvw2y"; + const events: NostrEvent[] = []; + + // figure out when the last post of wasved + const lastSavedPost = await pb.getLatestPostBySource("nostr"); + let since: number | undefined; + if (lastSavedPost) { + since = new Date(lastSavedPost.posted).getTime() / 1000; + } + // listen for new events for 30 seconds + logger.info("trying to connecting to nostr relay"); + const ws = new WebSocket("wss://travis-shears-nostr-relay-v2.fly.dev/"); + // Other Relay URLs // "wss://nos.lol", // "wss://nostr.wine", // "wss://nostr.einundzwanzig.space", - let client = new RelayClient([ - { - url: "wss://travis-shears-nostr-relay-v2.fly.dev/", - read: true, - write: false, - }, - // { - // url: "wss://nos.lol", - // read: true, - // write: false, - // }, - // { - // url: "wss://nostr.einundzwanzig.space", - // read: true, - // write: false, - // }, - ]); - const info = await client.getRelayInformation(); - logger.info({ info }, "relay info"); - - const filters = new NFilters(); - filters.addAuthor(npub); - const req: EventsRequest = { - filters, - type: CLIENT_MESSAGE_TYPE.REQ, - options: { - timeoutIn: 10000, - }, - }; - client.subscribe(req); - - client.listen((payload) => { - logger.info( - { info: payload.meta.info, url: payload.meta.url }, - "received event" - ); - // logRelayMessage(payload.data); + ws.on("error", logger.error); + ws.on("message", function message(data: Buffer) { + const decodedData = JSON.parse( + Buffer.from(data).toString("utf8") + ) as NostrEvent; + logger.info({ decodedData }, "recived a message from nostr relay"); + if (decodedData[0] === "EVENT") { + events.push(decodedData); + } }); - client.disconnect(); + ws.on("open", function open() { + logger.info("connection established"); + ws.send( + JSON.stringify([ + "REQ", + fetcherNpub, + { kinds: [1], authors: [myNpub], ...(since ? { since } : {}) }, + ]) + ); + }); + await new Promise((resolve) => setTimeout(resolve, 30000)); + logger.info("closing connection to nostr relay"); + ws.close(); + logger.info("saving nostr posts"); + for (const event of events) { + const post = await pb.savePost({ + remoteId: event[2].id, + fullPost: event[2], + posted: new Date(event[2].created_at * 1000).toISOString(), + source: "nostr", + authorId: event[1], + }); - // const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("mastodon"); - // console.log({ lastSavedPostId }); - // const posts = await getPostUntilId({ lastSavedId: lastSavedPostId }); - // const user = ndk.getUser({ - // npub, - // }); - // await user.fetchProfile(); - - // for (const post of posts) { - // logger.info({ post }, "saving post"); - // const savedNewPost = await savePost(post); - // await saveTags(post, savedNewPost.id); - // await saveImages(post, savedNewPost.id); - // } + for (const tag of event[2].tags) { + if (tag[0] === "t") { + await pb.setTag(tag[1], post.id); + } else if (tag[0] === "imeta") { + const value = tag[1]; + // remove "url " from the start of the string + const url = value.slice(4); + await pb.saveAndSetImage( + { + remoteURL: url, + }, + post.id + ); + } + } + } }; diff --git a/src/pocketbase.ts b/src/pocketbase.ts index 69d2053..8ef275e 100644 --- a/src/pocketbase.ts +++ b/src/pocketbase.ts @@ -18,7 +18,8 @@ export type MicroBlogPostSource = | "blue_sky" | "mastodon" | "pleroma" - | "pixelfed"; + | "pixelfed" + | "nostr"; export type MicroBlogPost = { source: MicroBlogPostSource; @@ -26,6 +27,7 @@ export type MicroBlogPost = { remoteId: string; authorId: string; id: string; + posted: string; expand: { images?: MicroBlogPostImage[]; tags?: { diff --git a/yarn.lock b/yarn.lock index ec6f704..078d11d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -19,94 +19,6 @@ __metadata: languageName: node linkType: hard -"@noble/ciphers@npm:^0.5.1": - version: 0.5.3 - resolution: "@noble/ciphers@npm:0.5.3" - checksum: 10c0/2303217304baf51ec6caa2d984f4e640a66d3d586162ed8fecf37a00268fbf362e22cd5bceae4b0ccda4fa06ad0cb294d6a6b158260bbd2eac6a3dc0448f5254 - languageName: node - linkType: hard - -"@noble/curves@npm:1.2.0": - version: 1.2.0 - resolution: "@noble/curves@npm:1.2.0" - dependencies: - "@noble/hashes": "npm:1.3.2" - checksum: 10c0/0bac7d1bbfb3c2286910b02598addd33243cb97c3f36f987ecc927a4be8d7d88e0fcb12b0f0ef8a044e7307d1844dd5c49bb724bfa0a79c8ec50ba60768c97f6 - languageName: node - linkType: hard - -"@noble/curves@npm:^1.1.0": - version: 1.4.2 - resolution: "@noble/curves@npm:1.4.2" - dependencies: - "@noble/hashes": "npm:1.4.0" - checksum: 10c0/65620c895b15d46e8087939db6657b46a1a15cd4e0e4de5cd84b97a0dfe0af85f33a431bb21ac88267e3dc508618245d4cb564213959d66a84d690fe18a63419 - languageName: node - linkType: hard - -"@noble/curves@npm:~1.1.0": - version: 1.1.0 - resolution: "@noble/curves@npm:1.1.0" - dependencies: - "@noble/hashes": "npm:1.3.1" - checksum: 10c0/81115c3ebfa7e7da2d7e18d44d686f98dc6d35dbde3964412c05707c92d0994a01545bc265d5c0bc05c8c49333f75b99c9acef6750f5a79b3abcc8e0546acf88 - languageName: node - linkType: hard - -"@noble/hashes@npm:1.3.1": - version: 1.3.1 - resolution: "@noble/hashes@npm:1.3.1" - checksum: 10c0/86512713aaf338bced594bc2046ab249fea4e1ba1e7f2ecd02151ef1b8536315e788c11608fafe1b56f04fad1aa3c602da7e5f8e5fcd5f8b0aa94435fe65278e - languageName: node - linkType: hard - -"@noble/hashes@npm:1.3.2": - version: 1.3.2 - resolution: "@noble/hashes@npm:1.3.2" - checksum: 10c0/2482cce3bce6a596626f94ca296e21378e7a5d4c09597cbc46e65ffacc3d64c8df73111f2265444e36a3168208628258bbbaccba2ef24f65f58b2417638a20e7 - languageName: node - linkType: hard - -"@noble/hashes@npm:1.4.0, @noble/hashes@npm:^1.3.1": - version: 1.4.0 - resolution: "@noble/hashes@npm:1.4.0" - checksum: 10c0/8c3f005ee72e7b8f9cff756dfae1241485187254e3f743873e22073d63906863df5d4f13d441b7530ea614b7a093f0d889309f28b59850f33b66cb26a779a4a5 - languageName: node - linkType: hard - -"@noble/hashes@npm:~1.3.0, @noble/hashes@npm:~1.3.1": - version: 1.3.3 - resolution: "@noble/hashes@npm:1.3.3" - checksum: 10c0/23c020b33da4172c988e44100e33cd9f8f6250b68b43c467d3551f82070ebd9716e0d9d2347427aa3774c85934a35fa9ee6f026fca2117e3fa12db7bedae7668 - languageName: node - linkType: hard - -"@nostr-ts/common@npm:0.0.5, @nostr-ts/common@npm:^0.0.5": - version: 0.0.5 - resolution: "@nostr-ts/common@npm:0.0.5" - dependencies: - "@noble/curves": "npm:^1.1.0" - "@noble/hashes": "npm:^1.3.1" - bech32: "npm:^2.0.0" - light-bolt11-decoder: "npm:^3.0.0" - nanoid: "npm:^3.0.0" - checksum: 10c0/94c28fcaa3d85cad04ce85076e76f5ed845ad0c4ec8456af500e3fee8172eb9f7a1a35e58c5f39194265d06d48f12f41fb350b7e9e22c882f8cc0e8dfa9881d0 - languageName: node - linkType: hard - -"@nostr-ts/node@npm:^0.0.5": - version: 0.0.5 - resolution: "@nostr-ts/node@npm:0.0.5" - dependencies: - "@noble/curves": "npm:^1.1.0" - "@noble/hashes": "npm:^1.3.1" - "@nostr-ts/common": "npm:0.0.5" - node-fetch: "npm:^3.3.2" - ws: "npm:^8.13.0" - checksum: 10c0/fb3a4fec815425871f9bf5bc6392ba240c8c6cdc85c9eb22af048407345ec8b741d482c250a8afd7b7ca3d9232cb24a664ded04b1a584f7895dd88ed99b50ef2 - languageName: node - linkType: hard - "@pkgjs/parseargs@npm:^0.11.0": version: 0.11.0 resolution: "@pkgjs/parseargs@npm:0.11.0" @@ -114,41 +26,6 @@ __metadata: languageName: node linkType: hard -"@scure/base@npm:1.1.1": - version: 1.1.1 - resolution: "@scure/base@npm:1.1.1" - checksum: 10c0/97d200da8915ca18a4eceb73c23dda7fc3a4b8509f620c9b7756ee451d7c9ebbc828c6662f9ffa047806fbe41f37bf236c6ef75692690688b7659196cb2dc804 - languageName: node - linkType: hard - -"@scure/base@npm:~1.1.0": - version: 1.1.7 - resolution: "@scure/base@npm:1.1.7" - checksum: 10c0/2d06aaf39e6de4b9640eb40d2e5419176ebfe911597856dcbf3bc6209277ddb83f4b4b02cb1fd1208f819654268ec083da68111d3530bbde07bae913e2fc2e5d - languageName: node - linkType: hard - -"@scure/bip32@npm:1.3.1": - version: 1.3.1 - resolution: "@scure/bip32@npm:1.3.1" - dependencies: - "@noble/curves": "npm:~1.1.0" - "@noble/hashes": "npm:~1.3.1" - "@scure/base": "npm:~1.1.0" - checksum: 10c0/9ff0ad56f512794aed1ed62e582bf855db829e688235420a116b210169dc31e3e2a8cc4a908126aaa07b6dcbcc4cd085eb12f9d0a8b507a88946d6171a437195 - languageName: node - linkType: hard - -"@scure/bip39@npm:1.2.1": - version: 1.2.1 - resolution: "@scure/bip39@npm:1.2.1" - dependencies: - "@noble/hashes": "npm:~1.3.0" - "@scure/base": "npm:~1.1.0" - checksum: 10c0/fe951f69dd5a7cdcefbe865bce1b160d6b59ba19bd01d09f0718e54fce37a7d8be158b32f5455f0e9c426a7fbbede3e019bf0baa99bacc88ef26a76a07e115d4 - languageName: node - linkType: hard - "@types/node@npm:*": version: 20.14.9 resolution: "@types/node@npm:20.14.9" @@ -263,13 +140,6 @@ __metadata: languageName: node linkType: hard -"bech32@npm:^2.0.0": - version: 2.0.0 - resolution: "bech32@npm:2.0.0" - checksum: 10c0/45e7cc62758c9b26c05161b4483f40ea534437cf68ef785abadc5b62a2611319b878fef4f86ddc14854f183b645917a19addebc9573ab890e19194bc8f521942 - languageName: node - linkType: hard - "brace-expansion@npm:^2.0.1": version: 2.0.1 resolution: "brace-expansion@npm:2.0.1" @@ -325,13 +195,6 @@ __metadata: languageName: node linkType: hard -"data-uri-to-buffer@npm:^4.0.0": - version: 4.0.1 - resolution: "data-uri-to-buffer@npm:4.0.1" - checksum: 10c0/20a6b93107597530d71d4cb285acee17f66bcdfc03fd81040921a81252f19db27588d87fc8fc69e1950c55cfb0bf8ae40d0e5e21d907230813eb5d5a7f9eb45b - languageName: node - linkType: hard - "delayed-stream@npm:~1.0.0": version: 1.0.0 resolution: "delayed-stream@npm:1.0.0" @@ -381,16 +244,6 @@ __metadata: languageName: node linkType: hard -"fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4": - version: 3.2.0 - resolution: "fetch-blob@npm:3.2.0" - dependencies: - node-domexception: "npm:^1.0.0" - web-streams-polyfill: "npm:^3.0.3" - checksum: 10c0/60054bf47bfa10fb0ba6cb7742acec2f37c1f56344f79a70bb8b1c48d77675927c720ff3191fa546410a0442c998d27ab05e9144c32d530d8a52fbe68f843b69 - languageName: node - linkType: hard - "follow-redirects@npm:^1.15.6": version: 1.15.6 resolution: "follow-redirects@npm:1.15.6" @@ -422,15 +275,6 @@ __metadata: languageName: node linkType: hard -"formdata-polyfill@npm:^4.0.10": - version: 4.0.10 - resolution: "formdata-polyfill@npm:4.0.10" - dependencies: - fetch-blob: "npm:^3.1.2" - checksum: 10c0/5392ec484f9ce0d5e0d52fb5a78e7486637d516179b0eb84d81389d7eccf9ca2f663079da56f761355c0a65792810e3b345dc24db9a8bbbcf24ef3c8c88570c6 - languageName: node - linkType: hard - "glob@npm:^10.3.7": version: 10.4.2 resolution: "glob@npm:10.4.2" @@ -481,15 +325,6 @@ __metadata: languageName: node linkType: hard -"light-bolt11-decoder@npm:^3.0.0": - version: 3.1.1 - resolution: "light-bolt11-decoder@npm:3.1.1" - dependencies: - "@scure/base": "npm:1.1.1" - checksum: 10c0/e89ff40fb617b7e96084b10278a18d12ba9ce741eca764dfcf7c07bf4b1966d79708e1f71a190cdd1674febda05bf505da1e0eedc9d4ee506f8af6dcccdf4db3 - languageName: node - linkType: hard - "lru-cache@npm:^10.2.0": version: 10.2.2 resolution: "lru-cache@npm:10.2.2" @@ -529,63 +364,6 @@ __metadata: languageName: node linkType: hard -"nanoid@npm:^3.0.0": - version: 3.3.7 - resolution: "nanoid@npm:3.3.7" - bin: - nanoid: bin/nanoid.cjs - checksum: 10c0/e3fb661aa083454f40500473bb69eedb85dc160e763150b9a2c567c7e9ff560ce028a9f833123b618a6ea742e311138b591910e795614a629029e86e180660f3 - languageName: node - linkType: hard - -"node-domexception@npm:^1.0.0": - version: 1.0.0 - resolution: "node-domexception@npm:1.0.0" - checksum: 10c0/5e5d63cda29856402df9472335af4bb13875e1927ad3be861dc5ebde38917aecbf9ae337923777af52a48c426b70148815e890a5d72760f1b4d758cc671b1a2b - languageName: node - linkType: hard - -"node-fetch@npm:^3.3.2": - version: 3.3.2 - resolution: "node-fetch@npm:3.3.2" - dependencies: - data-uri-to-buffer: "npm:^4.0.0" - fetch-blob: "npm:^3.1.4" - formdata-polyfill: "npm:^4.0.10" - checksum: 10c0/f3d5e56190562221398c9f5750198b34cf6113aa304e34ee97c94fd300ec578b25b2c2906edba922050fce983338fde0d5d34fcb0fc3336ade5bd0e429ad7538 - languageName: node - linkType: hard - -"nostr-tools@npm:^2.7.0": - version: 2.7.0 - resolution: "nostr-tools@npm:2.7.0" - dependencies: - "@noble/ciphers": "npm:^0.5.1" - "@noble/curves": "npm:1.2.0" - "@noble/hashes": "npm:1.3.1" - "@scure/base": "npm:1.1.1" - "@scure/bip32": "npm:1.3.1" - "@scure/bip39": "npm:1.2.1" - nostr-wasm: "npm:v0.1.0" - peerDependencies: - typescript: ">=5.0.0" - dependenciesMeta: - nostr-wasm: - optional: true - peerDependenciesMeta: - typescript: - optional: true - checksum: 10c0/a0ca770d4900a2134710ae477b6f6cf74fcad061c0bc5e92d6127a53024d3ecf44d0fc22ef1e8179b78265ac0dcbbc59683cd7b13f825c5746a15f71dffa8efa - languageName: node - linkType: hard - -"nostr-wasm@npm:v0.1.0": - version: 0.1.0 - resolution: "nostr-wasm@npm:0.1.0" - checksum: 10c0/a8a674c0e038d5f790840e442a80587f6eca0810e01f3101828c34517f5c3238f510ef49f53b3f596e8effb32eb64993c57248aa25b9ccfa9386e4421c837edb - languageName: node - linkType: hard - "on-exit-leak-free@npm:^2.1.0": version: 2.1.2 resolution: "on-exit-leak-free@npm:2.1.2" @@ -743,11 +521,8 @@ __metadata: version: 0.0.0-use.local resolution: "root-workspace-0b6124@workspace:." dependencies: - "@nostr-ts/common": "npm:^0.0.5" - "@nostr-ts/node": "npm:^0.0.5" "@types/node": "npm:^20.14.8" "@types/ws": "npm:^8" - nostr-tools: "npm:^2.7.0" pino: "npm:^9.2.0" pino-lambda: "npm:^4.4.0" pocketbase: "npm:^0.21.3" @@ -896,13 +671,6 @@ __metadata: languageName: node linkType: hard -"web-streams-polyfill@npm:^3.0.3": - version: 3.3.3 - resolution: "web-streams-polyfill@npm:3.3.3" - checksum: 10c0/64e855c47f6c8330b5436147db1c75cb7e7474d924166800e8e2aab5eb6c76aac4981a84261dd2982b3e754490900b99791c80ae1407a9fa0dcff74f82ea3a7f - languageName: node - linkType: hard - "which@npm:^2.0.1": version: 2.0.2 resolution: "which@npm:2.0.2" @@ -936,7 +704,7 @@ __metadata: languageName: node linkType: hard -"ws@npm:^8.13.0, ws@npm:^8.18.0": +"ws@npm:^8.18.0": version: 8.18.0 resolution: "ws@npm:8.18.0" peerDependencies: