travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

switch to simple ws backed nostr fetcher

Browse source
This commit is contained in:
Travis Shears 2024-07-05 17:27:41 +02:00
parent c5e5e24802
commit a481626e47
6 changed files with 84 additions and 375 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -2,3 +2,4 @@ node_modules
.serverless
.env
.yarn/
nostr/*

2
README.md
View file

@ -11,7 +11,7 @@ The Micro Blog Repo is made up of a PocketBase db and these lambda functions. To
- [X] Post with image support
- [X] [Pixelfed via gram.social](https://gram.social/i/web/profile/703621281309160235)
- [X] [Mastodon via dice.camp](https://dice.camp/@travisshears)
- [ ] [Nostr profile1qqs9....](https://snort.social/nprofile1qqs9udcv9uhqggjz87js9rtaph4lajlxnxsvwvm7zwdjt6etzyk52rgpypmhxue69uhkummnw3ezuetfde6kuer6wasku7nfvuh8xurpvdjj7qgkwaehxw309ajkgetw9ehx7um5wghxcctwvshszrnhwden5te0dehhxtnvdakz7z94haj)
- [X] [Nostr profile1qqs9....](https://snort.social/nprofile1qqs9udcv9uhqggjz87js9rtaph4lajlxnxsvwvm7zwdjt6etzyk52rgpypmhxue69uhkummnw3ezuetfde6kuer6wasku7nfvuh8xurpvdjj7qgkwaehxw309ajkgetw9ehx7um5wghxcctwvshszrnhwden5te0dehhxtnvdakz7z94haj)
## Deployment

3
package.json
View file

@ -10,9 +10,6 @@
"serverless": "4"
},
"dependencies": {
"@nostr-ts/common": "^0.0.5",
"@nostr-ts/node": "^0.0.5",
"nostr-tools": "^2.7.0",
"pino": "^9.2.0",
"pino-lambda": "^4.4.0",
"pocketbase": "^0.21.3",

215
src/nostr.ts
View file

@ -1,9 +1,7 @@
import pino from "pino";
import WebSocket from "ws";
import { lambdaRequestTracker, pinoLambdaDestination } from "pino-lambda";
import { MicroBlogBackend } from "./pocketbase";
import { RelayClient } from "@nostr-ts/node";
import { NFilters, EventsRequest, CLIENT_MESSAGE_TYPE } from "@nostr-ts/common";
import Client from "pocketbase";
// custom destination formatter
const destination = pinoLambdaDestination();
@ -12,147 +10,90 @@ const withRequest = lambdaRequestTracker();
const pb = new MicroBlogBackend(logger);
// const feedEvents = await npubA.feed();
const fetcherNpub =
"npub1qqqqqqp350slvdds7028l4yre5cuh8v38zseert25mxf7lkr2trsy0j2m8";
const myNpub =
"5e370c2f2e0422423fa5028d7d0debfecbe699a0c7337e139b25eb2b112d450d";
// type MastodonPost = {
// media_attachments: {
// type: string; //'image',
// url: string;
// description: string; // 'Blurry gate',
// }[];
// id: string;
// content: string;
// account: {
// id: string;
// };
// created_at: string;
// tags: { name: string }[];
// };
// const getPostUntilId = async ({
// lastSavedId,
// maxId,
// carryPosts = [],
// }: {
// lastSavedId?: string;
// maxId?: string;
// carryPosts?: MastodonPost[];
// }): Promise<MastodonPost[]> => {
// const params = new URLSearchParams();
// params.append("limit", "10");
// const urlWithParams = new URL(
// `${baseURL}/api/v1/accounts/${accountId}/statuses`
// );
// urlWithParams.search = params.toString();
// const res = await fetch(urlWithParams);
// const posts = (await res.json()) as MastodonPost[];
// const containsId = posts.some((post) => post.id === lastSavedId);
// if (!containsId && posts.length >= 5) {
// return getPostUntilId({
// lastSavedId,
// carryPosts: carryPosts?.concat(posts),
// maxId: posts[posts.length - 1]?.id,
// });
// }
// const allPosts = carryPosts?.concat(posts).reverse();
// if (lastSavedId) {
// const index = allPosts.findIndex((post) => post.id === lastSavedId);
// return allPosts.slice(index + 1);
// }
// return allPosts;
// };
// const savePost = async (post: MastodonPost) => {
// const postData = {
// remoteId: post.id,
// authorId: post.account.id,
// posted: post.created_at,
// source: "mastodon" as const,
// fullPost: post,
// };
// return await pb.savePost(postData);
// };
// const saveTags = async (post: MastodonPost, postId: string) => {
// logger.info({ tags: post.tags }, "saving tags");
// for (const tag of post.tags) {
// await pb.setTag(tag.name, postId);
// }
// };
// const saveImages = async (post: MastodonPost, postId: string) => {
// logger.info({ images: post.media_attachments }, "saving images");
// for (const image of post.media_attachments) {
// await pb.saveAndSetImage(
// { remoteURL: image.url, alt: image.description },
// postId
// );
// }
// };
type NostrTag = ["t" | "r" | "imeta", string];
type NostrEvent = [
"EVENT" | "EOSE",
string,
{
id: string;
pubkey: string;
created_at: number;
kind: number; // 1
tags: NostrTag[];
content: string;
sig: string;
},
];
exports.run = async (event: any, context: any) => {
withRequest(event, context);
logger.info("connecting to nostr relays");
const npub =
"npub1tcmsctewqs3yy0a9q2xh6r0tlm97dxdqcuehuyumyh4jkyfdg5xsmyvw2y";
const events: NostrEvent[] = [];
// figure out when the last post of wasved
const lastSavedPost = await pb.getLatestPostBySource("nostr");
let since: number | undefined;
if (lastSavedPost) {
since = new Date(lastSavedPost.posted).getTime() / 1000;
}
// listen for new events for 30 seconds
logger.info("trying to connecting to nostr relay");
const ws = new WebSocket("wss://travis-shears-nostr-relay-v2.fly.dev/");
// Other Relay URLs
// "wss://nos.lol",
// "wss://nostr.wine",
// "wss://nostr.einundzwanzig.space",
let client = new RelayClient([
{
url: "wss://travis-shears-nostr-relay-v2.fly.dev/",
read: true,
write: false,
},
// {
// url: "wss://nos.lol",
// read: true,
// write: false,
// },
// {
// url: "wss://nostr.einundzwanzig.space",
// read: true,
// write: false,
// },
]);
const info = await client.getRelayInformation();
logger.info({ info }, "relay info");
const filters = new NFilters();
filters.addAuthor(npub);
const req: EventsRequest = {
filters,
type: CLIENT_MESSAGE_TYPE.REQ,
options: {
timeoutIn: 10000,
},
};
client.subscribe(req);
client.listen((payload) => {
logger.info(
{ info: payload.meta.info, url: payload.meta.url },
"received event"
);
// logRelayMessage(payload.data);
ws.on("error", logger.error);
ws.on("message", function message(data: Buffer) {
const decodedData = JSON.parse(
Buffer.from(data).toString("utf8")
) as NostrEvent;
logger.info({ decodedData }, "recived a message from nostr relay");
if (decodedData[0] === "EVENT") {
events.push(decodedData);
}
});
client.disconnect();
ws.on("open", function open() {
logger.info("connection established");
ws.send(
JSON.stringify([
"REQ",
fetcherNpub,
{ kinds: [1], authors: [myNpub], ...(since ? { since } : {}) },
])
);
});
await new Promise((resolve) => setTimeout(resolve, 30000));
logger.info("closing connection to nostr relay");
ws.close();
logger.info("saving nostr posts");
for (const event of events) {
const post = await pb.savePost({
remoteId: event[2].id,
fullPost: event[2],
posted: new Date(event[2].created_at * 1000).toISOString(),
source: "nostr",
authorId: event[1],
});
// const lastSavedPostId = await pb.getLatestPostRemoteIDBySource("mastodon");
// console.log({ lastSavedPostId });
// const posts = await getPostUntilId({ lastSavedId: lastSavedPostId });
// const user = ndk.getUser({
// npub,
// });
// await user.fetchProfile();
// for (const post of posts) {
// logger.info({ post }, "saving post");
// const savedNewPost = await savePost(post);
// await saveTags(post, savedNewPost.id);
// await saveImages(post, savedNewPost.id);
// }
for (const tag of event[2].tags) {
if (tag[0] === "t") {
await pb.setTag(tag[1], post.id);
} else if (tag[0] === "imeta") {
const value = tag[1];
// remove "url " from the start of the string
const url = value.slice(4);
await pb.saveAndSetImage(
{
remoteURL: url,
},
post.id
);
}
}
}
};

4
src/pocketbase.ts
View file

@ -18,7 +18,8 @@ export type MicroBlogPostSource =
| "blue_sky"
| "mastodon"
| "pleroma"
| "pixelfed";
| "pixelfed"
| "nostr";
export type MicroBlogPost = {
source: MicroBlogPostSource;
@ -26,6 +27,7 @@ export type MicroBlogPost = {
remoteId: string;
authorId: string;
id: string;
posted: string;
expand: {
images?: MicroBlogPostImage[];
tags?: {

234
yarn.lock
View file

@ -19,94 +19,6 @@ __metadata:
languageName: node
linkType: hard
"@noble/ciphers@npm:^0.5.1":
version: 0.5.3
resolution: "@noble/ciphers@npm:0.5.3"
checksum: 10c0/2303217304baf51ec6caa2d984f4e640a66d3d586162ed8fecf37a00268fbf362e22cd5bceae4b0ccda4fa06ad0cb294d6a6b158260bbd2eac6a3dc0448f5254
languageName: node
linkType: hard
"@noble/curves@npm:1.2.0":
version: 1.2.0
resolution: "@noble/curves@npm:1.2.0"
dependencies:
"@noble/hashes": "npm:1.3.2"
checksum: 10c0/0bac7d1bbfb3c2286910b02598addd33243cb97c3f36f987ecc927a4be8d7d88e0fcb12b0f0ef8a044e7307d1844dd5c49bb724bfa0a79c8ec50ba60768c97f6
languageName: node
linkType: hard
"@noble/curves@npm:^1.1.0":
version: 1.4.2
resolution: "@noble/curves@npm:1.4.2"
dependencies:
"@noble/hashes": "npm:1.4.0"
checksum: 10c0/65620c895b15d46e8087939db6657b46a1a15cd4e0e4de5cd84b97a0dfe0af85f33a431bb21ac88267e3dc508618245d4cb564213959d66a84d690fe18a63419
languageName: node
linkType: hard
"@noble/curves@npm:~1.1.0":
version: 1.1.0
resolution: "@noble/curves@npm:1.1.0"
dependencies:
"@noble/hashes": "npm:1.3.1"
checksum: 10c0/81115c3ebfa7e7da2d7e18d44d686f98dc6d35dbde3964412c05707c92d0994a01545bc265d5c0bc05c8c49333f75b99c9acef6750f5a79b3abcc8e0546acf88
languageName: node
linkType: hard
"@noble/hashes@npm:1.3.1":
version: 1.3.1
resolution: "@noble/hashes@npm:1.3.1"
checksum: 10c0/86512713aaf338bced594bc2046ab249fea4e1ba1e7f2ecd02151ef1b8536315e788c11608fafe1b56f04fad1aa3c602da7e5f8e5fcd5f8b0aa94435fe65278e
languageName: node
linkType: hard
"@noble/hashes@npm:1.3.2":
version: 1.3.2
resolution: "@noble/hashes@npm:1.3.2"
checksum: 10c0/2482cce3bce6a596626f94ca296e21378e7a5d4c09597cbc46e65ffacc3d64c8df73111f2265444e36a3168208628258bbbaccba2ef24f65f58b2417638a20e7
languageName: node
linkType: hard
"@noble/hashes@npm:1.4.0, @noble/hashes@npm:^1.3.1":
version: 1.4.0
resolution: "@noble/hashes@npm:1.4.0"
checksum: 10c0/8c3f005ee72e7b8f9cff756dfae1241485187254e3f743873e22073d63906863df5d4f13d441b7530ea614b7a093f0d889309f28b59850f33b66cb26a779a4a5
languageName: node
linkType: hard
"@noble/hashes@npm:~1.3.0, @noble/hashes@npm:~1.3.1":
version: 1.3.3
resolution: "@noble/hashes@npm:1.3.3"
checksum: 10c0/23c020b33da4172c988e44100e33cd9f8f6250b68b43c467d3551f82070ebd9716e0d9d2347427aa3774c85934a35fa9ee6f026fca2117e3fa12db7bedae7668
languageName: node
linkType: hard
"@nostr-ts/common@npm:0.0.5, @nostr-ts/common@npm:^0.0.5":
version: 0.0.5
resolution: "@nostr-ts/common@npm:0.0.5"
dependencies:
"@noble/curves": "npm:^1.1.0"
"@noble/hashes": "npm:^1.3.1"
bech32: "npm:^2.0.0"
light-bolt11-decoder: "npm:^3.0.0"
nanoid: "npm:^3.0.0"
checksum: 10c0/94c28fcaa3d85cad04ce85076e76f5ed845ad0c4ec8456af500e3fee8172eb9f7a1a35e58c5f39194265d06d48f12f41fb350b7e9e22c882f8cc0e8dfa9881d0
languageName: node
linkType: hard
"@nostr-ts/node@npm:^0.0.5":
version: 0.0.5
resolution: "@nostr-ts/node@npm:0.0.5"
dependencies:
"@noble/curves": "npm:^1.1.0"
"@noble/hashes": "npm:^1.3.1"
"@nostr-ts/common": "npm:0.0.5"
node-fetch: "npm:^3.3.2"
ws: "npm:^8.13.0"
checksum: 10c0/fb3a4fec815425871f9bf5bc6392ba240c8c6cdc85c9eb22af048407345ec8b741d482c250a8afd7b7ca3d9232cb24a664ded04b1a584f7895dd88ed99b50ef2
languageName: node
linkType: hard
"@pkgjs/parseargs@npm:^0.11.0":
version: 0.11.0
resolution: "@pkgjs/parseargs@npm:0.11.0"
@ -114,41 +26,6 @@ __metadata:
languageName: node
linkType: hard
"@scure/base@npm:1.1.1":
version: 1.1.1
resolution: "@scure/base@npm:1.1.1"
checksum: 10c0/97d200da8915ca18a4eceb73c23dda7fc3a4b8509f620c9b7756ee451d7c9ebbc828c6662f9ffa047806fbe41f37bf236c6ef75692690688b7659196cb2dc804
languageName: node
linkType: hard
"@scure/base@npm:~1.1.0":
version: 1.1.7
resolution: "@scure/base@npm:1.1.7"
checksum: 10c0/2d06aaf39e6de4b9640eb40d2e5419176ebfe911597856dcbf3bc6209277ddb83f4b4b02cb1fd1208f819654268ec083da68111d3530bbde07bae913e2fc2e5d
languageName: node
linkType: hard
"@scure/bip32@npm:1.3.1":
version: 1.3.1
resolution: "@scure/bip32@npm:1.3.1"
dependencies:
"@noble/curves": "npm:~1.1.0"
"@noble/hashes": "npm:~1.3.1"
"@scure/base": "npm:~1.1.0"
checksum: 10c0/9ff0ad56f512794aed1ed62e582bf855db829e688235420a116b210169dc31e3e2a8cc4a908126aaa07b6dcbcc4cd085eb12f9d0a8b507a88946d6171a437195
languageName: node
linkType: hard
"@scure/bip39@npm:1.2.1":
version: 1.2.1
resolution: "@scure/bip39@npm:1.2.1"
dependencies:
"@noble/hashes": "npm:~1.3.0"
"@scure/base": "npm:~1.1.0"
checksum: 10c0/fe951f69dd5a7cdcefbe865bce1b160d6b59ba19bd01d09f0718e54fce37a7d8be158b32f5455f0e9c426a7fbbede3e019bf0baa99bacc88ef26a76a07e115d4
languageName: node
linkType: hard
"@types/node@npm:*":
version: 20.14.9
resolution: "@types/node@npm:20.14.9"
@ -263,13 +140,6 @@ __metadata:
languageName: node
linkType: hard
"bech32@npm:^2.0.0":
version: 2.0.0
resolution: "bech32@npm:2.0.0"
checksum: 10c0/45e7cc62758c9b26c05161b4483f40ea534437cf68ef785abadc5b62a2611319b878fef4f86ddc14854f183b645917a19addebc9573ab890e19194bc8f521942
languageName: node
linkType: hard
"brace-expansion@npm:^2.0.1":
version: 2.0.1
resolution: "brace-expansion@npm:2.0.1"
@ -325,13 +195,6 @@ __metadata:
languageName: node
linkType: hard
"data-uri-to-buffer@npm:^4.0.0":
version: 4.0.1
resolution: "data-uri-to-buffer@npm:4.0.1"
checksum: 10c0/20a6b93107597530d71d4cb285acee17f66bcdfc03fd81040921a81252f19db27588d87fc8fc69e1950c55cfb0bf8ae40d0e5e21d907230813eb5d5a7f9eb45b
languageName: node
linkType: hard
"delayed-stream@npm:~1.0.0":
version: 1.0.0
resolution: "delayed-stream@npm:1.0.0"
@ -381,16 +244,6 @@ __metadata:
languageName: node
linkType: hard
"fetch-blob@npm:^3.1.2, fetch-blob@npm:^3.1.4":
version: 3.2.0
resolution: "fetch-blob@npm:3.2.0"
dependencies:
node-domexception: "npm:^1.0.0"
web-streams-polyfill: "npm:^3.0.3"
checksum: 10c0/60054bf47bfa10fb0ba6cb7742acec2f37c1f56344f79a70bb8b1c48d77675927c720ff3191fa546410a0442c998d27ab05e9144c32d530d8a52fbe68f843b69
languageName: node
linkType: hard
"follow-redirects@npm:^1.15.6":
version: 1.15.6
resolution: "follow-redirects@npm:1.15.6"
@ -422,15 +275,6 @@ __metadata:
languageName: node
linkType: hard
"formdata-polyfill@npm:^4.0.10":
version: 4.0.10
resolution: "formdata-polyfill@npm:4.0.10"
dependencies:
fetch-blob: "npm:^3.1.2"
checksum: 10c0/5392ec484f9ce0d5e0d52fb5a78e7486637d516179b0eb84d81389d7eccf9ca2f663079da56f761355c0a65792810e3b345dc24db9a8bbbcf24ef3c8c88570c6
languageName: node
linkType: hard
"glob@npm:^10.3.7":
version: 10.4.2
resolution: "glob@npm:10.4.2"
@ -481,15 +325,6 @@ __metadata:
languageName: node
linkType: hard
"light-bolt11-decoder@npm:^3.0.0":
version: 3.1.1
resolution: "light-bolt11-decoder@npm:3.1.1"
dependencies:
"@scure/base": "npm:1.1.1"
checksum: 10c0/e89ff40fb617b7e96084b10278a18d12ba9ce741eca764dfcf7c07bf4b1966d79708e1f71a190cdd1674febda05bf505da1e0eedc9d4ee506f8af6dcccdf4db3
languageName: node
linkType: hard
"lru-cache@npm:^10.2.0":
version: 10.2.2
resolution: "lru-cache@npm:10.2.2"
@ -529,63 +364,6 @@ __metadata:
languageName: node
linkType: hard
"nanoid@npm:^3.0.0":
version: 3.3.7
resolution: "nanoid@npm:3.3.7"
bin:
nanoid: bin/nanoid.cjs
checksum: 10c0/e3fb661aa083454f40500473bb69eedb85dc160e763150b9a2c567c7e9ff560ce028a9f833123b618a6ea742e311138b591910e795614a629029e86e180660f3
languageName: node
linkType: hard
"node-domexception@npm:^1.0.0":
version: 1.0.0
resolution: "node-domexception@npm:1.0.0"
checksum: 10c0/5e5d63cda29856402df9472335af4bb13875e1927ad3be861dc5ebde38917aecbf9ae337923777af52a48c426b70148815e890a5d72760f1b4d758cc671b1a2b
languageName: node
linkType: hard
"node-fetch@npm:^3.3.2":
version: 3.3.2
resolution: "node-fetch@npm:3.3.2"
dependencies:
data-uri-to-buffer: "npm:^4.0.0"
fetch-blob: "npm:^3.1.4"
formdata-polyfill: "npm:^4.0.10"
checksum: 10c0/f3d5e56190562221398c9f5750198b34cf6113aa304e34ee97c94fd300ec578b25b2c2906edba922050fce983338fde0d5d34fcb0fc3336ade5bd0e429ad7538
languageName: node
linkType: hard
"nostr-tools@npm:^2.7.0":
version: 2.7.0
resolution: "nostr-tools@npm:2.7.0"
dependencies:
"@noble/ciphers": "npm:^0.5.1"
"@noble/curves": "npm:1.2.0"
"@noble/hashes": "npm:1.3.1"
"@scure/base": "npm:1.1.1"
"@scure/bip32": "npm:1.3.1"
"@scure/bip39": "npm:1.2.1"
nostr-wasm: "npm:v0.1.0"
peerDependencies:
typescript: ">=5.0.0"
dependenciesMeta:
nostr-wasm:
optional: true
peerDependenciesMeta:
typescript:
optional: true
checksum: 10c0/a0ca770d4900a2134710ae477b6f6cf74fcad061c0bc5e92d6127a53024d3ecf44d0fc22ef1e8179b78265ac0dcbbc59683cd7b13f825c5746a15f71dffa8efa
languageName: node
linkType: hard
"nostr-wasm@npm:v0.1.0":
version: 0.1.0
resolution: "nostr-wasm@npm:0.1.0"
checksum: 10c0/a8a674c0e038d5f790840e442a80587f6eca0810e01f3101828c34517f5c3238f510ef49f53b3f596e8effb32eb64993c57248aa25b9ccfa9386e4421c837edb
languageName: node
linkType: hard
"on-exit-leak-free@npm:^2.1.0":
version: 2.1.2
resolution: "on-exit-leak-free@npm:2.1.2"
@ -743,11 +521,8 @@ __metadata:
version: 0.0.0-use.local
resolution: "root-workspace-0b6124@workspace:."
dependencies:
"@nostr-ts/common": "npm:^0.0.5"
"@nostr-ts/node": "npm:^0.0.5"
"@types/node": "npm:^20.14.8"
"@types/ws": "npm:^8"
nostr-tools: "npm:^2.7.0"
pino: "npm:^9.2.0"
pino-lambda: "npm:^4.4.0"
pocketbase: "npm:^0.21.3"
@ -896,13 +671,6 @@ __metadata:
languageName: node
linkType: hard
"web-streams-polyfill@npm:^3.0.3":
version: 3.3.3
resolution: "web-streams-polyfill@npm:3.3.3"
checksum: 10c0/64e855c47f6c8330b5436147db1c75cb7e7474d924166800e8e2aab5eb6c76aac4981a84261dd2982b3e754490900b99791c80ae1407a9fa0dcff74f82ea3a7f
languageName: node
linkType: hard
"which@npm:^2.0.1":
version: 2.0.2
resolution: "which@npm:2.0.2"
@ -936,7 +704,7 @@ __metadata:
languageName: node
linkType: hard
"ws@npm:^8.13.0, ws@npm:^8.18.0":
"ws@npm:^8.18.0":
version: 8.18.0
resolution: "ws@npm:8.18.0"
peerDependencies: