travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

get nostr scraping working

Browse source
This commit is contained in:
Travis Shears 2025-05-22 12:27:45 +02:00
parent 022d18dc1f
commit 28470e0eba
3 changed files with 142 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

6
package.json
View file

@ -1,7 +1,8 @@
{
"name": "micro_blog_fetchers",
"version": "1.0.0",
"scripts": {
"deploy": "serverless deploy",
"dev": "serverless dev"
"dev:nostr": "node build.mjs && node ./dist/nostr.js | pino-pretty"
},
"devDependencies": {
"@types/node": "^20.14.8",
@ -12,6 +13,7 @@
"dependencies": {
"pino": "^9.2.0",
"pino-lambda": "^4.4.0",
"pino-pretty": "^13.0.0",
"pocketbase": "^0.21.3",
"ws": "^8.18.0"
},

28
src/nostr.ts
View file

@ -9,12 +9,10 @@ const fetcherNpub = process.env.NOSTR_FETCHER_NPUB;
const myNpub = process.env.NOSTR_ID;
if (!fetcherNpub) {
logger.error("NOSTR_FETCHER_NPUB is not set");
throw new Error("NOSTR_FETCHER_NPUB is not set");
}
if (!myNpub) {
logger.error("NOSTR_ID is not set");
throw new Error("NOSTR_ID is not set");
}
@ -35,21 +33,27 @@ type NostrEvent = [
(async () => {
logger.info("Starting Nostr Fetcher");
const events: NostrEvent[] = [];
// figure out when the last post of wasved
const lastSavedPost = await pb.getLatestPostBySource("nostr");
let since: number | undefined;
if (lastSavedPost) {
since = new Date(lastSavedPost.posted).getTime() / 1000;
if (!lastSavedPost) {
throw new Error("No last saved nostr post found");
}
let since: number | undefined;
since = new Date(lastSavedPost.posted).getTime() / 1000;
logger.info(
{ lastSavedPostId: lastSavedPost.id, since },
"lastSavedPost nostr post",
);
// listen for new events for 30 seconds
logger.info("trying to connecting to nostr relay");
const ws = new WebSocket(process.env.NOSTR_RELAY!);
// Other Relay URLs
// "wss://nos.lol",
// "wss://nostr.wine",
// "wss://nostr.einundzwanzig.space",
const relay = process.env.NOSTR_RELAY;
if (!relay) {
throw new Error("No NOSTR_RELAY environment variable found");
}
const events: NostrEvent[] = [];
const ws = new WebSocket(relay);
ws.on("error", logger.error);
ws.on("message", function message(data: Buffer) {
const decodedData = JSON.parse(
@ -73,7 +77,7 @@ type NostrEvent = [
await new Promise((resolve) => setTimeout(resolve, 30000));
logger.info("closing connection to nostr relay");
ws.close();
logger.info("saving nostr posts");
logger.info({ count: events.length }, "saving nostr posts");
for (const event of events) {
const post = await pb.savePost({
remoteId: event[2].id,

122
yarn.lock
View file

@ -214,6 +214,29 @@ __metadata:
languageName: node
linkType: hard
"colorette@npm:^2.0.7":
version: 2.0.20
resolution: "colorette@npm:2.0.20"
checksum: 10c0/e94116ff33b0ff56f3b83b9ace895e5bf87c2a7a47b3401b8c3f3226e050d5ef76cf4072fb3325f9dc24d1698f9b730baf4e05eeaf861d74a1883073f4c98a40
languageName: node
linkType: hard
"dateformat@npm:^4.6.3":
version: 4.6.3
resolution: "dateformat@npm:4.6.3"
checksum: 10c0/e2023b905e8cfe2eb8444fb558562b524807a51cdfe712570f360f873271600b5c94aebffaf11efb285e2c072264a7cf243eadb68f3eba0f8cc85fb86cd25df6
languageName: node
linkType: hard
"end-of-stream@npm:^1.1.0":
version: 1.4.4
resolution: "end-of-stream@npm:1.4.4"
dependencies:
once: "npm:^1.4.0"
checksum: 10c0/870b423afb2d54bb8d243c63e07c170409d41e20b47eeef0727547aea5740bd6717aca45597a9f2745525667a6b804c1e7bede41f856818faee5806dd9ff3975
languageName: node
linkType: hard
"esbuild@npm:^0.25.4":
version: 0.25.4
resolution: "esbuild@npm:0.25.4"
@ -300,6 +323,13 @@ __metadata:
languageName: node
linkType: hard
"fast-copy@npm:^3.0.2":
version: 3.0.2
resolution: "fast-copy@npm:3.0.2"
checksum: 10c0/02e8b9fd03c8c024d2987760ce126456a0e17470850b51e11a1c3254eed6832e4733ded2d93316c82bc0b36aeb991ad1ff48d1ba95effe7add7c3ab8d8eb554a
languageName: node
linkType: hard
"fast-redact@npm:^3.1.1":
version: 3.5.0
resolution: "fast-redact@npm:3.5.0"
@ -307,6 +337,34 @@ __metadata:
languageName: node
linkType: hard
"fast-safe-stringify@npm:^2.1.1":
version: 2.1.1
resolution: "fast-safe-stringify@npm:2.1.1"
checksum: 10c0/d90ec1c963394919828872f21edaa3ad6f1dddd288d2bd4e977027afff09f5db40f94e39536d4646f7e01761d704d72d51dce5af1b93717f3489ef808f5f4e4d
languageName: node
linkType: hard
"help-me@npm:^5.0.0":
version: 5.0.0
resolution: "help-me@npm:5.0.0"
checksum: 10c0/054c0e2e9ae2231c85ab5e04f75109b9d068ffcc54e58fb22079822a5ace8ff3d02c66fd45379c902ad5ab825e5d2e1451fcc2f7eab1eb49e7d488133ba4cacb
languageName: node
linkType: hard
"joycon@npm:^3.1.1":
version: 3.1.1
resolution: "joycon@npm:3.1.1"
checksum: 10c0/131fb1e98c9065d067fd49b6e685487ac4ad4d254191d7aa2c9e3b90f4e9ca70430c43cad001602bdbdabcf58717d3b5c5b7461c1bd8e39478c8de706b3fe6ae
languageName: node
linkType: hard
"minimist@npm:^1.2.6":
version: 1.2.8
resolution: "minimist@npm:1.2.8"
checksum: 10c0/19d3fcdca050087b84c2029841a093691a91259a47def2f18222f41e7645a0b7c44ef4b40e88a1e58a40c84d2ef0ee6047c55594d298146d0eb3f6b737c20ce6
languageName: node
linkType: hard
"on-exit-leak-free@npm:^2.1.0":
version: 2.1.2
resolution: "on-exit-leak-free@npm:2.1.2"
@ -314,6 +372,15 @@ __metadata:
languageName: node
linkType: hard
"once@npm:^1.3.1, once@npm:^1.4.0":
version: 1.4.0
resolution: "once@npm:1.4.0"
dependencies:
wrappy: "npm:1"
checksum: 10c0/5d48aca287dfefabd756621c5dfce5c91a549a93e9fdb7b8246bc4c4790aa2ec17b34a260530474635147aeb631a2dcc8b32c613df0675f96041cbb8244517d0
languageName: node
linkType: hard
"pino-abstract-transport@npm:^2.0.0":
version: 2.0.0
resolution: "pino-abstract-transport@npm:2.0.0"
@ -332,6 +399,29 @@ __metadata:
languageName: node
linkType: hard
"pino-pretty@npm:^13.0.0":
version: 13.0.0
resolution: "pino-pretty@npm:13.0.0"
dependencies:
colorette: "npm:^2.0.7"
dateformat: "npm:^4.6.3"
fast-copy: "npm:^3.0.2"
fast-safe-stringify: "npm:^2.1.1"
help-me: "npm:^5.0.0"
joycon: "npm:^3.1.1"
minimist: "npm:^1.2.6"
on-exit-leak-free: "npm:^2.1.0"
pino-abstract-transport: "npm:^2.0.0"
pump: "npm:^3.0.0"
secure-json-parse: "npm:^2.4.0"
sonic-boom: "npm:^4.0.1"
strip-json-comments: "npm:^3.1.1"
bin:
pino-pretty: bin.js
checksum: 10c0/015dac25006c1b9820b9e01fccb8a392a019e12b30e6bfc3f3f61ecca8dbabcd000a8f3f64410b620b7f5d08579ba85e6ef137f7fbeaad70d46397a97a5f75ea
languageName: node
linkType: hard
"pino-std-serializers@npm:^7.0.0":
version: 7.0.0
resolution: "pino-std-serializers@npm:7.0.0"
@ -383,6 +473,16 @@ __metadata:
languageName: node
linkType: hard
"pump@npm:^3.0.0":
version: 3.0.2
resolution: "pump@npm:3.0.2"
dependencies:
end-of-stream: "npm:^1.1.0"
once: "npm:^1.3.1"
checksum: 10c0/5ad655cb2a7738b4bcf6406b24ad0970d680649d996b55ad20d1be8e0c02394034e4c45ff7cd105d87f1e9b96a0e3d06fd28e11fae8875da26e7f7a8e2c9726f
languageName: node
linkType: hard
"quick-format-unescaped@npm:^4.0.3":
version: 4.0.4
resolution: "quick-format-unescaped@npm:4.0.4"
@ -406,6 +506,7 @@ __metadata:
esbuild: "npm:^0.25.4"
pino: "npm:^9.2.0"
pino-lambda: "npm:^4.4.0"
pino-pretty: "npm:^13.0.0"
pocketbase: "npm:^0.21.3"
prettier: "npm:^3.3.2"
ws: "npm:^8.18.0"
@ -419,6 +520,13 @@ __metadata:
languageName: node
linkType: hard
"secure-json-parse@npm:^2.4.0":
version: 2.7.0
resolution: "secure-json-parse@npm:2.7.0"
checksum: 10c0/f57eb6a44a38a3eeaf3548228585d769d788f59007454214fab9ed7f01fbf2e0f1929111da6db28cf0bcc1a2e89db5219a59e83eeaec3a54e413a0197ce879e4
languageName: node
linkType: hard
"sonic-boom@npm:^4.0.1":
version: 4.2.0
resolution: "sonic-boom@npm:4.2.0"
@ -435,6 +543,13 @@ __metadata:
languageName: node
linkType: hard
"strip-json-comments@npm:^3.1.1":
version: 3.1.1
resolution: "strip-json-comments@npm:3.1.1"
checksum: 10c0/9681a6257b925a7fa0f285851c0e613cc934a50661fa7bb41ca9cbbff89686bb4a0ee366e6ecedc4daafd01e83eee0720111ab294366fe7c185e935475ebcecd
languageName: node
linkType: hard
"thread-stream@npm:^3.0.0":
version: 3.1.0
resolution: "thread-stream@npm:3.1.0"
@ -458,6 +573,13 @@ __metadata:
languageName: node
linkType: hard
"wrappy@npm:1":
version: 1.0.2
resolution: "wrappy@npm:1.0.2"
checksum: 10c0/56fece1a4018c6a6c8e28fbc88c87e0fbf4ea8fd64fc6c63b18f4acc4bd13e0ad2515189786dd2c30d3eec9663d70f4ecf699330002f8ccb547e4a18231fc9f0
languageName: node
linkType: hard
"ws@npm:^8.18.0":
version: 8.18.2
resolution: "ws@npm:8.18.2"