travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

get nostr scraping working

Browse source
This commit is contained in:
Travis Shears 2025-05-22 12:27:45 +02:00
parent 022d18dc1f
commit 28470e0eba
3 changed files with 142 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/nostr.ts
View file

@ -9,12 +9,10 @@ const fetcherNpub = process.env.NOSTR_FETCHER_NPUB;
const myNpub = process.env.NOSTR_ID;
if (!fetcherNpub) {
logger.error("NOSTR_FETCHER_NPUB is not set");
throw new Error("NOSTR_FETCHER_NPUB is not set");
}
if (!myNpub) {
logger.error("NOSTR_ID is not set");
throw new Error("NOSTR_ID is not set");
}
@ -35,21 +33,27 @@ type NostrEvent = [
(async () => {
logger.info("Starting Nostr Fetcher");
const events: NostrEvent[] = [];
// figure out when the last post of wasved
const lastSavedPost = await pb.getLatestPostBySource("nostr");
let since: number | undefined;
if (lastSavedPost) {
since = new Date(lastSavedPost.posted).getTime() / 1000;
if (!lastSavedPost) {
throw new Error("No last saved nostr post found");
}
let since: number | undefined;
since = new Date(lastSavedPost.posted).getTime() / 1000;
logger.info(
{ lastSavedPostId: lastSavedPost.id, since },
"lastSavedPost nostr post",
);
// listen for new events for 30 seconds
logger.info("trying to connecting to nostr relay");
const ws = new WebSocket(process.env.NOSTR_RELAY!);
// Other Relay URLs
// "wss://nos.lol",
// "wss://nostr.wine",
// "wss://nostr.einundzwanzig.space",
const relay = process.env.NOSTR_RELAY;
if (!relay) {
throw new Error("No NOSTR_RELAY environment variable found");
}
const events: NostrEvent[] = [];
const ws = new WebSocket(relay);
ws.on("error", logger.error);
ws.on("message", function message(data: Buffer) {
const decodedData = JSON.parse(
@ -73,7 +77,7 @@ type NostrEvent = [
await new Promise((resolve) => setTimeout(resolve, 30000));
logger.info("closing connection to nostr relay");
ws.close();
logger.info("saving nostr posts");
logger.info({ count: events.length }, "saving nostr posts");
for (const event of events) {
const post = await pb.savePost({
remoteId: event[2].id,