From 3a0588dd65fdcf6702202c76484a7dec5c4fc782 Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Mon, 18 Aug 2025 10:05:16 +0200 Subject: [PATCH 1/7] Fix mastodon scrape when no img discription is set --- src/micro_blog/mastodon.clj | 4 ++-- src/micro_blog/pocket_base.clj | 25 +++++++++++++------------ src/micro_blog/utils.clj | 1 + 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/micro_blog/mastodon.clj b/src/micro_blog/mastodon.clj index 8b7f027..24b7d0d 100644 --- a/src/micro_blog/mastodon.clj +++ b/src/micro_blog/mastodon.clj @@ -17,7 +17,7 @@ [:media_attachments [:vector [:map [:url :string] [:type [:= "image"]] - [:description :string]]]]]]) + [:description [:maybe :string]]]]]]]) (defn get-posts-until-id [id] (let [limit 10 @@ -50,7 +50,7 @@ :remoteId (:id raw-post) :authorId (get-in raw-post [:account :id]) :tags (map :name (:tags raw-post)) - :images (map (fn [img] [(:url img) (:description img)]) (:media_attachments raw-post)) + :images (map (fn [img] [(:url img) (or (:description img) "")]) (:media_attachments raw-post)) :posted (:created_at raw-post))) (defn save-post [post] diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index a4dc520..f74e8c4 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -44,6 +44,10 @@ (defn valid-source? [source] (m/validate source-enum source)) +(def post-schema [:map + [:id :string] + [:remoteId :string]]) + (defn get-all-posts-by-source ([source] (get-all-posts-by-source source [] 1)) ([source carry page] @@ -68,14 +72,11 @@ (concat carry rows) (get-all-posts-by-source source (concat carry rows) (inc page)))))) -(defn get-latest-post-remote-id-by-source [source] +(defn get-latest-post-by-source [source] (let [res-schema [:map [:items - [:vector - [:map - [:id string?] - [:remoteId string?]]]]]] + [:vector post-schema]]]] (when (not (valid-source? source)) (throw (ex-info "Invalid source" {:source source}))) (as-> @@ -85,17 +86,17 @@ "perPage" 1 :sort "-posted" :filter (str "source = '" (name source) "'") - :fields (str/join "," ["remoteId" "id"]) + ;; :fields (str/join "," ["remoteId" "id"]) "skipTotal" true} :content-type :json :as :json}) x (:body x) - (if (m/validate res-schema x) - x - (do - (m/explain res-schema x) - (throw (ex-info "Res does not follow schema" {:res x})))) - (-> x :items first :remoteId)))) + (utils/validate-with-throw x res-schema) + (-> x :items first)))) + +(defn get-latest-post-remote-id-by-source [source] + (tel/log! {:level :info :data {:source source}} "Fetching latest post remote ID for source") + (:remoteId (get-latest-post-by-source source))) (defn post-with-remote-id-already-saved? [remote-id] (-> diff --git a/src/micro_blog/utils.clj b/src/micro_blog/utils.clj index be84ba5..a675c4c 100644 --- a/src/micro_blog/utils.clj +++ b/src/micro_blog/utils.clj @@ -4,6 +4,7 @@ [malli.core :as m])) (defn validate-with-throw [value schema] + (tel/log! {:level :info :data {:value value :schema schema}} "Validating value") (if (m/validate schema value) value (do From 2c3d3cef2ff64f3a9b1db8347fa3886e8075913b Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Mon, 18 Aug 2025 10:05:16 +0200 Subject: [PATCH 2/7] add debug logging to validate-with-throw --- src/micro_blog/utils.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/micro_blog/utils.clj b/src/micro_blog/utils.clj index a675c4c..931028d 100644 --- a/src/micro_blog/utils.clj +++ b/src/micro_blog/utils.clj @@ -4,7 +4,7 @@ [malli.core :as m])) (defn validate-with-throw [value schema] - (tel/log! {:level :info :data {:value value :schema schema}} "Validating value") + (tel/log! {:level :debug :data {:value value :schema schema}} "Validating value") (if (m/validate schema value) value (do From 911f2f03307c9aed9aa7f6cfb0ab176b53c73f92 Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Mon, 18 Aug 2025 09:39:23 +0200 Subject: [PATCH 3/7] init nostr with gniazdo --- deps.edn | 6 +++++- src/micro_blog/config.clj | 6 +++++- src/micro_blog/nostr.clj | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 src/micro_blog/nostr.clj diff --git a/deps.edn b/deps.edn index 0a35e3d..bbb74f0 100644 --- a/deps.edn +++ b/deps.edn @@ -15,7 +15,11 @@ ;; json cheshire/cheshire {:mvn/version "6.0.0"} ;; metosin/muuntaja {:mvn/version "0.6.11"} - org.clojure/clojure {:mvn/version "1.12.1"}} + org.clojure/clojure {:mvn/version "1.12.1"} + ;; websockets + stylefruits/gniazdo {:mvn/version "1.2.2"} + ;; hato/hato {:mvn/version "1.0.0"} + } :aliases {;; Run with clj -T:build function-in-build diff --git a/src/micro_blog/config.clj b/src/micro_blog/config.clj index c12eaef..1a5354a 100644 --- a/src/micro_blog/config.clj +++ b/src/micro_blog/config.clj @@ -8,7 +8,11 @@ :mastodon-host "MASTODON_BASE_URL" :mastodon-account-id "MASTODON_ACCOUNT_ID" :api-host "API_HOST" - :api-port "API_PORT"}) + :api-port "API_PORT" + :nostr-fetcher-npub "NOSTR_FETCHER_NPUB" + :nostr-id "NOSTR_ID" + :nostr-relay "NOSTR_RELAY" + }) (defn- load-config [] (merge (read-string (slurp "config.edn")) diff --git a/src/micro_blog/nostr.clj b/src/micro_blog/nostr.clj new file mode 100644 index 0000000..1d952b8 --- /dev/null +++ b/src/micro_blog/nostr.clj @@ -0,0 +1,28 @@ +(ns micro-blog.nostr + (:require + [gniazdo.core :as ws] + [cheshire.core :as json] + [micro-blog.config :refer [config]])) + +;; :nostr-fetcher-npub "NOSTR_FETCHER_NPUB" +;; :nostr-id "NOSTR_ID" +;; :nostr-relay "NOSTR_RELAY" +(def socket (atom nil)) + +(defn connect [] + (reset! socket + (ws/connect + (@config :nostr-relay) + :on-receive #(prn 'received %)))) + +(defn subscribe-to-author [pubkey] + (let [sub-id "sub-1" + filter {:kinds [1] :authors [pubkey] :limit 10} + msg (json/generate-string ["REQ" sub-id filter])] + (send-msg msg))) + +(defn send-msg [msg] + (ws/send-msg @socket msg)) + +(defn close [] + (ws/close @socket)) From 33a6ccd33b8c1b04dbf6ffc7bffd983c2ad88407 Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Mon, 18 Aug 2025 10:05:16 +0200 Subject: [PATCH 4/7] switch to hato for websockets --- deps.edn | 4 +--- src/micro_blog/config.clj | 6 +++--- src/micro_blog/nostr.clj | 45 +++++++++++++++++++++++++-------------- 3 files changed, 33 insertions(+), 22 deletions(-) diff --git a/deps.edn b/deps.edn index bbb74f0..47e2f36 100644 --- a/deps.edn +++ b/deps.edn @@ -17,9 +17,7 @@ ;; metosin/muuntaja {:mvn/version "0.6.11"} org.clojure/clojure {:mvn/version "1.12.1"} ;; websockets - stylefruits/gniazdo {:mvn/version "1.2.2"} - ;; hato/hato {:mvn/version "1.0.0"} - } + hato/hato {:mvn/version "1.0.0"}} :aliases {;; Run with clj -T:build function-in-build diff --git a/src/micro_blog/config.clj b/src/micro_blog/config.clj index 1a5354a..ad9372b 100644 --- a/src/micro_blog/config.clj +++ b/src/micro_blog/config.clj @@ -9,10 +9,10 @@ :mastodon-account-id "MASTODON_ACCOUNT_ID" :api-host "API_HOST" :api-port "API_PORT" - :nostr-fetcher-npub "NOSTR_FETCHER_NPUB" + ;; :nostr-fetcher-npub "NOSTR_FETCHER_NPUB" + :nostr-fetcher-id "NOSTR_FETCHER_PUB_HEX" :nostr-id "NOSTR_ID" - :nostr-relay "NOSTR_RELAY" - }) + :nostr-relay "NOSTR_RELAY"}) (defn- load-config [] (merge (read-string (slurp "config.edn")) diff --git a/src/micro_blog/nostr.clj b/src/micro_blog/nostr.clj index 1d952b8..ed9adbb 100644 --- a/src/micro_blog/nostr.clj +++ b/src/micro_blog/nostr.clj @@ -1,28 +1,41 @@ (ns micro-blog.nostr (:require - [gniazdo.core :as ws] - [cheshire.core :as json] - [micro-blog.config :refer [config]])) + [micro-blog.pocket-base :as pb] + [hato.websocket :as ws] + [cheshire.core :as json] + [clojure.string :as str] + [micro-blog.config :refer [config]]) + (:import + [java.time Instant OffsetDateTime ZoneOffset] + [java.time.format DateTimeFormatter])) + +(defn pb-date-to-unix-timestamp-seconds [date-str] + (-> date-str + (str/replace " " "T") + (Instant/parse) + (.getEpochSecond))) + +(defn last-post-timestamp [] + (pb-date-to-unix-timestamp-seconds (:posted (pb/get-latest-post-by-source :nostr)))) ;; :nostr-fetcher-npub "NOSTR_FETCHER_NPUB" ;; :nostr-id "NOSTR_ID" ;; :nostr-relay "NOSTR_RELAY" (def socket (atom nil)) - (defn connect [] - (reset! socket - (ws/connect - (@config :nostr-relay) - :on-receive #(prn 'received %)))) + (reset! socket @(ws/websocket (@config :nostr-relay) + {:on-message (fn [_ws msg _last?] + (println "Received message:" msg)) + :on-close (fn [_ws _status _reason] + (println "WebSocket closed!"))}))) -(defn subscribe-to-author [pubkey] - (let [sub-id "sub-1" - filter {:kinds [1] :authors [pubkey] :limit 10} +;; (last-post-timestamp []) + +(defn subscribe-to-author [pubkey since] + (let [sub-id (@config :nostr-fetcher-id) + filter {:kinds [1] :authors [pubkey] :since since} msg (json/generate-string ["REQ" sub-id filter])] - (send-msg msg))) - -(defn send-msg [msg] - (ws/send-msg @socket msg)) + (.get (ws/send! @socket msg)))) (defn close [] - (ws/close @socket)) + (ws/close! @socket)) From 0b89cb40abb540ad6ef4b37bc57545b52d5c3422 Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Thu, 21 Aug 2025 13:04:24 +0200 Subject: [PATCH 5/7] start nostr websockets on main app start --- src/micro_blog/main.clj | 3 +++ src/micro_blog/nostr.clj | 46 +++++++++++++++++++++++++++++----- src/micro_blog/pocket_base.clj | 1 - 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/micro_blog/main.clj b/src/micro_blog/main.clj index cb84b05..c4ec6b9 100644 --- a/src/micro_blog/main.clj +++ b/src/micro_blog/main.clj @@ -5,6 +5,7 @@ [taoensso.telemere :as tel] [micro-blog.logging.main :as logging] micro-blog.api + [micro-blog.nostr :as nostr] [micro-blog.blue-sky :as blue-sky] [micro-blog.mastodon :as masto])) @@ -15,6 +16,8 @@ (logging/setup-logging) (tel/log! :info "Setting up API") (micro-blog.api/start) + (tel/log! :info "Setting up nostr scraper") + (nostr/start) (tel/log! :info "Setting up crons") (doseq [[i cron] (map-indexed vector crons)] (let [start (.plus (Instant/now) (Duration/ofMinutes (* i 5)))] diff --git a/src/micro_blog/nostr.clj b/src/micro_blog/nostr.clj index ed9adbb..35afe68 100644 --- a/src/micro_blog/nostr.clj +++ b/src/micro_blog/nostr.clj @@ -1,9 +1,12 @@ (ns micro-blog.nostr (:require [micro-blog.pocket-base :as pb] + [micro-blog.is-tech] + [taoensso.telemere :as tel] [hato.websocket :as ws] [cheshire.core :as json] [clojure.string :as str] + [clojure.pprint :refer [pprint]] [micro-blog.config :refer [config]]) (:import [java.time Instant OffsetDateTime ZoneOffset] @@ -15,22 +18,49 @@ (Instant/parse) (.getEpochSecond))) +(defn nostr-date-to-pb [ts] + (let [instant (java.time.Instant/ofEpochSecond ts) + formatter (java.time.format.DateTimeFormatter/ofPattern "yyyy-MM-dd HH:mm:ss.SSSX") + zoned (.atZone instant java.time.ZoneOffset/UTC)] + (.format formatter zoned))) + (defn last-post-timestamp [] (pb-date-to-unix-timestamp-seconds (:posted (pb/get-latest-post-by-source :nostr)))) -;; :nostr-fetcher-npub "NOSTR_FETCHER_NPUB" -;; :nostr-id "NOSTR_ID" -;; :nostr-relay "NOSTR_RELAY" +(defn transform-post [post] + (tel/log! {:level :info :data {:post post}} "Transforming nostr post") + (hash-map :source :nostr + :fullPost post + :remoteId (get post "id") + :isTech (micro-blog.is-tech/is-tech? (get post "content")) + :authorId (get post "pubkey") + :tags (reduce (fn [acc tag] + (let [tag-type (first tag) + tag-value (second tag)] + (if (= tag-type "t") + (conj acc tag-value) + acc))) [] (get post "tags")) + :images [] + :posted (nostr-date-to-pb (get post "created_at")))) + +(defn process-msg [raw-msg] + (let [msg (json/parse-string (.toString raw-msg))] + (tel/log! {:level :info :data {:msg msg}} "Processing nostr message") + (let [msg-type (first msg) ;; ex: EVENT + event (nth msg 2)] + (when (and (= (get event "kind") 1) (= msg-type "EVENT")) + (-> event + transform-post + pb/save-post))))) + (def socket (atom nil)) (defn connect [] (reset! socket @(ws/websocket (@config :nostr-relay) {:on-message (fn [_ws msg _last?] - (println "Received message:" msg)) + (process-msg msg)) :on-close (fn [_ws _status _reason] (println "WebSocket closed!"))}))) -;; (last-post-timestamp []) - (defn subscribe-to-author [pubkey since] (let [sub-id (@config :nostr-fetcher-id) filter {:kinds [1] :authors [pubkey] :since since} @@ -39,3 +69,7 @@ (defn close [] (ws/close! @socket)) + +(defn start [] + (connect) + (subscribe-to-author (@config :nostr-id) (last-post-timestamp))) diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index f74e8c4..bd43e02 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -1,6 +1,5 @@ (ns micro-blog.pocket-base (:require - [clojure.pprint :refer [pprint]] [clojure.string :as str] [clj-http.client :as http-client] [malli.core :as m] From 7ccd356e8ca83038f7e3eef15460bd7b5ee926a2 Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Thu, 21 Aug 2025 14:57:06 +0200 Subject: [PATCH 6/7] fix blue sky is tech --- src/micro_blog/blue_sky.clj | 4 +++- src/micro_blog/is_tech.clj | 3 ++- src/micro_blog/pocket_base.clj | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/micro_blog/blue_sky.clj b/src/micro_blog/blue_sky.clj index 1c9579c..d491dac 100644 --- a/src/micro_blog/blue_sky.clj +++ b/src/micro_blog/blue_sky.clj @@ -3,6 +3,7 @@ [clj-http.client :as http-client] [micro-blog.pocket-base :as pb] [micro-blog.utils :as utils] + [cheshire.core :as json] [micro-blog.is-tech] [taoensso.telemere :as tel] [micro-blog.config :refer [config]])) @@ -82,10 +83,11 @@ (map #(vector (:fullsize %) (:alt %)) images))) (defn transform-post [post] + (tel/log! {:level :info :data {:post post}} "Transforming post") (hash-map :source :blue_sky :fullPost post :remoteId (:cid post) - :isTech (micro-blog.is-tech/is-tech? (:record post)) + :isTech (micro-blog.is-tech/is-tech? (json/generate-string (:record post))) :authorId (get-in post [:author :handle]) :tags (extract-tags post) :images (extract-images post) diff --git a/src/micro_blog/is_tech.clj b/src/micro_blog/is_tech.clj index 6a33047..49eb22d 100644 --- a/src/micro_blog/is_tech.clj +++ b/src/micro_blog/is_tech.clj @@ -5,6 +5,7 @@ [clj-http.client :as client])) (defn is-tech? [post-text] + (when (not (string? post-text)) (throw (ex-info "Post text must be a string" {:post-text post-text}))) (let [url (str (:mistral-host @config) "/v1/conversations") headers {"Content-Type" "application/json" "Accept" "application/json" @@ -12,7 +13,7 @@ body {:inputs post-text :stream false :agent_id (@config :mistral-agent-id)}] - (tel/log! {:level :info :data {:url url :agent_id (:agent_id body)}} "making request to mistral agent") + (tel/log! {:level :info :data {:url url :agent_id (:agent_id body) :post-text post-text}} "making request to mistral agent") (-> (client/post url {:headers headers :form-params body diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index bd43e02..ab3f3ae 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -185,7 +185,6 @@ [:remoteId :string] [:authorId :string] [:posted :string]]] - (utils/validate-with-throw post save-post-schema) (tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Post passed save validation") (if (post-with-remote-id-already-saved? (:remoteId post)) From 7fc8cd9dcdf1b480d462335cc6bd30b7d035abfa Mon Sep 17 00:00:00 2001 From: Travis Shears Date: Fri, 22 Aug 2025 15:18:22 +0200 Subject: [PATCH 7/7] add better websocker related logging --- src/micro_blog/nostr.clj | 9 +++++---- src/micro_blog/pocket_base.clj | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/micro_blog/nostr.clj b/src/micro_blog/nostr.clj index 35afe68..1aecdc5 100644 --- a/src/micro_blog/nostr.clj +++ b/src/micro_blog/nostr.clj @@ -6,7 +6,6 @@ [hato.websocket :as ws] [cheshire.core :as json] [clojure.string :as str] - [clojure.pprint :refer [pprint]] [micro-blog.config :refer [config]]) (:import [java.time Instant OffsetDateTime ZoneOffset] @@ -55,11 +54,13 @@ (def socket (atom nil)) (defn connect [] + (tel/log! :info "Opening websocket connection to nostr relay") (reset! socket @(ws/websocket (@config :nostr-relay) - {:on-message (fn [_ws msg _last?] + {:headers {"User-Agent" "micro-blog-fetcher"} + :on-message (fn [_ws msg _last?] (process-msg msg)) - :on-close (fn [_ws _status _reason] - (println "WebSocket closed!"))}))) + :on-close (fn [_ws status reason] + (tel/log! {:level :warn :data {:status status :reason reason}} "WebSocket connection closed"))}))) (defn subscribe-to-author [pubkey since] (let [sub-id (@config :nostr-fetcher-id) diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index ab3f3ae..515ac85 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -188,7 +188,7 @@ (utils/validate-with-throw post save-post-schema) (tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Post passed save validation") (if (post-with-remote-id-already-saved? (:remoteId post)) - (println "post already saved") + (tel/log! {:level :warn :data {:remoteId (:remoteId post)}} "Post already saved, skipping") (try (http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records") {:headers {"Authorization" (get-login-token-with-cache)}