init blue sky scraping

src/micro_blog/pocket_base.clj

@@ -0,0 +1,71 @@
+(ns micro-blog.pocket-base
+  (:require
+   [clojure.pprint :refer [pprint]]
+   [clojure.string :as str]
+   [clj-http.client :as http-client]
+   [malli.core :as m]
+   [micro-blog.config :refer [config]]))
+
+(defonce token-cache ^:private (atom {:token nil :fetched-at nil}))
+(defn- now []
+  (java.time.Instant/now))
+
+(defn older-then-a-day? [fetched-at]
+  (when fetched-at
+    (let [duration (java.time.Duration/between fetched-at (now))]
+      (> (.toHours duration) 23)))) ; 23 to be safe, or use 24
+
+(defn get-login-token []
+  (let [user-name (@config :pocket-base-user)
+        pw (@config :pocket-base-pw)
+        body {:identity user-name :password pw}
+        url (str (@config :pocket-base-host) "/api/collections/users/auth-with-password")]
+    (->
+     (http-client/post url
+                       {:form-params body
+                        :content-type :json
+                        :as :json})
+     :body
+     :token)))
+
+(defn get-login-token-with-cache []
+  (let [{:keys [token fetched-at]} @token-cache]
+    (if (and token (not (older-then-a-day? fetched-at)))
+      token
+      (let [new-token (get-login-token)]
+        (println "Getting new login token")
+        (reset! token-cache {:token new-token :fetched-at (now)})
+        new-token))))
+
+(def sources #{:pleroma :blue_sky :mastodon :pixelfed :nostr})
+(defn valid-source? [source]
+  (contains? sources source))
+
+(defn get-latest-post-remote-id-by-source [source]
+  (let [res-schema
+        [:map
+         [:items
+          [:vector
+           [:map
+            [:id string?]
+            [:remoteId string?]]]]]]
+    (when (not (valid-source? source))
+      (throw (ex-info "Invalid source" {:source source})))
+    (as->
+     (http-client/get  (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records")
+                       {:headers {"Authorization" (get-login-token-with-cache)}
+                        :query-params  {:page 1
+                                        "perPage" 1
+                                        :sort "-posted"
+                                        :filter (str "source = '" (name source) "'")
+                                        :fields (str/join "," ["remoteId" "id"])
+                                        "skipTotal" true}
+                        :content-type :json
+                        :as :json}) x
+      (:body x)
+      (if (m/validate res-schema x)
+        x
+        (do
+          (m/explain res-schema x)
+          (throw (ex-info "Res does not follow schema" {:res x}))))
+      (-> x :items first :remoteId))))