diff --git a/src/micro_blog/blue_sky.clj b/src/micro_blog/blue_sky.clj index 6f7bd8f..e350d24 100644 --- a/src/micro_blog/blue_sky.clj +++ b/src/micro_blog/blue_sky.clj @@ -32,7 +32,15 @@ [:author [:map [:handle :string]]] + [:embed {:optional true} + [:map [:images {:optional true} [:vector [:map + [:fullsize :string] + [:alt :string]]]]]] [:record [:map + [:facets {:optional true} [:vector [:map + [:features [:vector [:map + [:$type :string] + [:tag {:optional true} [:maybe :string]]]]]]]] [:createdAt :string]]]]]]]]]) (defn get-posts-until-id @@ -75,9 +83,27 @@ ;; "RELATION_RECORD_ID" ;; ] ;; }; +;; + +(defn extract-tags [post] + (let [facets (get (post :record) :facets []) + features (flatten (map :features facets)) + tag-features (filter #(= (:$type %) "app.bsky.richtext.facet#tag") features) + tags (map :tag tag-features)] + tags)) + +(defn extract-images [post] + (let [images (get-in post [:embed :images] [])] + (map #(hash-map :url (:fullsize %) :alt (:alt %)) images))) (defn run [] (let [session (create-session) last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky) - new-posts (get-posts-until-id session last-saved-id)] - {:session session :last-saved-id last-saved-id :new-posts new-posts})) + new-posts (reverse (get-posts-until-id session last-saved-id))] + (->> new-posts + (map #(hash-map :source :blue_sky + :fullPost % + :remoteId (:cid %) + :authorId (get-in % [:author :handle]) + :posted (get-in % [:record :createdAt]))) + (map pb/save-post)))) diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index 5a027ae..996b29d 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -4,6 +4,7 @@ [clojure.string :as str] [clj-http.client :as http-client] [malli.core :as m] + [micro-blog.utils :as utils] [micro-blog.config :refer [config]])) (defonce token-cache ^:private (atom {:token nil :fetched-at nil})) @@ -37,9 +38,9 @@ (reset! token-cache {:token new-token :fetched-at (now)}) new-token)))) -(def sources #{:pleroma :blue_sky :mastodon :pixelfed :nostr}) +(def source-enum [:enum :pleroma :blue_sky :mastodon :pixelfed :nostr]) (defn valid-source? [source] - (contains? sources source)) + (m/validate source-enum source)) (defn get-latest-post-remote-id-by-source [source] (let [res-schema @@ -69,3 +70,33 @@ (m/explain res-schema x) (throw (ex-info "Res does not follow schema" {:res x})))) (-> x :items first :remoteId)))) + +(defn post-with-remote-id-already-saved? [remote-id] + (-> + (http-client/get (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records") + {:headers {"Authorization" (get-login-token-with-cache)} + :query-params {:page 1 + "perPage" 1 + :filter (str "remoteId = '" remote-id "'") + :fields "id" + "skipTotal" true} + :content-type :json + :as :json}) + :body :items count (> 0))) + +(defn save-post [post] + (printf "Saving %s post with remoteId: %s\n" (name (:source post)) (:remoteId post)) + (let [save-post-schema [:map + [:source source-enum] + [:fullPost :any] + [:remoteId :string] + [:authorId :string] + [:posted :string]]] + (utils/validate-with-throw post save-post-schema) + (if (post-with-remote-id-already-saved? (:remoteId post)) + (println "post already saved") + (http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records") + {:headers {"Authorization" (get-login-token-with-cache)} + :form-params post + :content-type :json + :as :json})))) diff --git a/src/micro_blog/utils.clj b/src/micro_blog/utils.clj index 2d68e46..b9b503c 100644 --- a/src/micro_blog/utils.clj +++ b/src/micro_blog/utils.clj @@ -1,9 +1,11 @@ (ns micro-blog.utils - (:require [malli.core :as m])) + (:require + [clojure.pprint :refer [pprint]] + [malli.core :as m])) (defn validate-with-throw [value schema] (if (m/validate schema value) value (do - (m/explain schema value) + (pprint (m/explain schema value)) (throw (ex-info "Res does not follow schema" {:value value})))))