diff --git a/src/micro_blog/mastodon.clj b/src/micro_blog/mastodon.clj index dc06781..5ff5cf3 100644 --- a/src/micro_blog/mastodon.clj +++ b/src/micro_blog/mastodon.clj @@ -22,27 +22,43 @@ (let [limit 10 url (str (@config :mastodon-host) "/api/v1/accounts/" (@config :mastodon-account-id) "/statuses")] (tel/log! {:level :info :data {:url url :post-id id}} "Getting mastodon posts until id") - (->> - (-> (http-client/get - url - {:query-params {:limit limit} - :content-type :json - :as :json}) - :body - (utils/validate-with-throw post-res-schema)) - (take-while #(not= (:id %) id))))) + (let [new-posts + (->> + (-> (http-client/get + url + {:query-params {:limit limit} + :content-type :json + :as :json}) + :body + (utils/validate-with-throw post-res-schema)) + (take-while #(not= (:id %) id)))] + (if (nil? new-posts) + (do + (tel/log! {:level :info} "No new mastodon posts") + []) + (do + (tel/log! {:level :info :data {:count (count new-posts) :ids (map :id new-posts)}} "Got new mastodon posts") + new-posts))))) + +(defn transform-post [raw-post] + (tel/log! {:level :info :data {:remoteId (:id raw-post)}} "Transforming mastodon post") + (hash-map + :source :mastodon + :fullPost raw-post + :remoteId (:id raw-post) + :authorId (get-in raw-post [:account :id]) + :tags (map :name (:tags raw-post)) + :images (map (fn [img] [(:url img) (:description img)]) (:media_attachments raw-post)) + :posted (:created_at raw-post))) + +(defn save-post [post] + (tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Saving mastodon post") + (pb/save-post post)) (defn run [] (tel/log! :info "Running mastodon fetcher") (let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon) new-posts (reverse (get-posts-until-id last-saved-id))] - (tel/log! {:level :info :data {:count (if (nil? new-posts) 0 (count new-posts))}} "Found new posts to save") (->> new-posts - (map #(hash-map :source :mastodon - :fullPost % - :remoteId (:id %) - :authorId (get-in % [:account :id]) - :tags (map :name (:tags %)) - :images (map (fn [img] [(:url img) (:description img)]) (:media_attachments %)) - :posted (:created_at %))) - (map pb/save-post)))) + (map transform-post) + (map save-post)))) diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index e64cae1..35de871 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -150,7 +150,7 @@ (get-in [:body :id]))))) (defn save-post [post] - (tel/log! {:level :info :data {:source (:source post) :remoteId (:remoteId post)}}) + (tel/log! {:level :info :data {:source (:source post) :remoteId (:remoteId post)}} "Saving post to pocketbase") (let [save-post-schema [:map [:source source-enum] [:fullPost :any] @@ -160,8 +160,8 @@ [:authorId :string] [:posted :string]]] - ;; (tel/log! {:level :info :data {:post post}} "Saving post to pocketbase") (utils/validate-with-throw post save-post-schema) + (tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Post passed save validation") (if (post-with-remote-id-already-saved? (:remoteId post)) (println "post already saved") (http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records")