travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

add more logging

Browse source
This commit is contained in:
Travis Shears 2025-08-11 10:49:13 +02:00
parent edc74be72b
commit 32355ae894
2 changed files with 36 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

52
src/micro_blog/mastodon.clj
View file

@ -22,27 +22,43 @@
(let [limit 10
url (str (@config :mastodon-host) "/api/v1/accounts/" (@config :mastodon-account-id) "/statuses")]
(tel/log! {:level :info :data {:url url :post-id id}} "Getting mastodon posts until id")
(->>
(-> (http-client/get
url
{:query-params {:limit limit}
:content-type :json
:as :json})
:body
(utils/validate-with-throw post-res-schema))
(take-while #(not= (:id %) id)))))
(let [new-posts
(->>
(-> (http-client/get
url
{:query-params {:limit limit}
:content-type :json
:as :json})
:body
(utils/validate-with-throw post-res-schema))
(take-while #(not= (:id %) id)))]
(if (nil? new-posts)
(do
(tel/log! {:level :info} "No new mastodon posts")
[])
(do
(tel/log! {:level :info :data {:count (count new-posts) :ids (map :id new-posts)}} "Got new mastodon posts")
new-posts)))))
(defn transform-post [raw-post]
(tel/log! {:level :info :data {:remoteId (:id raw-post)}} "Transforming mastodon post")
(hash-map
:source :mastodon
:fullPost raw-post
:remoteId (:id raw-post)
:authorId (get-in raw-post [:account :id])
:tags (map :name (:tags raw-post))
:images (map (fn [img] [(:url img) (:description img)]) (:media_attachments raw-post))
:posted (:created_at raw-post)))
(defn save-post [post]
(tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Saving mastodon post")
(pb/save-post post))
(defn run []
(tel/log! :info "Running mastodon fetcher")
(let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon)
new-posts (reverse (get-posts-until-id last-saved-id))]
(tel/log! {:level :info :data {:count (if (nil? new-posts) 0 (count new-posts))}} "Found new posts to save")
(->> new-posts
(map #(hash-map :source :mastodon
:fullPost %
:remoteId (:id %)
:authorId (get-in % [:account :id])
:tags (map :name (:tags %))
:images (map (fn [img] [(:url img) (:description img)]) (:media_attachments %))
:posted (:created_at %)))
(map pb/save-post))))
(map transform-post)
(map save-post))))

4
src/micro_blog/pocket_base.clj
View file

@ -150,7 +150,7 @@
(get-in [:body :id])))))
(defn save-post [post]
(tel/log! {:level :info :data {:source (:source post) :remoteId (:remoteId post)}})
(tel/log! {:level :info :data {:source (:source post) :remoteId (:remoteId post)}} "Saving post to pocketbase")
(let [save-post-schema [:map
[:source source-enum]
[:fullPost :any]
@ -160,8 +160,8 @@
[:authorId :string]
[:posted :string]]]
;; (tel/log! {:level :info :data {:post post}} "Saving post to pocketbase")
(utils/validate-with-throw post save-post-schema)
(tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Post passed save validation")
(if (post-with-remote-id-already-saved? (:remoteId post))
(println "post already saved")
(http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records")