add doseq to fix scraping in docker

This commit is contained in:
Travis Shears 2025-08-13 11:13:08 +02:00
parent e18cc26ddd
commit 10a374c0c8
2 changed files with 18 additions and 12 deletions

View file

@ -45,6 +45,7 @@
(defn get-posts-until-id
([session id] (get-posts-until-id session id nil []))
([session id cursor prev-posts]
(tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id")
(let [limit 5
body
(-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
@ -79,17 +80,23 @@
(let [images (get-in post [:embed :images] [])]
(map #(vector (:fullsize %) (:alt %)) images)))
(defn transform-post [post]
(hash-map :source :blue_sky
:fullPost post
:remoteId (:cid post)
:authorId (get-in post [:author :handle])
:tags (extract-tags post)
:images (extract-images post)
:posted (get-in post [:record :createdAt])))
(defn save-post [post]
(tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post")
(pb/save-post post))
(defn run []
(tel/log! :info "Running blue sky fetcher")
(let [session (create-session)
last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
new-posts (reverse (get-posts-until-id session last-saved-id))]
(->> new-posts
(map #(hash-map :source :blue_sky
:fullPost %
:remoteId (:cid %)
:authorId (get-in % [:author :handle])
:tags (extract-tags %)
:images (extract-images %)
:posted (get-in % [:record :createdAt])))
(map pb/save-post))))
(doseq [post new-posts]
(-> post transform-post save-post))))

View file

@ -59,6 +59,5 @@
(tel/log! :info "Running mastodon fetcher")
(let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon)
new-posts (reverse (get-posts-until-id last-saved-id))]
(->> new-posts
(map transform-post)
(map save-post))))
(doseq [post new-posts]
(save-post (transform-post post)))))