add doseq to fix scraping in docker
This commit is contained in:
parent
e18cc26ddd
commit
10a374c0c8
2 changed files with 18 additions and 12 deletions
|
|
@ -45,6 +45,7 @@
|
|||
(defn get-posts-until-id
|
||||
([session id] (get-posts-until-id session id nil []))
|
||||
([session id cursor prev-posts]
|
||||
(tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id")
|
||||
(let [limit 5
|
||||
body
|
||||
(-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
|
||||
|
|
@ -79,17 +80,23 @@
|
|||
(let [images (get-in post [:embed :images] [])]
|
||||
(map #(vector (:fullsize %) (:alt %)) images)))
|
||||
|
||||
(defn transform-post [post]
|
||||
(hash-map :source :blue_sky
|
||||
:fullPost post
|
||||
:remoteId (:cid post)
|
||||
:authorId (get-in post [:author :handle])
|
||||
:tags (extract-tags post)
|
||||
:images (extract-images post)
|
||||
:posted (get-in post [:record :createdAt])))
|
||||
|
||||
(defn save-post [post]
|
||||
(tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post")
|
||||
(pb/save-post post))
|
||||
|
||||
(defn run []
|
||||
(tel/log! :info "Running blue sky fetcher")
|
||||
(let [session (create-session)
|
||||
last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
|
||||
new-posts (reverse (get-posts-until-id session last-saved-id))]
|
||||
(->> new-posts
|
||||
(map #(hash-map :source :blue_sky
|
||||
:fullPost %
|
||||
:remoteId (:cid %)
|
||||
:authorId (get-in % [:author :handle])
|
||||
:tags (extract-tags %)
|
||||
:images (extract-images %)
|
||||
:posted (get-in % [:record :createdAt])))
|
||||
(map pb/save-post))))
|
||||
(doseq [post new-posts]
|
||||
(-> post transform-post save-post))))
|
||||
|
|
|
|||
|
|
@ -59,6 +59,5 @@
|
|||
(tel/log! :info "Running mastodon fetcher")
|
||||
(let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon)
|
||||
new-posts (reverse (get-posts-until-id last-saved-id))]
|
||||
(->> new-posts
|
||||
(map transform-post)
|
||||
(map save-post))))
|
||||
(doseq [post new-posts]
|
||||
(save-post (transform-post post)))))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue