add doseq to fix scraping in docker
This commit is contained in:
parent
e18cc26ddd
commit
10a374c0c8
2 changed files with 18 additions and 12 deletions
|
|
@ -45,6 +45,7 @@
|
||||||
(defn get-posts-until-id
|
(defn get-posts-until-id
|
||||||
([session id] (get-posts-until-id session id nil []))
|
([session id] (get-posts-until-id session id nil []))
|
||||||
([session id cursor prev-posts]
|
([session id cursor prev-posts]
|
||||||
|
(tel/log! {:level :info :data {:postId (:remoteId id)}} "Getting posts until id")
|
||||||
(let [limit 5
|
(let [limit 5
|
||||||
body
|
body
|
||||||
(-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
|
(-> (http-client/get (str (@config :blue-sky-host) "/app.bsky.feed.getAuthorFeed")
|
||||||
|
|
@ -79,17 +80,23 @@
|
||||||
(let [images (get-in post [:embed :images] [])]
|
(let [images (get-in post [:embed :images] [])]
|
||||||
(map #(vector (:fullsize %) (:alt %)) images)))
|
(map #(vector (:fullsize %) (:alt %)) images)))
|
||||||
|
|
||||||
|
(defn transform-post [post]
|
||||||
|
(hash-map :source :blue_sky
|
||||||
|
:fullPost post
|
||||||
|
:remoteId (:cid post)
|
||||||
|
:authorId (get-in post [:author :handle])
|
||||||
|
:tags (extract-tags post)
|
||||||
|
:images (extract-images post)
|
||||||
|
:posted (get-in post [:record :createdAt])))
|
||||||
|
|
||||||
|
(defn save-post [post]
|
||||||
|
(tel/log! {:level :info :data {:postId (:remoteId post)}} "Saving post")
|
||||||
|
(pb/save-post post))
|
||||||
|
|
||||||
(defn run []
|
(defn run []
|
||||||
(tel/log! :info "Running blue sky fetcher")
|
(tel/log! :info "Running blue sky fetcher")
|
||||||
(let [session (create-session)
|
(let [session (create-session)
|
||||||
last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
|
last-saved-id (pb/get-latest-post-remote-id-by-source :blue_sky)
|
||||||
new-posts (reverse (get-posts-until-id session last-saved-id))]
|
new-posts (reverse (get-posts-until-id session last-saved-id))]
|
||||||
(->> new-posts
|
(doseq [post new-posts]
|
||||||
(map #(hash-map :source :blue_sky
|
(-> post transform-post save-post))))
|
||||||
:fullPost %
|
|
||||||
:remoteId (:cid %)
|
|
||||||
:authorId (get-in % [:author :handle])
|
|
||||||
:tags (extract-tags %)
|
|
||||||
:images (extract-images %)
|
|
||||||
:posted (get-in % [:record :createdAt])))
|
|
||||||
(map pb/save-post))))
|
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,5 @@
|
||||||
(tel/log! :info "Running mastodon fetcher")
|
(tel/log! :info "Running mastodon fetcher")
|
||||||
(let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon)
|
(let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon)
|
||||||
new-posts (reverse (get-posts-until-id last-saved-id))]
|
new-posts (reverse (get-posts-until-id last-saved-id))]
|
||||||
(->> new-posts
|
(doseq [post new-posts]
|
||||||
(map transform-post)
|
(save-post (transform-post post)))))
|
||||||
(map save-post))))
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue