65 lines
2.5 KiB
Clojure
65 lines
2.5 KiB
Clojure
(ns micro-blog.mastodon
|
|
(:require
|
|
[clj-http.client :as http-client]
|
|
[micro-blog.config :refer [config]]
|
|
[micro-blog.is-tech]
|
|
[micro-blog.pocket-base :as pb]
|
|
[micro-blog.utils :as utils]
|
|
[taoensso.telemere :as tel]))
|
|
|
|
(def post-res-schema [:sequential
|
|
[:map
|
|
[:id :string]
|
|
[:content :string]
|
|
[:account [:map [:id :string]]]
|
|
[:created_at :string]
|
|
[:tags [:vector [:map [:name :string]]]]
|
|
[:media_attachments [:vector [:map
|
|
[:url :string]
|
|
[:type [:= "image"]]
|
|
[:description [:maybe :string]]]]]]])
|
|
|
|
(defn get-posts-until-id [id]
|
|
(let [limit 10
|
|
url (str (@config :mastodon-host) "/api/v1/accounts/" (@config :mastodon-account-id) "/statuses")]
|
|
(tel/log! {:level :info :data {:url url :post-id id}} "Getting mastodon posts until id")
|
|
(let [new-posts
|
|
(->>
|
|
(-> (http-client/get
|
|
url
|
|
{:query-params {:limit limit}
|
|
:content-type :json
|
|
:as :json})
|
|
:body
|
|
(utils/validate-with-throw post-res-schema))
|
|
(take-while #(not= (:id %) id)))]
|
|
(if (nil? new-posts)
|
|
(do
|
|
(tel/log! {:level :info} "No new mastodon posts")
|
|
[])
|
|
(do
|
|
(tel/log! {:level :info :data {:count (count new-posts) :ids (map :id new-posts)}} "Got new mastodon posts")
|
|
new-posts)))))
|
|
|
|
(defn transform-post [raw-post]
|
|
(tel/log! {:level :info :data {:remoteId (:id raw-post)}} "Transforming mastodon post")
|
|
(hash-map
|
|
:source :mastodon
|
|
:isTech (micro-blog.is-tech/is-tech? (:content raw-post))
|
|
:fullPost raw-post
|
|
:remoteId (:id raw-post)
|
|
:authorId (get-in raw-post [:account :id])
|
|
:tags (map :name (:tags raw-post))
|
|
:images (map (fn [img] [(:url img) (or (:description img) "")]) (:media_attachments raw-post))
|
|
:posted (:created_at raw-post)))
|
|
|
|
(defn save-post [post]
|
|
(tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Saving mastodon post")
|
|
(pb/save-post post))
|
|
|
|
(defn run []
|
|
(tel/log! :info "Running mastodon fetcher")
|
|
(let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon)
|
|
new-posts (reverse (get-posts-until-id last-saved-id))]
|
|
(doseq [post new-posts]
|
|
(save-post (transform-post post)))))
|