(ns micro-blog.mastodon (:require [clj-http.client :as http-client] [micro-blog.config :refer [config]] [micro-blog.is-tech] [micro-blog.pocket-base :as pb] [micro-blog.utils :as utils] [taoensso.telemere :as tel])) (def post-res-schema [:sequential [:map [:id :string] [:content :string] [:account [:map [:id :string]]] [:created_at :string] [:tags [:vector [:map [:name :string]]]] [:media_attachments [:vector [:map [:url :string] [:type [:= "image"]] [:description [:maybe :string]]]]]]]) (defn get-posts-until-id [id] (let [limit 10 url (str (@config :mastodon-host) "/api/v1/accounts/" (@config :mastodon-account-id) "/statuses")] (tel/log! {:level :info :data {:url url :post-id id}} "Getting mastodon posts until id") (let [new-posts (->> (-> (http-client/get url {:query-params {:limit limit} :content-type :json :as :json}) :body (utils/validate-with-throw post-res-schema)) (take-while #(not= (:id %) id)))] (if (nil? new-posts) (do (tel/log! {:level :info} "No new mastodon posts") []) (do (tel/log! {:level :info :data {:count (count new-posts) :ids (map :id new-posts)}} "Got new mastodon posts") new-posts))))) (defn transform-post [raw-post] (tel/log! {:level :info :data {:remoteId (:id raw-post)}} "Transforming mastodon post") (hash-map :source :mastodon :isTech (micro-blog.is-tech/is-tech? (:content raw-post)) :fullPost raw-post :remoteId (:id raw-post) :authorId (get-in raw-post [:account :id]) :tags (map :name (:tags raw-post)) :images (map (fn [img] [(:url img) (or (:description img) "")]) (:media_attachments raw-post)) :posted (:created_at raw-post))) (defn save-post [post] (tel/log! {:level :info :data {:remoteId (:remoteId post)}} "Saving mastodon post") (pb/save-post post)) (defn run [] (tel/log! :info "Running mastodon fetcher") (let [last-saved-id (pb/get-latest-post-remote-id-by-source :mastodon) new-posts (reverse (get-posts-until-id last-saved-id))] (doseq [post new-posts] (save-post (transform-post post)))))