travisshears/micro_blog_repo_fetchers
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases Packages Wiki Activity Actions

get image saving / uploading working

Browse source
This commit is contained in:
Travis Shears 2025-07-22 18:00:14 +02:00
parent fd146d622b
commit a695f791c4
4 changed files with 56 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/micro_blog/blue_sky.clj
View file

@ -94,7 +94,7 @@
(defn extract-images [post]
(let [images (get-in post [:embed :images] [])]
(map #(hash-map :url (:fullsize %) :alt (:alt %)) images)))
(map #(vector (:fullsize %) (:alt %)) images)))
(defn run []
(let [session (create-session)
@ -106,5 +106,6 @@
:remoteId (:cid %)
:authorId (get-in % [:author :handle])
:tags (extract-tags %)
:images (extract-images %)
:posted (get-in % [:record :createdAt])))
(map pb/save-post))))

58
src/micro_blog/pocket_base.clj
View file

@ -4,6 +4,7 @@
[clojure.string :as str]
[clj-http.client :as http-client]
[malli.core :as m]
[taoensso.telemere :as tel]
[micro-blog.utils :as utils]
[micro-blog.config :refer [config]]))
@ -16,7 +17,8 @@
(let [duration (java.time.Duration/between fetched-at (now))]
(> (.toHours duration) 23)))) ; 23 to be safe, or use 24
(defn get-login-token []
(defn get-new-login-token []
(tel/log! :info "Getting new login token")
(let [user-name (@config :pocket-base-user)
pw (@config :pocket-base-pw)
body {:identity user-name :password pw}
@ -33,8 +35,7 @@
(let [{:keys [token fetched-at]} @token-cache]
(if (and token (not (older-then-a-day? fetched-at)))
token
(let [new-token (get-login-token)]
(println "Getting new login token")
(let [new-token (get-new-login-token)]
(reset! token-cache {:token new-token :fetched-at (now)})
new-token))))
@ -84,6 +85,46 @@
:as :json})
:body :items count (> 0)))
(defn get-existing-image-id [remote-url]
(tel/log! {:level :info, :id :pocket-base/get-existing-image-id, :data {:remote-url remote-url}} "Looking for existing image")
(->
(http-client/get (str (@config :pocket-base-host) "/api/collections/micro_blog_images/records")
{:headers {"Authorization" (get-login-token-with-cache)}
:query-params {:page 1
"perPage" 1
:filter (str "remoteURL = '" remote-url "'")
:fields "id"
"skipTotal" true}
:content-type :json
:as :json})
(get-in [:body :items 0 :id])))
(defn upload-new-image [remote-url alt-txt]
(let [res (http-client/get remote-url {:as :byte-array})
content-type (get-in res [:headers "Content-Type"])
body (:body res)]
(tel/log! {:level :info :data {:remote-url remote-url :alt-txt alt-txt :content-type content-type :body-length (count body)}} "uploading image to pocketbase")
(->
(http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_images/records")
{:as :json
:coerce :always
:throw-exceptions false
:headers {"Authorization" (get-login-token-with-cache)}
:multipart [{:name "image"
:filename (str/replace content-type "/" ".")
:content body
:mime-type content-type
:content-type content-type}
{:name "remoteURL" :content remote-url}
{:name "alt" :content alt-txt}]}) :body :id)))
(defn get-image-id
"returns the id of the image, creating if needed"
[remote-url alt-txt]
(let [existing-id (get-existing-image-id remote-url)]
(if (not (nil? existing-id)) existing-id
(upload-new-image remote-url alt-txt))))
(defn get-tag-id
"returns id of tag in pocketbase, creating it if it does not exist"
[tag]
@ -109,20 +150,25 @@
(get-in [:body :id])))))
(defn save-post [post]
(printf "Saving %s post with remoteId: %s\n" (name (:source post)) (:remoteId post))
(tel/log! {:level :info :data {:source (:source post) :remoteId (:remoteId post)}})
(let [save-post-schema [:map
[:source source-enum]
[:fullPost :any]
[:tags [:list :string]]
[:tags [:sequential :string]]
[:images [:sequential [:tuple :string :string]]]
[:remoteId :string]
[:authorId :string]
[:posted :string]]]
;; (pprint post)
;; (tel/log! {:level :info :data {:post post}} "Saving post to pocketbase")
(utils/validate-with-throw post save-post-schema)
(if (post-with-remote-id-already-saved? (:remoteId post))
(println "post already saved")
(http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records")
{:headers {"Authorization" (get-login-token-with-cache)}
:form-params (assoc post :tags (map get-tag-id (:tags post)))
:form-params (assoc post
:tags (map get-tag-id (:tags post))
:images (map #(apply get-image-id %) (:images post)))
:content-type :json
:as :json}))))

4
src/micro_blog/utils.clj
View file

@ -1,11 +1,11 @@
(ns micro-blog.utils
(:require
[clojure.pprint :refer [pprint]]
[taoensso.telemere :as tel]
[malli.core :as m]))
(defn validate-with-throw [value schema]
(if (m/validate schema value)
value
(do
(pprint (m/explain schema value))
(tel/log! {:level :error :data {:why (m/explain schema value)}} "Validation failed")
(throw (ex-info "Res does not follow schema" {:value value})))))