diff --git a/src/micro_blog/blue_sky.clj b/src/micro_blog/blue_sky.clj index 60aa37d..b8e2bef 100644 --- a/src/micro_blog/blue_sky.clj +++ b/src/micro_blog/blue_sky.clj @@ -94,7 +94,7 @@ (defn extract-images [post] (let [images (get-in post [:embed :images] [])] - (map #(hash-map :url (:fullsize %) :alt (:alt %)) images))) + (map #(vector (:fullsize %) (:alt %)) images))) (defn run [] (let [session (create-session) @@ -106,5 +106,6 @@ :remoteId (:cid %) :authorId (get-in % [:author :handle]) :tags (extract-tags %) + :images (extract-images %) :posted (get-in % [:record :createdAt]))) (map pb/save-post)))) diff --git a/src/micro_blog/pocket_base.clj b/src/micro_blog/pocket_base.clj index 51eabbd..5f0274f 100644 --- a/src/micro_blog/pocket_base.clj +++ b/src/micro_blog/pocket_base.clj @@ -4,6 +4,7 @@ [clojure.string :as str] [clj-http.client :as http-client] [malli.core :as m] + [taoensso.telemere :as tel] [micro-blog.utils :as utils] [micro-blog.config :refer [config]])) @@ -16,7 +17,8 @@ (let [duration (java.time.Duration/between fetched-at (now))] (> (.toHours duration) 23)))) ; 23 to be safe, or use 24 -(defn get-login-token [] +(defn get-new-login-token [] + (tel/log! :info "Getting new login token") (let [user-name (@config :pocket-base-user) pw (@config :pocket-base-pw) body {:identity user-name :password pw} @@ -33,8 +35,7 @@ (let [{:keys [token fetched-at]} @token-cache] (if (and token (not (older-then-a-day? fetched-at))) token - (let [new-token (get-login-token)] - (println "Getting new login token") + (let [new-token (get-new-login-token)] (reset! token-cache {:token new-token :fetched-at (now)}) new-token)))) @@ -84,6 +85,46 @@ :as :json}) :body :items count (> 0))) +(defn get-existing-image-id [remote-url] + (tel/log! {:level :info, :id :pocket-base/get-existing-image-id, :data {:remote-url remote-url}} "Looking for existing image") + (-> + (http-client/get (str (@config :pocket-base-host) "/api/collections/micro_blog_images/records") + {:headers {"Authorization" (get-login-token-with-cache)} + :query-params {:page 1 + "perPage" 1 + :filter (str "remoteURL = '" remote-url "'") + :fields "id" + "skipTotal" true} + :content-type :json + :as :json}) + (get-in [:body :items 0 :id]))) + +(defn upload-new-image [remote-url alt-txt] + (let [res (http-client/get remote-url {:as :byte-array}) + content-type (get-in res [:headers "Content-Type"]) + body (:body res)] + (tel/log! {:level :info :data {:remote-url remote-url :alt-txt alt-txt :content-type content-type :body-length (count body)}} "uploading image to pocketbase") + (-> + (http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_images/records") + {:as :json + :coerce :always + :throw-exceptions false + :headers {"Authorization" (get-login-token-with-cache)} + :multipart [{:name "image" + :filename (str/replace content-type "/" ".") + :content body + :mime-type content-type + :content-type content-type} + {:name "remoteURL" :content remote-url} + {:name "alt" :content alt-txt}]}) :body :id))) + +(defn get-image-id + "returns the id of the image, creating if needed" + [remote-url alt-txt] + (let [existing-id (get-existing-image-id remote-url)] + (if (not (nil? existing-id)) existing-id + (upload-new-image remote-url alt-txt)))) + (defn get-tag-id "returns id of tag in pocketbase, creating it if it does not exist" [tag] @@ -109,20 +150,25 @@ (get-in [:body :id]))))) (defn save-post [post] - (printf "Saving %s post with remoteId: %s\n" (name (:source post)) (:remoteId post)) + (tel/log! {:level :info :data {:source (:source post) :remoteId (:remoteId post)}}) (let [save-post-schema [:map [:source source-enum] [:fullPost :any] - [:tags [:list :string]] + [:tags [:sequential :string]] + [:images [:sequential [:tuple :string :string]]] [:remoteId :string] [:authorId :string] [:posted :string]]] + ;; (pprint post) + ;; (tel/log! {:level :info :data {:post post}} "Saving post to pocketbase") (utils/validate-with-throw post save-post-schema) (if (post-with-remote-id-already-saved? (:remoteId post)) (println "post already saved") (http-client/post (str (@config :pocket-base-host) "/api/collections/micro_blog_posts/records") {:headers {"Authorization" (get-login-token-with-cache)} - :form-params (assoc post :tags (map get-tag-id (:tags post))) + :form-params (assoc post + :tags (map get-tag-id (:tags post)) + :images (map #(apply get-image-id %) (:images post))) :content-type :json :as :json})))) diff --git a/src/micro_blog/utils.clj b/src/micro_blog/utils.clj index b9b503c..be84ba5 100644 --- a/src/micro_blog/utils.clj +++ b/src/micro_blog/utils.clj @@ -1,11 +1,11 @@ (ns micro-blog.utils (:require - [clojure.pprint :refer [pprint]] + [taoensso.telemere :as tel] [malli.core :as m])) (defn validate-with-throw [value schema] (if (m/validate schema value) value (do - (pprint (m/explain schema value)) + (tel/log! {:level :error :data {:why (m/explain schema value)}} "Validation failed") (throw (ex-info "Res does not follow schema" {:value value}))))) diff --git a/test.png b/test.png new file mode 100644 index 0000000..5eab283 Binary files /dev/null and b/test.png differ