From be55f44a877a233c9db0c13abd29102728c9d8ef Mon Sep 17 00:00:00 2001 From: Peter Taoussanis Date: Fri, 13 Sep 2024 17:20:36 +0200 Subject: [PATCH] [new] Add `clean-signal-fn` util --- projects/main/src/taoensso/telemere.cljc | 1 + .../main/src/taoensso/telemere/utils.cljc | 130 ++++++++++-------- .../main/test/taoensso/telemere_tests.cljc | 45 +++--- 3 files changed, 99 insertions(+), 77 deletions(-) diff --git a/projects/main/src/taoensso/telemere.cljc b/projects/main/src/taoensso/telemere.cljc index 23fa23d..25b023e 100644 --- a/projects/main/src/taoensso/telemere.cljc +++ b/projects/main/src/taoensso/telemere.cljc @@ -71,6 +71,7 @@ #?(:clj impl/signal-allowed?) ;; Utils + utils/clean-signal-fn utils/format-signal-fn utils/pr-signal-fn utils/error-signal?) diff --git a/projects/main/src/taoensso/telemere/utils.cljc b/projects/main/src/taoensso/telemere/utils.cljc index dcdcdcf..2c9364b 100644 --- a/projects/main/src/taoensso/telemere/utils.cljc +++ b/projects/main/src/taoensso/telemere/utils.cljc @@ -656,6 +656,75 @@ ((signal-content-fn) (tel/with-signal (tel/event! ::ev-id))) ((signal-content-fn) (tel/with-signal (tel/event! ::ev-id {:data {:k1 "v1"}})))) +(defn clean-signal-fn + "Experimental, subject to change. + Returns a (fn clean [signal]) that: + - Takes a Telemere signal (map). + - Returns a minimal signal (map) ready for printing, etc. + + Signals are optimized for cheap creation and easy handling, so tend to be + verbose and may contain things like nil values and duplicated content. + + This util efficiently cleans signals of such noise, helping reduce + storage/transmission size, and making key info easier to see. + + Options: + `:incl-nils?` - Include signal's keys with nil values? (default false) + `:incl-kvs?` - Include signal's app-level root kvs? (default false) + `:incl-keys` - Subset of signal keys to retain from those otherwise + excluded by default: #{:location :kvs :file :host :thread}" + ([] (clean-signal-fn nil)) + ([{:keys [incl-kvs? incl-nils? incl-keys] :as opts}] + (let [assoc!* + (if-not incl-nils? + (fn [m k v] (if (nil? v) m (assoc! m k v))) ; As `remove-signal-nils` + (do assoc!)) + + incl-location? (contains? incl-keys :location) + incl-kvs-key? (contains? incl-keys :kvs) + incl-file? (contains? incl-keys :file) + incl-host? (contains? incl-keys :host) + incl-thread? (contains? incl-keys :thread)] + + (fn clean-signal [signal] + (when (map? signal) + (persistent! + (reduce-kv + (fn [m k v] + (enc/case-eval k + ;; Main keys to always include as-is + (clojure.core/into () + (clojure.core/disj + taoensso.telemere.impl/standard-signal-keys + :msg_ :error :location :kvs :file :host :thread)) + (assoc!* m k v) + + ;; Main keys to include with modified val + :error (if-let [chain (enc/ex-chain :as-map v)] (assoc! m k chain) m) ; As `expand-signal-error` + :msg_ (assoc!* m k (force v)) ; As `force-signal-msg` + + ;; Implementation keys to always exclude + (clojure.core/into () + taoensso.telemere.impl/impl-signal-keys) m ; noop + + ;;; Other keys to exclude by default + :location (if incl-location? (assoc!* m k v) m) + :kvs (if incl-kvs-key? (assoc!* m k v) m) + :file (if incl-file? (assoc!* m k v) m) + :thread (if incl-thread? (assoc!* m k v) m) + :host (if incl-host? (assoc!* m k v) m) + + ;; Other (app-level) keys + (enc/cond + incl-kvs? (assoc!* m k v) ; Incl. all kvs + (contains? incl-keys k) (assoc!* m k v) ; Incl. specific kvs + :else m ; As `remove-signal-kvs` + ))) + + (transient {}) signal))))))) + +(comment ((clean-signal-fn {:incl-keys #{:a}}) {:level :info, :id nil, :a "a", :b "b", :msg_ (delay "hi")})) + (defn pr-signal-fn "Experimental, subject to change. Returns a (fn pr [signal]) that: @@ -683,14 +752,6 @@ #?(:cljs :json ; Use js/JSON.stringify :clj jsonista/write-value-as-string)}) - Motivation: - Why use this util instead of just directly using the print function - given to `:pr-fn`? Signals are optimized for cheap creation and easy handling, - so may contain things like nil values and duplicated content. - - This util efficiently clean signals of such noise, helping reduce - storage/transmission size, and making key info easier to see. - See also `format-signal-fn` for an alternative to `pr-signal-fn` that produces human-readable output." ([] (pr-signal-fn nil)) @@ -700,10 +761,10 @@ incl-newline? true}}] (let [nl newline + clean-fn (clean-signal-fn opts) pr-fn (or (case pr-fn - :none nil ; Undocumented, used for unit tests :edn pr-edn :json #?(:cljs pr-json @@ -719,56 +780,13 @@ :param 'pr-fn :expected #?(:clj '#{:edn unary-fn} - :cljs '#{:edn :json unary-fn})})))) - - assoc!* - (if-not incl-nils? - (fn [m k v] (if (nil? v) m (assoc! m k v))) ; As `remove-signal-nils` - (do assoc!)) - - incl-location? (contains? incl-keys :location) - incl-kvs-key? (contains? incl-keys :kvs) - incl-file? (contains? incl-keys :file) - incl-host? (contains? incl-keys :host) - incl-thread? (contains? incl-keys :thread)] + :cljs '#{:edn :json unary-fn})}))))] (fn pr-signal [signal] (when (map? signal) - (let [signal - (persistent! - (reduce-kv - (fn [m k v] - (enc/case-eval k - :msg_ (assoc!* m k (force v)) ; As force-signal-msg - :error (if-let [chain (enc/ex-chain :as-map v)] (assoc! m k chain) m) ; As expand-signal-error - - ;;; Keys excluded by default - :location (if incl-location? (assoc!* m k v) m) - :kvs (if incl-kvs-key? (assoc!* m k v) m) - :file (if incl-file? (assoc!* m k v) m) - :thread (if incl-thread? (assoc!* m k v) m) - :host (if incl-host? (assoc!* m k v) m) - - (clojure.core/into () - taoensso.telemere.impl/impl-signal-keys) m ; noop - - (clojure.core/into () - (clojure.core/disj - taoensso.telemere.impl/standard-signal-keys - :msg_ :error :location :kvs :file :host :thread)) - (assoc!* m k v) - - (if incl-kvs? (assoc!* m k v) m) ; As remove-signal-kvs - )) - - (transient {}) signal))] - - (if-not pr-fn - signal - (let [output (pr-fn signal)] - (if incl-newline? - (str output nl) - (do output)))))))))) + (if incl-newline? + (str (pr-fn (clean-fn signal)) nl) + (do (pr-fn (clean-fn signal))))))))) (comment (def s1 (tel/with-signal (tel/event! ::ev-id {:kvs {:k1 "v1"}}))) diff --git a/projects/main/test/taoensso/telemere_tests.cljc b/projects/main/test/taoensso/telemere_tests.cljc index a3ae1a9..54bad7e 100644 --- a/projects/main/test/taoensso/telemere_tests.cljc +++ b/projects/main/test/taoensso/telemere_tests.cljc @@ -760,6 +760,28 @@ (is (= (utils/error-signal? {:level :fatal}) true)) (is (= (utils/error-signal? {:error? true}) true))]) + (testing "clean-signal-fn" + (let [sig + {:level :info + :id nil + :msg_ (delay "msg") + :error ex2 + :location "loc" + :kvs "kvs" + :file "file" + :thread "thread" + :a "a" + :b "b"}] + + [(is (= ((utils/clean-signal-fn) sig) {:level :info, :msg_ "msg", :error ex2-chain})) + (is (= ((utils/clean-signal-fn {:incl-kvs? true}) sig) {:level :info, :msg_ "msg", :error ex2-chain, :a "a", :b "b"})) + (is (= ((utils/clean-signal-fn {:incl-nils? true}) sig) {:level :info, :msg_ "msg", :error ex2-chain, :id nil})) + (is (= ((utils/clean-signal-fn {:incl-keys #{:kvs}}) sig) {:level :info, :msg_ "msg", :error ex2-chain, :kvs "kvs"})) + (is (= ((utils/clean-signal-fn {:incl-keys #{:a}}) sig) {:level :info, :msg_ "msg", :error ex2-chain, :a "a"})) + (is (= ((utils/clean-signal-fn {:incl-keys + #{:location :kvs :file :thread}}) sig) {:level :info, :msg_ "msg", :error ex2-chain, + :location "loc", :kvs "kvs", :file "file", :thread "thread"}))])) + (testing "Misc utils" [(is (= (utils/remove-signal-kvs {:a :A, :b :B, :kvs {:b :B}}) {:a :A})) (is (= (utils/remove-signal-nils {:a :A, :b nil}) {:a :A})) @@ -836,27 +858,8 @@ "line" pnat-int? "column" pnat-int?}))))) - (testing "User pr-fn" - (is (= ((tel/pr-signal-fn {:pr-fn (fn [_] "str")}) sig) (str "str" utils/newline)))) - - (testing "Other options" - (let [sig - {:msg_ (delay "msg") - :error ex2 - :id nil - :location "loc" - :kvs "kvs" - :file "file" - :thread "thread" - :user-key "user-val"}] - - [(is (= ((tel/pr-signal-fn {:pr-fn :none}) sig) {:msg_ "msg", :error ex2-chain})) - (is (= ((tel/pr-signal-fn {:pr-fn :none, :incl-kvs? true}) sig) {:msg_ "msg", :error ex2-chain, :user-key "user-val"})) - (is (= ((tel/pr-signal-fn {:pr-fn :none, :incl-nils? true}) sig) {:msg_ "msg", :error ex2-chain, :id nil})) - (is (= ((tel/pr-signal-fn {:pr-fn :none, :incl-keys #{:kvs}}) sig) {:msg_ "msg", :error ex2-chain, :kvs "kvs"})) - (is (= ((tel/pr-signal-fn {:pr-fn :none, :incl-keys - #{:location :kvs :file :thread}}) sig) {:msg_ "msg", :error ex2-chain, - :location "loc", :kvs "kvs", :file "file", :thread "thread"}))]))])) + (testing "Custom pr-fn" + (is (= ((tel/pr-signal-fn {:pr-fn (fn [_] "str")}) sig) (str "str" utils/newline))))])) (testing "format-signal-fn" (let [sig (with-sig :raw :trap (tel/event! ::ev-id {:inst t1, :msg ["a" "b"]}))]