Provides functions that support formatting results data. In particular, customizing formatting for when timezone, column metadata, and visualization-settings are known. These functions can be used for uniform rendering of all artifacts such as generated CSV or image files that need consistent formatting across the board. | (ns metabase.formatter (:require [clojure.pprint :refer [cl-format]] [clojure.string :as str] [hiccup.util] [metabase.formatter.datetime :as datetime] [metabase.models.visualization-settings :as mb.viz] [metabase.public-settings :as public-settings] [metabase.query-processor.streaming.common :as common] [metabase.types :as types] [metabase.util.currency :as currency] [metabase.util.json :as json] [metabase.util.malli :as mu] [metabase.util.malli.schema :as ms] [metabase.util.ui-logic :as ui-logic] [potemkin :as p] [potemkin.types :as p.types]) (:import (java.math RoundingMode) (java.text DecimalFormat DecimalFormatSymbols))) |
(set! *warn-on-reflection* true) | |
Fool Eastwood into thinking this namespace is used | (comment hiccup.util/keep-me) |
(p/import-vars [datetime make-temporal-str-formatter temporal-string?]) | |
(p.types/defrecord+ NumericWrapper [^String num-str ^Number num-value] hiccup.util/ToString (to-str [_] num-str) Object (toString [_] num-str)) | |
(defn- strip-trailing-zeroes
[num-as-string decimal]
(if (str/includes? num-as-string (str decimal))
(let [pattern (re-pattern (str/escape (str decimal \$) {\. "\\."}))]
(-> num-as-string
(str/split #"0+$")
first
(str/split pattern)
first))
num-as-string)) | |
(defn- digits-after-decimal
([value] (digits-after-decimal value "."))
([value decimal]
(if (zero? value)
0
(let [val-string (-> (condp = (type value)
java.math.BigDecimal (.toPlainString ^BigDecimal value)
java.lang.Double (format "%.20f" value)
java.lang.Float (format "%.20f" value)
(str value))
(strip-trailing-zeroes (str decimal)))
decimal-idx (str/index-of val-string decimal)]
(if decimal-idx
(- (count val-string) decimal-idx 1)
0))))) | |
(defn- sig-figs-after-decimal
[value decimal]
(if (zero? value)
0
(let [val-string (-> (condp = (type value)
java.math.BigDecimal (.toPlainString ^BigDecimal value)
java.lang.Double (format "%.20f" value)
java.lang.Float (format "%.20f" value)
(str value))
(strip-trailing-zeroes (str decimal)))
figs (last (str/split val-string #"[\.0]+"))]
(count figs)))) | |
(defn- qualify-keys
[m]
(update-keys m (fn [k] (keyword
"metabase.models.visualization-settings"
(name k))))) | |
Return a function that will take a number and format it according to its column viz settings. Useful to compute the format string once and then apply it over many values. TODO: use | (defn number-formatter
[{:keys [semantic_type effective_type base_type]
col-id :id field-ref :field_ref col-name :name col-settings :settings :as col}
viz-settings]
(let [global-type-settings (try
(common/global-type-settings col viz-settings)
(catch Exception _e
(common/global-type-settings (dissoc col :base_type :effective_type) viz-settings)))
col-id (or col-id (second field-ref))
column-settings (-> (get viz-settings ::mb.viz/column-settings)
(update-keys #(select-keys % [::mb.viz/field-id ::mb.viz/column-name])))
column-settings (merge
(or (get column-settings {::mb.viz/field-id col-id})
(get column-settings {::mb.viz/column-name col-name}))
(qualify-keys col-settings)
global-type-settings)
global-settings (merge
global-type-settings
(::mb.viz/global-column-settings viz-settings))
currency? (boolean (or (= (::mb.viz/number-style column-settings) "currency")
(= (::mb.viz/number-style viz-settings) "currency")
(and (nil? (::mb.viz/number-style column-settings))
(or
(::mb.viz/currency-style column-settings)
(::mb.viz/currency column-settings)))))
{::mb.viz/keys [number-separators decimals scale number-style
prefix suffix currency-style currency]} (merge
(when currency?
(:type/Currency global-settings))
(:type/Number global-settings)
column-settings)
currency (when currency?
(keyword (or currency "USD")))
integral? (and (isa? (or effective_type base_type) :type/Integer) (integer? (or scale 1)))
relation? (isa? semantic_type :Relation/*)
percent? (or (isa? semantic_type :type/Percentage) (= number-style "percent"))
scientific? (= number-style "scientific")
[decimal grouping] (or number-separators
(get-in (public-settings/custom-formatting) [:type/Number :number_separators])
".,")
symbols (doto (DecimalFormatSymbols.)
(cond-> decimal (.setDecimalSeparator decimal))
(cond-> grouping (.setGroupingSeparator grouping)))
base (cond-> (if (or scientific? relation?) "0" "#,##0")
(not grouping) (str/replace #"," ""))
;; A small cache of decimal-digits->formatter to avoid creating new ones all the time. This cache is bound by
;; the maximum number of decimal digits we can format which is 20 (constrained by `digits-after-decimal`).
fmtr-cache (volatile! {})]
(fn [value]
(if (number? value)
(let [scaled-value (cond-> (* value (or scale 1))
percent?
(* 100))
decimals-in-value (digits-after-decimal scaled-value)
decimal-digits (cond
decimals decimals ;; if user ever specifies # of decimals, use that
integral? 0
scientific? (min 2 (max decimals-in-value
;; Scientific representation can introduce its own decimal
;; digits even in integer numbers. Count how many integer
;; digits are in the number (but limit to 2).
(int (Math/log10 (abs scaled-value)))))
currency? (get-in currency/currency [currency :decimal_digits])
percent? (min 2 decimals-in-value) ;; 5.5432 -> %554.32
:else (if (>= (abs scaled-value) 1)
(min 2 decimals-in-value) ;; values greater than 1 round to 2 decimal places
(let [n-figs (sig-figs-after-decimal scaled-value decimal)]
(if (> n-figs 2)
(max 2 (- decimals-in-value (- n-figs 2))) ;; values less than 1 round to 2 sig-dig
decimals-in-value))))
fmtr (or (@fmtr-cache decimal-digits)
(let [fmt-str (cond-> base
(not (zero? decimal-digits)) (str "." (apply str (repeat decimal-digits "0")))
scientific? (str "E0"))
fmtr (doto (DecimalFormat. fmt-str symbols) (.setRoundingMode RoundingMode/HALF_UP))]
(vswap! fmtr-cache assoc decimal-digits fmtr)
fmtr))]
(->NumericWrapper
(let [inline-currency? (and currency?
(false? (::mb.viz/currency-in-header column-settings)))
sb (StringBuilder.)]
;; Using explicit StringBuilder to avoid touching the slow `clojure.core/str` multi-arity.
(when prefix (.append sb prefix))
(when (and inline-currency? (or (nil? currency-style)
(= currency-style "symbol")))
(.append sb (get-in currency/currency [currency :symbol])))
(when (and inline-currency? (= currency-style "code"))
(.append sb (get-in currency/currency [currency :code]))
(.append sb \space))
(.append sb (cond-> (.format ^DecimalFormat fmtr scaled-value)
(and (not currency?) (not decimals))
(strip-trailing-zeroes decimal)))
(when percent?
(.append sb "%"))
(when (and inline-currency? (= currency-style "name"))
(.append sb \space)
(.append sb (get-in currency/currency [currency :name_plural])))
(when suffix (.append sb suffix))
(str sb))
value))
value)))) |
(mu/defn format-number :- (ms/InstanceOfClass NumericWrapper)
"Format a number `n` and return it as a NumericWrapper; this type is used to do special formatting in other
`pulse.render` namespaces."
([n :- number?]
(map->NumericWrapper {:num-str (cl-format nil (if (integer? n) "~:d" "~,2f") n)
:num-value n}))
([value column viz-settings]
(let [fmttr (number-formatter column viz-settings)]
(fmttr value)))) | |
Return a pair of | (defn graphing-column-row-fns
[card data]
[(or (ui-logic/x-axis-rowfn card data)
first)
(or (ui-logic/y-axis-rowfn card data)
second)]) |
Graal polyglot system (not the JS machine itself, the polyglot system) is not happy with BigInts or BigDecimals. For more information, this is the GraalVM issue, open a while https://github.com/oracle/graal/issues/2737 Because of this unfortunately they all have to get smushed into normal ints and decimals in JS land. | (defn coerce-bignum-to-int
[row]
(for [member row]
(cond
;; this returns true for bigint only, not normal int or long
(instance? clojure.lang.BigInt member)
(int member)
;; this returns true for bigdec only, not actual normal decimals
;; not the clearest clojure native function in the world
(decimal? member)
(double member)
:else
member))) |
Preprocess rows.
| (defn row-preprocess
[x-axis-fn y-axis-fn rows]
(->> rows
(filter (every-pred x-axis-fn y-axis-fn))
(map coerce-bignum-to-int))) |
Format longitude/latitude values as 0.00000000° N|S|E|W | (defn format-geographic-coordinates
[lon-or-lat v]
(str (when (number? v)
(let [v (double v)
dir (case lon-or-lat
:type/Latitude (if (neg? v) "S" "N")
:type/Longitude (if (neg? v) "W" "E")
nil)]
(if dir
(format "%.8f° %s" (Math/abs v) dir)
v))))) |
Format dictionaries as json. The value if a dictionary is Clojure edn on the backend, but displays as JSON in When exporting, the map must be encoded as json so that exports match the app's output. | (defn- dictionary-formatter
[value]
(cond-> value
(not (string? value)) json/encode)) |
Create a formatter for a column based on its timezone, column metadata, and visualization-settings | (mu/defn create-formatter
([timezone-id :- [:maybe :string] col visualization-settings]
(create-formatter timezone-id col visualization-settings true))
([timezone-id :- [:maybe :string] col visualization-settings apply-formatting?]
(cond
;; for numbers, return a format function that has already computed the differences.
;; todo: do the same for temporal strings
(and apply-formatting? (types/temporal-field? col))
(datetime/make-temporal-str-formatter timezone-id col visualization-settings)
(and apply-formatting? (isa? (:semantic_type col) :type/Coordinate))
(partial format-geographic-coordinates (:semantic_type col))
;; todo integer columns with a unit
(and apply-formatting?
(or (isa? (:effective_type col) :type/Number)
(isa? (:base_type col) :type/Number)))
(number-formatter col visualization-settings)
(or (isa? (:semantic_type col) :type/SerializedJSON)
(isa? ((some-fn :effective_type :base_type) col) :type/Dictionary))
dictionary-formatter
:else
(if apply-formatting? str identity)))) |