(ns metabase.models.secret (:require [clojure.core.memoize :as memoize] [clojure.java.io :as io] [clojure.string :as str] [java-time.api :as t] [metabase.api.common :as api] [metabase.driver :as driver] [metabase.driver.util :as driver.u] [metabase.models.interface :as mi] [metabase.public-settings.premium-features :as premium-features] [metabase.util :as u] [metabase.util.i18n :refer [tru]] [metabase.util.log :as log] [methodical.core :as methodical] [toucan2.core :as t2]) (:import (java.io File) (java.nio.charset StandardCharsets))) | |
(set! *warn-on-reflection* true) | |
----------------------------------------------- Entity & Lifecycle ----------------------------------------------- | |
Used to be the toucan1 model name defined using [[toucan.models/defmodel]], now it's a reference to the toucan2 model name. We'll keep this till we replace all the symbols in our codebase. | (def Secret :model/Secret) |
(methodical/defmethod t2/table-name :model/Secret [_model] :secret) | |
(doto Secret (derive :metabase/model) (derive :hook/timestamped?) (derive ::mi/read-policy.superuser) (derive ::mi/write-policy.superuser)) | |
(t2/deftransforms :model/Secret
{:value mi/transform-secret-value
:kind mi/transform-keyword
:source mi/transform-keyword}) | |
---------------------------------------------- Hydration / Util Fns ---------------------------------------------- | |
Returns the value of the given | (defn value->string
{:added "0.42.0"}
^String [{:keys [value] :as _secret}]
(cond (string? value)
value
(bytes? value)
(String. ^bytes value StandardCharsets/UTF_8))) |
For the given | (defn conn-props->secret-props-by-name
{:added "0.42.0"}
[conn-props]
(->> (filter #(= :secret (keyword (:type %))) conn-props)
(reduce (fn [acc prop] (assoc acc (:name prop) prop)) {}))) |
Returns the value of the given
| (defn value->file!*
{:added "0.42.0"}
(^File [secret]
(value->file!* secret nil))
(^File [secret driver?]
(value->file!* secret driver? nil))
(^File [{:keys [connection-property-name id value] :as secret} driver? ext?]
(if (= :file-path (:source secret))
(let [secret-val (value->string secret)
^File existing-file (File. secret-val)]
(if (.exists existing-file)
existing-file
(let [error-source (cond
id
(tru "Secret ID {0}" id)
(and connection-property-name driver?)
(let [secret-props (-> (driver/connection-properties driver?)
conn-props->secret-props-by-name)]
(tru "File path for {0}" (-> (get secret-props connection-property-name)
:display-name)))
:else
(tru "Path"))]
(throw (ex-info (tru "{0} points to non-existent file: {1}" error-source secret-val)
{:file-path secret-val
:secret secret})))))
(let [^File tmp-file (doto (File/createTempFile "metabase-secret_" ext?)
;; make the file only readable by owner
(.setReadable false false)
(.setReadable true true)
(.deleteOnExit))]
(log/tracef "Creating temp file for secret %s value at %s" (or id "") (.getAbsolutePath tmp-file))
(with-open [out (io/output-stream tmp-file)]
(let [^bytes v (cond
(string? value)
(.getBytes ^String value "UTF-8")
(bytes? value)
^bytes value)]
(.write out v)))
tmp-file)))) |
Returns the value of the given
| (def
^java.io.File
^{:arglists '([{:keys [connection-property-name id value] :as secret} & [driver? ext?]])}
value->file!
(memoize/memo
(with-meta value->file!*
{::memoize/args-fn (fn [[secret _driver? ext?]]
;; not clear if value->string could return nil due to the cond so we'll just cache on a key
;; that is unique
[(vec (:value secret)) ext?])}))) |
Return a map of secret subproperties for the property | (defn ->sub-props
[connection-property-name]
(let [sub-prop-types [:path :value :options :id]
sub-prop #(keyword (str connection-property-name "-" (name %)))]
(zipmap sub-prop-types (map sub-prop sub-prop-types)))) |
Regex for parsing base64 encoded file uploads. | (def uploaded-base-64-prefix-pattern #"^data:application/([^;]*);base64,") |
Returns the latest Secret instance for the given | (defn latest-for-id
{:added "0.42.0"}
[id]
(t2/select-one Secret :id id {:order-by [[:version :desc]]})) |
Returns a map containing This returned map represents a partial Secret model instance (having some of the required properties set), but also
represents a discrete property that can be used in connection testing (even without the Secret needing to be
persisted). In addition to possibly having
| (defn db-details-prop->secret-map
{:added "0.42.0"}
[details conn-prop-nm]
(let [{path-kw :path, value-kw :value, options-kw :options, id-kw :id}
(->sub-props conn-prop-nm)
value (cond
;; ssl-root-certs will need their prefix removed, and to be base 64 decoded (#20319)
(and (value-kw details) (#{"ssl-client-cert" "ssl-root-cert"} conn-prop-nm)
(re-find uploaded-base-64-prefix-pattern (value-kw details)))
(-> (value-kw details) (str/replace-first uploaded-base-64-prefix-pattern "") u/decode-base64)
(and (value-kw details) (#{"ssl-key"} conn-prop-nm)
(re-find uploaded-base-64-prefix-pattern (value-kw details)))
(.decode (java.util.Base64/getDecoder)
(str/replace-first (value-kw details) uploaded-base-64-prefix-pattern ""))
;; the -value suffix was specified; use that
(value-kw details)
(value-kw details)
;; the -path suffix was specified; this is actually a :file-path
(path-kw details)
(u/prog1 (path-kw details)
(when (premium-features/is-hosted?)
(throw (ex-info
(tru "{0} (a local file path) cannot be used in Metabase hosted environment" path-kw)
{:invalid-db-details-entry (select-keys details [path-kw])}))))
(id-kw details)
(:value (latest-for-id (id-kw details))))
source (cond
;; set the :source due to the -path suffix (see above))
(and (not= "uploaded" (options-kw details)) (path-kw details))
:file-path
(id-kw details)
(:source (latest-for-id (id-kw details))))]
(cond-> {:connection-property-name conn-prop-nm, :subprops [path-kw value-kw id-kw]}
value
(assoc :value value
:source source)))) |
Get the value of a secret property from the database details as a string. | (defn get-secret-string
[details secret-property]
(let [{path-kw :path, value-kw :value, options-kw :options, id-kw :id} (->sub-props secret-property)
id (id-kw details)
;; When a secret is updated, we get both a new value as well as the ID of old secret.
value (or (when-let [value (value-kw details)]
(if (string? value)
value
(String. ^bytes value "UTF-8")))
(when id
(String. ^bytes (:value (latest-for-id id)) "UTF-8")))]
(case (options-kw details)
"uploaded" (try
;; When a secret is updated, the value has already been decoded
;; instead of checking if the string is base64 encoded, we just
;; try to decoded it and leave it as is if the attempt fails.
(String. ^bytes (driver.u/decode-uploaded value) "UTF-8")
(catch IllegalArgumentException _
value))
"local" (slurp (if id value (path-kw details)))
value))) |
The attributes of a secret which, if changed, will result in a version bump | (def
^{:doc :private true}
bump-version-keys
[:kind :source :value]) |
Inserts a new secret value, or updates an existing one, for the given parameters. * if there is no existing Secret instance, inserts with the given field values * if there is an existing latest Secret instance, and the value (or any of the supporting fields, like kind or source) has changed, then inserts a new version with the given parameters. * if there is an existing latest Secret instance, but none of the aforementioned fields changed, then update it | (defn upsert-secret-value!
{:added "0.42.0"}
[existing-id nm kind src value]
(let [insert-new (fn [id v]
(let [inserted (first (t2/insert-returning-instances! Secret (cond-> {:version v
:name nm
:kind kind
:source src
:value value
:creator_id api/*current-user-id*}
id
(assoc :id id))))]
;; Toucan doesn't support composite primary keys, so adding a new record with incremented
;; version for an existing ID won't return a result from t2/insert!, hence we may need to
;; manually select it here
(t2/select-one Secret :id (or id (u/the-id inserted)) :version v)))
latest-version (when existing-id (latest-for-id existing-id))]
(if latest-version
(if (= (select-keys latest-version bump-version-keys) [kind src value])
(pos? (t2/update! Secret {:id existing-id :version (:version latest-version)}
{:name nm}))
(insert-new (u/the-id latest-version) (inc (:version latest-version))))
(insert-new nil 1)))) |
Reduces over the given
In essence, this is a utility function to provide a generic mechanism for transforming db-details containing secret values. | (defn reduce-over-details-secret-values
{:added "0.42.0"}
[driver db-details reduce-fn]
(let [conn-props-fn (get-method driver/connection-properties driver)]
(if (and (map? db-details) (fn? conn-props-fn))
(let [conn-props (conn-props-fn driver)
conn-secrets-by-name (conn-props->secret-props-by-name conn-props)]
(reduce-kv reduce-fn db-details conn-secrets-by-name))
db-details))) |
Expand certain secret sub-properties in the
The keys/value pairs that may be added into
| (defn expand-inferred-secret-values
{:added "0.42.0"}
[db-details conn-prop-nm _conn-prop & [secret-or-id]]
(let [subprop (fn [prop-nm]
(keyword (str conn-prop-nm prop-nm)))
secret* (cond (int? secret-or-id)
(latest-for-id secret-or-id)
(mi/instance-of? Secret secret-or-id)
secret-or-id
:else ; default; app DB look up from the ID in db-details
(latest-for-id (get db-details (subprop "-id"))))
src (:source secret*)]
;; always populate the -source, -creator-id, and -created-at sub properties
(cond-> (assoc db-details (subprop "-source") src
(subprop "-creator-id") (:creator_id secret*))
(some? (:created_at secret*))
(assoc (subprop "-created-at") (t/format :iso-offset-date-time (:created_at secret*)))
(= :file-path src) ; for file path sources only, populate the value
(assoc (subprop "-value") (value->string secret*))))) |
Expand certain inferred secret sub-properties in the | (defn expand-db-details-inferred-secret-values
{:added "0.42.0"}
[database]
(update database :details (fn [details]
(reduce-over-details-secret-values (driver.u/database->driver database)
details
expand-inferred-secret-values)))) |
(methodical/defmethod mi/to-json Secret "Never include the secret value in JSON." [secret json-generator] (next-method (dissoc secret :value) json-generator)) | |