Metabase Backend Developer Documentation

Welcome to Metabase! Here are links to useful resources.

Project Management

Dev Environment

Important Parts of the Codebase

Important Libraries

Other Helpful Things

Tips on our Github wiki

The Dev Debug Page

If you want an easy way to GET/POST to an endpoint and display the results in a webpage, check out the Dev Debug Page. Cherry-pick the commit from that PR, modify DevDebug.jsx as you see fit (here is an example from the ParseSQL project), and then play with the results at /dev_debug. *Don't forget to remove the commit before merging to master!*

Lifecycle of a Query

Dan wrote a nice guide here


Put everything needed for REPL development within easy reach

(ns dev
  (:require
   [clojure.core.async :as a]
   [clojure.string :as str]
   [clojure.test]
   [dev.debug-qp :as debug-qp]
   [dev.explain :as dev.explain]
   [dev.migrate :as dev.migrate]
   [dev.model-tracking :as model-tracking]
   [dev.render-png :as render-png]
   [hashp.core :as hashp]
   [honey.sql :as sql]
   [java-time.api :as t]
   [malli.dev :as malli-dev]
   [metabase.api.common :as api]
   [metabase.config :as config]
   [metabase.core :as mbc]
   [metabase.db :as mdb]
   [metabase.db.env :as mdb.env]
   [metabase.driver :as driver]
   [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn]
   [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute]
   [metabase.email :as email]
   [metabase.models.database :refer [Database]]
   [metabase.models.setting :as setting]
   [metabase.query-processor.compile :as qp.compile]
   [metabase.query-processor.timezone :as qp.timezone]
   [metabase.server.handler :as handler]
   [metabase.server.instance :as server]
   [metabase.sync :as sync]
   [metabase.test :as mt]
   [metabase.test-runner]
   [metabase.test.data.impl :as data.impl]
   [metabase.util :as u]
   [metabase.util.log :as log]
   [methodical.core :as methodical]
   [potemkin :as p]
   [toucan2.connection :as t2.connection]
   [toucan2.core :as t2]
   [toucan2.pipeline :as t2.pipeline]
   [toucan2.tools.hydrate :as t2.hydrate]))
(set! *warn-on-reflection* true)
(comment
  debug-qp/keep-me
  model-tracking/keep-me)
#_:clj-kondo/ignore
(defn tap>-spy [x]
  (doto x tap>))
(p/import-vars
 [debug-qp
  pprint-sql]
 [dev.explain
  explain-query]
 [dev.migrate
  migrate!
  rollback!
  migration-sql-by-id]
 [render-png
  open-html
  open-png-bytes
  open-hiccup-as-html]
 [model-tracking
  track!
  untrack!
  untrack-all!
  reset-changes!
  changes]
 [mt
  set-ns-log-level!])

Was Metabase already initialized? Used in init! to prevent calling core/init! more than once (during start!, for example).

(def initialized?
  (atom nil))

Trigger general initialization, but only once.

(defn init!
  []
  (when-not @initialized?
    (mbc/init!)
    (reset! initialized? true)))

Returns a UTC timestamp in format yyyy-MM-dd'T'HH:mm:ss that you can used to postfix for migration ID.

(defn migration-timestamp
  []
  (t/format (t/formatter "yyyy-MM-dd'T'HH:mm:ss") (t/zoned-date-time (t/zone-id "UTC"))))

Finds in-memory Databases for which the underlying in-mem h2 db no longer exists.

(defn deleted-inmem-databases
  []
  (let [h2-dbs (t2/select :model/Database :engine :h2)
        in-memory? (fn [db] (some-> db :details :db (str/starts-with? "mem:")))
        can-connect? (fn [db]
                       #_:clj-kondo/ignore
                       (binding [metabase.driver.h2/*allow-testing-h2-connections* true]
                         (try
                           (driver/can-connect? :h2 (:details db))
                           (catch org.h2.jdbc.JdbcSQLNonTransientConnectionException _
                             false)
                           (catch Exception e
                             (log/error e "Error checking in-memory database for deletion")
                             ;; we don't want to delete these, so just pretend we could connect
                             true))))]
    (remove can-connect? (filter in-memory? h2-dbs))))

Delete any in-memory Databases to which we can't connect (in order to trigger cleanup of their related tasks, which will otherwise spam logs).

(defn prune-deleted-inmem-databases!
  []
  (when-let [outdated-ids (seq (map :id (deleted-inmem-databases)))]
    (t2/delete! :model/Database :id [:in outdated-ids])))

Start Metabase

(defn start!
  []
  (server/start-web-server! #'handler/app)
  (init!)
  (when config/is-dev?
    (prune-deleted-inmem-databases!)
    (with-out-str (malli-dev/start!))))

Stop Metabase

(defn stop!
  []
  (malli-dev/stop!)
  (server/stop-web-server!))

Restart Metabase

(defn restart!
  []
  (stop!)
  (start!))

Unmap all interned vars in a namespace. Reset the namespace to a blank slate! Perfect for when you rename everything and want to make sure you didn't miss a reference or when you redefine a multimethod.

(ns-unmap-all ns)

(defn ns-unmap-all
  ([]
   (ns-unmap-all *ns*))
  ([a-namespace]
   (doseq [[symb] (ns-interns a-namespace)]
     (ns-unmap a-namespace symb))
   (doseq [[symb varr] (ns-refers a-namespace)
           :when (not= (the-ns (:ns (meta varr)))
                       (the-ns 'clojure.core))]
     (ns-unmap a-namespace symb))))

Remove all aliases for other namespaces from the current namespace.

(ns-unalias-all ns)

(defn ns-unalias-all
  ([]
   (ns-unalias-all *ns*))
  ([a-namespace]
   (doseq [[symb] (ns-aliases a-namespace)]
     (ns-unalias a-namespace symb))))

Rather than requiring all models in the ns declaration, make it easy to require the ones you need for your current session

(defmacro require-model
  [model-sym]
  `(require [(symbol (str "metabase.models." (quote ~model-sym))) :as (quote ~model-sym)]))

Execute the body with the given permissions.

(defmacro with-permissions
  [permissions & body]
  `(binding [api/*current-user-permissions-set* (delay ~permissions)]
     ~@body))

Execute a SQL query against a JDBC database. Useful for testing SQL syntax locally.

(query-jdbc-db :oracle "SELECT to_date('1970-01-01', 'YYYY-MM-DD') FROM dual")

(query-jdbc-db :h2 "SELECT name FROM people WHERE name LIKE '%Ken%'")

sql-args can be either a SQL string or a tuple with a SQL string followed by any prepared statement args. By default this method uses the same methods to set prepared statement args and read columns from results as used by the :sql-jdbc Query Processor, but you pass the optional third arg options, as nil to use the driver's default behavior.

You can query against a dataset other than the default test data DB by passing in a [driver dataset] tuple as the first arg:

(dev/query-jdbc-db [:sqlserver 'time-test-data] ["SELECT * FROM dbo.users WHERE dbo.users.lastlogintime > ?" (java-time/offset-time "16:00Z")])

(defn query-jdbc-db
  {:arglists '([driver sql]            [[driver dataset] sql]
                                       [driver honeysql-form]  [[driver dataset] honeysql-form]
                                       [driver [sql & params]] [[driver dataset] [sql & params]])}
  [driver-or-driver+dataset sql-args]
  (let [[driver dataset] (u/one-or-many driver-or-driver+dataset)
        [sql & params]   (if (map? sql-args)
                           (sql/format sql-args)
                           (u/one-or-many sql-args))
        canceled-chan    (a/promise-chan)]
    (try
      (driver/with-driver driver
        (letfn [(thunk []
                  (let [db (mt/db)]
                    (sql-jdbc.execute/do-with-connection-with-options
                     driver
                     db
                     {:session-timezone (qp.timezone/report-timezone-id-if-supported driver db)}
                     (fn [conn]
                       (with-open [stmt (sql-jdbc.execute/prepared-statement driver conn sql params)
                                   rs   (sql-jdbc.execute/execute-prepared-statement! driver stmt)]
                         (let [rsmeta (.getMetaData rs)]
                           {:cols (sql-jdbc.execute/column-metadata driver rsmeta)
                            :rows (reduce conj [] (sql-jdbc.execute/reducible-rows driver rs rsmeta canceled-chan))}))))))]
          (if dataset
            (data.impl/do-with-dataset (data.impl/resolve-dataset-definition *ns* dataset) thunk)
            (thunk))))
      (catch InterruptedException e
        (a/>!! canceled-chan :cancel)
        (throw e)))))
(methodical/defmethod t2.connection/do-with-connection :model/Database
  "Support running arbitrary queries against data warehouse DBs for easy REPL debugging. Only works for SQL+JDBC drivers
  right now!
    ;; use Honey SQL
    (t2/query (t2/select-one Database :engine :postgres, :name \"test-data\")
              {:select [:*], :from [:venues]})
    ;; use it with `select`
    (t2/select :conn (t2/select-one Database :engine :postgres, :name \"test-data\")
               \"venues\")
    ;; use it with raw SQL
    (t2/query (t2/select-one Database :engine :postgres, :name \"test-data\")
              \"SELECT * FROM venues;\")
    ;; use it with the Sample Database
    (t2/query (t2/select-one Database :engine :h2, :name \"Sample Database\")
              \"SELECT * FROM people LIMIT 1;\")"
  [database f]
  (t2.connection/do-with-connection (sql-jdbc.conn/db->pooled-connection-spec database) f))
(methodical/defmethod t2.pipeline/build [#_query-type     :default
                                         #_model          :default
                                         #_resolved-query :mbql]
  [_query-type _model _parsed-args resolved-query]
  resolved-query)
(methodical/defmethod t2.pipeline/compile [#_query-type  :default
                                           #_model       :default
                                           #_built-query :mbql]
  "Run arbitrary MBQL queries. Only works for SQL right now!
    ;; Run a query against a Data warehouse DB
    (t2/query (t2/select-one Database :name \"test-data\")
              (mt/mbql-query venues))
    ;; Run MBQL queries against the application database
    (t2/query (dev/with-app-db (mt/mbql-query core_user {:aggregation [[:min [:get-year $date_joined]]]})))
    =>
    [{:min 2023}]"
  [_query-type _model built-query]
  ;; make sure we use the application database when compiling the query and not something goofy like a connection for a
  ;; Data warehouse DB, if we're using this in combination with a Database as connectable
  (let [{:keys [query params]} (binding [t2.connection/*current-connectable* nil]
                                 (qp.compile/compile built-query))]
    (into [query] params)))

Realize a lazy sequence if it's a lazy sequence. Otherwise, return the value as is.

(defn- maybe-realize
  [x]
  (if (instance? clojure.lang.LazySeq x)
    (doall x)
    x))
(methodical/defmethod t2.hydrate/hydrate-with-strategy :around ::t2.hydrate/multimethod-simple
  "Throws an error if simple hydrations make DB calls (which is an easy way to accidentally introduce an N+1 bug)."
  [model strategy k instances]
  (if (or config/is-prod?
          (< (count instances) 2))
    (next-method model strategy k instances)
    (do
      ;; prevent things like dereferencing metabase.api.common/*current-user-permissions-set* from triggering the check
      ;; by calling `next-method` *twice*. To reduce the performance impact, just call it with the first instance.
      (maybe-realize (next-method model strategy k [(first instances)]))
      ;; Now we can actually run the hydration with the full set of instances and make sure no more DB calls happened.
      (t2/with-call-count [call-count]
        (let [res (maybe-realize (next-method model strategy k instances))]
          ;; only throws an exception if the simple hydration makes a DB call
          (when (pos-int? (call-count))
            (throw (ex-info (format "N+1 hydration detected!!! Model %s, key %s]" (pr-str model) k)
                            {:model model :strategy strategy :k k :items-count (count instances) :db-calls (call-count)})))
          res)))))

Add the application database as a Database. Currently only works if your app DB uses broken-out details!

(defn app-db-as-data-warehouse
  []
  (binding [t2.connection/*current-connectable* nil]
    (or (t2/select-one Database :name "Application Database")
        #_:clj-kondo/ignore
        (let [details (#'metabase.db.env/broken-out-details
                       (mdb/db-type)
                       @#'metabase.db.env/env)
              app-db  (first (t2/insert-returning-instances! Database
                                                             {:name    "Application Database"
                                                              :engine  (mdb/db-type)
                                                              :details details}))]
          (sync/sync-database! app-db)
          app-db))))

Use the app DB as a Database and bind it so [[metabase.test/db]], [[metabase.test/mbql-query]], and the like use it.

(defmacro with-app-db
  [& body]
  `(let [db# (app-db-as-data-warehouse)]
     (mt/with-driver (:engine db#)
       (mt/with-db db#
         ~@body))))

p, but to use in pipelines like `(-> 1 inc dev/p inc)`.

See https://github.com/weavejester/hashp

(defmacro p
  [form]
  (hashp/p* form))

tap, but to use in pipelines like `(-> 1 inc dev/tap prn inc)`.

#_:clj-kondo/ignore
(defn tap
  [form]
  (u/prog1 form
    (tap> <>)))
(defn- tests-in-var-ns [test-var]
  (->> test-var meta :ns ns-interns vals
       (filter (comp :test meta))))

Sometimes tests fail due to another test not cleaning up after itself properly (e.g. leaving permissions in a dirty state). This is a common cause of tests failing in CI, or when run via find-and-run-tests, but not when run alone.

This helper allows you to pass in a test var for a test that fails only after other tests run. It finds and runs all tests, running your passed test after each.

When the passed test starts failing, it throws an exception notifying you of the test that caused it to start failing. At that point, you can start investigating what pleasant surprises that test is leaving behind in the database.

(defn find-root-test-failure!
  [failing-test-var & {:keys [scope] :or {scope :same-ns}}]
  (let [failed? (fn []
                  (not= [0 0] ((juxt :fail :error) (clojure.test/run-test-var failing-test-var))))]
    (when (failed?)
      (throw (ex-info "Test is already failing! Better go fix it." {:failed-test failing-test-var})))
    (let [tests (case scope
                  :same-ns (tests-in-var-ns failing-test-var)
                  :full-suite (metabase.test-runner/find-tests))]
      (doseq [test tests]
        (clojure.test/run-test-var test)
        (when (failed?)
          (throw (ex-info (format "Test failed after running: `%s`" test)
                          {:test test})))))))

Set up email settings for sending emails from Metabase. This is useful for testing email sending in the REPL.

(defn setup-email!
  [& settings]
  (let [settings (merge {:host     "localhost"
                         :port     1025
                         :user     "metabase"
                         :pass     "metabase@secret"
                         :security :none}
                        settings)]
    (when (::email/error (email/test-smtp-connection settings))
      (throw (ex-info "Failed to connect to SMTP server" {:settings settings})))
    (setting/set-many! (update-keys settings
                                    {:host        :email-smtp-host,
                                     :user        :email-smtp-username,
                                     :pass        :email-smtp-password,
                                     :port        :email-smtp-port,
                                     :security    :email-smtp-security,
                                     :sender-name :email-from-name,
                                     :sender      :email-from-address,
                                     :reply-to    :email-reply-to}))))