metabase-enterprise.audit-app.audit

(ns metabase-enterprise.audit-app.audit
  (:require
   [babashka.fs :as fs]
   [clojure.java.io :as io]
   [clojure.string :as str]
   [metabase-enterprise.serialization.cmd :as serialization.cmd]
   [metabase.audit :as audit]
   [metabase.db :as mdb]
   [metabase.models.serialization :as serdes]
   [metabase.models.setting :refer [defsetting]]
   [metabase.plugins :as plugins]
   [metabase.premium-features.core :refer [defenterprise]]
   [metabase.sync.util :as sync-util]
   [metabase.util :as u]
   [metabase.util.files :as u.files]
   [metabase.util.log :as log]
   [toucan2.core :as t2])
  (:import
   (java.nio.file Path)
   (java.util.jar JarEntry JarFile)))

(set! *warn-on-reflection* true)

(defn- running-from-jar?
  []
  (-> (Thread/currentThread)
      (.getContextClassLoader)
      (.getResource "")
      (str/starts-with? "jar:")))

(defn- get-jar-path
  []
  (assert (running-from-jar?) "Can only get-jar-path when running from a jar.")
  (-> (class {})
      (.getProtectionDomain)
      (.getCodeSource)
      (.getLocation)
      (.toURI) ;; avoid problems with special characters in path.
      (.getPath)))

(defn copy-from-jar!
  [jar-path resource-path out-dir]
  (let [jar-file (JarFile. (str jar-path))
        entries (.entries jar-file)]
    (doseq [^JarEntry entry (iterator-seq entries)
            :let [entry-name (.getName entry)]
            :when (str/starts-with? entry-name resource-path)
            :let [out-file (fs/path out-dir entry-name)]]
      (if (.isDirectory entry)
        (fs/create-dirs out-file)
        (do
          (-> out-file fs/parent fs/create-dirs)
          (with-open [in (.getInputStream jar-file entry)
                      out (io/output-stream (str out-file))]
            (io/copy in out)))))))

(def default-question-overview-entity-id
  "jm7KgY6IuS6pQjkBZ7WUI")

(def default-dashboard-overview-entity-id
  "bJEYb0o5CXlfWFcIztDwJ")

(defn- install-database!
  [engine id]
  (t2/insert! :model/Database {:is_audit         true
                               :id               id
                               :name             "Internal Metabase Database"
                               :description      "Internal Audit DB used to power metabase analytics."
                               :engine           engine
                               :is_full_sync     true
                               :is_on_demand     false
                               :creator_id       nil
                               :auto_run_queries true})
  ;; guard against someone manually deleting the audit-db entry, but not removing the audit-db permissions.
  (t2/delete! :model/Permissions {:where [:like :object (str "%/db/" id "/%")]}))

(defn- adjust-audit-db-to-source!
  [{audit-db-id :id}]
  ;; We need to move back to a schema that matches the serialized data
  (when (contains? #{:mysql :h2} (mdb/db-type))
    (t2/update! :model/Database audit-db-id {:engine "postgres"})
    (when (= :mysql (mdb/db-type))
      (t2/update! :model/Table {:db_id audit-db-id} {:schema "public"}))
    (when (= :h2 (mdb/db-type))
      (t2/update! :model/Table {:db_id audit-db-id} {:schema [:lower :schema] :name [:lower :name]})
      (t2/update! :model/Field
                  {:table_id
                   [:in
                    {:select [:id]
                     :from [(t2/table-name :model/Table)]
                     :where [:= :db_id audit-db-id]}]}
                  {:name [:lower :name]}))
    (log/info "Adjusted Audit DB for loading Analytics Content")))

(defn- adjust-audit-db-to-host!
  [{audit-db-id :id :keys [engine]}]
  (when (not= engine (mdb/db-type))
    ;; We need to move the loaded data back to the host db
    (t2/update! :model/Database audit-db-id {:engine (name (mdb/db-type))})
    (when (= :mysql (mdb/db-type))
      (t2/update! :model/Table {:db_id audit-db-id} {:schema nil}))
    (when (= :h2 (mdb/db-type))
      (t2/update! :model/Table {:db_id audit-db-id} {:schema [:upper :schema] :name [:upper :name]})
      (t2/update! :model/Field
                  {:table_id
                   [:in
                    {:select [:id]
                     :from [(t2/table-name :model/Table)]
                     :where [:= :db_id audit-db-id]}]}
                  {:name [:upper :name]}))
    (log/infof "Adjusted Audit DB to match host engine: %s" (name (mdb/db-type)))))

(def ^:private analytics-dir-resource
  (io/resource "instance_analytics"))

(defn- instance-analytics-plugin-dir
  [plugins-dir]
  (fs/path (fs/absolutize plugins-dir) "instance_analytics"))

(def ^:private jar-resource-path "instance_analytics/")

(defn- ia-content->plugins
  [plugins-dir]
  (let [ia-dir (instance-analytics-plugin-dir plugins-dir)]
    (when (fs/exists? (u.files/relative-path ia-dir))
      (fs/delete-tree (u.files/relative-path ia-dir)))
    (if (running-from-jar?)
      (let [path-to-jar (get-jar-path)]
        (log/info "The app is running from a jar, starting copy...")
        (log/info (str "Copying " path-to-jar "::" jar-resource-path " -> " plugins-dir))
        (copy-from-jar! path-to-jar jar-resource-path plugins-dir)
        (log/info "Copying complete."))
      (let [in-path (fs/path analytics-dir-resource)]
        (log/info "The app is not running from a jar, starting copy...")
        (log/info (str "Copying " in-path " -> " ia-dir))
        (fs/copy-tree (u.files/relative-path in-path)
                      (u.files/relative-path ia-dir)
                      {:replace-existing true})
        (log/info "Copying complete.")))))

(defsetting load-analytics-content
  :type       :boolean
  :default    true
  :visibility :internal
  :setter     :none
  :audit      :never
  :doc        "Setting this environment variable to false can also come in handy when migrating environments, as it can simplify the migration process.")

(def ^:constant SKIP_CHECKSUM_FLAG
  -1)

(defn- should-skip-checksum? [last-checksum]
  (= SKIP_CHECKSUM_FLAG last-checksum))

(defn analytics-checksum
  []
  (->> ^Path (instance-analytics-plugin-dir (plugins/plugins-dir))
       (.toFile)
       file-seq
       (remove fs/directory?)
       (pmap #(hash (slurp %)))
       (reduce +)))

(defn- should-load-audit?
  [load-analytics-content? last-checksum current-checksum]
  (and load-analytics-content?
       (or (should-skip-checksum? last-checksum)
           (not= last-checksum current-checksum))))

(defn- get-last-and-current-checksum
  []
  (let [last-checksum (audit/last-analytics-checksum)]
    (if (should-skip-checksum? last-checksum)
      [SKIP_CHECKSUM_FLAG SKIP_CHECKSUM_FLAG]
      [last-checksum (analytics-checksum)])))

(defn- maybe-load-analytics-content!
  [audit-db]
  (when analytics-dir-resource
    (adjust-audit-db-to-source! audit-db)
    (ia-content->plugins (plugins/plugins-dir))
    (let [[last-checksum current-checksum] (get-last-and-current-checksum)]
      (when (should-load-audit? (load-analytics-content) last-checksum current-checksum)
        (log/info (str "Loading Analytics Content from: " (instance-analytics-plugin-dir (plugins/plugins-dir))))
        ;; The EE token might not have :serialization enabled, but audit features should still be able to use it.
        (let [report (log/with-no-logs
                       (serialization.cmd/v2-load-internal! (str (instance-analytics-plugin-dir (plugins/plugins-dir)))
                                                            {:backfill? false}
                                                            :token-check? false
                                                            :require-initialized-db? false))]
          (if (not-empty (:errors report))
            (log/info (str "Error Loading Analytics Content: " (pr-str report)))
            (do
              (log/info (str "Loading Analytics Content Complete (" (count (:seen report)) ") entities loaded."))
              (audit/last-analytics-checksum! current-checksum))))))
    (when-let [audit-db (t2/select-one :model/Database :is_audit true)]
      (adjust-audit-db-to-host! audit-db))))

(def ^:private audit-db-entity-id
  "audit__rP75CiURKZ-0pq")

(defn- entity-id-for-table [table]
  (-> [audit-db-entity-id
       ;; The hard-coded entity_ids saved in the serdes export used a schema of "public", so that's now hard-coded.
       ;; The schema (and spelling) used for the AppDB tables varies by engine, so it should not influence the idents.
       "public"
       ;; We use inconsistent upper and lower case table and field names for audit across AppDB engines; this uses
       ;; lower case everywhere to make the entity IDs effectively hard-coded.
       (u/lower-case-en (:name table))]
      serdes/raw-hash
      u/generate-nano-id))

(defn- entity-id-for-field [table-eid field]
  ;; We use inconsistent upper and lower case table and field names for audit across AppDB engines; this uses lower
  ;; case to make the entity IDs effectively hard-coded.
  (-> [table-eid (u/lower-case-en (:name field))]
      serdes/raw-hash
      u/generate-nano-id))

(defn- backfill-entity-ids!
  [db]
  (when db
    (t2/update! :model/Database (:id db) {:entity_id audit-db-entity-id})
    (let [tables (t2/select :model/Table :db_id (:id db))
          eids   (into {} (map (juxt :id (some-fn :entity_id entity-id-for-table))) tables)]
      (doseq [table tables
              :when (not (:entity_id table))]
        (t2/update! :model/Table (:id table) {:entity_id (get eids (:id table))}))
      (when (seq tables)
        (doseq [field (t2/select :model/Field :table_id [:in (map :id tables)] :entity_id nil)]
          (t2/update! :model/Field (:id field)
                      {:entity_id (entity-id-for-field (get eids (:table_id field)) field)}))))))

(defn- maybe-install-audit-db
  []
  (let [audit-db (t2/select-one :model/Database :is_audit true)]
    (when audit-db
      (backfill-entity-ids! audit-db))
    (cond
      (nil? audit-db)
      (u/prog1 ::installed
        (log/info "Installing Audit DB...")
        (install-database! (mdb/db-type) audit/audit-db-id))
      (not= (mdb/db-type) (:engine audit-db))
      (u/prog1 ::updated
        (log/infof "App DB change detected. Changing Audit DB source to match: %s." (name (mdb/db-type)))
        (adjust-audit-db-to-host! audit-db))
      :else
      ::no-op)))

(defenterprise ensure-audit-db-installed!
  :feature :none
  []
  (u/prog1 (maybe-install-audit-db)
    (let [audit-db (t2/select-one :model/Database :is_audit true)]
      ;; prevent sync while loading
      ((sync-util/with-duplicate-ops-prevented
        :sync-database audit-db
        (fn []
          (maybe-load-analytics-content! audit-db)))))))