corrplot/src/tservice/plugins/corrplot.clj

(ns tservice.plugins.corrplot
  (:require [clojure.data.json :as json]
            [clojure.spec.alpha :as s]
            [clojure.tools.logging :as log]
            [spec-tools.core :as st]
            [tservice.util :as u]
            [tservice.lib.files :as ff]
            [tservice.lib.fs :as fs-lib]
            [tservice.plugins.corrplot.common :as corrplot]
            [tservice.api.task :refer [publish-event! make-plugin-metadata make-events-init create-task! update-process!]]))

;;; ------------------------------------------------ Event Specs ------------------------------------------------
(s/def ::datafile
  (st/spec
   {:spec                string?
    :type                :string
    :description         "A path for data file."
    :swagger/default     ""
    :reason              "The datafile must be string."}))

(s/def ::corr_vars
  (st/spec
   {:spec                (s/coll-of string?)
    :type                :array
    :description         "Variables."
    :swagger/default     nil
    :reason              "The corr_vars must be a vector."}))

(s/def ::method
  (st/spec
   {:spec                #{"square" "circle"}
    :type                :string
    :description         "The visualization method of correlation matrix to be used. Allowed values are square (default), circle."
    :swagger/default     "square"
    :reason              "The corr_vars must be a vector."}))

(s/def ::corr_type
  (st/spec
   {:spec                #{"full" "lower" "upper"}
    :type                :string
    :description         "full (default), lower or upper display."
    :swagger/default     "full"
    :reason              "The corr_type must be one of full, lower, upper."}))

(s/def ::hc_method
  (st/spec
   {:spec                #{"ward.D" "ward.D2" "single" "complete" "average" "mcquitty" "median" "centroid"}
    :type                :string
    :description         "The agglomeration method to be used in hclust (see ?hclust)."
    :swagger/default     "complete"
    :reason              "The hc_method must be (an unambiguous abbreviation of) one of ward.D, ward.D2,
                          single, complete, average, mcquitty, median or centroid."}))

(s/def ::hc_order
  (st/spec
   {:spec                #{true false}
    :type                :bool
    :description         "Logical value. If TRUE, correlation matrix will be hc.ordered using hclust function."
    :swagger/default     true
    :reason              "The hc_order must be one of true, false."}))

(s/def ::sig_level
  (st/spec
   {:spec                #(and (>= % 0) (<= % 1))
    :type                :float
    :description         "Significant level, greater than 0 and less than 1."
    :swagger/default     0.05
    :reason              "The sig_level must be a float."}))

(def corrplot-params-body
  "A spec for the body parameters."
  (s/keys :req-un [::datafile ::corr_vars]
          :opt-un [::sig_level ::hc_order ::hc_method ::corr_type ::method]))

;;; ------------------------------------------------ Event Metadata ------------------------------------------------
(def metadata
  (make-plugin-metadata
   {:name "corrplot"
    :summary "It is used to investigate the dependence between multiple variables at the same time and to highlight the most correlated variables in a data table."
    :params-schema corrplot-params-body
    :handler (fn [{:keys [datafile corr_vars sig_level hc_order hc_method corr_type method plugin-env]
                   :or {sig_level 0.05
                        hc_order true
                        hc_method "complete"
                        corr_type "full"
                        method "square"}
                   :as payload}]
               (log/info "Make a correlation plot with %s" payload)
               (let [workdir (ff/get-workdir)
                     log-path (fs-lib/join-paths workdir "log")
                     response {:files [(fs-lib/join-paths workdir "plotly.json")
                                       (fs-lib/join-paths workdir "result.md")]
                               :log log-path
                               :response-type :data2files}
                     task-id (create-task! {:name (str "corrplot" (u/datetime))
                                            :description "Make a correlation plot."
                                            :payload payload
                                            :plugin-name (:plugin-name plugin-env)
                                            :plugin-type (:plugin-type plugin-env)
                                            :plugin-version (:plugin-version plugin-env)
                                            :response response})]
                 (fs-lib/create-directories! workdir)
                 (spit log-path (json/write-str {:status "Running" :msg ""}))
                 (update-process! task-id 0)
                 (publish-event! "corrplot"
                                 {:context {:datafile datafile
                                            :corr_vars corr_vars
                                            :sig_level sig_level
                                            :hc_order hc_order
                                            :hc_method hc_method
                                            :corr_type corr_type
                                            :method method
                                            :title "Correlation Plot"}
                                  :template-dir (fs-lib/join-paths (:config-dir plugin-env) "templates")
                                  :env-dir (:env-dir plugin-env)
                                  :dest-dir workdir
                                  :task-id task-id})
                 response))
    :plugin-type :ChartPlugin
    :response-type :data2files}))

;;; ------------------------------------------------ Event Processing ------------------------------------------------
(defn- corrplot!
  "Make a correlation plot"
  [{:keys [context dest-dir template-dir env-dir task-id]}]
  (let [log-path (fs-lib/join-paths dest-dir "log")
        args-json (fs-lib/join-paths dest-dir "arguments.json")
        args-template (fs-lib/join-paths template-dir "args.json.template")
        output-file (fs-lib/join-paths dest-dir "plotly.json")]
    (corrplot/make-args-json! args-template context args-json)
    (log/info "Make an argument json file: " args-json)
    (update-process! task-id 30)
    (let [result (corrplot/call-corrplot! args-json output-file env-dir)
          status (:status result)
          process (if (= status "Success") 100 -1)]
      (spit log-path (json/write-str result))
      (update-process! task-id process)
      (if (= status "Success")
        (log/info "The task is finished, result file is " output-file)
        (log/error "The task is failed, error msg is located in " log-path)))))

;;; --------------------------------------------------- Lifecycle ----------------------------------------------------
(def events-init
  "Automatically called during startup; start event listener for corrplot events."
  (make-events-init "corrplot" corrplot!))