tservice-plugins
/
quartet-metqc-report
Vērot 1
Pievienot zvaigznīti 0
Atdalīts 0
Kods Problēmas 0 Izmaiņu pieprasījumi 0 Laidieni 4 Vikivietne Aktivitāte
Pārlūkot izejas kodu

It's a new version and not compatible with previous version.

master
YJC pirms 3 gadiem
vecāks
5403ee3815
revīzija
0421a5c8a8
11 mainītis faili ar 302 papildinājumiem un 215 dzēšanām
Dalītais skats
Diff Options
Show Stats Download Patch File Download Diff File
  1. +1
    -0
      .gitignore
  2. +19
    -8
      project.clj
  3. +0
    -0
      resources/bin/metqc.sh
  4. +25
    -9
      resources/tservice-plugin.yaml
  5. +15
    -0
      src/quartet_metqc_report/core.clj
  6. +92
    -0
      src/quartet_metqc_report/metqc.clj
  7. +41
    -0
      src/quartet_metqc_report/spec.clj
  8. +105
    -0
      src/quartet_metqc_report/task.clj
  9. +4
    -0
      src/quartet_metqc_report/version.clj
  10. +0
    -144
      src/tservice/plugins/quartet_metqc_report.clj
  11. +0
    -54
      src/tservice/plugins/quartet_metqc_report/metqc.clj

+ 1
- 0
.gitignore Parādīt failu

@@ -9,3 +9,4 @@ pom.xml.asc
/.nrepl-port
.hgignore
.hg/
.clj-kondo

+ 19
- 8
project.clj Parādīt failu

@@ -1,6 +1,6 @@
(defproject tservice-plugins/quartet-metqc-report "v0.1.3"
(defproject quartet-metqc-report "0.1.4"
:description "Visualizes Quality Control(QC) results for Quartet Project."
:url "https://github.com/tservice-plugins/quartet-metqc-report"
:url "https://github.com/chinese-quartet/quartet-metqc-report"
:license {:name "Eclipse Public License"
:url "http://www.eclipse.org/legal/epl-v10.html"}
:min-lein-version "2.5.0"
@@ -8,16 +8,28 @@

:dependencies
[[org.clojure/data.csv "1.0.0"]
[me.raynes/fs "1.4.6"]
[com.github.yjcyxky/local-fs "0.1.5"]
[org.clojure/tools.logging "1.1.0"]
[org.clojure/core.async "0.4.500"
:exclusions [org.clojure/tools.reader]]]
[metosin/spec-tools "0.10.5"]]

:test-paths ["test"]
:plugins [[lein-cloverage "1.0.13"]
[lein-shell "0.5.0"]
[lein-changelog "0.3.2"]]

:aliases {"update-version" ["shell" "sed" "-i" "" "s/version \"[0-9.]*\"/version \"${:version}\"/" "src/quartet_metqc_report/version.clj"]}
:deploy-repositories [["releases" :clojars]]
:release-tasks [["change" "version" "leiningen.release/bump-version"]
["change" "version" "leiningen.release/bump-version" "release"]
["changelog" "release"]
["update-version"]
["deploy"]]

:profiles
{:provided
{:dependencies
[[org.clojure/clojure "1.10.1"]
[org.clojars.yjcyxky/tservice "0.6.0"]]}
[com.github.yjcyxky/tservice-core "0.2.0"]]}

:uberjar
{:auto-clean true
@@ -25,5 +37,4 @@
:omit-source true
:javac-options ["-target" "1.8", "-source" "1.8"]
:target-path "target/%s"
:resource-paths ["resources"]
:uberjar-name "quartet-metqc-report.tservice-plugin.jar"}})
:resource-paths ["resources"]}})

resources/quartet-metqc-report/bin/metqc.sh → resources/bin/metqc.sh Parādīt failu


+ 25
- 9
resources/tservice-plugin.yaml Parādīt failu

@@ -1,16 +1,22 @@
info:
name: QC Report for Quartet Metabolomics
version: v0.1.3
version: v0.1.4
description: Generate the QC Report for Quartet Metabolomics data.
category: Report
home: https://github.com/tservice-plugins/quartet-metqc-report
source: PGx
home: https://github.com/chinese-quartet/quartet-metqc-report
source: Chinese Quartet
short_name: quartet-metqc-report
icons:
- src: ""
type: image/png
sizes: 192x192
author: PGx
author: Jingcheng Yang
maintainers:
- Jingcheng Yang
tags:
- R
- Chart
readme: https://github.com/chinese-quartet/quartet-metqc-report/blob/master/README.md
plugin:
name: quartet-metqc-report
display-name: QC Report for Quartet Metabolomics
@@ -18,11 +24,21 @@ plugin:
init:
# Unpack environment file to the directory, repository/envs/quartet-metqc-report
- step: unpack-env
envname: quartet-metqc-report
postunpack: chmod a+x {{ENV_DEST_DIR}}/bin/metqc.sh
envtype: environment
envname: bin
postunpack: chmod a+x {{ENV_DIR}}/bin/metqc.sh
- step: unpack-env
envtype: environment
envname: renv
- step: unpack-env
envtype: environment
envname: requirements.txt
- step: unpack-env
envtype: environment
envname: renv.lock
- step: load-namespace
namespace: tservice.plugins.quartet-metqc-report
namespace: quartet-metqc-report.core
- step: register-plugin
entrypoint: tservice.plugins.quartet-metqc-report/metadata
entrypoint: quartet-metqc-report.core/metadata
- step: init-event
entrypoint: tservice.plugins.quartet-metqc-report/events-init
entrypoint: quartet-metqc-report.core/events-init

+ 15
- 0
src/quartet_metqc_report/core.clj Parādīt failu

@@ -0,0 +1,15 @@
(ns quartet-metqc-report.core
(:require [tservice-core.tasks.http :as http-task]
[quartet-metqc-report.spec :as spec]
[quartet-metqc-report.task :as task]))

(def metadata
(http-task/make-routes "quartet-metqc-report" :ReportPlugin
{:method-type :post
:endpoint "quartet-metqc-report"
:summary "Generate the QC Report for Quartet Metabolomics data."
:body-schema spec/quartet-metqc-report-params-body
:response-schema any?
:handler task/post-handler}))

(def events-init task/events-init)

+ 92
- 0
src/quartet_metqc_report/metqc.clj Parādīt failu

@@ -0,0 +1,92 @@
(ns quartet-metqc-report.metqc
"A wrapper for metqc tool."
(:require [clojure.string :as clj-str]
[local-fs.core :as fs-lib]
[clojure.java.shell :as shell :refer [sh]]
[tservice-core.plugins.util :refer [call-command!]]
[tservice-core.plugins.env :refer [get-context-path add-env-to-path]]
[clojure.tools.logging :as log]
[quartet-metqc-report.version :as v]))

(defn call-metqc!
"Call metqc bash script. more details on https://github.com/chinese-quartet/MetQC
exp-file: metabolomics profiled data.
meta-file: metabolomics metadata.
result-dir: A directory for result files."
[exp-file meta-file result-dir]
(let [command ["bash" "-c"
(format "metqc.sh -d %s -m %s -o %s" exp-file meta-file result-dir)]
path-var (add-env-to-path v/plugin-name)
rprofile (fs-lib/join-paths (get-context-path :env v/plugin-name) "Rprofile")]
(log/info "PATH variable: " path-var)
(log/info "Rprofile file is in " rprofile)
(shell/with-sh-env {:PATH path-var
:R_PROFILE_USER rprofile
:LC_ALL "en_US.utf-8"
:LANG "en_US.utf-8"}
(let [result (apply sh command)]
{:status (if (= (:exit result) 0) "Success" "Error")
:msg (str (:out result) "\n" (:err result))}))))

(defn multiqc
"A multiqc wrapper for generating multiqc report:
TODO: set the absolute path of multiqc binary instead of environment variable

Required:
analysis-dir: Analysis directory, e.g. data directory from project
outdir: Create report in the specified output directory.

Options:
| key | description |
| -------------------|-------------|
| :dry-run? | Dry run mode |
| :filename | Report filename. Use 'stdout' to print to standard out. |
| :comment | Custom comment, will be printed at the top of the report. |
| :title | Report title. Printed as page header, used for filename if not otherwise specified. |
| :force? | Overwrite any existing reports |
| :prepend-dirs? | Prepend directory to sample names |
| :template | default, other custom template |
| :config | Where is the config file |
| :env | An environemnt map for running multiqc, such as {:PATH (get-path-variable)} |

Example:
(multiqc 'XXX' 'YYY' {:filename 'ZZZ'
:comment ''
:title ''
:force? true
:prepend-dirs? true})"
[analysis-dir outdir {:keys [dry-run? filename comment title force? prepend-dirs? template config env]
:or {dry-run? false
force? true
prepend-dirs? false
filename "multiqc_report.html"
comment ""
template "default"
title "iSEQ Analyzer Report"}}]
(let [force-arg (if force? "--force" "")
dirs-arg (if prepend-dirs? "--dirs" "")
config-arg (if config (str "-c " config) "")
multiqc-command (filter #(> (count %) 0) ["multiqc"
force-arg dirs-arg config-arg
"--title" (format "'%s'" title)
"--comment" (format "'%s'" comment)
"--filename" filename
"--outdir" outdir
"-t" template
analysis-dir])
command (clj-str/join " " multiqc-command)]
(if dry-run?
(log/info command)
(if env
(call-command! command env)
(call-command! command)))))

(defn is-localpath?
[filepath]
(re-matches #"^file:\/\/.*" filepath))

(defn correct-filepath
[filepath]
(if (is-localpath? filepath)
(clj-str/replace filepath #"^file:\/\/" "")
filepath))

+ 41
- 0
src/quartet_metqc_report/spec.clj Parādīt failu

@@ -0,0 +1,41 @@
(ns quartet-metqc-report.spec
(:require [clojure.spec.alpha :as s]
[spec-tools.core :as st]))

;;; ------------------------------------------------ Event Specs ------------------------------------------------
(s/def ::metadata_file
(st/spec
{:spec (s/and string? #(re-matches #"^[a-zA-Z0-9]+:\/\/(\/|\.\/)[a-zA-Z0-9_]+.*" %))
:type :string
:description "File path for metadata file, such as file:///xxx/xxx/metadata.csv"
:swagger/default nil
:reason "The filepath must be string."}))

(s/def ::data_file
(st/spec
{:spec (s/and string? #(re-matches #"^[a-zA-Z0-9]+:\/\/(\/|\.\/)[a-zA-Z0-9_]+.*" %))
:type :string
:description "File path for metabolomics profiled data, such as file:///xxx/xxx/data.csv"
:swagger/default nil
:reason "The filepath must be string."}))

(s/def ::name
(st/spec
{:spec string?
:type :string
:description "The name of the report"
:swagger/default ""
:reason "Not a valid name"}))

(s/def ::description
(st/spec
{:spec string?
:type :string
:description "Description of the report"
:swagger/default ""
:reason "Not a valid description."}))

(def quartet-metqc-report-params-body
"A spec for the body parameters."
(s/keys :req-un [::name ::data_file ::metadata_file]
:opt-un [::description]))

+ 105
- 0
src/quartet_metqc_report/task.clj Parādīt failu

@@ -0,0 +1,105 @@
(ns quartet-metqc-report.task
(:require [quartet-metqc-report.metqc :as metqc]
[local-fs.core :as fs-lib]
[tservice-core.plugins.env :refer [add-env-to-path create-task! update-task!]]
[tservice-core.plugins.util :as util]
[clojure.data.json :as json]
[clojure.tools.logging :as log]
[tservice-core.tasks.async :refer [publish-event! make-events-init]]))

(defn date
[]
(.format (java.text.SimpleDateFormat. "yyyy-MM-dd")
(new java.util.Date)))

(defn update-process!
[^String task-id ^Integer percentage]
(let [record (cond
(= percentage 100) {:status "Finished"
:percentage 100
:finished_time (util/time->int (util/now))}
(= percentage -1) {:status "Failed"
:finished_time (util/time->int (util/now))}
:else {:percentage percentage})
record (merge {:id task-id} record)]
(update-task! record)))

(defn update-log-process!
"Update message into log file and process into database."
[log-path coll task-id process]
(spit log-path (json/write-str coll))
(update-process! task-id process))

(defn post-handler
[{{:keys [name data_file metadata_file description owner plugin-context]
:or {description (format "Quality control report for %s" name)}
:as payload} :body}]
(log/info (format "Create a report %s with %s" name payload))
(let [payload (merge {:description description} payload)
data-file (metqc/correct-filepath data_file)
metadata-file (metqc/correct-filepath metadata_file)
workdir (fs-lib/dirname data-file)
log-path (fs-lib/join-paths workdir "log")
response {:report (format "%s/multiqc_report.html" workdir)
:log log-path}
task-id (create-task! {:name name
:description description
:payload payload
:owner owner
:plugin-name "quartet-metqc-report"
:plugin-type "ReportPlugin"
:plugin-version (:plugin-version plugin-context)
:response response})
result-dir (fs-lib/join-paths workdir "results")]
(fs-lib/create-directories! result-dir)
(spit log-path (json/write-str {:status "Running"
:msg ""}))
(update-process! task-id 0)
(publish-event! "quartet_metqc_report"
{:data-file data-file
:metadata-file metadata-file
:dest-dir workdir
:task-id task-id
:metadata {:name name
:description description
:plugin-name "quartet-metqc-report"
:plutin-type "ReportPlugin"
:plugin-version (:plugin-version plugin-context)}})
response))

(defn- make-report!
[{:keys [data-file metadata-file dest-dir metadata task-id]}]
(let [log-path (fs-lib/join-paths dest-dir "log")
result-dir (fs-lib/join-paths dest-dir "results")
parameters-file (fs-lib/join-paths result-dir "general_information.json")
results (util/chain-fn-coll [(fn []
(update-process! task-id 20)
(metqc/call-metqc! data-file metadata-file result-dir))
(fn []
(update-process! task-id 50)
(spit parameters-file (json/write-str {"Report Name" (:name metadata)
"Description" (:description metadata)
"Report Tool" (format "%s-%s"
(:plugin-name metadata)
(:plugin-version metadata))
"Team" "Quartet Team"
"Date" (date)}))
{:status "Success" :msg ""})
(fn []
(update-process! task-id 80)
(metqc/multiqc result-dir dest-dir
{:template "quartet_metabolite_report"
:title "Quartet Report for Metabolomics"
:env {:PATH (add-env-to-path "quartet-metqc-report")}}))]
(fn [result] (= (:status result) "Success")))
status (:status (last results))
msg (apply str (map :msg results))
process (if (= status "Success") 100 -1)]
(log/info (format "Running batch command: %s" (pr-str results)))
(update-log-process! log-path {:status status
:msg msg}
task-id process)))

(def events-init
"Automatically called during startup; start event listener for quartet_metqc_report events."
(make-events-init "quartet_metqc_report" make-report!))

+ 4
- 0
src/quartet_metqc_report/version.clj Parādīt failu

@@ -0,0 +1,4 @@
(ns quartet-metqc-report.version)

(def plugin-name "quartet-metqc-report")
(def version "0.1.4")

+ 0
- 144
src/tservice/plugins/quartet_metqc_report.clj Parādīt failu

@@ -1,144 +0,0 @@
(ns tservice.plugins.quartet-metqc-report
(:require [clojure.data.json :as json]
[clojure.spec.alpha :as s]
[spec-tools.core :as st]
[clojure.tools.logging :as log]
[tservice.lib.files :as ff]
[tservice.api.config :refer [add-env-to-path]]
[tservice.lib.fs :as fs-lib]
[tservice.vendor.multiqc :as mq]
[tservice.plugins.quartet-metqc-report.metqc :as metqc]
[tservice.api.task :refer [make-events-init publish-event! make-plugin-metadata create-task! update-process!]]))

;;; ------------------------------------------------ Event Specs ------------------------------------------------
(s/def ::metadata_file
(st/spec
{:spec (s/and string? #(re-matches #"^[a-zA-Z0-9]+:\/\/(\/|\.\/)[a-zA-Z0-9_]+.*" %))
:type :string
:description "File path for metadata file, such as file:///xxx/xxx/metadata.csv"
:swagger/default nil
:reason "The filepath must be string."}))

(s/def ::data_file
(st/spec
{:spec (s/and string? #(re-matches #"^[a-zA-Z0-9]+:\/\/(\/|\.\/)[a-zA-Z0-9_]+.*" %))
:type :string
:description "File path for metabolomics profiled data, such as file:///xxx/xxx/data.csv"
:swagger/default nil
:reason "The filepath must be string."}))

(s/def ::name
(st/spec
{:spec string?
:type :string
:description "The name of the report"
:swagger/default ""
:reason "Not a valid name"}))

(s/def ::description
(st/spec
{:spec string?
:type :string
:description "Description of the report"
:swagger/default ""
:reason "Not a valid description."}))

(def quartet-metqc-report-params-body
"A spec for the body parameters."
(s/keys :req-un [::name ::data_file ::metadata_file]
:opt-un [::description]))

;;; ------------------------------------------------ Event Metadata ------------------------------------------------
(def metadata
(make-plugin-metadata
{:name "quartet-metqc-report"
:summary "Visualizes Quality Control(QC) results from metabolomics data for Quartet Project."
:params-schema quartet-metqc-report-params-body
:handler (fn [{:keys [name data_file metadata_file description owner plugin-context]
:or {description (format "Quality control report for %s" name)}
:as payload}]
(let [payload (merge {:description description} payload)
data-file (metqc/correct-filepath data_file)
metadata-file (metqc/correct-filepath metadata_file)
workdir (metqc/dirname data-file)
log-path (fs-lib/join-paths workdir "log")
response {:report (format "%s/multiqc_report.html" workdir)
:log log-path
:response-type :data2report}
task-id (create-task! {:name name
:description description
:payload payload
:owner owner
:plugin-name "quartet-metqc-report"
:plugin-type "ReportPlugin"
:plugin-version (:plugin-version plugin-context)
:response response})
result-dir (fs-lib/join-paths workdir "results")]
(fs-lib/create-directories! result-dir)
(log/info (format "Create a report %s with %s in %s" name payload workdir))
(spit log-path (json/write-str {:status "Running"
:msg ""}))
(update-process! task-id 0)
(publish-event! "quartet_metqc_report"
{:data-file data-file
:metadata-file metadata-file
:dest-dir workdir
:task-id task-id
:metadata {:name name
:description description
:plugin-name "quartet-metqc-report"
:plutin-type "ReportPlugin"
:plugin-version (:plugin-version plugin-context)}})
response))
:plugin-type :ReportPlugin
:response-type :data2report}))

(defn update-log-process!
"Update message into log file and process into database."
[log-path coll task-id process]
(spit log-path (json/write-str coll))
(update-process! task-id process))

(defn date
[]
(.format (java.text.SimpleDateFormat. "yyyy-MM-dd")
(new java.util.Date)))

(defn- make-report!
[{:keys [data-file metadata-file dest-dir metadata task-id]}]
(let [log-path (fs-lib/join-paths dest-dir "log")
result-dir (fs-lib/join-paths dest-dir "results")
parameters-file (fs-lib/join-paths result-dir "general_information.json")
results (ff/chain-fn-coll [(fn []
(update-process! task-id 20)
(metqc/call-metqc! data-file metadata-file result-dir))
(fn []
(update-process! task-id 50)
(spit parameters-file (json/write-str {"Report Name" (:name metadata)
"Description" (:description metadata)
"Report Tool" (format "%s-%s"
(:plugin-name metadata)
(:plugin-version metadata))
"Team" "Quartet Team"
"Date" (date)}))
{:status "Success" :msg ""})
(fn []
(update-process! task-id 80)
(mq/multiqc result-dir dest-dir
{:template "quartet_metabolite_report"
:title "Quartet Report for Metabolomics"
:env {:PATH (add-env-to-path "quartet-metqc-report")}}))]
(fn [result] (= (:status result) "Success")))
status (:status (last results))
msg (apply str (map :msg results))
process (if (= status "Success") 100 -1)]
(log/info (format "Running batch command: %s" (pr-str results)))
(update-log-process! log-path {:status status
:msg msg}
task-id process)))

;;; --------------------------------------------------- Lifecycle ----------------------------------------------------

(def events-init
"Automatically called during startup; start event listener for quartet_metqc_report events."
(make-events-init "quartet_metqc_report" make-report!))

+ 0
- 54
src/tservice/plugins/quartet_metqc_report/metqc.clj Parādīt failu

@@ -1,54 +0,0 @@
(ns tservice.plugins.quartet-metqc-report.metqc
"A wrapper for metqc tool."
(:require [tservice.api.config :refer [add-env-to-path]]
[tservice.lib.files :refer [is-localpath? get-plugin-jar-env-dir]]
[clojure.string :as clj-str]
[tservice.lib.fs :as fs-lib]
[clojure.java.shell :as shell :refer [sh]]
[clojure.java.io :refer [file]]))

(defn call-metqc!
"Call metqc bash script. more details on https://github.com/chinese-quartet/MetQC
exp-file: Proteomics profiled data.
meta-file: proteomics metadata.
result-dir: A directory for result files.
"
[exp-file meta-file result-dir]
(shell/with-sh-env {:PATH (add-env-to-path "quartet-metqc-report")
:R_PROFILE_USER (fs-lib/join-paths (get-plugin-jar-env-dir "quartet-metqc-report") "Rprofile")
:LC_ALL "en_US.utf-8"
:LANG "en_US.utf-8"}
(let [command ["bash" "-c"
(format "metqc.sh -d %s -m %s -o %s" exp-file meta-file result-dir)]
result (apply sh command)
status (if (= (:exit result) 0) "Success" "Error")
msg (str (:out result) "\n" (:err result))]
{:status status
:msg msg})))

(defn correct-filepath
[filepath]
(if (is-localpath? filepath)
(clj-str/replace filepath #"^file:\/\/" "")
filepath))

(defn ^String basename
"Returns the basename of 'path'.
This works by calling getName() on a java.io.File instance. It's prefered
over last-dir-in-path for that reason.
Parameters:
path - String containing the path for an item in iRODS.
Returns:
String containing the basename of path."
[^String path]
(.getName (file path)))

(defn ^String dirname
"Returns the dirname of 'path'.
This works by calling getParent() on a java.io.File instance.
Parameters:
path - String containing the path for an item in iRODS.
Returns:
String containing the dirname of path."
[^String path]
(when path (.getParent (file path))))

Rakstīt Priekšskatītījums
Notiek ielāde…
Atcelt
Saglabāt