chenhaonan
/
quartet_dna_quality_control_wes_big_pipeline
forked from renluyao/quartet_dna_quality_control_wes_big_pipeline
Watch 1
Star 0
Fork 0
Code Issues 0 Pull Requests 0 Releases 1 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
16 Commits
1 Branch
Tree: ddead72efc
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'ddead72efc'
${ noResults }
quartet_dna_quality_control.../codescripts/extract_tables.py

152 lines
9.2KB
Raw Blame History 

  1. import json

  2. import pandas as pd

  3. from functools import reduce

  4. import sys, argparse, os

  5. 

  6. parser = argparse.ArgumentParser(description="This script is to get information from multiqc and sentieon, output the raw fastq, bam and variants calling (precision and recall) quality metrics")

  7. 

  8. 

  9. parser.add_argument('-quality', '--quality_yield', type=str, help='*.quality_yield.txt')

  10. parser.add_argument('-depth', '--wgs_metrics', type=str, help='*deduped_WgsMetricsAlgo.txt')

  11. parser.add_argument('-aln', '--aln_metrics', type=str, help='*_deduped_aln_metrics.txt')

  12. parser.add_argument('-is', '--is_metrics', type=str, help='*_deduped_is_metrics.txt')

  13. parser.add_argument('-hs', '--hs_metrics', type=str, help='*_deduped_hs_metrics.txt')

  14. 

  15. 

  16. parser.add_argument('-fastqc', '--fastqc', type=str, help='multiqc_fastqc.txt')

  17. parser.add_argument('-fastqscreen', '--fastqscreen', type=str, help='multiqc_fastq_screen.txt')

  18. parser.add_argument('-hap', '--happy', type=str, help='multiqc_happy_data.json',  required=True)

  19. 

  20. parser.add_argument('-project', '--project_name', type=str, help='project_name')

  21. 

  22. args = parser.parse_args()

  23. 

  24. if args.quality_yield:

  25. 	# Rename input:

  26. 	quality_yield_file = args.quality_yield

  27. 	wgs_metrics_file = args.wgs_metrics

  28. 	aln_metrics_file = args.aln_metrics

  29. 	is_metrics_file = args.is_metrics

  30. 	hs_metrics_file = args.hs_metrics

  31. 	fastqc_file = args.fastqc

  32. 	fastqscreen_file = args.fastqscreen

  33. 	hap_file = args.happy

  34. 	project_name = args.project_name

  35. 	#############################################

  36. 	# fastqc

  37. 	fastqc = pd.read_table(fastqc_file)

  38. 	# fastqscreen

  39. 	dat = pd.read_table(fastqscreen_file)

  40. 	fastqscreen = dat.loc[:, dat.columns.str.endswith('percentage')]

  41. 	dat['Sample'] = [i.replace('_screen','') for i in dat['Sample']]

  42. 	fastqscreen.insert(loc=0, column='Sample', value=dat['Sample'])

  43. 	# pre-alignment

  44. 	pre_alignment_dat = pd.merge(fastqc,fastqscreen,how="outer",left_on=['Sample'],right_on=['Sample'])

  45. 	pre_alignment_dat['FastQC_mqc-generalstats-fastqc-total_sequences'] = pre_alignment_dat['FastQC_mqc-generalstats-fastqc-total_sequences']/1000000

  46. 	del pre_alignment_dat['FastQC_mqc-generalstats-fastqc-percent_fails']

  47. 	del pre_alignment_dat['FastQC_mqc-generalstats-fastqc-avg_sequence_length']

  48. 	del pre_alignment_dat['ERCC percentage']

  49. 	del pre_alignment_dat['Phix percentage']

  50. 	del pre_alignment_dat['Mouse percentage']

  51. 	pre_alignment_dat = pre_alignment_dat.round(2)

  52. 	pre_alignment_dat.columns = ['Sample','%Dup','%GC','Total Sequences (million)','%Human','%EColi','%Adapter','%Vector','%rRNA','%Virus','%Yeast','%Mitoch','%No hits']

  53. 	pre_alignment_dat.to_csv('pre_alignment.txt',sep="\t",index=0)

  54. 	############################

  55. 	dat = pd.read_table(aln_metrics_file,index_col=False)

  56. 	dat['PCT_ALIGNED_READS'] = dat["PF_READS_ALIGNED"]/dat["TOTAL_READS"]

  57. 	aln_metrics = dat[["Sample", "PCT_ALIGNED_READS","PF_MISMATCH_RATE"]]

  58. 	aln_metrics = aln_metrics * 100

  59. 	aln_metrics['Sample'] = [x[-1] for x in aln_metrics['Sample'].str.split('/')]

  60. 	dat = pd.read_table(is_metrics_file,index_col=False)

  61. 	is_metrics = dat[['Sample', 'MEDIAN_INSERT_SIZE']]

  62. 	is_metrics['Sample'] = [x[-1] for x in is_metrics['Sample'].str.split('/')]

  63. 	dat = pd.read_table(quality_yield_file,index_col=False)

  64. 	dat['%Q20'] = dat['Q20_BASES']/dat['TOTAL_BASES']

  65. 	dat['%Q30'] = dat['Q30_BASES']/dat['TOTAL_BASES']

  66. 	quality_yield = dat[['Sample','%Q20','%Q30']]

  67. 	quality_yield = quality_yield * 100

  68. 	quality_yield['Sample'] = [x[-1] for x in quality_yield['Sample'].str.split('/')]

  69. 	dat = pd.read_table(wgs_metrics_file,index_col=False)

  70. 	wgs_metrics = dat[['Sample','MEDIAN_COVERAGE','PCT_1X', 'PCT_5X', 'PCT_10X','PCT_20X','PCT_30X']]

  71. 	wgs_metrics['PCT_1X'] = wgs_metrics['PCT_1X'] * 100

  72. 	wgs_metrics['PCT_5X'] = wgs_metrics['PCT_5X'] * 100

  73. 	wgs_metrics['PCT_10X'] = wgs_metrics['PCT_10X'] * 100

  74. 	wgs_metrics['PCT_20X'] = wgs_metrics['PCT_20X'] * 100

  75. 	wgs_metrics['PCT_30X'] = wgs_metrics['PCT_30X'] * 100

  76. 	wgs_metrics['Sample'] = [x[-1] for x in wgs_metrics['Sample'].str.split('/')]

  77. 	dat = pd.read_table(hs_metrics_file,index_col=False)

  78. 	hs_metrics = dat[['Sample','FOLD_80_BASE_PENALTY','PCT_USABLE_BASES_ON_TARGET']]

  79. 	data_frames = [aln_metrics, is_metrics, quality_yield, wgs_metrics, hs_metrics]

  80. 	post_alignment_dat = reduce(lambda  left,right: pd.merge(left,right,on=['Sample'],how='outer'), data_frames)

  81. 	post_alignment_dat.columns = ['Sample', '%Mapping', '%Mismatch Rate', 'Mendelian Insert Size','%Q20', '%Q30', 'Median Coverage', 'PCT_1X', 'PCT_5X', 'PCT_10X','PCT_20X','PCT_30X','Fold-80','On target bases rate']

  82. 	post_alignment_dat = post_alignment_dat.round(2)

  83. 	post_alignment_dat.to_csv('post_alignment.txt',sep="\t",index=0)

  84. 	#########################################

  85. 	# variants calling

  86. 	with open(hap_file) as hap_json:

  87. 		happy = json.load(hap_json)

  88. 	dat =pd.DataFrame.from_records(happy)

  89. 	dat = dat.loc[:, dat.columns.str.endswith('ALL')]

  90. 	dat_transposed = dat.T

  91. 	dat_transposed = dat_transposed.loc[:,['sample_id','TRUTH.FN','QUERY.TOTAL','QUERY.FP','QUERY.UNK','METRIC.Precision','METRIC.Recall','METRIC.F1_Score']]

  92. 	dat_transposed['QUERY.TP'] = dat_transposed['QUERY.TOTAL'].astype(int) - dat_transposed['QUERY.UNK'].astype(int) - dat_transposed['QUERY.FP'].astype(int)

  93. 	dat_transposed['QUERY'] =dat_transposed['QUERY.TOTAL'].astype(int) - dat_transposed['QUERY.UNK'].astype(int)

  94. 	indel = dat_transposed[['INDEL' in s for s in dat_transposed.index]]

  95. 	snv = dat_transposed[['SNP' in s for s in dat_transposed.index]]

  96. 	indel.reset_index(drop=True, inplace=True)

  97. 	snv.reset_index(drop=True, inplace=True)

  98. 	benchmark = pd.concat([snv, indel], axis=1)

  99. 	benchmark = benchmark[["sample_id", 'QUERY.TOTAL', 'QUERY','QUERY.TP','QUERY.FP','TRUTH.FN','METRIC.Precision', 'METRIC.Recall','METRIC.F1_Score']]

  100. 	benchmark.columns = ['Sample','sample_id','SNV number','INDEL number','SNV query','INDEL query','SNV TP','INDEL TP','SNV FP','INDEL FP','SNV FN','INDEL FN','SNV precision','INDEL precision','SNV recall','INDEL recall','SNV F1','INDEL F1']

  101. 	benchmark = benchmark[['Sample','SNV number','INDEL number','SNV query','INDEL query','SNV TP','INDEL TP','SNV FP','INDEL FP','SNV FN','INDEL FN','SNV precision','INDEL precision','SNV recall','INDEL recall','SNV F1','INDEL F1']]

  102. 	benchmark['SNV precision'] = benchmark['SNV precision'].astype(float)

  103. 	benchmark['INDEL precision'] = benchmark['INDEL precision'].astype(float)

  104. 	benchmark['SNV recall'] = benchmark['SNV recall'].astype(float)

  105. 	benchmark['INDEL recall'] = benchmark['INDEL recall'].astype(float)

  106. 	benchmark['SNV F1'] = benchmark['SNV F1'].astype(float)

  107. 	benchmark['INDEL F1'] = benchmark['INDEL F1'].astype(float)

  108. 	benchmark['SNV precision'] = benchmark['SNV precision'] * 100

  109. 	benchmark['INDEL precision'] = benchmark['INDEL precision'] * 100

  110. 	benchmark['SNV recall'] = benchmark['SNV recall'] * 100

  111. 	benchmark['INDEL recall'] = benchmark['INDEL recall']* 100

  112. 	benchmark['SNV F1'] = benchmark['SNV F1'] * 100

  113. 	benchmark['INDEL F1'] = benchmark['INDEL F1'] * 100

  114. 	benchmark = benchmark.round(2)

  115. 	benchmark.to_csv('variants.calling.qc.txt',sep="\t",index=0)

  116. else:

  117. 	hap_file = args.happy

  118. 	with open(hap_file) as hap_json:

  119. 		happy = json.load(hap_json)

  120. 	dat =pd.DataFrame.from_records(happy)

  121. 	dat = dat.loc[:, dat.columns.str.endswith('ALL')]

  122. 	dat_transposed = dat.T

  123. 	dat_transposed = dat_transposed.loc[:,['sample_id','TRUTH.FN','QUERY.TOTAL','QUERY.FP','QUERY.UNK','METRIC.Precision','METRIC.Recall','METRIC.F1_Score']]

  124. 	dat_transposed['QUERY.TP'] = dat_transposed['QUERY.TOTAL'].astype(int) - dat_transposed['QUERY.UNK'].astype(int) - dat_transposed['QUERY.FP'].astype(int)

  125. 	dat_transposed['QUERY'] =dat_transposed['QUERY.TOTAL'].astype(int) - dat_transposed['QUERY.UNK'].astype(int)

  126. 	indel = dat_transposed[['INDEL' in s for s in dat_transposed.index]]

  127. 	snv = dat_transposed[['SNP' in s for s in dat_transposed.index]]

  128. 	indel.reset_index(drop=True, inplace=True)

  129. 	snv.reset_index(drop=True, inplace=True)

  130. 	benchmark = pd.concat([snv, indel], axis=1)

  131. 	benchmark = benchmark[["sample_id", 'QUERY.TOTAL', 'QUERY','QUERY.TP','QUERY.FP','TRUTH.FN','METRIC.Precision', 'METRIC.Recall','METRIC.F1_Score']]

  132. 	benchmark.columns = ['Sample','sample_id','SNV number','INDEL number','SNV query','INDEL query','SNV TP','INDEL TP','SNV FP','INDEL FP','SNV FN','INDEL FN','SNV precision','INDEL precision','SNV recall','INDEL recall','SNV F1','INDEL F1']

  133. 	benchmark = benchmark[['Sample','SNV number','INDEL number','SNV query','INDEL query','SNV TP','INDEL TP','SNV FP','INDEL FP','SNV FN','INDEL FN','SNV precision','INDEL precision','SNV recall','INDEL recall','SNV F1','INDEL F1']]

  134. 	benchmark['SNV precision'] = benchmark['SNV precision'].astype(float)

  135. 	benchmark['INDEL precision'] = benchmark['INDEL precision'].astype(float)

  136. 	benchmark['SNV recall'] = benchmark['SNV recall'].astype(float)

  137. 	benchmark['INDEL recall'] = benchmark['INDEL recall'].astype(float)

  138. 	benchmark['SNV F1'] = benchmark['SNV F1'].astype(float)

  139. 	benchmark['INDEL F1'] = benchmark['INDEL F1'].astype(float)

  140. 	benchmark['SNV precision'] = benchmark['SNV precision'] * 100

  141. 	benchmark['INDEL precision'] = benchmark['INDEL precision'] * 100

  142. 	benchmark['SNV recall'] = benchmark['SNV recall'] * 100

  143. 	benchmark['INDEL recall'] = benchmark['INDEL recall']* 100

  144. 	benchmark['SNV F1'] = benchmark['SNV F1'] * 100

  145. 	benchmark['INDEL F1'] = benchmark['INDEL F1'] * 100

  146. 	benchmark = benchmark.round(2)

  147. 	benchmark.to_csv('variants.calling.qc.txt',sep="\t",index=0)

  148. 	

  149. 

  150.