Skip to content

Commit 92fb751

Browse files
fgvieirafgvieira
authored
test: QC upload (#27)
1 parent 84e8a9f commit 92fb751

23 files changed

Lines changed: 766 additions & 33 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ jobs:
3838
strategy:
3939
fail-fast: false
4040
runs-on: ubuntu-latest
41+
env:
42+
SQL_DRIVER: sqlite
4143

4244
steps:
4345
- uses: actions/checkout@v4

‎examples/empty/.tests/unit/common.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44

55
import os
66
from pathlib import Path
7-
from subprocess import check_output
7+
from subprocess import check_output, PIPE
88

99

1010
cmp_cmds = {
@@ -28,21 +28,22 @@ def check(self, cmp_cmds=cmp_cmds):
2828
for path, subdirs, files in os.walk(self.data_path)
2929
for f in files
3030
)
31-
print(f"input: {input_files}") # DEBUG
32-
# Workdir files
31+
print(f"input_files: {input_files}") # DEBUG
32+
# Workdir files (ignoring '.snakemake/' and 'config/' folders)
3333
workdir_files = set(
3434
(Path(path) / f).relative_to(self.workdir)
3535
for path, subdirs, files in os.walk(self.workdir)
3636
for f in files
37+
if "/.snakemake" not in path and "/config" not in path
3738
)
38-
print(f"workdir: {workdir_files}") # DEBUG
39+
print(f"workdir_files: {workdir_files}") # DEBUG
3940
# Expected files
4041
expected_files = set(
4142
(Path(path) / f).relative_to(self.expected_path)
4243
for path, subdirs, files in os.walk(self.expected_path)
4344
for f in files
4445
)
45-
print(f"expected: {expected_files}") # DEBUG
46+
print(f"expected_files: {expected_files}") # DEBUG
4647

4748
assert expected_files.issubset(
4849
workdir_files
@@ -55,5 +56,6 @@ def check(self, cmp_cmds=cmp_cmds):
5556
def compare_files(self, expected_file, generated_file, cmp_cmds):
5657
check_output(
5758
cmp_cmds.get(expected_file.suffix, ["cmp"])
58-
+ [expected_file, generated_file]
59+
+ [expected_file, generated_file],
60+
stderr=PIPE,
5961
)

‎examples/empty/.tests/unit/multiqc_taxon_upload/config/config/config.yaml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
2+
# - Only tested with Phred33 quality scores
3+
4+
samples: config/samples.tsv
5+
6+
units: config/units.tsv
7+
8+
9+
10+
#############
11+
### READS ###
12+
#############
13+
trim:
14+
trim:
15+
activate: true
16+
tool: adapterremoval
17+
params: "--trimns --maxns 10 --trimqualities --minlength 30 --mask-degenerate-bases --seed 12345"
18+
19+
# Ignored for SE
20+
collapse:
21+
activate: true
22+
params: "--collapse-conservatively"
23+
24+
derep:
25+
extension:
26+
activate: false
27+
k: 16
28+
params: "ibb=t prefilter=0 el=100 er=100 ecc=f ecco=f ignorebadquality extendrollback=0"
29+
30+
derep:
31+
activate: true
32+
# vsearch or seqkit
33+
tool: seqkit
34+
params: ""
35+
36+
low_complex:
37+
params: "entropy=0.7 entropywindow=30 entropyk=4"
38+
39+
40+
41+
#############
42+
### ALIGN ###
43+
#############
44+
prefilter:
45+
taxa: "Bacteria,Archaea,Viruses"
46+
47+
ref:
48+
prok:
49+
n_shards: 2
50+
path: "data/prok.{n_shard}-of-2.fas.gz"
51+
map:
52+
tool: bowtie2
53+
params: "-k 10 -L 22 -i S,1,1.15 --mp 1,1 --rdg 0,1 --rfg 0,1 --score-min L,0,-0.1 --no-unal -N 1"
54+
bt2l: False
55+
acc2taxid: "data/prok.acc2taxid.gz"
56+
virus:
57+
n_shards: 1
58+
path: "data/virus.1-of-1.fas.gz"
59+
map:
60+
tool: bowtie2
61+
params: "-k 10 -L 22 -i S,1,1.15 --mp 1,1 --rdg 0,1 --rfg 0,1 --score-min L,0,-0.1 --no-unal"
62+
bt2l: False
63+
acc2taxid: "data/virus.acc2taxid.gz"
64+
65+
filter:
66+
saturated_reads:
67+
activate: true
68+
n_alns: 10
69+
70+
bam_filter:
71+
reassign:
72+
activate: false
73+
params: "--iters 0 --min-read-ani 92 --min-read-count 3 --scale 0 --reference-lengths genomes.len.map"
74+
75+
filter:
76+
activate: false
77+
params: "--min-read-ani 92 --min-read-count 3 --min-normalized-entropy 0.6 --min-normalized-gini 0.4 --min-avg-read-ani 94 --reference-lengths genomes.len.map"
78+
79+
lca:
80+
activate: false
81+
params: "--lca-rank genus --reference-lengths genomes.len.map"
82+
83+
taxonomy:
84+
nodes: "data/taxdump/nodes.dmp"
85+
names: "data/taxdump/names.dmp"
86+
87+
metadmg:
88+
damage:
89+
params: "--print_length 15"
90+
91+
lca:
92+
params: "--fix_ncbi 0 --how_many 25 --weight_type 1 --edit_dist_max 10000 --lca_rank genus"
93+
94+
dfit:
95+
params: "--nopt 5 --showfits 2"
96+
97+
98+
euk:
99+
ref:
100+
mitoch:
101+
n_shards: 1
102+
path: "data/mitoch.1-of-1.fas.gz"
103+
map:
104+
tool: bowtie2
105+
params: "-k 10 -L 22 -i S,1,1.15 --mp 1,1 --rdg 0,1 --rfg 0,1 --score-min L,0,-0.1 --no-unal"
106+
bt2l: False
107+
acc2taxid: "data/mitoch.acc2taxid.gz"
108+
plastid:
109+
n_shards: 1
110+
path: "data/plastid.1-of-1.fas.gz"
111+
map:
112+
tool: bowtie2
113+
params: "-k 10 -L 22 -i S,1,1.15 --mp 1,1 --rdg 0,1 --rfg 0,1 --score-min L,0,-0.1 --no-unal"
114+
bt2l: False
115+
acc2taxid: "data/plastid.acc2taxid.gz"
116+
117+
filter:
118+
saturated_reads:
119+
activate: true
120+
n_alns: 10
121+
122+
bam_filter:
123+
reassign:
124+
activate: false
125+
params: "--iters 0 --min-read-ani 92 --min-read-count 3 --scale 0"
126+
127+
filter:
128+
activate: false
129+
params: "--min-read-ani 92 --min-read-count 3 --min-normalized-entropy 0.6 --min-normalized-gini 0.4 --min-avg-read-ani 92"
130+
131+
lca:
132+
activate: false
133+
params: "--lca-rank genus"
134+
135+
taxonomy:
136+
nodes: "data/taxdump/nodes.dmp"
137+
names: "data/taxdump/names.dmp"
138+
139+
metadmg:
140+
damage:
141+
params: "--print_length 15"
142+
143+
lca:
144+
params: "--fix_ncbi 0 --how_many 15 --sim_score_low 0.95 --weight_type 0 --lca_rank genus"
145+
146+
dfit:
147+
params: "--nopt 5 --showfits 2 --seed 12345"
148+
149+
150+
############
151+
## REPORT ##
152+
############
153+
report:
154+
multiqc: "--verbose --cl-config 'custom_logo: data/KU_long.png' --cl-config 'custom_logo_title: CAEG - Center for Ancient Environmental Genomics' --cl-config 'custom_logo_url: https://globe.ku.dk/research/caeg/'"
155+
multiqc_db_url: "sqlite:///test_qc.sqlite"

‎examples/empty/.tests/unit/multiqc_taxon_upload/config/config/samples.tsv‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
sample alias group condition
2+
HD827sonic_1 NA NA NA

‎examples/empty/.tests/unit/multiqc_taxon_upload/config/config/units.tsv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Adapter list: https://gist.github.com/photocyte/3edd9401d0b13476e60f8b104c2575f8
2+
sample library barcode flowcell lane seq_type library_type material data machine run_n center platform adapters sample_n date
3+
HD827sonic_1 lib1 ACGGAACAxACGAGAAC HKTG2BGXG L001 PE ds DNA data/empty_L001_R{Read}.fq.gz NDX550220 98 HYDRA_GEN ILLUMINA AGATCGGAAGAGCACACGTCTGAACTCCAGTCA,AGATCGGAAGAGCGTCGTGTAGGGAAAGAGTGT S1 21-09-2021

‎examples/empty/.tests/unit/multiqc_taxon_upload/data/reports/multiqc_data.taxon.zip‎

519 KB
Binary file not shown.

‎examples/empty/.tests/unit/multiqc_taxon_upload/expected/stats/reports/multiqc_taxon.upload.flag‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
INFO:DB sqlite:///test_qc.sqlite does not exist.
2+
INFO:Reading file reports/multiqc_data.taxon.zip
3+
INFO:Uploading report v1.33 to DB...
4+
INFO:Adding report record to DB
5+
INFO:Adding report metadata to DB
6+
INFO:Parsing section multiqc_fastqc_prefilter-fastqc
7+
INFO:Parsing section multiqc_fastqc_low_complexity-fastqc
8+
INFO:Parsing section multiqc_fastqc_derep-fastqc
9+
INFO:Parsing section multiqc_nonpareil_merge_lanes-nonpareil
10+
INFO:Parsing section multiqc_fastqc_merge_lanes-fastqc
11+
INFO:Parsing section multiqc_fastqc_trim-fastqc
12+
INFO:Parsing section multiqc_adapter_removal
13+
INFO:Parsing section multiqc_fastqc_raw-fastqc
14+
INFO:Parsing section multiqc_general_stats
15+
INFO:Parsing plot prefilter-fastqc_sequence_counts_plot
16+
INFO:Parsing dataset prefilter-fastqc_sequence_counts_plot
17+
INFO:Parsing plot prefilter-fastqc_per_sequence_gc_content_plot
18+
INFO:Parsing dataset prefilter-fastqc_per_sequence_gc_content_plot_Percentages
19+
INFO:Parsing dataset prefilter-fastqc_per_sequence_gc_content_plot_Counts
20+
INFO:Parsing plot prefilter-fastqc_sequence_duplication_levels_plot
21+
INFO:Parsing dataset prefilter-fastqc_sequence_duplication_levels_plot
22+
INFO:Parsing plot prefilter-fastqc-status-check-heatmap
23+
WARNING:Plot type heatmap is not supported
24+
INFO:Parsing plot low_complexity-fastqc_sequence_counts_plot
25+
INFO:Parsing dataset low_complexity-fastqc_sequence_counts_plot
26+
INFO:Parsing plot low_complexity-fastqc_per_sequence_gc_content_plot
27+
INFO:Parsing dataset low_complexity-fastqc_per_sequence_gc_content_plot_Percentages
28+
INFO:Parsing dataset low_complexity-fastqc_per_sequence_gc_content_plot_Counts
29+
INFO:Parsing plot low_complexity-fastqc_sequence_duplication_levels_plot
30+
INFO:Parsing dataset low_complexity-fastqc_sequence_duplication_levels_plot
31+
INFO:Parsing plot low_complexity-fastqc-status-check-heatmap
32+
WARNING:Plot type heatmap is not supported
33+
INFO:Parsing plot derep-fastqc_sequence_counts_plot
34+
INFO:Parsing dataset derep-fastqc_sequence_counts_plot
35+
INFO:Parsing plot derep-fastqc_per_sequence_gc_content_plot
36+
INFO:Parsing dataset derep-fastqc_per_sequence_gc_content_plot_Percentages
37+
INFO:Parsing dataset derep-fastqc_per_sequence_gc_content_plot_Counts
38+
INFO:Parsing plot derep-fastqc_sequence_duplication_levels_plot
39+
INFO:Parsing dataset derep-fastqc_sequence_duplication_levels_plot
40+
INFO:Parsing plot derep-fastqc-status-check-heatmap
41+
WARNING:Plot type heatmap is not supported
42+
INFO:Parsing plot nonpareil-table
43+
WARNING:Plot type violin plot is not supported
44+
INFO:Parsing plot nonpareil-redundancy-plot
45+
INFO:Parsing dataset nonpareil-redundancy-plot_Combined
46+
INFO:Parsing dataset nonpareil-redundancy-plot_Observed
47+
INFO:Parsing plot merge_lanes-fastqc_sequence_counts_plot
48+
INFO:Parsing dataset merge_lanes-fastqc_sequence_counts_plot
49+
INFO:Parsing plot merge_lanes-fastqc_per_sequence_gc_content_plot
50+
INFO:Parsing dataset merge_lanes-fastqc_per_sequence_gc_content_plot_Percentages
51+
INFO:Parsing dataset merge_lanes-fastqc_per_sequence_gc_content_plot_Counts
52+
INFO:Parsing plot merge_lanes-fastqc_sequence_duplication_levels_plot
53+
INFO:Parsing dataset merge_lanes-fastqc_sequence_duplication_levels_plot
54+
INFO:Parsing plot merge_lanes-fastqc-status-check-heatmap
55+
WARNING:Plot type heatmap is not supported
56+
INFO:Parsing plot trim-fastqc_sequence_counts_plot
57+
INFO:Parsing dataset trim-fastqc_sequence_counts_plot
58+
INFO:Parsing plot trim-fastqc_per_sequence_gc_content_plot
59+
INFO:Parsing dataset trim-fastqc_per_sequence_gc_content_plot_Percentages
60+
INFO:Parsing dataset trim-fastqc_per_sequence_gc_content_plot_Counts
61+
INFO:Parsing plot trim-fastqc_sequence_duplication_levels_plot
62+
INFO:Parsing dataset trim-fastqc_sequence_duplication_levels_plot
63+
INFO:Parsing plot trim-fastqc-status-check-heatmap
64+
WARNING:Plot type heatmap is not supported
65+
INFO:Parsing plot ar_retained_plot
66+
INFO:Parsing dataset ar_retained_plot
67+
INFO:Parsing plot raw-fastqc_sequence_counts_plot
68+
INFO:Parsing dataset raw-fastqc_sequence_counts_plot
69+
INFO:Parsing plot raw-fastqc_per_sequence_gc_content_plot
70+
INFO:Parsing dataset raw-fastqc_per_sequence_gc_content_plot_Percentages
71+
INFO:Parsing dataset raw-fastqc_per_sequence_gc_content_plot_Counts
72+
INFO:Parsing plot raw-fastqc_sequence_duplication_levels_plot
73+
INFO:Parsing dataset raw-fastqc_sequence_duplication_levels_plot
74+
INFO:Parsing plot raw-fastqc-status-check-heatmap
75+
WARNING:Plot type heatmap is not supported
76+
INFO:Parsing plot general_stats_table
77+
WARNING:Plot type violin plot is not supported

‎examples/empty/.tests/unit/test_multiqc_taxon_upload.py‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
"""
2+
Rule test code for unit testing of rules generated with Snakemake 9.16.4.dev3.
3+
"""
4+
5+
import os
6+
import sys
7+
import shutil
8+
import tempfile
9+
from pathlib import Path
10+
from subprocess import check_output
11+
12+
sys.path.insert(0, os.path.dirname(__file__))
13+
14+
15+
def test_multiqc_taxon_upload(conda_prefix):
16+
17+
with tempfile.TemporaryDirectory() as tmpdir:
18+
workdir = Path(tmpdir) / "workdir"
19+
config_path = Path(".tests/unit/multiqc_taxon_upload/config")
20+
data_path = Path(".tests/unit/multiqc_taxon_upload/data")
21+
expected_path = Path(".tests/unit/multiqc_taxon_upload/expected")
22+
23+
# Copy config to the temporary workdir.
24+
shutil.copytree(config_path, workdir)
25+
26+
# Copy data to the temporary workdir.
27+
shutil.copytree(data_path, workdir, dirs_exist_ok=True)
28+
29+
# Run the test job.
30+
check_output(
31+
[
32+
"python",
33+
"-m",
34+
"snakemake",
35+
"stats/reports/multiqc_taxon.upload.flag",
36+
"--snakefile",
37+
"../../workflow/Snakefile",
38+
"-f",
39+
"--notemp",
40+
"--show-failed-logs",
41+
"-j1",
42+
"--target-files-omit-workdir-adjustment",
43+
"--allowed-rules",
44+
"multiqc_taxon_upload",
45+
"--configfile",
46+
"config/config.yaml",
47+
"--software-deployment-method",
48+
"conda",
49+
"--directory",
50+
workdir,
51+
]
52+
+ conda_prefix
53+
)
54+
55+
# Check the output byte by byte using cmp/zmp/bzcmp/xzcmp.
56+
# To modify this behavior, you can inherit from common.OutputChecker in here
57+
# and overwrite the method `compare_files(generated_file, expected_file),
58+
# also see common.py.
59+
import common
60+
61+
common.OutputChecker(data_path, expected_path, workdir).check(
62+
{
63+
".flag": [
64+
"diff",
65+
"--ignore-matching-lines=Uploading",
66+
"--ignore-matching-lines=tzname",
67+
]
68+
}
69+
)

‎examples/empty/.tests/unit/test_taxon_align_stats.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,5 +59,11 @@ def test_taxon_align_stats(conda_prefix):
5959
import common
6060

6161
common.OutputChecker(data_path, expected_path, workdir).check(
62-
{".txt": ["diff", "--ignore-matching-lines=\\#"]}
62+
{
63+
".txt": [
64+
"diff",
65+
"--ignore-matching-lines=samtools",
66+
"--ignore-matching-lines=command",
67+
]
68+
}
6369
)

‎examples/empty/config/config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,3 +152,4 @@ euk:
152152
############
153153
report:
154154
multiqc: "--verbose --cl-config 'custom_logo: data/KU_long.png' --cl-config 'custom_logo_title: CAEG - Center for Ancient Environmental Genomics' --cl-config 'custom_logo_url: https://globe.ku.dk/research/caeg/'"
155+
multiqc_db_url: "test_qc.sqlite"

0 commit comments

Comments
 (0)