fixed sra convert functionality

Author: khoroshevskyi

.looper.yaml

@@ -0,0 +1,4 @@
+pep_config: /home/bnt4me/virginia/repos/geofetch/red_algae/GSE67303_PEP/GSE67303_PEP.yaml
+output_dir: /home/bnt4me/virginia/repos/geofetch/red_algae/GSE67303_PEP
+pipeline_interfaces:
+  - /home/bnt4me/virginia/repos/geofetch/pipeline_interface_convert.yaml

MANIFEST.in

@@ -1,6 +1,9 @@
 include requirements/*
 include README.md
 include docs/img/geofetch_logo.svg
-include geofetch/config_template.yaml
-include geofetch/config_processed_template.yaml
-include geofetch/looper_sra_convert.yaml
+include geofetch/templates/*
+include geofetch/templates/config_template.yaml
+include geofetch/templates/config_processed_template.yaml
+include geofetch/templates/looper_sra_convert.yaml
+include geofetch/templates/looper_config_template.yaml
+include geofetch/templates/pipeline_interface_convert.yaml

geofetch/_version.py

@@ -1 +1 @@
-__version__ = "0.12.7"
+__version__ = "0.12.8"

geofetch/const.py

@@ -44,9 +44,14 @@
 
 NEW_GENOME_COL_NAME = "ref_genome"
 
+TEMPLATES_DIR = "templates"
 CONFIG_PROCESSED_TEMPLATE_NAME = "config_processed_template.yaml"
 CONFIG_RAW_TEMPLATE_NAME = "config_template.yaml"
-CONFIG_SRA_TEMPLATE = "looper_sra_convert.yaml"
+CONFIG_SRA_TEMPLATE_NAME = "looper_sra_convert.yaml"
+PIPELINE_INTERFACE_CONVERT_TEMPLATE_NAME = "pipeline_interface_convert.yaml"
+LOOPER_SRA_CONVERT = "looper_config_template.yaml"
+# SRA_CONVERT_SCHEMA_NAME = "sra_convert_schema.yaml"
+# RESOURCES_NAME = "resources.tsv"
 
 # const for Finder:
 RETMAX = 10000000  # once it should be increased
@@ -63,3 +68,5 @@
     '+AND+("{start_date}"[Publication%20Date]%20:%20"{end_date}"[Publication%20Date])'
 )
 THREE_MONTH_FILTER = '+AND+"published+last+3+months"[Filter]'
+
+LOOPER_CONFIG_FILE_NAME = "looper_config.yaml"

geofetch/geofetch.py

@@ -20,7 +20,7 @@
 from geofetch.const import (
     CONFIG_PROCESSED_TEMPLATE_NAME,
     CONFIG_RAW_TEMPLATE_NAME,
-    CONFIG_SRA_TEMPLATE,
+    CONFIG_SRA_TEMPLATE_NAME,
     EXP_SUPP_METADATA_FILE,
     EXPERIMENT_PATTERN,
     FILE_RAW_NAME_SAMPLE_PATTERN,
@@ -34,6 +34,10 @@
     SAMPLE_SUPP_METADATA_FILE,
     SER_SUPP_FILE_PATTERN,
     SUPP_FILE_PATTERN,
+    TEMPLATES_DIR,
+    PIPELINE_INTERFACE_CONVERT_TEMPLATE_NAME,
+    LOOPER_SRA_CONVERT,
+    LOOPER_CONFIG_FILE_NAME,
 )
 from geofetch.utils import (
     Accession,
@@ -1171,11 +1175,43 @@ def _write_raw_annotation_new(
             if len(subannot_dict) > 0:
                 self._write_subannotation(subannot_dict, proj_root_subsample)
 
-            self._write(proj_root_yaml, template, msg_pre="  Config file: ")
+            self._write(proj_root_yaml, template, msg_pre="Config file: ")
 
             if self.add_dotfile:
                 _create_dot_yaml(dot_yaml_path, yaml_name)
 
+            if self.add_convert_modifier:
+                geofetchdir = os.path.dirname(__file__)
+                pipeline_interface_convert_path = os.path.join(
+                    geofetchdir, TEMPLATES_DIR, PIPELINE_INTERFACE_CONVERT_TEMPLATE_NAME
+                )
+
+                looper_config_template_path = os.path.join(
+                    geofetchdir, TEMPLATES_DIR, LOOPER_SRA_CONVERT
+                )
+
+                with open(looper_config_template_path, "r") as template_file:
+                    template_looper = template_file.read()
+
+                template_values = {
+                    "pep_config": proj_root_yaml,
+                    "output_dir": os.path.join(self.metadata_root_full, "output_dir"),
+                    "pipeline_interface_convert": pipeline_interface_convert_path,
+                }
+
+                for k, v in template_values.items():
+                    placeholder = "{" + str(k) + "}"
+                    template_looper = template_looper.replace(placeholder, str(v))
+
+                looper_config_file = os.path.join(
+                    self.metadata_root_full,
+                    LOOPER_CONFIG_FILE_NAME,
+                )
+
+                self._write(
+                    looper_config_file, template_looper, msg_pre="Looper config file: "
+                )
+
         else:
             meta_df = pd.DataFrame.from_dict(metadata_dict, orient="index")
 
@@ -1214,8 +1250,11 @@ def _create_config_processed(
         :param meta_in_series:
         :return: generated, complete config file content
         """
+
         geofetchdir = os.path.dirname(__file__)
-        config_template = os.path.join(geofetchdir, CONFIG_PROCESSED_TEMPLATE_NAME)
+        config_template = os.path.join(
+            geofetchdir, TEMPLATES_DIR, CONFIG_PROCESSED_TEMPLATE_NAME
+        )
         with open(config_template, "r") as template_file:
             template = template_file.read()
         meta_list_str = [
@@ -1270,9 +1309,13 @@ def _create_config_raw(
         else:
             sample_modifier_str = ""
         if not self.config_template:
-            self.config_template = os.path.join(geofetchdir, CONFIG_RAW_TEMPLATE_NAME)
+            self.config_template = os.path.join(
+                geofetchdir, TEMPLATES_DIR, CONFIG_RAW_TEMPLATE_NAME
+            )
         if self.add_convert_modifier:
-            sra_convert_path = os.path.join(geofetchdir, CONFIG_SRA_TEMPLATE)
+            sra_convert_path = os.path.join(
+                geofetchdir, TEMPLATES_DIR, CONFIG_SRA_TEMPLATE_NAME
+            )
             with open(sra_convert_path, "r") as template_file:
                 sra_convert_template = template_file.read()
         else:
@@ -1301,6 +1344,7 @@ def _create_config_raw(
         for k, v in template_values.items():
             placeholder = "{" + str(k) + "}"
             template = template.replace(placeholder, str(v))
+
         return template
 
     @staticmethod

geofetch/config_processed_template.yaml …templates/config_processed_template.yamlgeofetch/config_processed_template.yaml renamed to geofetch/templates/config_processed_template.yaml geofetch/config_processed_template.yaml renamed to geofetch/templates/config_processed_template.yaml

@@ -0,0 +1,4 @@
+pep_config: {pep_config}
+output_dir: {output_dir}
+pipeline_interfaces:
+  - {pipeline_interface_convert}

geofetch/config_template.yaml geofetch/templates/config_template.yamlgeofetch/config_template.yaml renamed to geofetch/templates/config_template.yaml

@@ -4,7 +4,7 @@
   derive:
     attributes: [read1, read2, SRR_files]
     sources:
-      SRA: "${SRABAM}/{srr}.bam"
+      SRA: "${SRARAW}/{srr}/{srr}.sra"
       FQ: "${SRAFQ}/{srr}.fastq.gz"
       FQ1: "${SRAFQ}/{srr}_1.fastq.gz"
       FQ2: "${SRAFQ}/{srr}_2.fastq.gz"
@@ -26,20 +26,3 @@
         read_type: "SINGLE"
       then:
         read1: FQ1
-
-project_modifiers:
-  amend:
-    sra_convert:
-      looper:
-        results_subdir: sra_convert_results
-      sample_modifiers:
-        append:
-          SRR_files: SRA
-          pipeline_interfaces: ${CODE}/geofetch/pipeline_interface_convert.yaml
-        derive:
-          attributes: [read1, read2, SRR_files]
-          sources:
-            SRA: "${SRARAW}/{srr}/{srr}.sra"
-            FQ: "${SRAFQ}/{srr}.fastq.gz"
-            FQ1: "${SRAFQ}/{srr}_1.fastq.gz"
-            FQ2: "${SRAFQ}/{srr}_2.fastq.gz"

geofetch/templates/looper_config_template.yaml

@@ -0,0 +1,12 @@
+pipeline_name: sra_convert
+path: sraconvert
+input_schema: ./sra_convert_schema.yaml
+sample_interface:
+  command_template: >
+    {pipeline.path} --srr {sample.SRR_files}
+    {% if sample.SRX is defined %} --sample-name {sample.SRX} {% endif %}
+    {% if project.fqfolder is defined %} --fqfolder {project.fqfolder} {% endif %}
+    -O {looper.results_subdir}
+compute:
+  bulker_crate: databio/sra_convert
+  size_dependent_variables: ./resources.tsv