Merge pull request #416 from lcdb/v1.12rc

V1.12rc

Author: daler

.circleci/config.yml

@@ -5,7 +5,7 @@ variables:
   # default settings for all steps
   defaults: &defaults
     docker:
-      - image: ubuntu:20.04
+      - image: ubuntu:latest
 
   # --------------------------------------------------------------------------
   # The caching dramatically speeds up testing time, because we can do the
@@ -28,7 +28,7 @@ variables:
     save_cache:
       key: v5-{{ checksum "env.yml" }}-{{ checksum "env-r.yml" }}
       paths:
-        - /opt/mambaforge
+        - /opt/miniforge
 
         # this file is created by sra-tools upon installation by conda, and so
         # needs to be included in the cache otherwise fastq-dump thinks it's
@@ -48,6 +48,7 @@ variables:
         name: Set path
         command: |
           # x11-utils required to avoid R::png() segfaulting
+          export DEBIAN_FRONTEND=noninteractive
           apt update && apt install -y \
             curl \
             git \
@@ -73,7 +74,7 @@ variables:
 
           # Note that if we don't escape \$PATH, we'll be stuck with the exact
           # PATH defined here, which will break anything needing conda envs.
-          echo "export PATH=\$PATH:/opt/mambaforge/bin" >> $BASH_ENV
+          echo "export PATH=\$PATH:/opt/miniforge/bin" >> $BASH_ENV
           source $BASH_ENV
 
 
@@ -85,28 +86,16 @@ variables:
       command: |
         source $BASH_ENV
         echo $PATH
-        # /opt/mambaforge will only exist if there was a cache restore; otherwise we'll make it here.
+        # /opt/miniforge will only exist if there was a cache restore; otherwise we'll make it here.
         #
-        # Use mambaforge which comes with mamba.
-        if [ ! -e /opt/mambaforge ]; then
-            curl -L https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh > mambaforge.sh
-            bash mambaforge.sh -b -p /opt/mambaforge
-            source "/opt/mambaforge/etc/profile.d/conda.sh"
-            source "/opt/mambaforge/etc/profile.d/mamba.sh"
+        if [ ! -e /opt/miniforge ]; then
+            curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh"
+            bash Miniforge3-$(uname)-$(uname -m).sh -b -p /opt/miniforge
+            source "/opt/miniforge/etc/profile.d/conda.sh"
             conda activate
 
             which conda
-            which mamba
-            mamba --version
-
-            # Note that mambaforge doesn't come with the defaults channel, but
-            # we're adding it here at the beginning to simulate what most users
-            # probably have locally (and following the bioconda docs). Using
-            # strict channel priority means we should [theoretically] never
-            # pull packages from defaults because they all exist on
-            # conda-forge.
-            conda config --system --add channels defaults
-
+            conda --version
             conda config --system --add channels bioconda
             conda config --system --add channels conda-forge
             conda config --system --set channel_priority strict
@@ -115,10 +104,10 @@ variables:
             # https://docs.conda.io/projects/conda-build/en/latest/resources/link-scripts.html,
             # post-link scripts should not depend on any installed or
             # to-be-installed conda packages...but they do.
-            mamba install -n base r-base yq
+            conda install -n base r-base yq
 
-            time mamba env create -n $LCDBWF_ENV --file env.yml
-            time mamba env create -n $LCDBWF_ENV_R --file env-r.yml
+            time conda env create -n $LCDBWF_ENV --file env.yml
+            time conda env create -n $LCDBWF_ENV_R --file env-r.yml
         fi
 
   # --------------------------------------------------------------------------
@@ -127,7 +116,7 @@ variables:
     run:
       name: Download example data
       command: |
-        source /opt/mambaforge/etc/profile.d/conda.sh
+        source /opt/miniforge/etc/profile.d/conda.sh
         conda activate $LCDBWF_ENV
         conda info --envs
         conda config --show
@@ -151,7 +140,7 @@ variables:
         cp $ORIG/workflows/rnaseq/run_test.sh $DEPLOY/workflows/rnaseq/run_test.sh
         cp $ORIG/workflows/rnaseq/run_downstream_test.sh $DEPLOY/workflows/rnaseq/run_downstream_test.sh
         cp $ORIG/workflows/references/run_test.sh $DEPLOY/workflows/references/run_test.sh
-        cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/colocalization/run_test.sh
+        # cp $ORIG/workflows/colocalization/run_test.sh $DEPLOY/workflows/colocalization/run_test.sh
 
         mkdir $DEPLOY/ci
         mkdir $DEPLOY/test
@@ -172,7 +161,7 @@ variables:
     run:
       name: Run pytest suite and testthat suite
       command: |
-        source /opt/mambaforge/etc/profile.d/conda.sh
+        source /opt/miniforge/etc/profile.d/conda.sh
         conda activate $LCDBWF_ENV
         # run unit tests and doctests for the modules in lib
         test/lcdb-wf-test unit_tests --pytest
@@ -194,9 +183,9 @@ variables:
         name: chipseq workflow
         command: |
           cd $DEPLOY/workflows/chipseq
-          source /opt/mambaforge/etc/profile.d/conda.sh
+          source /opt/miniforge/etc/profile.d/conda.sh
           conda activate $LCDBWF_ENV
-          $DEPLOY/test/lcdb-wf-test chipseq --run-workflow --use-conda -j2 -k -p -r
+          $DEPLOY/test/lcdb-wf-test chipseq --run-workflow --use-conda -j2 -k -p
           $DEPLOY/test/lcdb-wf-test chipseq --trackhub
 
   # --------------------------------------------------------------------------
@@ -208,10 +197,10 @@ variables:
         name: chipseq misc
         command: |
           cd $DEPLOY/workflows/chipseq
-          source /opt/mambaforge/etc/profile.d/conda.sh
+          source /opt/miniforge/etc/profile.d/conda.sh
           conda activate $LCDBWF_ENV
 
-          ./run_test.sh --use-conda -j2 -k -p -r \
+          ./run_test.sh --use-conda -j2 -k -p \
             --configfile $ORIG/test/test_configs/test_chipseq_regression.yaml \
             --config sampletable=$ORIG/test/test_configs/chipseq_one_run.tsv \
             merged_bigwigs="{}" \
@@ -237,9 +226,9 @@ variables:
       run:
         name: references workflow
         command: |
-          source /opt/mambaforge/etc/profile.d/conda.sh
+          source /opt/miniforge/etc/profile.d/conda.sh
           conda activate $LCDBWF_ENV
-          $DEPLOY/test/lcdb-wf-test references --run-workflow --configfile=config/config.yaml -j2 -p -r -k --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test references --run-workflow --configfile=config/config.yaml -j2 -p -k --orig $ORIG
 
   # --------------------------------------------------------------------------
   # Standard RNA-seq workflow
@@ -248,10 +237,10 @@ variables:
         name: rnaseq workflow
         command: |
           cd $DEPLOY
-          source /opt/mambaforge/etc/profile.d/conda.sh
+          source /opt/miniforge/etc/profile.d/conda.sh
           conda activate $LCDBWF_ENV
           $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow -n
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --use-conda -j2 -k -p -r --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --use-conda -j2 -k -p --orig $ORIG
 
           $DEPLOY/test/lcdb-wf-test rnaseq --trackhub --orig $ORIG
 
@@ -276,32 +265,32 @@ variables:
         command: |
           ORIG=$(pwd)
           cd $DEPLOY
-          source /opt/mambaforge/etc/profile.d/conda.sh
+          source /opt/miniforge/etc/profile.d/conda.sh
           conda activate $LCDBWF_ENV
 
           # Check the help for test/lcdb-wf-test to see what args these
           # provide; some of them use the --until argument to restrict the
           # rules that are run. Note the use of --orig $ORIG to use the test
           # configs from the original clone rather than the deployed directory.
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --sra-pe          -k -r -p -j2 --use-conda --orig $ORIG
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --sra-se          -k -r -p -j2 --use-conda --orig $ORIG
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --strandedness-pe -k -r -p -j2 --use-conda --orig $ORIG
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --star-2pass      -k -r -p -j2 --use-conda --orig $ORIG
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --star-1pass      -k -r -p -j2 --use-conda --orig $ORIG
-          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --pe              -k -r -p -j2 --use-conda --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --sra-pe          -k -p -j2 --use-conda --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --sra-se          -k -p -j2 --use-conda --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --strandedness-pe -k -p -j2 --use-conda --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --star-2pass      -k -p -j2 --use-conda --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --star-1pass      -k -p -j2 --use-conda --orig $ORIG
+          $DEPLOY/test/lcdb-wf-test rnaseq --run-workflow --pe              -k -p -j2 --use-conda --orig $ORIG
 
 
 
   # --------------------------------------------------------------------------
   # Standard colocalization workflow
-  colocalization-step: &colocalization-step
-      run:
-        name: colocalization workflow
-        command: |
-          cd $DEPLOY/workflows/colocalization
-          source /opt/mambaforge/etc/profile.d/conda.sh
-          conda activate $LCDBWF_ENV
-          $DEPLOY/test/lcdb-wf-test colocalization --run-workflow -k -r -p -j2 --use-conda --orig $ORIG
+  # colocalization-step: &colocalization-step
+  #     run:
+  #       name: colocalization workflow
+  #       command: |
+  #         cd $DEPLOY/workflows/colocalization
+  #         source /opt/miniforge/etc/profile.d/conda.sh
+  #         conda activate $LCDBWF_ENV
+  #         $DEPLOY/test/lcdb-wf-test colocalization --run-workflow -k -p -j2 --use-conda --orig $ORIG
 
 # --------------------------------------------------------------------------
 # Syntax note: All of the steps above, with their "&step-name" labels, can be
@@ -410,14 +399,14 @@ jobs:
       - *get-data
       - *rnaseq-misc-step
 
-  colocalization:
-    <<: *defaults
-    steps:
-      - checkout
-      - *restore_cache
-      - *set-path
-      - *get-data
-      - *colocalization-step
+  # colocalization:
+  #   <<: *defaults
+  #   steps:
+  #     - checkout
+  #     - *restore_cache
+  #     - *set-path
+  #     - *get-data
+  #     - *colocalization-step
 
   references:
     <<: *defaults
@@ -438,9 +427,9 @@ jobs:
       - run:
           name: Install sphinx
           command: |
-            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/miniforge/etc/profile.d/conda.sh
             conda activate lcdb-wf-test
-            mamba install -y sphinx make yaml
+            conda install -y sphinx make yaml
       - run:
           name: OK for unknown github host
           command: mkdir -p ~/.ssh/ && echo -e "Host github.com\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
@@ -450,7 +439,7 @@ jobs:
       - run:
           name: Build and upload docs
           command: |
-            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/miniforge/etc/profile.d/conda.sh
             conda activate lcdb-wf-test
             ci/build-docs.sh
       - store_artifacts:
@@ -466,7 +455,7 @@ jobs:
       - run:
           name: Report environment
           command: |
-            source /opt/mambaforge/etc/profile.d/conda.sh
+            source /opt/miniforge/etc/profile.d/conda.sh
             conda env export -n lcdb-wf-test > /tmp/env.yaml
             conda env export -n lcdb-wf-test-r > /tmp/env-r.yaml
       - store_artifacts:
@@ -504,10 +493,10 @@ workflows:
           requires:
             - initial-setup
             - pytest
-      - colocalization:
-          requires:
-            - initial-setup
-            - pytest
+      # - colocalization:
+      #     requires:
+      #       - initial-setup
+      #       - pytest
       - build-docs:
           requires:
             - initial-setup
@@ -518,4 +507,4 @@ workflows:
             - chipseq
             - chipseq-misc
             - references
-            - colocalization
+            # - colocalization

docs/changelog.rst

@@ -1,6 +1,51 @@
 Changelog
 =========
 
+v1.12
+-----
+
+- Fixed render_r1_r2 function(s) in Snakefiles (#413) which cleans up some
+  Snakefile functions
+
+- add Plodia interpunctella reference config (#417) which adds some
+  postprocessing functionality for dealing with references in zip files, and
+  adds references for Plodia interpunctella
+
+- Make ``strand_arg`` a param (#415) which moves strandedness arguments to
+  params: directive, letting jobs re-run if strandedness changed in config.
+  Previously, rules needed to be manually re-run after changing strandedness in
+  config.
+
+- Update ``plotting.R`` (#423) to fix a typo in the (infrequently-used)
+  lcdbwf::vargenes_heatmap() function
+
+- Change SRA fastq directory (#418) which improves how SRA-configured runs are
+  handled, addressing #408
+
+- Use tempdir, if configured, for STAR (#425) which allows STAR to use a temp
+  dir if configured
+
+v1.11
+-----
+
+- Add samtools stats and samtools flagstat (#397) for additional QC options
+
+- Generate RDS object ready for carnation (#399), which overhauls the way
+  output Rds objects are made from RNA-seq downstream RMarkdown files. This
+  prepares for integration with Carnation, an interactive web interface
+  currently still in development but being used internally Rds files are
+  dramatically smaller after stripping out superfluous data Better support for
+  running rnaseq.Rmd followed by functional-enrichment.Rmd, and/or
+  gene-patterns.Rmd without overwriting existing Rds files.
+
+- Limit background genes in overrepresentation analysis in
+  functional-enrichment.Rmd (#401) to support universe of expressed genes by
+  default when running functional enrichment, while also supporting
+  customization if needed.
+
+- Implemented LRT functionality (#396) to support LRT in the lcdb::make_results
+  machinery
+
 v1.10.3
 -------