Hi,
I am running my program on Nextflow version 24.10.5. The content of my nextflow.config is as follows:
params {
// Workflow flags
// TODO nf-core: Specify your pipeline's command line flags
genome = "GRCh38"
input = null
input_csv = "/workspace/ieee/CancerDataset_phase2_sample/GSM8462129/GSM8462129.csv"
single_end = false
outdir = '/workspace/ieee/CancerDataset_phase2_sample/GSM8462129/starsolo_v2/'
publish_dir_mode = 'copy'
// UMI tools options
umitools_bc_pattern = "CCCCCCCCCCCCCCCCNNNNNNNNNN"
save_umi_intermeds = true
skip_umitools = true
// Alignment options
skip_star = true
min_mapped_reads = 5
save_align_intermeds = true
star_ignore_sjdbgtf = false
// Inputs for SICILIAN after STAR alignment
star_bam = true
star_sj_out_tab = true
reads_per_gene = true
star_chimeric_junction = true
// Sicilian configs
skip_classinput = true
skip_glm = false
smartseq2 = false
tenx = true
stranded = true
domain = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/domain_file/ucscGenePfam.txt"
annotator = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/annotator_file/hg38_refseq.pkl"
exon_bounds = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/exon_pickle_file/hg38_refseq_exon_bounds.pkl"
splices = "/workspace/ieee/Project/alternative_splicing/nf-sicilian/SICILIAN_human_hg38_Refs/splice_pickle_file/hg38_refseq_splices.pkl"
// Inputs for SICILIAN after class_input (e.g. if ran partially and restarting)
sicilian_class_input = false
// Inputs for SICILIAN after GLM (e.g. if ran partially and restarting)
sicilian_glm_output = false
// Genome inputs
star_index = "/workspace/ieee/Project/alternative_splicing/starsolo/genome_100/"
gtf = "/workspace/ieee/Project/alternative_splicing/starsolo/refdata-gex-GRCh38-2024-A/genes/genes.gtf"
save_reference = true
// MultiQC options
skip_multiqc = false
multiqc_title = null
// Boilerplate options
seq_center = false
enable_conda = false
multiqc_config = false
email = false
email_on_fail = false
max_multiqc_email_size = 25.MB
plaintext_email = false
monochrome_logs = false
help = false
igenomes_base = '/workspace/ieee/Project/alternative_splicing/nf-sicilian'
tracedir = "${params.outdir}/pipeline_info"
igenomes_ignore = false
custom_config_version = 'master'
custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}"
hostnames = false
config_profile_name = false
config_profile_description = false
config_profile_contact = false
config_profile_url = false
validate_params = true
show_hidden_params = false
schema_ignore_params = 'genomes,input_paths,modules,sicilian_glm_output_paths,sicilian_glm_output,sicilian_class_input_paths,sicilian_class_input,star_chimeric_junction_paths,star_reads_per_gene_paths,star_sj_out_tab_paths,star_bam_paths,star_chimeric_junction,reads_per_gene,star_sj_out_tab,star_bam'
// Defaults only, expecting to be overwritten
max_memory = 1024.GB
max_cpus = 70
max_time = 24.h
}
process {
withName: STAR_ALIGN {
cpus = 60
memory = 1024.GB
time = 4.h
}
withName: SICILIAN:SICILIAN_CLASSINPUT {
cpus = 60
memory = 1024.GB
time = 8.h
}
withName: SICILIAN:SICILIAN_GLM {
cpus = 60
memory = 1024.GB
time = 8.h
}
withName: SICILIAN:SICILIAN_ANNSPLICES {
cpus = 60
memory = 1024.GB
time = 8.h
}
withName: SICILIAN:SICILIAN_CONSOLIDATE {
cpus = 60
memory = 1024.GB
time = 8.h
}
withName: SICILIAN:SICILIAN_PROCESS_CI_10X {
cpus = 60
memory = 1024.GB
time = 8.h
}
withName: SICILIAN:SICILIAN_POSTPROCESS {
cpus = 60
memory = 1024.GB
time = 8.h
}
withName: MULTIQC {
cpus = 60
memory = 1024.GB
time = 8.h
}
}
// Container slug. Stable releases should specify release tag!
// Developmental code should specify :dev
// process.container = 'nfcore/sicilian:dev'
// Load base.config by default for all pipelines
includeConfig 'conf/base.config'
// Load modules.config for DSL2 module specific options
includeConfig 'conf/modules.config'
// Load nf-core custom profiles from different Institutions
try {
includeConfig "${params.custom_config_base}/nfcore_custom.config"
} catch (Exception e) {
System.err.println("WARNING: Could not load nf-core/config profiles: ${params.custom_config_base}/nfcore_custom.config")
}
profiles {
conda {
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud = false
process.conda = "$projectDir/environment.yml"
}
debug { process.beforeScript = 'echo $HOSTNAME' }
docker {
docker.enabled = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
// Avoid this error:
// WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap.
// Testing this in nf-core after discussion here https://github.com/nf-core/tools/pull/351
// once this is established and works well, nextflow might implement this behavior as new default.
docker.runOptions = '-u \$(id -u):\$(id -g)'
}
singularity {
docker.enabled = false
singularity.enabled = true
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
singularity.autoMounts = true
}
podman {
singularity.enabled = false
docker.enabled = false
podman.enabled = true
shifter.enabled = false
charliecloud = false
}
shifter {
singularity.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = true
charliecloud.enabled = false
}
charliecloud {
singularity.enabled = false
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = true
}
test { includeConfig 'conf/test.config' }
test_samplesheet { includeConfig 'conf/test_samplesheet.config' }
test_skip_umitools { includeConfig 'conf/test_skip_umitools.config' }
test_smartseq2 { includeConfig 'conf/test_smartseq2.config' }
test_full { includeConfig 'conf/test_full.config' }
}
// Load igenomes.config if required
if (!params.igenomes_ignore) {
includeConfig 'conf/igenomes.config'
}
// Export these variables to prevent local Python/R libraries from conflicting with those in the container
env {
PYTHONNOUSERSITE = 1
R_PROFILE_USER = "/.Rprofile"
R_ENVIRON_USER = "/.Renviron"
}
// Capture exit codes from upstream processes when piping
process.shell = ['/bin/bash', '-euo', 'pipefail']
def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss')
timeline {
enabled = true
file = "${params.tracedir}/execution_timeline_${trace_timestamp}.html"
}
report {
enabled = true
file = "${params.tracedir}/execution_report_${trace_timestamp}.html"
}
trace {
enabled = true
file = "${params.tracedir}/execution_trace_${trace_timestamp}.txt"
}
dag {
enabled = true
file = "${params.tracedir}/pipeline_dag_${trace_timestamp}.svg"
}
manifest {
name = 'nf-core/sicilian'
author = 'Julia Salzman Lab'
homePage = 'https://github.com/nf-core/sicilian'
description = 'Run SICILIAN alternative splicing analysis pipeline for single-cell RNA-seq data'
mainScript = 'main.nf'
nextflowVersion = '>=21.04.0'
version = '1.0dev'
}
The base.config is as follows:
process {
// TODO nf-core: Check the defaults for all processes
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
memory = { check_max( 7.GB * task.attempt, 'memory' ) }
time = { check_max( 4.h * task.attempt, 'time' ) }
errorStrategy = { task.exitStatus in [143,137,104,134,139] ? 'retry' : 'finish' }
maxRetries = 1
maxErrors = '-1'
// Process-specific resource requirements
// NOTE - Only one of the labels below are used in the fastqc process in the main script.
// If possible, it would be nice to keep the same label naming convention when
// adding in your processes.
// TODO nf-core: Customise requirements for specific processes.
// See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
withLabel:process_low {
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
memory = { check_max( 14.GB * task.attempt, 'memory' ) }
time = { check_max( 6.h * task.attempt, 'time' ) }
}
withLabel:process_medium {
cpus = { check_max( 6 * task.attempt, 'cpus' ) }
memory = { check_max( 42.GB * task.attempt, 'memory' ) }
time = { check_max( 8.h * task.attempt, 'time' ) }
}
withLabel:process_high {
cpus = { check_max( 12 * task.attempt, 'cpus' ) }
memory = { check_max( 84.GB * task.attempt, 'memory' ) }
time = { check_max( 10.h * task.attempt, 'time' ) }
}
withLabel:process_higher {
cpus = 40
memory = 1024.GB
time = 8.h
}
withLabel:process_long {
time = { check_max( 20.h * task.attempt, 'time' ) }
}
withLabel:cpu_2 {
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
}
withLabel:process_super_highmem {
memory = { check_max( 200.GB * task.attempt, 'memory' ) }
}
withLabel:process_super_long {
time = { check_max( 48.h * task.attempt, 'time' ) }
}
}
I create a new label process_higher to hard code the parameters of cpu and memory. However, when I execute the tasks, no matter how many CPUs I set, it only uses one core to run.
The following file is my nextflow.log
nextflow.log (64.1 KB)