/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    Nextflow config for Minerva HPC at Icahn School of Medicine at Mount Sinai
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    Author: Christopher Tastad - Judy Cho Lab
    Contact: christopher.tastad@mssm.edu
    HPC Support: hpchelp@hpc.mssm.edu
 
    IMPORTANT: Before running this pipeline, set the MINERVA_ALLOCATION environment
    variable in your master submission script:
 
    export MINERVA_ALLOCATION="acc_YOUR-PROJECT-NAME"
 
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
*/
 
// Global default params
params {
    config_profile_description = 'Minerva HPC at Icahn School of Medicine at Mount Sinai'
    config_profile_contact     = 'Christopher Tastad (@ctastad)'
    config_profile_url         = 'https://labs.icahn.mssm.edu/minervalab/'
 
    // Cluster-specific parameters
    minerva_allocation         = System.getenv('MINERVA_ALLOCATION') ?: 'default_allocation'
    max_cpus = 64
    max_memory = '1.5.TB'
    max_time = '336.h'
}
 
 
// Singularity configuration
env {
    SINGULARITY_CACHEDIR = "/sc/arion/work/${System.getenv('USER')}/singularity_cache"
    SINGULARITY_TMPDIR = "/sc/arion/work/${System.getenv('USER')}/singularity_tmp"
    SINGULARITY_LOCALCACHEDIR = "/sc/arion/work/${System.getenv('USER')}/singularity_cache"
    SINGULARITY_PULLFOLDER = "/sc/arion/work/${System.getenv('USER')}/singularity_cache/pull"
    SINGULARITY_DISABLE_CACHE = "no"
 
}
 
singularity {
    enabled = true
    autoMounts = true
    cacheDir = "/sc/arion/work/${System.getenv('USER')}/singularity_cache"
    pullTimeout = '120 min'
 
    // Pass proxy settings to container
    envWhitelist = ['http_proxy', 'https_proxy', 'all_proxy', 'no_proxy']
}
 
 
// LSF executor configuration
executor {
    name = 'lsf'
    submitRateLimit = '2 sec'
    // Specific LSF settings for proper memory handling
    perJobMemLimit = false
    perTaskReserve = true
    max_time = '336.h'
}
 
// Process configuration
process {
    executor = 'lsf'
    resourceLimits = [
        cpus: 64,
        memory: 1.5.TB,
        time: 336.h
    ]
 
    // Dynamic queue selection based on job requirements
    queue = {
        if (task.time > 144.h) {
            return 'long'
        } else if (task.label && task.label.toString().contains('gpu') && task.time <= 30.min) {
            return 'gpuexpress'
        } else if (task.label && task.label.toString().contains('gpu')) {
            return 'gpu'
        } else if (task.time <= 12.h && task.cpus <= 8) {
            return 'express'
        } else {
            return 'premium'
        }
    }
 
    // Cluster options with proper memory handling
    clusterOptions = {
        def options = "-P ${System.getenv('MINERVA_ALLOCATION') ?: params.minerva_allocation}"
 
        // Handle memory requests - ensure consistency between -M and rusage
        if (task.memory) {
            def mem = task.memory.toMega()
            options += " -M ${mem}"
        }
 
        // Add GPU-specific options
        if (task.label && task.label.toString().contains('gpu')) {
            def gpuNum = task.label.toString().contains('high_gpu') ? 2 : 1
            options += " -gpu num=${gpuNum}"
        }
 
        return options
    }
 
    // Add gpu awareness to container
    withLabel: 'gpu|.*gpu.*' {
        containerOptions = '--nv'
    }
 
    // ERROR HANDLING CONFIGURATION
    // Default dynamic error strategy for most processes
    errorStrategy = {
        if (task.exitStatus in [130, 137, 140] && task.attempt <= 3)
            return 'retry'
        else if (task.exitStatus in [131..145] && task.attempt <= 1)
            return 'retry'
        else
            return 'finish'
    }
    maxRetries = 3
 
    // Special error handling labels (these override the dynamic strategy above)
    withLabel:error_ignore {
        errorStrategy = 'ignore'
        maxRetries = 0
    }
 
    withLabel:error_retry {
        errorStrategy = 'retry'
        maxRetries = 2
    }
 
    // Set default failOnMissingField behavior (optional)
    shell = ['/bin/bash', '-euo', 'pipefail']
}