Commit a1b5c452 authored by Jean-Francois Rey's avatar Jean-Francois Rey

change pipeline to use Super Cluster Slurm

parent 41c5e4fe
Pipeline #2026 failed
......@@ -8,70 +8,37 @@ stages:
- build_img
- deploy
.buildData_template: &builddata_definition
tags:
- r-base-biosp
GenData:
tags:
- "cluster"
- "docker"
stage: gen_data
rules:
- if: '$GENDATA == "gen_data" && $CI_COMMIT_BRANCH == "master"'
when: always
- when : manual
allow_failure: true
BuildData0:
<<: *builddata_definition
script:
- 'cd R && Rscript main_forecast_plotly.R "0"'
artifacts:
paths:
- R/list_figure0.rds
expire_in: 2 days
BuildData1:
<<: *builddata_definition
script:
- 'cd R && Rscript main_forecast_plotly.R "1"'
artifacts:
paths:
- R/list_figure1.rds
expire_in: 2 days
- "mkdir -p /mnt/biomstock/Cluster/biometrie/parsi_covid/COVIDSHINYAPP/"
- "cp R/* /mnt/biomstock/Cluster/biometrie/parsi_covid/COVIDSHINYAPP/"
- 'sed -i -e "s/TOKEN/$TRIGGERTOKEN/g" /mnt/biomstock/Cluster/biometrie/parsi_covid/COVIDSHINYAPP/clusterbash.sh'
- export SSHPASS=$CLUSTER_PWD
- sshpass -e ssh -o StrictHostKeyChecking=no $CLUSTER_USER@$CLUSTER_IP
"sbatch -p pcbiom9x -D /mnt/biomstock/Cluster/biometrie/parsi_covid/COVIDSHINYAPP/ clusterbash.sh"
BuildData2:
<<: *builddata_definition
script:
- 'cd R && Rscript main_forecast_plotly.R "2"'
artifacts:
paths:
- R/list_figure2.rds
expire_in: 2 days
BuildData3:
<<: *builddata_definition
script:
- 'cd R && Rscript main_forecast_plotly.R "3"'
artifacts:
paths:
- R/list_figure3.rds
expire_in: 2 days
mergeData:
GetData:
tags:
- r-base-biosp
- "cluster"
- "docker"
stage: merge_data
dependencies:
- BuildData0
- BuildData1
- BuildData2
- BuildData3
rules:
- if: '$GENDATA == "gen_data" && $CI_COMMIT_BRANCH == "master"'
- if: '$GENDATA != "gen_data" && $CI_COMMIT_BRANCH == "master"'
when: always
- when : manual
allow_failure: true
script:
- 'cd R && Rscript main_forecast_plotly.R "merge"'
- 'cp /mnt/biomstock/Cluster/biometrie/parsi_covid/COVIDSHINYAPP/list_figures.rds R/list_figures.rds'
- "Rscript R/immunity.R && mv immun_M3_k.rds data/"
- "curl --request POST --form token=$TRIGGERTOKEN --form ref=master https://gitlab.paca.inrae.fr/api/v4/projects/105/trigger/pipeline"
artifacts:
paths:
- R/list_figures.rds
......@@ -91,10 +58,10 @@ buildImageDocker:
DOCKER_DRIVER: overlay2
services:
- docker:19.03.0-dind
dependencies:
- "GetData"
before_script:
- 'apk add --update curl'
- 'curl --header "PRIVATE-TOKEN: $PTOKEN" -o artifacts.zip "https://gitlab.paca.inrae.fr/api/v4/projects/105/jobs/artifacts/master/download?job=mergeData"'
- "unzip -o artifacts.zip && mv R/list_figures.rds www/list_figures.rds"
- "mv R/list_figures.rds www/list_figures.rds"
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
stage: build_img
rules:
......
#!/bin/bash
#
#SBATCH -J ShinyCovid19
#SBATCH -o logshinycovid19_%J_%2t.out
#SBATCH -e errorshinycovid19_%J_%2t.out
#
#SBATCH --ntasks=4
#SBATCH --cpus-per-task=3
#SBATCH --time=04:10:00
#SBATCH --mem-per-cpu=2048
#SBATCH --mail-type=ALL
#SBATCH --mail-user=jean-francois.rey@inrae.fr
module purge
module load R/4.0.0
srun -n1 -N1 -o logshinycovid19_%J.out -e errorshinycovid19_%J.out Rscript ./main_forecast_plotly.R "0" &
srun -n1 -N1 -o logshinycovid19_%J.out -e errorshinycovid19_%J.out Rscript ./main_forecast_plotly.R "1" &
srun -n1 -N1 -o logshinycovid19_%J.out -e errorshinycovid19_%J.out Rscript ./main_forecast_plotly.R "2" &
srun -n1 -N1 -o logshinycovid19_%J.out -e errorshinycovid19_%J.out Rscript ./main_forecast_plotly.R "3" &
echo "# on attend les 4 builds..."
wait
echo "# 4 builds fini !"
srun Rscript ./main_forecast_plotly.R "merge"
curl --request POST --form token=TOKEN --form ref=master https://gitlab.paca.inrae.fr/api/v4/projects/105/trigger/pipeline
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment