change playbook to roles

README.md

@@ -78,7 +78,7 @@ tags:
 Every things happen in the [.gitlab-ci.xml](.gitlab-ci.xml) file.  
 
 Mainly the [VagrantFile](VagrantFile) set up a virtual machine from __Vagrant Cloud__ [GAEV/win10](https://app.vagrantup.com/GAEV/boxes/win10) box (a windows10 professional evaluation version with vbBoxGuestAditions).  
-Files in [ansible\_provisioning/](ansible_provisioning/) are playbooks for __Ansible__ that will install/configure R, Rtools, packages, MiKTeX, Pandoc and GitLab-Runner.  
+Files in [ansible\_provisioning/](ansible_provisioning/) are playbooks for __Ansible__ that will install/configure R, Rtools, packages, MiKTeX, Pandoc and GitLab-Runner using 'roles'.  
 After the Virtual Machine creation and configuration, it registered as a new __gitlab-runner__ (by replacing the existing virtualbox machine "WIN10R4").  
 Bonus : The VM (box) is save/backup in a __OpenStack Object Store__ [Swift](https://wiki.openstack.org/wiki/Swift).  
 
@@ -93,7 +93,7 @@ vagrant halt
 
 ## Customization
 
-* To add R packages : edit file [ansible\_provisioning/files/install-r-packages.sh](ansible\_provisioning/files/install-r-packages.sh) and add systems libraries and R packages to install.  
-* Make artifacts work on your GitLab instance : edit file [ansible\_provisioning/files/install-gitlab-runner.sh](ansible\_provisioning/files/install-gitlab-runner.sh) and change the gitlab URL and IP.
+* To add R packages : add _r\_pkg\_cran_ variable in playbook.yml or edit [ansible\_provisioning/roles/R-packages/default.yml](ansible\_provisioning/roles/R-packages/default.yml) and add systems libraries and R packages to install.  
+* Make artifacts work on your GitLab instance : edit file [ansible\_provisioning/roles/cicd/files/install-gitlab-runner.sh](ansible\_provisioning/roles/cicd/files/install-gitlab-runner.sh) and change the gitlab URL and IP.
 * Custom VirtualBox : edit file [VagrantFile](VagrantFile) and modify parameters in virtualbox provider.
 

ansible_provisioning/playbook.yml

 ---
-- name: Windows R tools
+- name: Windows host ping
   hosts: windows10
   tasks:
     - name: ping
@@ -9,72 +9,30 @@
     #    category_names:
     #      - SecurityUpdates
     #    reboot: yes
-    - name: copy MiKTeX install script
-      win_copy:
-        src: ./files/install-miktex.ps1
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install MiKTeX
-      win_command: powershell.exe -File C:\Windows\Temp\install-miktex.ps1
-    - name: copy R script
-      win_copy:
-        src: ./files/install-r.ps1
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install R
-      win_command: powershell.exe -File C:\Windows\Temp\install-r.ps1
-    - name: copy Rtools script
-      win_copy:
-        src: ./files/install-rtools.ps1
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install Rtools4
-      win_command: powershell.exe -File C:\Windows\Temp\install-rtools.ps1
-    - name: copy msys2 config script
-      win_copy:
-        src: ./files/configure-msys2.sh
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Configure MSYS2
-      win_command: C:\Rtools\usr\bin\bash.exe -l C:\Windows\Temp\configure-msys2.sh
-    - name: copy msys2 sshd install script
-      win_copy:
-        src: ./files/install-msys2-sshd.sh
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install MSYS2 ssh deamon
-      win_command: C:\Rtools\usr\bin\bash.exe -l C:\Windows\Temp\install-msys2-sshd.sh
-    - name: Enable Firewall for sshd
-      win_shell: |
-        netsh advfirewall firewall add rule name="SSHD" dir=in action=allow service=msys2_sshd enable=yes
-        netsh advfirewall firewall add rule name="SSHD" dir=in action=allow program="C:\Rtools\usr\bin\sshd.exe" enable=yes
-        netsh advfirewall firewall add rule name="ssh" dir=in action=allow protocol=TCP localport=22
-    - name: copy R packages script
-      win_copy:
-        src: ./files/install-r-packages.sh
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install R packages
-      win_command: C:\Rtools\usr\bin\bash.exe -l C:\Windows\Temp\install-r-packages.sh
-    - name: copy GitLab install script
-      win_copy:
-        src: ./files/install-gitlab-runner.ps1
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install GitLab Runner
-      win_command: powershell.exe -File C:\Windows\Temp\install-gitlab-runner.ps1
-    - name: copy pandoc install script
-      win_copy:
-        src: ./files/install-pandoc.ps1
-        dest: 'C:\Windows\Temp\'
-        remote_src: no
-    - name: Install Pandoc
-      win_command: powershell.exe -File C:\Windows\Temp\install-pandoc.ps1
-    - name: Clean
+- name: install R, Rtools, dependencies and sshd
+  hosts: windows10
+  vars:
+    - r_cran_mirror: https://cloud.r-project.org
+  roles:
+    - R
+    - Rtools
+    - msys2
+    - R-packages
+    - MikTex
+    - Pandoc
+
+- name: Clean
+  hosts: windows10
+  tasks:
+    - name: clean Temp dir
       win_command: cmd.exe /c DEL /F/Q/S C:\Windows\Temp\*.*
-    - name: Reboot after install
+
+- name: Reboot after install
+  hosts: windows10
+  tasks: 
+    - name: Reboot windows
       win_reboot:
-        reboot_timeout: 3600
-        post_reboot_delay: 1200
+        reboot_timeout: 360
+        post_reboot_delay: 120
 
 

ansible_provisioning/roles/MikTex/defaults/main.yml

+---
+
+miktex_version: "21.1"

ansible_provisioning/files/install-miktex.ps1 → ansible_provisioning/roles/MikTex/files/install-miktex.ps1

 # Install miktex for latex support
 Write-Output "Installing MiKTeX"
 
+#### Now done in tasks ####
 ## install from basic ##
-Write-Output "Downloading installer"
-$url = "https://miktex.org/download/ctan/systems/win32/miktex/setup/windows-x64/basic-miktex-20.11-x64.exe"
-$output = "C:\Windows\Temp\miktex.exe"
-$wcR = New-Object System.Net.WebClient
+#Write-Output "Downloading installer"
+#$url = "https://miktex.org/download/ctan/systems/win32/miktex/setup/windows-x64/basic-miktex-20.11-x64.exe"
+#$output = "C:\Windows\Temp\miktex.exe"
+#$wcR = New-Object System.Net.WebClient
 
-Write-Output "Downloading $url"
+#Write-Output "Downloading $url"
 #Download the zip
-$wcR.DownloadFile($url, $output)
-Write-Output "Download completed $url"
+#$wcR.DownloadFile($url, $output)
+#Write-Output "Download completed $url"
+####
+
 
 C:\Windows\Temp\miktex.exe --auto-install=yes --common-install=C:\miktex --shared --unattended | Out-Null
 

ansible_provisioning/roles/MikTex/tasks/main.yml

+- name: Download MiKTek installer
+  win_get_url:
+    url : 'https://miktex.org/download/ctan/systems/win32/miktex/setup/windows-x64/basic-miktex-{{ miktex_version }}-x64.exe'
+    dest: 'C:\Windows\Temp\miktex.exe'
+
+## should work with ansible > 2.10
+#- name: MiKTex Install
+#  ansible.windows.win_package:
+#    path: C:\Windows\Temp\miktex.exe
+#    arguments: --auto-install=yes --common-install=C:\miktex --shared --unattended | Out-Null
+#    state: present
+
+## Old verison 
+- name: copy MiKTeX install script
+  win_copy:
+    src: ./files/install-miktex.ps1
+    dest: 'C:\Windows\Temp\'
+    remote_src: no
+- name: Install MiKTeX
+  win_command: powershell.exe -File C:\Windows\Temp\install-miktex.ps1

ansible_provisioning/roles/Pandoc/defaults/main.yml

+---
+
+pandoc_version: 2.11.4

ansible_provisioning/files/install-pandoc.ps1 → ansible_provisioning/roles/Pandoc/files/install-pandoc.ps1

+$pandocversion=$args[0]
+
 # Install miktex for latex support
 Write-Output "Installing pandoc"
 
-Write-Output "Downloading installer"
+#### Done in tasks now
+#Write-Output "Downloading installer"
 ## download 
-$url = "https://github.com/jgm/pandoc/releases/download/2.10.1/pandoc-2.10.1-windows-x86_64.zip"
-$output = 'c:\Windows\Temp\pandoc.zip'
-$wcR = New-Object System.Net.WebClient
+#$url = "https://github.com/jgm/pandoc/releases/download/2.10.1/pandoc-2.10.1-windows-x86_64.zip"
+#$output = 'c:\Windows\Temp\pandoc.zip'
+#$wcR = New-Object System.Net.WebClient
 
-Write-Output "Downloading $url"
+#Write-Output "Downloading $url"
 #Download the exe
-$wcR.DownloadFile($url, $output)
-Write-Output "Download completed $url"
+#$wcR.DownloadFile($url, $output)
+#Write-Output "Download completed $url"
+
+#$pandocDir = 'C:\pandoc\'
+#Write-Output "Install $output into $pandocDir"
+#Expand-Archive -LiteralPath $output -DestinationPath $pandocDir -Force
 
-$pandocDir = 'C:\pandoc\'
-Write-Output "Install $output into $pandocDir"
-Expand-Archive -LiteralPath $output -DestinationPath $pandocDir -Force
+$pandocDir="C:\\pandoc"
+Write-Output "Install into $pandocDir"
+msiexec /i c:\Windows\Temp\pandoc.msi APPLICATIONFOLDER="$pandocDir" ALLUSERS=1 /qn
+Start-Sleep -Seconds 3
 
-if (Test-Path 'C:\pandoc\pandoc-2.10.1\pandoc.exe')
+if (Test-Path "$pandocDir\\pandoc.exe")
 {
 	Write-Output "pandoc installed"
 }
@@ -26,6 +34,6 @@ else
 }
 # Add pandoc to PATH
 $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path
-$newPath = "$oldpath;C:\pandoc\pandoc-2.10.1\\"
+$newPath = "$oldpath;$pandocDir"
 Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath
 Remove-Item $output -Force

ansible_provisioning/roles/Pandoc/tasks/main.yml

+- name: Download Pandoc installer
+  win_get_url:
+    url : 'https://github.com/jgm/pandoc/releases/download/{{ pandoc_version }}/pandoc-{{ pandoc_version }}-windows-x86_64.msi'
+    dest: 'C:\Windows\Temp\pandoc.msi'
+
+## may work with ansible > 2.10
+#- name: Pandoc Install
+#  ansible.windows.win_package:
+#    path: https://github.com/jgm/pandoc/releases/download/{{ pandoc_version }}/pandoc-{{ pandoc_version }}-windows-x86_64.msi
+#    arguments: TARGETDIR='C:\pandoc\' /qn /quiet /norestart
+#    state: present
+
+
+- name: copy pandoc install script
+  win_copy:
+    src: ./files/install-pandoc.ps1
+    dest: 'C:\Windows\Temp\'
+    remote_src: no
+- name: Install Pandoc
+  win_command: powershell.exe -File C:\Windows\Temp\install-pandoc.ps1 {{ pandoc_version }}

ansible_provisioning/roles/R-packages/defaults/main.yml

+r_cran_mirror: https://cloud.r-projet.org
+
+## Bioconductor package list
+r_pkg_bioc: 
+  - Biostrings
+
+## CRAN package list
+r_pkg_cran:
+  - testthat
+  - pkgload
+  - devtools
+  - deSolve
+  - sp-rgeos
+  - fields
+  - MASS
+  - Matrix
+  - deldir
+  - pracma
+  - raster
+  - fftwtools
+  - mvtnorm
+  - rgdal
+  - sf
+  - splancs
+  - knitr
+  - rmarkdown
+  - testthat
+  - Rcpp
+  - RcppArmadillo
+  - RcppEigen
+  - fasterize
+  - roxygen2
+  - shiny
+  - htmltools
+  - shinydashboard
+  - ggplot2
+  - dplyr
+  - dbplyr
+  - DT
+  - magrittr
+  - devtools
+  - shinyjs
+  - stringi
+  - stringr
+  - jsonlite
+  - data.table
+  - htmlwidgets
+  - RSQLite
+  - BiocManager
+  - foreach
+  - doParallel
+  - mvtnorm
+  - mapdata
+  - proj4
+  - gstat
+  - automap
+  - RCALI
+  - rgenoud
+  - purrr
+
+## github package list to install
+#r_pkg_github:
+
+## local package list to install
+#r_pkg_local:
+#
+
+msys2_system_lib:
+  - !unsafe 'mingw-w64-{i686,x86_64}-fftw'
+  - !unsafe "mingw-w64-{i686,x86_64}-gdal"
+  - !unsafe 'mingw-w64-{i686,x86_64}-geos'
+  - !unsafe 'mingw-w64-{i686,x86_64}-libxml2'
+  - !unsafe 'mingw-w64-{i686,x86_64}-tools-git'
+  - !unsafe 'mingw-w64-{i686,x86_64}-tools'
+  - !unsafe 'msys/vim'
+

ansible_provisioning/roles/R-packages/tasks/main.yml

+## Old version using script
+#- name: copy R packages script
+#  win_copy:
+#    src: ./files/install-r-packages.sh
+#    dest: 'C:\Windows\Temp\'
+#    remote_src: no
+#- name: Install R packages
+#  win_command: C:\Rtools\usr\bin\bash.exe -l C:\Windows\Temp\install-r-packages.sh
+
+- name: install msys2 system libraries
+  win_command: C:\Rtools\usr\bin\bash.exe -lc "pacman --noconfirm -yS {{ item }}"
+  loop: "{{ msys2_system_lib }}"
+  when: msys2_system_lib is defined
+
+- name: install BiocManager
+  win_command: Rscript -e "install.packages(c('BiocManager'), repo='{{ r_cran_mirror }}', ask=FALSE)"
+
+- name: install Bioconductor pacakges
+  win_command: Rscript -e "BiocManager::install(c('{{ item }}'))"
+  loop: "{{ r_pkg_bioc }}"
+  when: r_pkg_bioc is defined
+
+- name: install CRAN packages
+  win_command: Rscript -e "install.packages(c('{{ item }}'), repo='{{ r_cran_mirror }}', ask=FALSE)"
+  loop: "{{ r_pkg_cran }}"
+  when: r_pkg_cran is defined
+
+- name: install github packages
+  win_command: Rscript -e "devtools::install_github(c('{{ item }}'))"
+  loop: "{{ r_pkg_github }}"
+  when: r_pkg_github is defined
+
+- name: install local R packages
+  win_command: R CMD INSTALL "{{ item }}"
+  loop: "{{ r_pkg_local }}"
+  when: r_pkg_local is defined
+

ansible_provisioning/roles/R/defaults/main.yml

+---
+r_version: "4.0.3"
+
+r_cran_mirror: https://cloud.r-project.org
+
+r_install_devel: false
+

ansible_provisioning/files/install-r.ps1 → ansible_provisioning/roles/R/files/install-r.ps1

 # Install R 
 Write-Output "Installing R"
 
-Write-Output "Downloading R Last Version"
+#### this is done in tasks now ####
+#Write-Output "Downloading R Last Version"
 ## Download R engine for Windows machine
 
 ## download last relase name
-$urlR = "https://cran.biotools.fr/bin/windows/base/release.html"
-$outputR = 'c:\Windows\Temp\R-win.exe'
-$wcR = New-Object System.Net.WebClient
-$res = $wcR.DownloadString($urlR)
+#$urlR = "https://cran.biotools.fr/bin/windows/base/release.html"
+#$outputR = 'c:\Windows\Temp\R-win.exe'
+#$wcR = New-Object System.Net.WebClient
+#$res = $wcR.DownloadString($urlR)
 
 ## get release name in the HTML page
-$pos1 = $res.IndexOf("URL=")
-$pos2 = $res.LastIndexOf('"')
-$Rexe = $res.Substring($pos1+4, $pos2-$pos1-4)
-$urlR = "https://cran.biotools.fr/bin/windows/base/" + $Rexe
+#$pos1 = $res.IndexOf("URL=")
+#$pos2 = $res.LastIndexOf('"')
+#$Rexe = $res.Substring($pos1+4, $pos2-$pos1-4)
+#$urlR = "https://cran.biotools.fr/bin/windows/base/" + $Rexe
 
-Write-Output "Downloading $($urlR)"
+#Write-Output "Downloading $($urlR)"
 #Download the exe
-$wcR.DownloadFile($urlR, $outputR)
+#$wcR.DownloadFile($urlR, $outputR)
 
-Write-Output "Download completed $Rexe"
+#Write-Output "Download completed $Rexe"
+####
 
+$outputR = 'c:\Windows\Temp\R-win.exe'
 $RDir = 'C:\R\'
 Write-Output "Install $outputR into $RDir"
 Start-Process -FilePath $outputR -ArgumentList "/VERYSILENT /DIR=$RDir" -Wait -NoNewWindow
@@ -31,7 +34,7 @@ if ( Test-Path "HKLM:\Software\R-core\R" ) {
 	Write-Output "R version $($Rver) installed"
 }
 else {
-	throw "ERROR: Can't install R $($urlR)"
+	throw "ERROR: Can't install R "
 }
 
 # Add R to PATH
@@ -40,12 +43,3 @@ $newPath = "$oldpath;C:\R\bin"
 Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath
 Remove-Item $outputR -Force
 
-# configure LANG
-$renv = "C:\Users\vagrant\.Renviron"
-New-Item -Path $renv -ItemType File
-Add-Content -Path $renv -Value 'LC_COLLATE  = "French_France.1252"'
-Add-Content -Path $renv -Value 'LC_CTYPE    = "French_France.1252"'
-Add-Content -Path $renv -Value 'LC_MONETARY = "French_France.1252"'
-Add-Content -Path $renv -Value 'LC_NUMERIC  = "French_France.1252"'
-Add-Content -Path $renv -Value 'LC_TIME     = "French_France.1252"'
-

ansible_provisioning/roles/R/tasks/main.yml

+- name: Download R
+  win_get_url:
+    url : 'https://cran.biotools.fr/bin/windows/base/old/{{ r_version }}/R-{{ r_version }}-win.exe'
+    dest: 'C:\Windows\Temp\R-win.exe'
+
+# should work with ansible > 2.10
+#- name: Install R Project
+#  ansible.windows.win_package:
+#    path: C:\Windows\Temp\R-win.exe
+#    arguments: /VERYSILENT /DIR='C:\R\'
+#    state: present
+ 
+# Old version
+- name: copy R script
+  win_copy:
+    src: ./files/install-r.ps1
+    dest: 'C:\Windows\Temp\'
+    remote_src: no
+- name: Install R
+  win_command: powershell.exe -File C:\Windows\Temp\install-r.ps1
+
+- name: Configure R env
+  win_command: Rscript -e "dir.create(Sys.getenv('R_LIBS_USER'), recursive=TRUE)"

ansible_provisioning/roles/Rtools/tasks/main.yml

+- name: copy Rtools script
+  win_copy:
+    src: ./files/install-rtools.ps1
+    dest: 'C:\Windows\Temp\'
+    remote_src: no
+- name: Install Rtools4
+  win_command: powershell.exe -File C:\Windows\Temp\install-rtools.ps1

ansible_provisioning/roles/cicd/tasks/main.yml

+---
+- name: copy GitLab install script
+  win_copy:
+    src: ./files/install-gitlab-runner.ps1
+    dest: 'C:\Windows\Temp\'
+    remote_src: no
+
+- name: Install GitLab-Runner
+  win_command: powershell.exe -File C:\Windows\Temp\install-gitlab-runner.ps1
+

ansible_provisioning/files/configure-msys2.sh → ansible_provisioning/roles/msys2/files/configure-msys2.sh

@@ -4,19 +4,17 @@
 
 # Add repository
 sed -i.bak -e "s/^#\[msys\]/\[msys\]/" -e "s/^#Include/Include/" /etc/pacman.conf
-echo -e "Server = http://repo.msys2.org/mingw/\$arch/" >> /etc/pacman.d/mirrorlist.msys
-echo -e "Server = http://repo.msys2.org/mingw/x86_64/" >> /etc/pacman.d/mirrorlist.mingw64
-#echo -e "Server = https://mirror.selfnet.de/msys2/msys/\$arch/" >> /etc/pacman.d/mirrorlist.msys
-#echo -e "Server = https://mirror.selfnet.de/msys2/msys/x86_64/" >> /etc/pacman.d/mirrorlist.mingw64
+#echo -e "Server = http://repo.msys2.org/mingw/\$arch/" >> /etc/pacman.d/mirrorlist.msys
+#echo -e "Server = http://repo.msys2.org/mingw/x86_64/" >> /etc/pacman.d/mirrorlist.mingw64
 
 # Update server gpg key
 curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
 curl -O http://repo.msys2.org/msys/x86_64/msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig
 pacman-key --verify msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz.sig
-pacman -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz --noconfirm
 pacman --noconfirm -U --config <(echo) msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz
+pacman -U msys2-keyring-r21.b39fb11-1-any.pkg.tar.xz --noconfirm
 
 # Update msys2 (just one time to keep Rtools dependences versions)
-pacman -Syuu --noconfirm
+pacman -Syy --noconfirm
 
 #pacman -S openssh cygrunsrv mingw-w64-x86_64-editrights --noconfirm

ansible_provisioning/files/install-msys2-sshd.sh → ansible_provisioning/roles/msys2/files/install-msys2-sshd.sh

@@ -29,9 +29,12 @@
 #   24 Aug 2015 — run server with -e to redirect logs to /var/log/sshd.log
 #
 
-pacman -Syy
-pacman -S openssh cygrunsrv mingw-w64-x86_64-editrights --noconfirm
-pacman -S git --noconfirm
+## install dependencies : openssh ans cygrunsrv
+pacman -S openssh cygrunsrv --noconfirm
+## get editrights and install it
+curl -O https://repo.msys2.org/mingw/x86_64/mingw-w64-x86_64-editrights-1.03-3-any.pkg.tar.xz
+pacman -U mingw-w64-x86_64-editrights-1.03-3-any.pkg.tar.xz --noconfirm
+
 
 set -e