The task is to run our backend PHP tests using SonarQube from a jenkins Pipeline job.
Jenkins running in Docker and all its builds also uses Docker.
The main issue I faced during this setup was the fact that SonarQube’s container inside spawns another process with Elastisearch (while Docker concept says “1 service per one container”).
During that Elasticsearch cannot be started from root user, so had to do play a bit with UIDs and mount points.
Docker Compose for SonarQube
Check UID of the default user in the SonarQube Docker image:
root@jenkins-dev:/opt/sonarqube# docker run -ti sonarqube id
uid=999(sonarqube) gid=999(sonarqube) groups=999(sonarqube)
Create directories to keep SonarQube’s data:
root@jenkins-dev:~# mkdir -p /data/sonarqube/{conf,logs,temp,data,extensions,bundled_plugins,postgresql,postgresql_data}
Create a new user and change those directories owner:
root@jenkins-dev:~# adduser sonarqube
root@jenkins-dev:~# usermod -aG docker sonarqube
root@jenkins-dev:~# chown -R sonarqube:sonarqube /data/sonarqube/
Find UID of the user created above:
root@jenkins-dev:/opt/sonarqube# id sonarqube
uid=1004(sonarqube) gid=1004(sonarqube) groups=1004(sonarqube),999(docker)
Create a Docker Compose file using the UID in user
:
version: "3"
networks:
sonarnet:
driver: bridge
services:
sonarqube:
// use UID here
user: 1004:1004
image: sonarqube
ports:
- "9000:9000"
networks:
- sonarnet
environment:
- sonar.jdbc.url=jdbc:postgresql://db:5432/sonar
volumes:
- /data/sonarqube/conf:/opt/sonarqube/conf
- /data/sonarqube/logs:/opt/sonarqube/logs
- /data/sonarqube/temp:/opt/sonarqube/temp
- /data/sonarqube/data:/opt/sonarqube/data
- /data/sonarqube/extensions:/opt/sonarqube/extensions
- /data/sonarqube/bundled_plugins:/opt/sonarqube/lib/bundled-plugins
db:
image: postgres
networks:
- sonarnet
environment:
- POSTGRES_USER=sonar
- POSTGRES_PASSWORD=sonar
volumes:
- /data/sonarqube/postgresql:/var/lib/postgresql
- /data/sonarqube/postgresql_data:/var/lib/postgresql/data
Check:
root@jenkins-dev:/opt/sonarqube# docker-compose -f sonarqube-compose.yml up
Starting sonarqube_db_1 ... done
Recreating sonarqube_sonarqube_1 ... done
...
sonarqube_1 | 2019.06.14 15:33:46 INFO app[][o.s.a.SchedulerImpl] Process[ce] is up
sonarqube_1 | 2019.06.14 15:33:46 INFO app[][o.s.a.SchedulerImpl] SonarQube is up
NGINX
Now need to configure NGINX and SSL to have the ability to access SonarQube’s web-interface.
Stop NGINX, install Let’s Encrypt client, obtain a certificate (see more details in the Bitwarden: an organization’s password manager self-hosted version installation on an AWS EC2 post):
root@jenkins-dev:/opt/sonarqube# systemctl stop nginx
root@jenkins-dev:/opt/sonarqube# /opt/letsencrypt/letsencrypt-auto certonly -d sonar.example.com
root@jenkins-dev:/opt/sonarqube# systemctl start nginx
Create a virtual host config, here is just by copying already existing:
root@jenkins-dev:/opt/sonarqube# cp /etc/nginx/conf.d/ci.example.com.conf /etc/nginx/conf.d/sonar.example.com.conf
Update it so it will look like next:
upstream sonar {
server 127.0.0.1:9000;
}
server {
listen 80;
server_name dev.sonar.example.com;
# Lets Encrypt Webroot
location ~ /.well-known {
root /var/www/html;
allow all;
}
location / {
return 301 https://dev.sonar.example.com;
}
}
server {
listen 443 ssl;
server_name dev.sonar.example.com;
access_log /var/log/nginx/dev.sonar.example.com-access.log proxy;
error_log /var/log/nginx/dev.sonar.example.com-error.log warn;
ssl_certificate /etc/letsencrypt/live/dev.sonar.example.com/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/dev.sonar.example.com/privkey.pem;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/nginx/dhparams.pem;
ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:ECDHE-RSA-AES128-GCM-SHA256:AES256+EECDH:DHE-RSA-AES128-GCM-SHA256:AES256+EDH:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4";
ssl_session_timeout 1d;
ssl_stapling on;
ssl_stapling_verify on;
location / {
proxy_http_version 1.1;
proxy_request_buffering off;
proxy_buffering off;
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_pass http://sonar$request_uri;
}
}
Check syntax and reload NGINX’s configs:
root@jenkins-dev:/home/admin# nginx -t && systemctl start nginx
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
Jenkins Docker Compose
Now time to add SonarQube to the Jenkins Docker Compose file (although can be used as a dedicated service) – move SonarQube and PostgreSQL to this file:
version: '3'
networks:
jenkins:
services:
jenkins:
user: root
image: jenkins/jenkins:2.164.3
networks:
- jenkins
ports:
- '8080:8080'
- '50000:50000'
volumes:
- /data/jenkins:/var/lib/jenkins
- /var/run/docker.sock:/var/run/docker.sock
- /usr/bin/docker:/usr/bin/docker
- /usr/lib/x86_64-linux-gnu/libltdl.so.7:/usr/lib/x86_64-linux-gnu/libltdl.so.7
environment:
- JENKINS_HOME=/var/lib/jenkins
- JAVA_OPTS=-Duser.timezone=Europe/Kiev
- JENKINS_JAVA_OPTIONS="-Djava.awt.headless=true -Dhudson.model.DirectoryBrowserSupport.CSP="default-src 'self' 'unsafe-inline' 'unsafe-eval'; img-src 'self' 'unsafe-inline' data:;""
logging:
driver: "journald"
sonarqube:
user: 1004:1004
image: sonarqube
ports:
- "9000:9000"
networks:
- jenkins
environment:
- sonar.jdbc.url=jdbc:postgresql://db:5432/sonar
volumes:
- /data/sonarqube/conf:/opt/sonarqube/conf
- /data/sonarqube/logs:/opt/sonarqube/logs
- /data/sonarqube/temp:/opt/sonarqube/temp
- /data/sonarqube/data:/opt/sonarqube/data
- /data/sonarqube/extensions:/opt/sonarqube/extensions
- /data/sonarqube/bundled_plugins:/opt/sonarqube/lib/bundled-plugins
logging:
driver: "journald"
db:
image: postgres
networks:
- jenkins
environment:
- POSTGRES_USER=sonar
- POSTGRES_PASSWORD=sonar
volumes:
- /data/sonarqube/postgresql:/var/lib/postgresql
- /data/sonarqube/postgresql_data:/var/lib/postgresql/data
logging:
driver: "journald"
Restart Jenkins service (see the Linux: systemd сервис для Docker Compose post (Rus)):
root@jenkins-dev:/opt/jenkins# systemctl restart jenkins
Check containers:
root@jenkins-dev:/home/admin# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
fc2662391c45 sonarqube "./bin/run.sh" 48 seconds ago Up 46 seconds 0.0.0.0:9000->9000/tcp jenkins_sonarqube_1
3ac2bb5f0e87 postgres "docker-entrypoint.s…" 48 seconds ago Up 46 seconds 5432/tcp jenkins_db_1
113496304b0f jenkins/jenkins:2.164.3 "/sbin/tini -- /usr/…" 48 seconds ago Up 46 seconds 0.0.0.0:8080->8080/tcp, 0.0.0.0:50000->50000/tcp jenkins_jenkins_1
Go to the Sonar:
Log in with admin:admin.
Jenkins configuration
Usually, for Jenkins, the SonarQube Scanner plugin is used, but we will run Scanner from a Docker container, so no need to install this plugin.
Job configuration
Create a Pipeline job:
Create a pipeline script:
node {
stage('Clone repo') {
git branch: "develop", url: "git@github.com:example-dev/example-me.git", credentialsId: "jenkins-github"
}
stage('SonarTests') {
docker.image('newtmitch/sonar-scanner').inside('-v /var/run/docker.sock:/var/run/docker.sock') {
sh "--version"
}
}
}
And again face with the ENTRYPOINT error:
$ docker top d2269fe30970490ba9957ac57701ec6091f3c9cbf78f957e903a3319fa1445bd -eo pid,comm
ERROR: The container started but didn’t run the expected command. Please double check your ENTRYPOINT does execute the command passed as docker run argument, as required by official docker images (see https://github.com/docker-library/official-images#consistency for entrypoint consistency requirements).
Alternatively you can force image entrypoint to be disabled by adding option–entrypoint=”
.
Just let’s use the solution from the Jenkins: running PHPUnit from Codeception by a Pull Request in Github and Allure-reports post.
Run container, set entrypoint
to bash
:
$ docker run -ti --entrypoint="bash" newtmitch/sonar-scanner
root@20baaae7de9a:/usr/src#
Find the scanner’s executable:
root@20baaae7de9a:/usr/src# which sonar-scanner
/usr/local/bin/sonar-scanner
Update Jenkins script – set --entrypoint=""
and full path in the sh
– /usr/local/bin/sonar-scanner
:
node {
stage('Clone repo') {
git branch: "develop", url: "git@github.com:example-dev/example-me.git", credentialsId: "jenkins-github"
}
stage('SonarTests') {
docker.image('newtmitch/sonar-scanner').inside('-v /var/run/docker.sock:/var/run/docker.sock --entrypoint=""') {
sh "/usr/local/bin/sonar-scanner --version"
}
}
}
Run the build, check results:
Nice – it works.
But now another issue is coming:
- Jenkins is running as Docker container in the jenkins_jenkins network
- SonarQube is running as Docker container in the jenkins_jenkins network
- Jenkins creates a new container with the
sonar-scanner
which has to have access to the SonarQube container which in its turn is running in an “external” network jenkins_jenkins (from the scanner’s container point of view)
Check existing networks in Docker on the Jenkins host:
root@jenkins-dev:/home/admin# docker network ls
NETWORK ID NAME DRIVER SCOPE
67900babbbc4 bridge bridge local
d51bc8ee54d0 host host local
30b091d801d6 jenkins_jenkins bridge local
16ab0c37234e none null local
The solution will be to update the scanner’s container settings – set its network as jenkins_jenkins (--net jenkins_jenkins
):
node {
stage('Clone repo') {
git branch: "develop", url: "git@github.com:example-dev/example-me.git", credentialsId: "jenkins-github"
}
stage('SonarTests') {
docker.image('newtmitch/sonar-scanner').inside('-v /var/run/docker.sock:/var/run/docker.sock --entrypoint="" --net jenkins_jenkins') {
sh "/usr/local/bin/sonar-scanner -Dsonar.host.url=http://sonarqube:9000 -Dsonar.sources=."
}
}
}
Run it:
Great – Sonar Scanner is running and is able to connect to the SonarQube container.
A project configuration in SonarQube
Add a new project:
Create its token:
Chose platform and SonarQube will display necessary settings:
Update script, run the job, and:
ERROR: Error during SonarQube Scanner execution
ERROR: No quality profiles have been found, you probably don’t have any language plugin installed.
Okay…
Go back to the SonarQube to the Administration > Marketplace and install the SonarPHP:
Restart Sonar:
Check installed plugins now:
root@jenkins-dev:/home/admin# curl localhost:9000/api/plugins/installed
{"plugins":[{"key":"php","name":"SonarPHP","description":"Code Analyzer for PHP","version":"3.0.0.4537","license":"GNU LGPL v3","organizationName":"SonarSource and Akram Ben Aissi","editionBundled":false,"homepageUrl":"http://redirect.sonarsource.com/plugins/php.html","issueTrackerUrl":"http://jira.codehaus.org/browse/SONARPHP","implementationBuild":"026dee08c29a3689ab1228552e14bfefda9ae57e","updatedAt":1560775404315,"filename":"sonar-php-plugin-3.0.0.4537.jar","sonarLintSupported":true,"hash":"c80c0d053f074a9147d341cf1357d994"}]}
“name”:”SonarPHP”,”description”:”Code Analyzer for PHP”
Good, installed, run the job again:
And results in the SonarQube:
Sonar Scanner: 0 files indexed
But why 0 scanned?
…
INFO: Indexing files…
INFO: Project configuration:
INFO: 0 files indexed
…
Something wrong with directories mapping from Jenkins to its Scanner container.
Let’s try to set sonar.sources
and sonar.projectBaseDir
explicitly:
...
stage('SonarTests') {
docker.image('newtmitch/sonar-scanner').inside('-v /var/run/docker.sock:/var/run/docker.sock --entrypoint="" --net jenkins_jenkins') {
sh "/usr/local/bin/sonar-scanner -Dsonar.host.url=http://sonarqube:9000 -Dsonar.sources=/var/lib/jenkins/workspace/SonarTest -Dsonar.projectBaseDir=/var/lib/jenkins/workspace/SonarTest -Dsonar.projectKey=ProjectName -Dsonar.login=91a691e7c91154d3fee69a05a8fa6e2b10bc82a6 -Dsonar.verbose=true"
}
}
Run:
Works.
And again – results in the SonarQube:
Actually – that’s all, now need to make all of this cleaner.
So the issue of the files looks like appear because Jenkins Docker plugin maps a current working dir to a container with the same name and then sets it as a --workdir
:
…
$ docker run […] -w /var/lib/jenkins/workspace/SonarTest
…
While sonar-scanner
tries to find it in the default Based dir:
…
09:08:56.666 INFO: Base dir: /usr/src
…
Let’s try to set sonar.projectBaseDir
with the “.
” value, i.e. – current directory.
Now the script looks like next:
node {
stage('Clone repo') {
git branch: "develop", url: "git@github.com:example-dev/example-me.git", credentialsId: "jenkins-github"
}
stage('SonarTests') {
docker.image('newtmitch/sonar-scanner').inside('-v /var/run/docker.sock:/var/run/docker.sock --entrypoint="" --net jenkins_jenkins') {
sh "/usr/local/bin/sonar-scanner -Dsonar.host.url=http://sonarqube:9000 -Dsonar.projectBaseDir=. -Dsonar.projectKey=ProjectName -Dsonar.login=91a691e7c91154d3fee69a05a8fa6e2b10bc82a6"
}
}
}
Run it:
…
INFO: Base dir: /var/lib/jenkins/workspace/SonarTest
INFO: Working dir: /var/lib/jenkins/workspace/SonarTest/.scannerwork …
INFO: EXECUTION SUCCESS
A project’s settings can be moved to a sonar-project.properties
file in a project’s Github repository root:
sonar.host.url=http://sonarqube:9000
sonar.projectBaseDir=.
sonar.projectKey=ProjectName
sonar.login=91a691e7c91154d3fee69a05a8fa6e2b10bc82a6
And update our script to removed all parameters – Scanner will look for the sonar-project.properties
to use its settings:
node {
stage('Clone repo') {
git branch: "develop", url: "git@github.com:example-dev/example-me.git", credentialsId: "jenkins-github"
}
stage('SonarTests') {
docker.image('newtmitch/sonar-scanner').inside('-v /var/run/docker.sock:/var/run/docker.sock --entrypoint="" --net jenkins_jenkins') {
sh "/usr/local/bin/sonar-scanner"
}
}
}
Done.
Top comments (0)