From e747a0b25037ea16546e1120d7608652c485dcd6 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 19:55:03 +0200 Subject: [PATCH 01/10] Updated how database tests are run in travis, so that all DB engines are covered --- .travis.yml | 5 +---- composer.json | 10 ++-------- config/test/test_config.global.php | 9 ++++++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b232c76..d53d2ed0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ php: - '7.4' services: - - mysql - - postgresql - docker cache: @@ -26,8 +24,7 @@ install: - composer install --no-interaction --prefer-dist before_script: - - mysql -e 'CREATE DATABASE shlink_test;' - - psql -c 'create database shlink_test;' -U postgres + - docker-compose up shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms -d - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) diff --git a/composer.json b/composer.json index 39c1e6db..111b60c4 100644 --- a/composer.json +++ b/composer.json @@ -109,7 +109,7 @@ ], "test:ci": [ "@test:unit:ci", - "@test:db:ci", + "@test:db", "@test:api:ci" ], "test:unit": "phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", @@ -121,11 +121,6 @@ "@test:db:postgres", "@test:db:ms" ], - "test:db:ci": [ - "@test:db:sqlite", - "@test:db:mysql", - "@test:db:postgres" - ], "test:db:sqlite": "APP_ENV=test phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-db.cov --testdox -c phpunit-db.xml", "test:db:mysql": "DB_DRIVER=mysql composer test:db:sqlite", "test:db:maria": "DB_DRIVER=maria composer test:db:sqlite", @@ -152,8 +147,7 @@ "test:ci": "Runs all test suites, generating all needed reports and logs for CI envs", "test:unit": "Runs unit test suites", "test:unit:ci": "Runs unit test suites, generating all needed reports and logs for CI envs", - "test:db": "Runs database test suites on a SQLite, MySQL, MariaDB and PostgreSQL", - "test:db:ci": "Runs database test suites on a SQLite, MySQL and PostgreSQL", + "test:db": "Runs database test suites on a SQLite, MySQL, MariaDB, PostgreSQL and MsSQL", "test:db:sqlite": "Runs database test suites on a SQLite database", "test:db:mysql": "Runs database test suites on a MySQL database", "test:db:maria": "Runs database test suites on a MariaDB database", diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index 6aad5375..0086bcd0 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -20,6 +20,7 @@ $buildDbConnection = function (): array { $driver = env('DB_DRIVER', 'sqlite'); $isCi = env('TRAVIS', false); $getMysqlHost = fn (string $driver) => sprintf('shlink_db%s', $driver === 'mysql' ? '' : '_maria'); + $getCiMysqlPort = fn (string $driver) => $driver === 'mysql' ? '3307' : '3308'; $driverConfigMap = [ 'sqlite' => [ @@ -29,8 +30,9 @@ $buildDbConnection = function (): array { 'mysql' => [ 'driver' => 'pdo_mysql', 'host' => $isCi ? '127.0.0.1' : $getMysqlHost($driver), + 'port' => $isCi ? $getCiMysqlPort($driver) : '3306', 'user' => 'root', - 'password' => $isCi ? '' : 'root', + 'password' => 'root', 'dbname' => 'shlink_test', 'charset' => 'utf8', 'driverOptions' => [ @@ -41,8 +43,9 @@ $buildDbConnection = function (): array { 'postgres' => [ 'driver' => 'pdo_pgsql', 'host' => $isCi ? '127.0.0.1' : 'shlink_db_postgres', + 'port' => $isCi ? '5433' : '5432', 'user' => 'postgres', - 'password' => $isCi ? '' : 'root', + 'password' => 'root', 'dbname' => 'shlink_test', 'charset' => 'utf8', ], @@ -50,7 +53,7 @@ $buildDbConnection = function (): array { 'driver' => 'pdo_sqlsrv', 'host' => $isCi ? '127.0.0.1' : 'shlink_db_ms', 'user' => 'sa', - 'password' => $isCi ? '' : 'Passw0rd!', + 'password' => 'Passw0rd!', 'dbname' => 'shlink_test', ], ]; From 1e8c36b5f1679bfea2f778a1be515e3c2952e0ca Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 19:55:52 +0200 Subject: [PATCH 02/10] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f50e81d2..0358e791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Changed * [#692](https://github.com/shlinkio/shlink/issues/692) Drastically improved performance when loading visits. Specially noticeable when loading big result sets. +* [#657](https://github.com/shlinkio/shlink/issues/657) Updated how DB tests are run in travis by using docker containers which allow all engines to be covered. #### Deprecated From 9f13063b1f1da8056e0540eafeac2d5d34528436 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 20:02:48 +0200 Subject: [PATCH 03/10] Fixed docker-compose command run in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d53d2ed0..426f55f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - composer install --no-interaction --prefer-dist before_script: - - docker-compose up shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms -d + - docker-compose up -d shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) From dfdae96da5bacf2bcff5c2834e536db1476518b0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 20:34:28 +0200 Subject: [PATCH 04/10] Added commands to initially create all testing database for all database engines in travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 426f55f6..cb180c03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,10 @@ install: before_script: - docker-compose up -d shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms + - docker-compose exec shlink_db mysql -u root -proot -e "CREATE DATABASE shlink_test" + - docker-compose exec -e PGPASSWORD=root shlink_db_postgres psql -U postgres -c 'create database shlink_test;' + - docker-compose exec shlink_db_maria mysql -u root -proot -e "CREATE DATABASE shlink_test" + - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) From f6d9a83202abbc5d3816fab8f0e19917cc506675 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 20:42:48 +0200 Subject: [PATCH 05/10] Moved initial ci databases to specific docker-compose file --- .travis.yml | 6 ++---- docker-compose.ci.yml | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 docker-compose.ci.yml diff --git a/.travis.yml b/.travis.yml index cb180c03..b9b8c6b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,8 @@ install: - composer install --no-interaction --prefer-dist before_script: - - docker-compose up -d shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms - - docker-compose exec shlink_db mysql -u root -proot -e "CREATE DATABASE shlink_test" - - docker-compose exec -e PGPASSWORD=root shlink_db_postgres psql -U postgres -c 'create database shlink_test;' - - docker-compose exec shlink_db_maria mysql -u root -proot -e "CREATE DATABASE shlink_test" + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria + - sleep 10 # Let MS database start-up before trying to interact with it - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 00000000..3783fef2 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,14 @@ +version: '3' + +services: + shlink_db: + environment: + MYSQL_DATABASE: shlink_test + + shlink_db_postgres: + environment: + POSTGRES_DB: shlink_test + + shlink_db_maria: + environment: + MYSQL_DATABASE: shlink_test From 13bb48d06886e2cba93766ce87974889c6af79b7 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:12:43 +0200 Subject: [PATCH 06/10] Installed pdo_sqlsrv extension in travis --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9b8c6b3..20ebd912 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,9 @@ cache: - $HOME/.composer/cache/files before_install: + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - yes | pecl install swoole-4.4.15 + - yes | pecl install swoole-4.4.15 pdo_sqlsrv - phpenv config-rm xdebug.ini || return 0 install: @@ -24,8 +25,6 @@ install: - composer install --no-interaction --prefer-dist before_script: - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - - sleep 10 # Let MS database start-up before trying to interact with it - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) From 78359c28c7226181356b63d6722a9c04fd1df9ff Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:22:41 +0200 Subject: [PATCH 07/10] Added MS ODBC package installation to travis --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 20ebd912..e59423a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,10 @@ cache: before_install: - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - yes | pecl install swoole-4.4.15 pdo_sqlsrv + - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + - curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + - sudo ACCEPT_EULA=Y apt-get install msodbcsql17 + - yes | pecl install pdo_sqlsrv swoole-4.4.15 - phpenv config-rm xdebug.ini || return 0 install: From 522d8ed236d34687962b754cc438ae267a88110f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:28:53 +0200 Subject: [PATCH 08/10] Ensured some commands are run as sudo during travis CI --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e59423a2..b8c131b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,14 @@ cache: - $HOME/.composer/cache/files before_install: - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + - sudo su - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - - curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + - curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + - exit + - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install msodbcsql17 + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 - phpenv config-rm xdebug.ini || return 0 From 5f42266cf2b41b5e0aec9adffca699454e016261 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:39:48 +0200 Subject: [PATCH 09/10] Moved ms odbc commands to a script --- .travis.yml | 10 +++------- data/infra/ci/install-ms-odbc.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100755 data/infra/ci/install-ms-odbc.sh diff --git a/.travis.yml b/.travis.yml index b8c131b6..36bf55cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,15 +15,11 @@ cache: - $HOME/.composer/cache/files before_install: - - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - sudo su - - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - - curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list - - exit - - sudo apt-get update - - sudo ACCEPT_EULA=Y apt-get install msodbcsql17 + - sudo ./data/infra/ci/install-ms-odbc.sh - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 + - echo 'extension = pdo_sqlsrv.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 install: diff --git a/data/infra/ci/install-ms-odbc.sh b/data/infra/ci/install-ms-odbc.sh new file mode 100755 index 00000000..8cd60580 --- /dev/null +++ b/data/infra/ci/install-ms-odbc.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -ex + +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list +apt-get update +ACCEPT_EULA=Y apt-get install msodbcsql17 +apt-get install unixodbc-dev From cc41c51f77a35b801b2da7f3f4c81f5eb9f720bd Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:55:18 +0200 Subject: [PATCH 10/10] Removed duplicated pdo_sqlsrv enabling on travis config --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 36bf55cf..a95bf67d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ before_install: - sudo ./data/infra/ci/install-ms-odbc.sh - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 - - echo 'extension = pdo_sqlsrv.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0