diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1ce0d0b..c37524ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,13 +34,16 @@ jobs: strategy: matrix: php-version: ['8.1'] - test-group: ['unit', 'api'] + test-group: ['unit', 'api', 'cli'] steps: - name: Checkout code uses: actions/checkout@v2 - - name: Start database server + - name: Start postgres database server if: ${{ matrix.test-group == 'api' }} run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_postgres + - name: Start maria database server + if: ${{ matrix.test-group == 'cli' }} + run: docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_maria - name: Use PHP uses: shivammathur/setup-php@v2 with: @@ -109,7 +112,7 @@ jobs: strategy: matrix: php-version: ['8.1'] - test-group: ['unit', 'db', 'api'] + test-group: ['unit', 'db', 'api', 'cli'] steps: - name: Checkout code uses: actions/checkout@v2 @@ -156,6 +159,7 @@ jobs: - run: mv build/coverage-unit/coverage-unit.cov build/coverage-unit.cov - run: mv build/coverage-db/coverage-db.cov build/coverage-db.cov - run: mv build/coverage-api/coverage-api.cov build/coverage-api.cov + - run: mv build/coverage-cli/coverage-cli.cov build/coverage-cli.cov - run: wget https://phar.phpunit.de/phpcov-8.2.1.phar - run: php phpcov-8.2.1.phar merge build --clover build/clover.xml - name: Publish coverage @@ -175,6 +179,7 @@ jobs: coverage-unit coverage-db coverage-api + coverage-cli build-docker-image: runs-on: ubuntu-22.04 diff --git a/CHANGELOG.md b/CHANGELOG.md index 71842087..93701f61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). +## [Unreleased] +### Added +* *Nothing* + +### Changed +* [#1339](https://github.com/shlinkio/shlink/issues/1339) Added new test suite for CLI E2E tests. + +### Deprecated +* *Nothing* + +### Removed +* *Nothing* + +### Fixed +* *Nothing* + + ## [3.2.1] - 2022-08-08 ### Added * *Nothing* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2024adca..5f620893 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -102,7 +102,9 @@ In order to ensure stability and no regressions are introduced while developing Since the app instance is run on a process different from the one running the tests, when a test fails it might not be obvious why. To help debugging that, the app will dump all its logs inside `data/log/api-tests`, where you will find the `shlink.log` and `access.log` files. -* **CLI tests**: *TBD. Once included, its purpose will be the same as API tests, but running through the command line* +* **CLI tests**: These are E2E tests too, but they test console commands instead of REST endpoints. + + They use Maria DB as the database engine, and include the same fixtures as the API tests, that ensure the same data exists at the beginning of the execution. Depending on the kind of contribution, maybe not all kinds of tests are needed, but the more you provide, the better. @@ -119,6 +121,7 @@ Depending on the kind of contribution, maybe not all kinds of tests are needed, For example, `test:db:postgres`. * Run `./indocker composer test:api` to run API E2E tests. For these, the Postgres database engine is used. +* Run `./indocker composer test:cli` to run CLI E2E tests. For these, the Maria DB database engine is used. * Run `./indocker composer infect:test` to run both unit and database tests (over sqlite) and then apply mutations to them with [infection](https://infection.github.io/). * Run `./indocker composer ci` to run all previous commands together. This command is run during the project's continuous integration. * Run `./indocker composer ci:parallel` to do the same as in previous case, but parallelizing non-conflicting tasks as much as possible. diff --git a/composer.json b/composer.json index 3cba02fe..627829aa 100644 --- a/composer.json +++ b/composer.json @@ -58,7 +58,7 @@ "require-dev": { "cebe/php-openapi": "^1.7", "devster/ubench": "^2.1", - "dms/phpunit-arraysubset-asserts": "^0.3.0", + "dms/phpunit-arraysubset-asserts": "^0.4.0", "infection/infection": "^0.26.5", "openswoole/ide-helper": "~4.11.1", "phpspec/prophecy-phpunit": "^2.0", @@ -87,6 +87,7 @@ "autoload-dev": { "psr-4": { "ShlinkioTest\\Shlink\\CLI\\": "module/CLI/test", + "ShlinkioCliTest\\Shlink\\CLI\\": "module/CLI/test-cli", "ShlinkioTest\\Shlink\\Rest\\": "module/Rest/test", "ShlinkioApiTest\\Shlink\\Rest\\": "module/Rest/test-api", "ShlinkioTest\\Shlink\\Core\\": "module/Core/test", @@ -106,7 +107,7 @@ ], "ci:parallel": [ "@parallel cs stan swagger:validate test:unit:ci test:db:sqlite:ci test:db:mysql test:db:maria test:db:postgres test:db:ms", - "@parallel infect:test:api infect:ci:unit infect:ci:db" + "@parallel infect:test:api infect:test:cli infect:ci:unit infect:ci:db" ], "cs": "phpcs", "cs:fix": "phpcbf", @@ -114,12 +115,14 @@ "test": [ "@test:unit", "@test:db", - "@test:api" + "@test:api", + "@test:cli" ], "test:ci": [ "@test:unit:ci", "@test:db", - "@test:api:ci" + "@test:api:ci", + "@test:cli:ci" ], "test:unit": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", "test:unit:ci": "@test:unit --coverage-xml=build/coverage-unit/coverage-xml --log-junit=build/coverage-unit/junit.xml", @@ -133,11 +136,14 @@ "test:db:ms": "DB_DRIVER=mssql composer test:db:sqlite", "test:api": "bin/test/run-api-tests.sh", "test:api:ci": "GENERATE_COVERAGE=yes composer test:api", + "test:cli": "APP_ENV=test DB_DRIVER=maria TEST_ENV=cli php vendor/bin/phpunit --order-by=random --colors=always --testdox -c phpunit-cli.xml", + "test:cli:ci": "GENERATE_COVERAGE=yes composer test:cli -- --log-junit=build/coverage-cli/junit.xml", "infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --only-covering-test-cases --skip-initial-tests", "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=84", "infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json", "infect:ci:api": "@infect:ci:base --coverage=build/coverage-api --min-msi=80 --configuration=infection-api.json", - "infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api", + "infect:ci:cli": "@infect:ci:base --coverage=build/coverage-cli --min-msi=80 --configuration=infection-cli.json", + "infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api infect:ci:cli", "infect:test": [ "@parallel test:unit:ci test:db:sqlite:ci test:api:ci", "@infect:ci" @@ -150,6 +156,10 @@ "@test:api:ci", "@infect:ci:api" ], + "infect:test:cli": [ + "@test:cli:ci", + "@infect:ci:cli" + ], "swagger:validate": "php-openapi validate docs/swagger/swagger.json", "swagger:inline": "php-openapi inline docs/swagger/swagger.json docs/swagger/swagger-inlined.json", "clean:dev": "rm -f data/database.sqlite && rm -f config/params/generated_config.php" diff --git a/config/test/bootstrap_cli_tests.php b/config/test/bootstrap_cli_tests.php new file mode 100644 index 00000000..893bdfd7 --- /dev/null +++ b/config/test/bootstrap_cli_tests.php @@ -0,0 +1,28 @@ +get(Helper\TestHelper::class); +$config = $container->get('config'); +$em = $container->get(EntityManager::class); + +// Delete old coverage in PHP, to avoid merging older executions with current one +$covFile = __DIR__ . '/../../build/coverage-cli.cov'; +if (file_exists($covFile)) { + unlink($covFile); +} + +$testHelper->createTestDb(['bin/cli', 'db:create'], ['bin/cli', 'db:migrate']); +CliTest\CliTestCase::setSeedFixturesCallback( + static fn () => $testHelper->seedFixtures($em, $config['data_fixtures'] ?? []), +); diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index b9bac12d..e82d9da5 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -8,8 +8,10 @@ use GuzzleHttp\Client; use Laminas\ConfigAggregator\ConfigAggregator; use Laminas\Diactoros\Response\EmptyResponse; use Laminas\ServiceManager\Factory\InvokableFactory; +use League\Event\EventDispatcher; use Monolog\Level; use PHPUnit\Runner\Version; +use Psr\Container\ContainerInterface; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; use Psr\Http\Server\RequestHandlerInterface; @@ -20,7 +22,12 @@ use SebastianBergmann\CodeCoverage\Report\Html\Facade as Html; use SebastianBergmann\CodeCoverage\Report\PHP; use SebastianBergmann\CodeCoverage\Report\Xml\Facade as Xml; use Shlinkio\Shlink\Common\Logger\LoggerType; +use Symfony\Component\Console\Application; +use Symfony\Component\Console\Event\ConsoleCommandEvent; +use Symfony\Component\Console\Event\ConsoleTerminateEvent; +use Symfony\Contracts\EventDispatcher\EventDispatcherInterface; +use function file_exists; use function Laminas\Stratigility\middleware; use function Shlinkio\Shlink\Config\env; use function sprintf; @@ -30,14 +37,40 @@ use const ShlinkioTest\Shlink\SWOOLE_TESTING_HOST; use const ShlinkioTest\Shlink\SWOOLE_TESTING_PORT; $isApiTest = env('TEST_ENV') === 'api'; +$isCliTest = env('TEST_ENV') === 'cli'; +$isE2eTest = $isApiTest || $isCliTest; $generateCoverage = env('GENERATE_COVERAGE') === 'yes'; -if ($isApiTest && $generateCoverage) { + +$coverage = null; +if ($isE2eTest && $generateCoverage) { $filter = new Filter(); $filter->includeDirectory(__DIR__ . '/../../module/Core/src'); - $filter->includeDirectory(__DIR__ . '/../../module/Rest/src'); + $filter->includeDirectory(__DIR__ . '/../../module/' . ($isApiTest ? 'Rest' : 'CLI') . '/src'); $coverage = new CodeCoverage((new Selector())->forLineCoverage($filter), $filter); } +/** + * @param 'api'|'cli' $type + */ +$exportCoverage = static function (string $type = 'api') use (&$coverage): void { + if ($coverage === null) { + return; + } + + $basePath = __DIR__ . '/../../build/coverage-' . $type; + $covPath = $basePath . '.cov'; + + // Every CLI test runs on its own process and dumps the coverage afterwards. + // Try to load it and merge it, so that we end up with the whole coverage at the end. + if ($type === 'cli' && file_exists($covPath)) { + $coverage->merge(require $covPath); + } + + (new PHP())->process($coverage, $covPath); + (new Xml(Version::getVersionString()))->process($coverage, $basePath . '/coverage-xml'); + (new Html())->process($coverage, $basePath . '/coverage-html'); +}; + $buildDbConnection = static function (): array { $driver = env('DB_DRIVER', 'sqlite'); $isCi = env('CI', false); @@ -113,17 +146,10 @@ return [ [ 'name' => 'dump_coverage', 'path' => '/api-tests/stop-coverage', - 'middleware' => middleware(static function () use (&$coverage) { + 'middleware' => middleware(static function () use ($exportCoverage) { // TODO I have tried moving this block to a listener so that it's invoked automatically, // but then the coverage is generated empty ¯\_(ツ)_/¯ - if ($coverage) { // @phpstan-ignore-line - $basePath = __DIR__ . '/../../build/coverage-api'; - - (new PHP())->process($coverage, $basePath . '.cov'); - (new Xml(Version::getVersionString()))->process($coverage, $basePath . '/coverage-xml'); - (new Html())->process($coverage, $basePath . '/coverage-html'); - } - + $exportCoverage(); return new EmptyResponse(); }), 'allowed_methods' => ['GET'], @@ -164,6 +190,62 @@ return [ 'factories' => [ TestUtils\Helper\TestHelper::class => InvokableFactory::class, ], + 'delegators' => $isCliTest ? [ + Application::class => [ + static function ( + ContainerInterface $c, + string $serviceName, + callable $callback, + ) use ( + &$coverage, + $exportCoverage, + ) { + /** @var Application $app */ + $app = $callback(); + $wrappedEventDispatcher = new EventDispatcher(); + + // When the command starts, start collecting coverage + $wrappedEventDispatcher->subscribeTo( + ConsoleCommandEvent::class, + static function () use (&$coverage): void { + $id = env('COVERAGE_ID'); + if ($id === null) { + return; + } + + $coverage?->start($id); + }, + ); + // When the command ends, stop collecting coverage + $wrappedEventDispatcher->subscribeTo( + ConsoleTerminateEvent::class, + static function () use (&$coverage, $exportCoverage): void { + $id = env('COVERAGE_ID'); + if ($id === null) { + return; + } + + $coverage?->stop(); + $exportCoverage('cli'); + }, + ); + + $app->setDispatcher(new class ($wrappedEventDispatcher) implements EventDispatcherInterface { + public function __construct(private EventDispatcher $wrappedDispatcher) + { + } + + public function dispatch(object $event, ?string $eventName = null): object + { + $this->wrappedDispatcher->dispatch($event); + return $event; + } + }); + + return $app; + }, + ], + ] : [], ], 'entity_manager' => [ @@ -172,6 +254,7 @@ return [ 'data_fixtures' => [ 'paths' => [ + // TODO These are used for CLI tests too, so maybe should be somewhere else __DIR__ . '/../../module/Rest/test-api/Fixtures', ], ], diff --git a/infection-cli.json b/infection-cli.json new file mode 100644 index 00000000..60552d11 --- /dev/null +++ b/infection-cli.json @@ -0,0 +1,24 @@ +{ + "source": { + "directories": [ + "module/*/src" + ] + }, + "timeout": 5, + "logs": { + "text": "build/infection-cli/infection-log.txt", + "html": "build/infection-cli/infection-log.html", + "summary": "build/infection-cli/summary-log.txt", + "debug": "build/infection-cli/debug-log.txt" + }, + "tmpDir": "build/infection-cli/temp", + "phpUnit": { + "configDir": "." + }, + "testFrameworkOptions": "--configuration=phpunit-cli.xml", + "mutators": { + "@default": true, + "IdenticalEqual": false, + "NotIdenticalNotEqual": false + } +} diff --git a/module/CLI/test-cli/Command/GenerateApiKeyTest.php b/module/CLI/test-cli/Command/GenerateApiKeyTest.php new file mode 100644 index 00000000..b14462c0 --- /dev/null +++ b/module/CLI/test-cli/Command/GenerateApiKeyTest.php @@ -0,0 +1,21 @@ +exec([GenerateKeyCommand::NAME]); + + self::assertStringContainsString('[OK] Generated API key', $output); + self::assertEquals(ExitCodes::EXIT_SUCCESS, $exitCode); + } +} diff --git a/module/CLI/test-cli/Command/ListApiKeysTest.php b/module/CLI/test-cli/Command/ListApiKeysTest.php new file mode 100644 index 00000000..96b60e92 --- /dev/null +++ b/module/CLI/test-cli/Command/ListApiKeysTest.php @@ -0,0 +1,63 @@ +exec([ListKeysCommand::NAME, ...$flags]); + + self::assertEquals($expectedOutput, $output); + self::assertEquals(ExitCodes::EXIT_SUCCESS, $exitCode); + } + + public function provideFlags(): iterable + { + $expiredApiKeyDate = Chronos::now()->subDay()->startOfDay()->toAtomString(); + $enabledOnlyOutput = << [[], << [['-e'], $enabledOnlyOutput]; + yield '--enabled-only' => [['--enabled-only'], $enabledOnlyOutput]; + } +} diff --git a/module/Rest/test-api/Fixtures/ApiKeyFixture.php b/module/Rest/test-api/Fixtures/ApiKeyFixture.php index ef6d1781..54797bb4 100644 --- a/module/Rest/test-api/Fixtures/ApiKeyFixture.php +++ b/module/Rest/test-api/Fixtures/ApiKeyFixture.php @@ -25,7 +25,7 @@ class ApiKeyFixture extends AbstractFixture implements DependentFixtureInterface { $manager->persist($this->buildApiKey('valid_api_key', true)); $manager->persist($this->buildApiKey('disabled_api_key', false)); - $manager->persist($this->buildApiKey('expired_api_key', true, Chronos::now()->subDay())); + $manager->persist($this->buildApiKey('expired_api_key', true, Chronos::now()->subDay()->startOfDay())); $authorApiKey = $this->buildApiKey('author_api_key', true); $authorApiKey->registerRole(RoleDefinition::forAuthoredShortUrls()); diff --git a/phpunit-cli.xml b/phpunit-cli.xml new file mode 100644 index 00000000..49ba781e --- /dev/null +++ b/phpunit-cli.xml @@ -0,0 +1,20 @@ + + + + + ./module/*/test-cli + + + + + + ./module/CLI/src + ./module/Core/src + + +