From f915b97606f8d7d92641f717fc66ba55ecc10cda Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Apr 2020 18:00:29 +0200 Subject: [PATCH 01/87] Created decorator for database connection closing and reopening for swoole tasks --- composer.json | 2 +- .../Core/config/event_dispatcher.config.php | 6 ++ .../CloseDbConnectionEventListener.php | 32 ++++++++ ...loseDbConnectionEventListenerDelegator.php | 24 ++++++ .../EventDispatcher/LocateShortUrlVisit.php | 40 ++++------ .../EventDispatcher/NotifyVisitToWebHooks.php | 15 ++-- ...DbConnectionEventListenerDelegatorTest.php | 43 +++++++++++ .../CloseDbConnectionEventListenerTest.php | 75 +++++++++++++++++++ .../LocateShortUrlVisitTest.php | 5 -- .../NotifyVisitToWebHooksTest.php | 2 +- 10 files changed, 204 insertions(+), 40 deletions(-) create mode 100644 module/Core/src/EventDispatcher/CloseDbConnectionEventListener.php create mode 100644 module/Core/src/EventDispatcher/CloseDbConnectionEventListenerDelegator.php create mode 100644 module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php create mode 100644 module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php rename module/{Rest => Core}/test/EventDispatcher/NotifyVisitToWebHooksTest.php (98%) diff --git a/composer.json b/composer.json index cd0d699a..164d17df 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.5", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "^3.0", + "shlinkio/shlink-common": "dev-master#aafa221ec979271713f87e23f17f6a6b5ae5ee67 as 3.0.1", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", "shlinkio/shlink-installer": "^4.3.2", diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index 29dbbd11..e885a283 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -29,6 +29,12 @@ return [ EventDispatcher\LocateShortUrlVisit::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class, ], + + 'delegators' => [ + EventDispatcher\LocateShortUrlVisit::class => [ + EventDispatcher\CloseDbConnectionEventListenerDelegator::class, + ], + ], ], ConfigAbstractFactory::class => [ diff --git a/module/Core/src/EventDispatcher/CloseDbConnectionEventListener.php b/module/Core/src/EventDispatcher/CloseDbConnectionEventListener.php new file mode 100644 index 00000000..7f2c7297 --- /dev/null +++ b/module/Core/src/EventDispatcher/CloseDbConnectionEventListener.php @@ -0,0 +1,32 @@ +em = $em; + $this->wrapped = $wrapped; + } + + public function __invoke(object $event): void + { + $this->em->open(); + + try { + ($this->wrapped)($event); + } finally { + $this->em->getConnection()->close(); + $this->em->clear(); + } + } +} diff --git a/module/Core/src/EventDispatcher/CloseDbConnectionEventListenerDelegator.php b/module/Core/src/EventDispatcher/CloseDbConnectionEventListenerDelegator.php new file mode 100644 index 00000000..cbfc7208 --- /dev/null +++ b/module/Core/src/EventDispatcher/CloseDbConnectionEventListenerDelegator.php @@ -0,0 +1,24 @@ +get('em'); + + return new CloseDbConnectionEventListener($em, $wrapped); + } +} diff --git a/module/Core/src/EventDispatcher/LocateShortUrlVisit.php b/module/Core/src/EventDispatcher/LocateShortUrlVisit.php index ff382770..6abbe02b 100644 --- a/module/Core/src/EventDispatcher/LocateShortUrlVisit.php +++ b/module/Core/src/EventDispatcher/LocateShortUrlVisit.php @@ -9,7 +9,6 @@ use Psr\EventDispatcher\EventDispatcherInterface; use Psr\Log\LoggerInterface; use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException; use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface; -use Shlinkio\Shlink\Common\Doctrine\ReopeningEntityManager; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; @@ -42,35 +41,22 @@ class LocateShortUrlVisit public function __invoke(ShortUrlVisited $shortUrlVisited): void { - // FIXME Temporarily handling DB connection reset here to fix https://github.com/shlinkio/shlink/issues/717 - // Remove when https://github.com/shlinkio/shlink-event-dispatcher/issues/23 is implemented - if ($this->em instanceof ReopeningEntityManager) { - $this->em->open(); - } - $visitId = $shortUrlVisited->visitId(); - try { - /** @var Visit|null $visit */ - $visit = $this->em->find(Visit::class, $visitId); - if ($visit === null) { - $this->logger->warning('Tried to locate visit with id "{visitId}", but it does not exist.', [ - 'visitId' => $visitId, - ]); - return; - } - - if ($this->downloadOrUpdateGeoLiteDb($visitId)) { - $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit); - } - - $this->eventDispatcher->dispatch(new VisitLocated($visitId)); - } finally { - // FIXME Temporarily handling DB connection reset here to fix https://github.com/shlinkio/shlink/issues/717 - // Remove when https://github.com/shlinkio/shlink-event-dispatcher/issues/23 is implemented - $this->em->getConnection()->close(); - $this->em->clear(); + /** @var Visit|null $visit */ + $visit = $this->em->find(Visit::class, $visitId); + if ($visit === null) { + $this->logger->warning('Tried to locate visit with id "{visitId}", but it does not exist.', [ + 'visitId' => $visitId, + ]); + return; } + + if ($this->downloadOrUpdateGeoLiteDb($visitId)) { + $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit); + } + + $this->eventDispatcher->dispatch(new VisitLocated($visitId)); } private function downloadOrUpdateGeoLiteDb(string $visitId): bool diff --git a/module/Core/src/EventDispatcher/NotifyVisitToWebHooks.php b/module/Core/src/EventDispatcher/NotifyVisitToWebHooks.php index de07d5ef..b3923b86 100644 --- a/module/Core/src/EventDispatcher/NotifyVisitToWebHooks.php +++ b/module/Core/src/EventDispatcher/NotifyVisitToWebHooks.php @@ -9,6 +9,7 @@ use Doctrine\ORM\EntityManagerInterface; use Fig\Http\Message\RequestMethodInterface; use GuzzleHttp\ClientInterface; use GuzzleHttp\Promise\Promise; +use GuzzleHttp\Promise\PromiseInterface; use GuzzleHttp\RequestOptions; use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Entity\Visit; @@ -89,12 +90,14 @@ class NotifyVisitToWebHooks */ private function performRequests(array $requestOptions, string $visitId): array { - return map($this->webhooks, function (string $webhook) use ($requestOptions, $visitId) { - $promise = $this->httpClient->requestAsync(RequestMethodInterface::METHOD_POST, $webhook, $requestOptions); - return $promise->otherwise( - partial_left(Closure::fromCallable([$this, 'logWebhookFailure']), $webhook, $visitId), - ); - }); + $logWebhookFailure = Closure::fromCallable([$this, 'logWebhookFailure']); + + return map( + $this->webhooks, + fn (string $webhook): PromiseInterface => $this->httpClient + ->requestAsync(RequestMethodInterface::METHOD_POST, $webhook, $requestOptions) + ->otherwise(partial_left($logWebhookFailure, $webhook, $visitId)), + ); } private function logWebhookFailure(string $webhook, string $visitId, Throwable $e): void diff --git a/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php new file mode 100644 index 00000000..41fe94fa --- /dev/null +++ b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php @@ -0,0 +1,43 @@ +container = $this->prophesize(ContainerInterface::class); + $this->delegator = new CloseDbConnectionEventListenerDelegator(); + } + + /** @test */ + public function properDependenciesArePassed(): void + { + $callbackInvoked = false; + $callback = function () use (&$callbackInvoked): callable { + $callbackInvoked = true; + + return function (): void { + }; + }; + + $em = $this->prophesize(ReopeningEntityManagerInterface::class); + $getEm = $this->container->get('em')->willReturn($em->reveal()); + + ($this->delegator)($this->container->reveal(), '', $callback); + + $this->assertTrue($callbackInvoked); + $getEm->shouldHaveBeenCalledOnce(); + } +} diff --git a/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php new file mode 100644 index 00000000..349e7724 --- /dev/null +++ b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php @@ -0,0 +1,75 @@ +em = $this->prophesize(ReopeningEntityManagerInterface::class); + } + + /** + * @test + * @dataProvider provideWrapped + */ + public function connectionIsOpenedBeforeAndClosedAfter(callable $wrapped, bool &$wrappedWasCalled): void + { + $conn = $this->prophesize(Connection::class); + $close = $conn->close()->will(function (): void { + }); + $getConn = $this->em->getConnection()->willReturn($conn->reveal()); + $clear = $this->em->clear()->will(function (): void { + }); + $open = $this->em->open()->will(function (): void { + }); + + $eventListener = new CloseDbConnectionEventListener($this->em->reveal(), $wrapped); + + try { + ($eventListener)(new stdClass()); + } catch (Throwable $e) { + // Ignore exceptions + } + + $this->assertTrue($wrappedWasCalled); + $close->shouldHaveBeenCalledOnce(); + $getConn->shouldHaveBeenCalledOnce(); + $clear->shouldHaveBeenCalledOnce(); + $open->shouldHaveBeenCalledOnce(); + } + + public function provideWrapped(): iterable + { + yield 'does not throw exception' => (function (): array { + $wrappedWasCalled = false; + $wrapped = function () use (&$wrappedWasCalled): void { + $wrappedWasCalled = true; + }; + + return [$wrapped, &$wrappedWasCalled]; + })(); + yield 'throws exception' => (function (): array { + $wrappedWasCalled = false; + $wrapped = function () use (&$wrappedWasCalled): void { + $wrappedWasCalled = true; + throw new RuntimeException('Some error'); + }; + + return [$wrapped, &$wrappedWasCalled]; + })(); + } +} diff --git a/module/Core/test/EventDispatcher/LocateShortUrlVisitTest.php b/module/Core/test/EventDispatcher/LocateShortUrlVisitTest.php index e35e7921..087c0e0b 100644 --- a/module/Core/test/EventDispatcher/LocateShortUrlVisitTest.php +++ b/module/Core/test/EventDispatcher/LocateShortUrlVisitTest.php @@ -4,7 +4,6 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\EventDispatcher; -use Doctrine\DBAL\Connection; use Doctrine\ORM\EntityManagerInterface; use PHPUnit\Framework\TestCase; use Prophecy\Argument; @@ -38,10 +37,6 @@ class LocateShortUrlVisitTest extends TestCase { $this->ipLocationResolver = $this->prophesize(IpLocationResolverInterface::class); $this->em = $this->prophesize(EntityManagerInterface::class); - $conn = $this->prophesize(Connection::class); - $this->em->getConnection()->willReturn($conn->reveal()); - $this->em->clear()->will(function (): void { - }); $this->logger = $this->prophesize(LoggerInterface::class); $this->dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class); $this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class); diff --git a/module/Rest/test/EventDispatcher/NotifyVisitToWebHooksTest.php b/module/Core/test/EventDispatcher/NotifyVisitToWebHooksTest.php similarity index 98% rename from module/Rest/test/EventDispatcher/NotifyVisitToWebHooksTest.php rename to module/Core/test/EventDispatcher/NotifyVisitToWebHooksTest.php index 369960e1..7a138960 100644 --- a/module/Rest/test/EventDispatcher/NotifyVisitToWebHooksTest.php +++ b/module/Core/test/EventDispatcher/NotifyVisitToWebHooksTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace ShlinkioTest\Shlink\Rest\EventDispatcher; +namespace ShlinkioTest\Shlink\Core\EventDispatcher; use Doctrine\ORM\EntityManagerInterface; use Exception; From 0c9deca3f8cd861b90811507168ca14e003bdd38 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 14 Mar 2020 19:47:22 +0100 Subject: [PATCH 02/87] Added symfony/mercure package and a container for development --- composer.json | 1 + docker-compose.yml | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/composer.json b/composer.json index 164d17df..c0f3514f 100644 --- a/composer.json +++ b/composer.json @@ -57,6 +57,7 @@ "symfony/console": "^5.0", "symfony/filesystem": "^5.0", "symfony/lock": "^5.0", + "symfony/mercure": "^0.3.0", "symfony/process": "^5.0" }, "require-dev": { diff --git a/docker-compose.yml b/docker-compose.yml index c78cf85f..d7867f32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: - shlink_db_maria - shlink_db_ms - shlink_redis + - shlink_mercure environment: LC_ALL: C @@ -47,6 +48,7 @@ services: - shlink_db_maria - shlink_db_ms - shlink_redis + - shlink_mercure environment: LC_ALL: C @@ -102,3 +104,12 @@ services: image: redis:5.0-alpine ports: - "6380:6379" + + shlink_mercure: + container_name: shlink_mercure + image: dunglas/mercure:v0.8 + ports: + - "3080:80" + environment: + CORS_ALLOWED_ORIGINS: "*" + JWT_KEY: "super_secret_key" From 10cad3324828b249125f200ec553dbbb4f925eab Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 14 Mar 2020 20:19:16 +0100 Subject: [PATCH 03/87] Added configuration for mercure integration --- config/autoload/mercure.global.php | 13 +++++++++++++ config/autoload/mercure.local.php.dist | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 config/autoload/mercure.global.php create mode 100644 config/autoload/mercure.local.php.dist diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php new file mode 100644 index 00000000..c70c9f06 --- /dev/null +++ b/config/autoload/mercure.global.php @@ -0,0 +1,13 @@ + [ + 'public_hub_url' => null, + 'internal_hub_url' => null, + 'jwt_secret' => null, + ], + +]; diff --git a/config/autoload/mercure.local.php.dist b/config/autoload/mercure.local.php.dist new file mode 100644 index 00000000..508d0f63 --- /dev/null +++ b/config/autoload/mercure.local.php.dist @@ -0,0 +1,13 @@ + [ + 'public_hub_url' => 'http://localhost:3080', + 'internal_hub_url' => 'http://shlink_mercure', + 'jwt_secret' => 'super_secret_key', + ], + +]; From 69962f1fe8cc9e42e7f68a57c661cc0f39c29507 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 15 Mar 2020 17:13:57 +0100 Subject: [PATCH 04/87] Added package to handle JWTs --- composer.json | 1 + 1 file changed, 1 insertion(+) diff --git a/composer.json b/composer.json index c0f3514f..987da145 100644 --- a/composer.json +++ b/composer.json @@ -34,6 +34,7 @@ "laminas/laminas-paginator": "^2.8", "laminas/laminas-servicemanager": "^3.4", "laminas/laminas-stdlib": "^3.2", + "lcobucci/jwt": "^4.0@alpha", "lstrojny/functional-php": "^1.9", "mezzio/mezzio": "^3.2", "mezzio/mezzio-fastroute": "^3.0", From 85440c1c5ff24cfbeb9c918026f329d6dd78e342 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 12:21:05 +0200 Subject: [PATCH 05/87] Improved mercure-related configs --- composer.json | 2 +- config/autoload/mercure.global.php | 2 ++ config/autoload/mercure.local.php.dist | 2 +- config/config.php | 2 -- docker-compose.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index 987da145..d71f0610 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.5", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "dev-master#aafa221ec979271713f87e23f17f6a6b5ae5ee67 as 3.0.1", + "shlinkio/shlink-common": "dev-master#3178fd18ce729d1dd63f4fb54f6a3696a11fef3f as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", "shlinkio/shlink-installer": "^4.3.2", diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php index c70c9f06..e04336f3 100644 --- a/config/autoload/mercure.global.php +++ b/config/autoload/mercure.global.php @@ -8,6 +8,8 @@ return [ 'public_hub_url' => null, 'internal_hub_url' => null, 'jwt_secret' => null, + 'jwt_days_duration' => 5, + 'jwt_issuer' => 'Shlink', ], ]; diff --git a/config/autoload/mercure.local.php.dist b/config/autoload/mercure.local.php.dist index 508d0f63..c760e771 100644 --- a/config/autoload/mercure.local.php.dist +++ b/config/autoload/mercure.local.php.dist @@ -7,7 +7,7 @@ return [ 'mercure' => [ 'public_hub_url' => 'http://localhost:3080', 'internal_hub_url' => 'http://shlink_mercure', - 'jwt_secret' => 'super_secret_key', + 'jwt_secret' => 'mercure_jwt_key', ], ]; diff --git a/config/config.php b/config/config.php index 98b4552b..5ab429f0 100644 --- a/config/config.php +++ b/config/config.php @@ -5,7 +5,6 @@ declare(strict_types=1); namespace Shlinkio\Shlink; use Laminas\ConfigAggregator; -use Laminas\ZendFrameworkBridge; use Mezzio; use Mezzio\ProblemDetails; @@ -30,7 +29,6 @@ return (new ConfigAggregator\ConfigAggregator([ ? new ConfigAggregator\PhpFileProvider('config/test/*.global.php') : new ConfigAggregator\LaminasConfigProvider('config/params/{generated_config.php,*.config.{php,json}}'), ], 'data/cache/app_config.php', [ - ZendFrameworkBridge\ConfigPostProcessor::class, Core\Config\SimplifiedConfigParser::class, Core\Config\BasePathPrefixer::class, Core\Config\DeprecatedConfigParser::class, diff --git a/docker-compose.yml b/docker-compose.yml index d7867f32..51a8a83b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,4 +112,4 @@ services: - "3080:80" environment: CORS_ALLOWED_ORIGINS: "*" - JWT_KEY: "super_secret_key" + JWT_KEY: "mercure_jwt_key" From 2ffbf03cf81fbd891e458f49a605ea81d4e29940 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 13:59:10 +0200 Subject: [PATCH 06/87] Created action to get mercure integration info --- config/autoload/mercure.global.php | 16 +++++ docs/swagger/definitions/MercureInfo.json | 18 +++++ docs/swagger/paths/v2_mercure-info.json | 67 +++++++++++++++++++ docs/swagger/swagger.json | 4 ++ module/Rest/config/dependencies.config.php | 3 + module/Rest/config/routes.config.php | 2 + module/Rest/src/Action/MercureAction.php | 56 ++++++++++++++++ .../Rest/src/Exception/MercureException.php | 30 +++++++++ 8 files changed, 196 insertions(+) create mode 100644 docs/swagger/definitions/MercureInfo.json create mode 100644 docs/swagger/paths/v2_mercure-info.json create mode 100644 module/Rest/src/Action/MercureAction.php create mode 100644 module/Rest/src/Exception/MercureException.php diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php index e04336f3..3466ce64 100644 --- a/config/autoload/mercure.global.php +++ b/config/autoload/mercure.global.php @@ -2,6 +2,9 @@ declare(strict_types=1); +use Laminas\ServiceManager\Proxy\LazyServiceFactory; +use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider; + return [ 'mercure' => [ @@ -12,4 +15,17 @@ return [ 'jwt_issuer' => 'Shlink', ], + 'dependencies' => [ + 'delegators' => [ + LcobucciJwtProvider::class => [ + LazyServiceFactory::class, + ], + ], + 'lazy_services' => [ + 'class_map' => [ + LcobucciJwtProvider::class => LcobucciJwtProvider::class, + ], + ], + ], + ]; diff --git a/docs/swagger/definitions/MercureInfo.json b/docs/swagger/definitions/MercureInfo.json new file mode 100644 index 00000000..ac1f273a --- /dev/null +++ b/docs/swagger/definitions/MercureInfo.json @@ -0,0 +1,18 @@ +{ + "type": "object", + "required": ["mercureHubUrl", "jwt", "jwtExpiration"], + "properties": { + "mercureHubUrl": { + "type": "string", + "description": "The public URL of the mercure hub that can be used to get real-time updates published by Shlink" + }, + "jwt": { + "type": "string", + "description": "A JWT with subscribe permissions which is valid with the mercure hub" + }, + "jwtExpiration": { + "type": "string", + "description": "The date (in ISO-8601 format) in which the JWT will expire" + } + } +} diff --git a/docs/swagger/paths/v2_mercure-info.json b/docs/swagger/paths/v2_mercure-info.json new file mode 100644 index 00000000..24f7fb5f --- /dev/null +++ b/docs/swagger/paths/v2_mercure-info.json @@ -0,0 +1,67 @@ +{ + "get": { + "operationId": "mercureInfo", + "tags": [ + "Integrations" + ], + "summary": "Get mercure integration info", + "description": "Returns information to consume updates published by Shlink on a mercure hub. https://mercure.rocks/", + "parameters": [ + { + "$ref": "../parameters/version.json" + } + ], + "security": [ + { + "ApiKey": [] + } + ], + "responses": { + "200": { + "description": "The mercure integration info", + "content": { + "application/json": { + "schema": { + "$ref": "../definitions/MercureInfo.json" + } + } + }, + "examples": { + "application/json": { + "mercureHubUrl": "https://example.com/.well-known/mercure", + "jwt": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJTaGxpbmsiLCJpYXQiOjE1ODY2ODY3MzIsImV4cCI6MTU4Njk0NTkzMiwibWVyY3VyZSI6eyJzdWJzY3JpYmUiOltdfX0.P-519lgU7dFz0bbNlRG1CXyqugGbaHon4kw6fu4QBdQ", + "jwtExpiration": "2020-04-15T12:18:52+02:00" + } + } + }, + "501": { + "description": "This Shlink instance is not integrated with a mercure hub", + "content": { + "application/problem+json": { + "schema": { + "$ref": "../definitions/Error.json" + } + } + }, + "examples": { + "application/json": { + "title": "Mercure integration not configured", + "type": "MERCURE_NOT_CONFIGURED", + "detail": "This Shlink instance is not integrated with a mercure hub.", + "status": 501 + } + } + }, + "500": { + "description": "Unexpected error.", + "content": { + "application/problem+json": { + "schema": { + "$ref": "../definitions/Error.json" + } + } + } + } + } + } +} diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 32e0caf3..c30bab97 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -82,6 +82,10 @@ "$ref": "paths/v1_short-urls_{shortCode}_visits.json" }, + "/rest/v{version}/mercure-info": { + "$ref": "paths/v2_mercure-info.json" + }, + "/rest/health": { "$ref": "paths/health.json" }, diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index b24ec1ee..e9fcbfa5 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -9,6 +9,7 @@ use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Laminas\ServiceManager\Factory\InvokableFactory; use Mezzio\Router\Middleware\ImplicitOptionsMiddleware; use Psr\Log\LoggerInterface; +use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider; use Shlinkio\Shlink\Core\Options\AppOptions; use Shlinkio\Shlink\Core\Service; use Shlinkio\Shlink\Rest\Service\ApiKeyService; @@ -20,6 +21,7 @@ return [ ApiKeyService::class => ConfigAbstractFactory::class, Action\HealthAction::class => ConfigAbstractFactory::class, + Action\MercureAction::class => ConfigAbstractFactory::class, Action\ShortUrl\CreateShortUrlAction::class => ConfigAbstractFactory::class, Action\ShortUrl\SingleStepCreateShortUrlAction::class => ConfigAbstractFactory::class, Action\ShortUrl\EditShortUrlAction::class => ConfigAbstractFactory::class, @@ -46,6 +48,7 @@ return [ ApiKeyService::class => ['em'], Action\HealthAction::class => [Connection::class, AppOptions::class, 'Logger_Shlink'], + Action\MercureAction::class => [LcobucciJwtProvider::class, 'config.mercure', 'Logger_Shlink'], Action\ShortUrl\CreateShortUrlAction::class => [ Service\UrlShortener::class, 'config.url_shortener.domain', diff --git a/module/Rest/config/routes.config.php b/module/Rest/config/routes.config.php index b104d81b..afb44249 100644 --- a/module/Rest/config/routes.config.php +++ b/module/Rest/config/routes.config.php @@ -33,6 +33,8 @@ return [ Action\Tag\DeleteTagsAction::getRouteDef(), Action\Tag\CreateTagsAction::getRouteDef(), Action\Tag\UpdateTagAction::getRouteDef(), + + Action\MercureAction::getRouteDef(), ], ]; diff --git a/module/Rest/src/Action/MercureAction.php b/module/Rest/src/Action/MercureAction.php new file mode 100644 index 00000000..7c33fa31 --- /dev/null +++ b/module/Rest/src/Action/MercureAction.php @@ -0,0 +1,56 @@ +jwtProvider = $jwtProvider; + $this->mercureConfig = $mercureConfig; + } + + public function handle(ServerRequestInterface $request): ResponseInterface + { + $hubUrl = $this->mercureConfig['public_hub_url'] ?? null; + if ($hubUrl === null) { + throw MercureException::mercureNotConfigured(); + } + + $days = $this->mercureConfig['jwt_days_duration'] ?? 3; + $expiresAt = Chronos::now()->addDays($days); + + try { + $jwt = $this->jwtProvider->buildSubscriptionToken($expiresAt); + } catch (Throwable $e) { + throw MercureException::mercureNotConfigured($e); + } + + return new JsonResponse([ + 'mercureHubUrl' => $hubUrl, + 'token' => $jwt, + 'jwtExpiration' => $expiresAt->toAtomString(), + ]); + } +} diff --git a/module/Rest/src/Exception/MercureException.php b/module/Rest/src/Exception/MercureException.php new file mode 100644 index 00000000..6c318e93 --- /dev/null +++ b/module/Rest/src/Exception/MercureException.php @@ -0,0 +1,30 @@ +detail = $e->getMessage(); + $e->title = self::TITLE; + $e->type = self::TYPE; + $e->status = StatusCodeInterface::STATUS_NOT_IMPLEMENTED; + + return $e; + } +} From 31db97228d885176bb0542ba78adad57ce206d9e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 14:19:51 +0200 Subject: [PATCH 07/87] Created MercureInfoActionTest --- module/Rest/config/dependencies.config.php | 4 +- module/Rest/config/routes.config.php | 2 +- ...ercureAction.php => MercureInfoAction.php} | 2 +- .../test/Action/MercureInfoActionTest.php | 107 ++++++++++++++++++ 4 files changed, 111 insertions(+), 4 deletions(-) rename module/Rest/src/Action/{MercureAction.php => MercureInfoAction.php} (96%) create mode 100644 module/Rest/test/Action/MercureInfoActionTest.php diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index e9fcbfa5..2434c88b 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -21,7 +21,7 @@ return [ ApiKeyService::class => ConfigAbstractFactory::class, Action\HealthAction::class => ConfigAbstractFactory::class, - Action\MercureAction::class => ConfigAbstractFactory::class, + Action\MercureInfoAction::class => ConfigAbstractFactory::class, Action\ShortUrl\CreateShortUrlAction::class => ConfigAbstractFactory::class, Action\ShortUrl\SingleStepCreateShortUrlAction::class => ConfigAbstractFactory::class, Action\ShortUrl\EditShortUrlAction::class => ConfigAbstractFactory::class, @@ -48,7 +48,7 @@ return [ ApiKeyService::class => ['em'], Action\HealthAction::class => [Connection::class, AppOptions::class, 'Logger_Shlink'], - Action\MercureAction::class => [LcobucciJwtProvider::class, 'config.mercure', 'Logger_Shlink'], + Action\MercureInfoAction::class => [LcobucciJwtProvider::class, 'config.mercure', 'Logger_Shlink'], Action\ShortUrl\CreateShortUrlAction::class => [ Service\UrlShortener::class, 'config.url_shortener.domain', diff --git a/module/Rest/config/routes.config.php b/module/Rest/config/routes.config.php index afb44249..3ced8357 100644 --- a/module/Rest/config/routes.config.php +++ b/module/Rest/config/routes.config.php @@ -34,7 +34,7 @@ return [ Action\Tag\CreateTagsAction::getRouteDef(), Action\Tag\UpdateTagAction::getRouteDef(), - Action\MercureAction::getRouteDef(), + Action\MercureInfoAction::getRouteDef(), ], ]; diff --git a/module/Rest/src/Action/MercureAction.php b/module/Rest/src/Action/MercureInfoAction.php similarity index 96% rename from module/Rest/src/Action/MercureAction.php rename to module/Rest/src/Action/MercureInfoAction.php index 7c33fa31..906a0b45 100644 --- a/module/Rest/src/Action/MercureAction.php +++ b/module/Rest/src/Action/MercureInfoAction.php @@ -13,7 +13,7 @@ use Shlinkio\Shlink\Common\Mercure\JwtProviderInterface; use Shlinkio\Shlink\Rest\Exception\MercureException; use Throwable; -class MercureAction extends AbstractRestAction +class MercureInfoAction extends AbstractRestAction { protected const ROUTE_PATH = '/mercure-info'; protected const ROUTE_ALLOWED_METHODS = [self::METHOD_GET]; diff --git a/module/Rest/test/Action/MercureInfoActionTest.php b/module/Rest/test/Action/MercureInfoActionTest.php new file mode 100644 index 00000000..f9489c99 --- /dev/null +++ b/module/Rest/test/Action/MercureInfoActionTest.php @@ -0,0 +1,107 @@ +provider = $this->prophesize(JwtProviderInterface::class); + } + + /** + * @test + * @dataProvider provideNoHostConfigs + */ + public function throwsExceptionWhenConfigDoesNotHavePublicHost(array $mercureConfig): void + { + $buildToken = $this->provider->buildSubscriptionToken(Argument::any())->willReturn('abc.123'); + + $action = new MercureInfoAction($this->provider->reveal(), $mercureConfig); + + $this->expectException(MercureException::class); + $buildToken->shouldNotBeCalled(); + + $action->handle(ServerRequestFactory::fromGlobals()); + } + + public function provideNoHostConfigs(): iterable + { + yield 'host not defined' => [[]]; + yield 'host is null' => [['public_hub_url' => null]]; + } + + /** + * @test + * @dataProvider provideValidConfigs + */ + public function throwsExceptionWhenBuildingTokenFails(array $mercureConfig): void + { + $buildToken = $this->provider->buildSubscriptionToken(Argument::any())->willThrow( + new RuntimeException('Error'), + ); + + $action = new MercureInfoAction($this->provider->reveal(), $mercureConfig); + + $this->expectException(MercureException::class); + $buildToken->shouldBeCalledOnce(); + + $action->handle(ServerRequestFactory::fromGlobals()); + } + + public function provideValidConfigs(): iterable + { + yield 'days not defined' => [['public_hub_url' => 'http://foobar.com']]; + yield 'days defined' => [['public_hub_url' => 'http://foobar.com', 'jwt_days_duration' => 20]]; + } + + /** + * @test + * @dataProvider provideDays + */ + public function returnsExpectedInfoWhenEverythingIsOk(?int $days): void + { + $buildToken = $this->provider->buildSubscriptionToken(Argument::any())->willReturn('abc.123'); + + $action = new MercureInfoAction($this->provider->reveal(), [ + 'public_hub_url' => 'http://foobar.com', + 'jwt_days_duration' => $days, + ]); + + /** @var JsonResponse $resp */ + $resp = $action->handle(ServerRequestFactory::fromGlobals()); + $payload = $resp->getPayload(); + + $this->assertArrayHasKey('mercureHubUrl', $payload); + $this->assertEquals('http://foobar.com', $payload['mercureHubUrl']); + $this->assertArrayHasKey('token', $payload); + $this->assertArrayHasKey('jwtExpiration', $payload); + $this->assertEquals( + Chronos::now()->addDays($days ?? 3)->startOfDay(), + Chronos::parse($payload['jwtExpiration'])->startOfDay(), + ); + $buildToken->shouldHaveBeenCalledOnce(); + } + + public function provideDays(): iterable + { + yield 'days not defined' => [null]; + yield 'days defined' => [10]; + } +} From 72d8edf4ff578139bcfe9a323c3b44e0c960c984 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 17:05:59 +0200 Subject: [PATCH 08/87] Created event listener that notifies mercure hub for new visits --- composer.json | 2 +- config/autoload/mercure.global.php | 6 +++ module/Core/config/dependencies.config.php | 4 ++ .../Core/config/event_dispatcher.config.php | 9 ++++ .../EventDispatcher/NotifyVisitToMercure.php | 54 +++++++++++++++++++ .../src/Mercure/MercureUpdatesGenerator.php | 33 ++++++++++++ .../MercureUpdatesGeneratorInterface.php | 13 +++++ 7 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 module/Core/src/EventDispatcher/NotifyVisitToMercure.php create mode 100644 module/Core/src/Mercure/MercureUpdatesGenerator.php create mode 100644 module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php diff --git a/composer.json b/composer.json index d71f0610..324d83b2 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.5", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "dev-master#3178fd18ce729d1dd63f4fb54f6a3696a11fef3f as 3.1.0", + "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", "shlinkio/shlink-installer": "^4.3.2", diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php index 3466ce64..04a698c3 100644 --- a/config/autoload/mercure.global.php +++ b/config/autoload/mercure.global.php @@ -4,6 +4,8 @@ declare(strict_types=1); use Laminas\ServiceManager\Proxy\LazyServiceFactory; use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider; +use Symfony\Component\Mercure\Publisher; +use Symfony\Component\Mercure\PublisherInterface; return [ @@ -20,10 +22,14 @@ return [ LcobucciJwtProvider::class => [ LazyServiceFactory::class, ], + Publisher::class => [ + LazyServiceFactory::class, + ], ], 'lazy_services' => [ 'class_map' => [ LcobucciJwtProvider::class => LcobucciJwtProvider::class, + Publisher::class => PublisherInterface::class, ], ], ], diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index a055e34b..63e6cfed 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -38,6 +38,8 @@ return [ Action\QrCodeAction::class => ConfigAbstractFactory::class, Resolver\PersistenceDomainResolver::class => ConfigAbstractFactory::class, + + Mercure\MercureUpdatesGenerator::class => ConfigAbstractFactory::class, ], ], @@ -83,6 +85,8 @@ return [ ], Resolver\PersistenceDomainResolver::class => ['em'], + + Mercure\MercureUpdatesGenerator::class => ['config.url_shortener.domain'], ], ]; diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index e885a283..c72e2d7a 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -8,12 +8,14 @@ use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Psr\EventDispatcher\EventDispatcherInterface; use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; +use Symfony\Component\Mercure\Publisher; return [ 'events' => [ 'regular' => [ EventDispatcher\VisitLocated::class => [ + EventDispatcher\NotifyVisitToMercure::class, EventDispatcher\NotifyVisitToWebHooks::class, ], ], @@ -28,6 +30,7 @@ return [ 'factories' => [ EventDispatcher\LocateShortUrlVisit::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class, + EventDispatcher\NotifyVisitToMercure::class => ConfigAbstractFactory::class, ], 'delegators' => [ @@ -53,6 +56,12 @@ return [ 'config.url_shortener.domain', Options\AppOptions::class, ], + EventDispatcher\NotifyVisitToMercure::class => [ + Publisher::class, + Mercure\MercureUpdatesGenerator::class, + 'em', + 'Logger_Shlink', + ], ], ]; diff --git a/module/Core/src/EventDispatcher/NotifyVisitToMercure.php b/module/Core/src/EventDispatcher/NotifyVisitToMercure.php new file mode 100644 index 00000000..69527413 --- /dev/null +++ b/module/Core/src/EventDispatcher/NotifyVisitToMercure.php @@ -0,0 +1,54 @@ +publisher = $publisher; + $this->em = $em; + $this->logger = $logger; + $this->updatesGenerator = $updatesGenerator; + } + + public function __invoke(VisitLocated $shortUrlLocated): void + { + $visitId = $shortUrlLocated->visitId(); + + /** @var Visit|null $visit */ + $visit = $this->em->find(Visit::class, $visitId); + if ($visit === null) { + $this->logger->warning('Tried to notify mercure for visit with id "{visitId}", but it does not exist.', [ + 'visitId' => $visitId, + ]); + return; + } + + try { + ($this->publisher)($this->updatesGenerator->newVisitUpdate($visit)); + } catch (Throwable $e) { + $this->logger->debug('Error while trying to notify mercure hub with new visit. {e}', [ + 'e' => $e, + ]); + } + } +} diff --git a/module/Core/src/Mercure/MercureUpdatesGenerator.php b/module/Core/src/Mercure/MercureUpdatesGenerator.php new file mode 100644 index 00000000..47d4b7be --- /dev/null +++ b/module/Core/src/Mercure/MercureUpdatesGenerator.php @@ -0,0 +1,33 @@ +transformer = new ShortUrlDataTransformer($domainConfig); + } + + public function newVisitUpdate(Visit $visit): Update + { + return new Update(self::NEW_VISIT_TOPIC, json_encode([ + 'shortUrl' => $this->transformer->transform($visit->getShortUrl()), + 'visit' => $visit->jsonSerialize(), + ], JSON_THROW_ON_ERROR)); + } +} diff --git a/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php b/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php new file mode 100644 index 00000000..af539803 --- /dev/null +++ b/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php @@ -0,0 +1,13 @@ + Date: Sun, 12 Apr 2020 17:50:09 +0200 Subject: [PATCH 09/87] Fixed mercure hub URL returned by MercureInfoAction --- module/Rest/src/Action/MercureInfoAction.php | 4 +++- module/Rest/test/Action/MercureInfoActionTest.php | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/module/Rest/src/Action/MercureInfoAction.php b/module/Rest/src/Action/MercureInfoAction.php index 906a0b45..73526d9f 100644 --- a/module/Rest/src/Action/MercureInfoAction.php +++ b/module/Rest/src/Action/MercureInfoAction.php @@ -13,6 +13,8 @@ use Shlinkio\Shlink\Common\Mercure\JwtProviderInterface; use Shlinkio\Shlink\Rest\Exception\MercureException; use Throwable; +use function sprintf; + class MercureInfoAction extends AbstractRestAction { protected const ROUTE_PATH = '/mercure-info'; @@ -48,7 +50,7 @@ class MercureInfoAction extends AbstractRestAction } return new JsonResponse([ - 'mercureHubUrl' => $hubUrl, + 'mercureHubUrl' => sprintf('%s/.well-known/mercure', $hubUrl), 'token' => $jwt, 'jwtExpiration' => $expiresAt->toAtomString(), ]); diff --git a/module/Rest/test/Action/MercureInfoActionTest.php b/module/Rest/test/Action/MercureInfoActionTest.php index f9489c99..5cb5e41b 100644 --- a/module/Rest/test/Action/MercureInfoActionTest.php +++ b/module/Rest/test/Action/MercureInfoActionTest.php @@ -89,7 +89,7 @@ class MercureInfoActionTest extends TestCase $payload = $resp->getPayload(); $this->assertArrayHasKey('mercureHubUrl', $payload); - $this->assertEquals('http://foobar.com', $payload['mercureHubUrl']); + $this->assertEquals('http://foobar.com/.well-known/mercure', $payload['mercureHubUrl']); $this->assertArrayHasKey('token', $payload); $this->assertArrayHasKey('jwtExpiration', $payload); $this->assertEquals( From e97dfbfdda923cfe34aae4dffdf77f976a470198 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 17:50:25 +0200 Subject: [PATCH 10/87] Created NotifyVisitToMercureTest --- .../NotifyVisitToMercureTest.php | 115 ++++++++++++++++++ .../test/Action/MercureInfoActionTest.php | 1 - 2 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php diff --git a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php new file mode 100644 index 00000000..ff5fb7c3 --- /dev/null +++ b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php @@ -0,0 +1,115 @@ +publisher = $this->prophesize(PublisherInterface::class); + $this->updatesGenerator = $this->prophesize(MercureUpdatesGeneratorInterface::class); + $this->em = $this->prophesize(EntityManagerInterface::class); + $this->logger = $this->prophesize(LoggerInterface::class); + + $this->listener = new NotifyVisitToMercure( + $this->publisher->reveal(), + $this->updatesGenerator->reveal(), + $this->em->reveal(), + $this->logger->reveal(), + ); + } + + /** @test */ + public function notificationIsNotSentWhenVisitCannotBeFound(): void + { + $visitId = '123'; + $findVisit = $this->em->find(Visit::class, $visitId)->willReturn(null); + $logWarning = $this->logger->warning( + 'Tried to notify mercure for visit with id "{visitId}", but it does not exist.', + ['visitId' => $visitId], + ); + $logDebug = $this->logger->debug(Argument::cetera()); + $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate(Argument::type(Visit::class))->willReturn( + new Update('', ''), + ); + $publish = $this->publisher->__invoke(Argument::type(Update::class)); + + ($this->listener)(new VisitLocated($visitId)); + + $findVisit->shouldHaveBeenCalledOnce(); + $logWarning->shouldHaveBeenCalledOnce(); + $logDebug->shouldNotHaveBeenCalled(); + $buildNewVisitUpdate->shouldNotHaveBeenCalled(); + $publish->shouldNotHaveBeenCalled(); + } + + /** @test */ + public function notificationIsSentWhenVisitIsFound(): void + { + $visitId = '123'; + $visit = new Visit(new ShortUrl(''), Visitor::emptyInstance()); + $update = new Update('', ''); + + $findVisit = $this->em->find(Visit::class, $visitId)->willReturn($visit); + $logWarning = $this->logger->warning(Argument::cetera()); + $logDebug = $this->logger->debug(Argument::cetera()); + $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); + $publish = $this->publisher->__invoke($update); + + ($this->listener)(new VisitLocated($visitId)); + + $findVisit->shouldHaveBeenCalledOnce(); + $logWarning->shouldNotHaveBeenCalled(); + $logDebug->shouldNotHaveBeenCalled(); + $buildNewVisitUpdate->shouldHaveBeenCalledOnce(); + $publish->shouldHaveBeenCalledOnce(); + } + + /** @test */ + public function debugIsLoggedWhenExceptionIsThrown(): void + { + $visitId = '123'; + $visit = new Visit(new ShortUrl(''), Visitor::emptyInstance()); + $update = new Update('', ''); + $e = new RuntimeException('Error'); + + $findVisit = $this->em->find(Visit::class, $visitId)->willReturn($visit); + $logWarning = $this->logger->warning(Argument::cetera()); + $logDebug = $this->logger->debug('Error while trying to notify mercure hub with new visit. {e}', [ + 'e' => $e, + ]); + $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); + $publish = $this->publisher->__invoke($update)->willThrow($e); + + ($this->listener)(new VisitLocated($visitId)); + + $findVisit->shouldHaveBeenCalledOnce(); + $logWarning->shouldNotHaveBeenCalled(); + $logDebug->shouldHaveBeenCalledOnce(); + $buildNewVisitUpdate->shouldHaveBeenCalledOnce(); + $publish->shouldHaveBeenCalledOnce(); + } +} diff --git a/module/Rest/test/Action/MercureInfoActionTest.php b/module/Rest/test/Action/MercureInfoActionTest.php index 5cb5e41b..5300fbaa 100644 --- a/module/Rest/test/Action/MercureInfoActionTest.php +++ b/module/Rest/test/Action/MercureInfoActionTest.php @@ -9,7 +9,6 @@ use Laminas\Diactoros\Response\JsonResponse; use Laminas\Diactoros\ServerRequestFactory; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\Argument\ArgumentsWildcard; use Prophecy\Prophecy\ObjectProphecy; use RuntimeException; use Shlinkio\Shlink\Common\Mercure\JwtProviderInterface; From 7f888c49b45a18bf7a6bcc1cd1d95b25be18e250 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 18:01:13 +0200 Subject: [PATCH 11/87] Created MercureUpdatesGeneratorTest --- ...DbConnectionEventListenerDelegatorTest.php | 2 +- .../CloseDbConnectionEventListenerTest.php | 2 +- .../NotifyVisitToMercureTest.php | 2 +- .../Mercure/MercureUpdatesGeneratorTest.php | 56 +++++++++++++++++++ .../test/Action/MercureInfoActionTest.php | 2 +- .../DefaultShortCodesLengthMiddlewareTest.php | 2 +- 6 files changed, 61 insertions(+), 5 deletions(-) create mode 100644 module/Core/test/Mercure/MercureUpdatesGeneratorTest.php diff --git a/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php index 41fe94fa..60113fc7 100644 --- a/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php +++ b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerDelegatorTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace ShlinkioApiTest\Shlink\Rest\EventDispatcher; +namespace ShlinkioTest\Shlink\Rest\EventDispatcher; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; diff --git a/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php index 349e7724..acc1784f 100644 --- a/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php +++ b/module/Core/test/EventDispatcher/CloseDbConnectionEventListenerTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace ShlinkioApiTest\Shlink\Core\EventDispatcher; +namespace ShlinkioTest\Shlink\Core\EventDispatcher; use Doctrine\DBAL\Connection; use PHPUnit\Framework\TestCase; diff --git a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php index ff5fb7c3..07f8bd1c 100644 --- a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php +++ b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace ShlinkioApiTest\Shlink\Core\EventDispatcher; +namespace ShlinkioTest\Shlink\Core\EventDispatcher; use Doctrine\ORM\EntityManagerInterface; use PHPUnit\Framework\TestCase; diff --git a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php new file mode 100644 index 00000000..361d3b2f --- /dev/null +++ b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php @@ -0,0 +1,56 @@ +generator = new MercureUpdatesGenerator([]); + } + + /** @test */ + public function visitIsProperlySerializedIntoUpdate(): void + { + $shortUrl = new ShortUrl(''); + $visit = new Visit($shortUrl, Visitor::emptyInstance()); + + $update = $this->generator->newVisitUpdate($visit); + + $this->assertEquals(['https://shlink.io/new_visit'], $update->getTopics()); + $this->assertEquals([ + 'shortUrl' => [ + 'shortCode' => $shortUrl->getShortCode(), + 'shortUrl' => 'http:/' . $shortUrl->getShortCode(), + 'longUrl' => '', + 'dateCreated' => $shortUrl->getDateCreated()->toAtomString(), + 'visitsCount' => 0, + 'tags' => [], + 'meta' => [ + 'validSince' => null, + 'validUntil' => null, + 'maxVisits' => null, + ], + 'domain' => null, + ], + 'visit' => [ + 'referer' => '', + 'userAgent' => '', + 'visitLocation' => null, + 'date' => $visit->getDate()->toAtomString(), + ], + ], json_decode($update->getData())); + } +} diff --git a/module/Rest/test/Action/MercureInfoActionTest.php b/module/Rest/test/Action/MercureInfoActionTest.php index 5300fbaa..df4b7260 100644 --- a/module/Rest/test/Action/MercureInfoActionTest.php +++ b/module/Rest/test/Action/MercureInfoActionTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace ShlinkioApiTest\Shlink\Rest\Action; +namespace ShlinkioTest\Shlink\Rest\Action; use Cake\Chronos\Chronos; use Laminas\Diactoros\Response\JsonResponse; diff --git a/module/Rest/test/Middleware/ShortUrl/DefaultShortCodesLengthMiddlewareTest.php b/module/Rest/test/Middleware/ShortUrl/DefaultShortCodesLengthMiddlewareTest.php index 38d875d9..c14a2f5c 100644 --- a/module/Rest/test/Middleware/ShortUrl/DefaultShortCodesLengthMiddlewareTest.php +++ b/module/Rest/test/Middleware/ShortUrl/DefaultShortCodesLengthMiddlewareTest.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace ShlinkioApiTest\Shlink\Rest\Middleware\ShortUrl; +namespace ShlinkioTest\Shlink\Rest\Middleware\ShortUrl; use Laminas\Diactoros\Response; use Laminas\Diactoros\ServerRequestFactory; From 8d888cb43de0251f838eede91aacf9fd6ca9d229 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 18:39:28 +0200 Subject: [PATCH 12/87] Documented how to use a mercure hub when using the docker image --- CHANGELOG.md | 31 ++++++++++++ config/test/test_config.global.php | 8 ++- docker/README.md | 78 ++++++++++++++++++++++++++---- 3 files changed, 105 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af2297d4..429c521a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,37 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). +## [Unreleased] + +#### Added + +* [#712](https://github.com/shlinkio/shlink/issues/712) Added support to integrate Shlink with a [mercure hub](https://mercure.rocks/) server. + + Thanks to that, Shlink will be able to publish events that can be consumed in real time. + + For now, only one topic (event) is published, identified by `https://shlink.io/new_visit`, which includes a payload with the visit and the shortUrl, every time a new visit occurs. + + The updates are only published when serving Shlink with swoole. + + Also, Shlink exposes a new endpoint `GET /rest/v2/mercure-info`, which returns the public URL of the mercure hub, and a valid JWT that can be used to subsribe to updates. + +#### Changed + +* *Nothing* + +#### Deprecated + +* *Nothing* + +#### Removed + +* *Nothing* + +#### Fixed + +* *Nothing* + + ## 2.1.3 - 2020-04-09 #### Added diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index fa51c240..5d57e981 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -79,13 +79,17 @@ return [ 'process-name' => 'shlink_test', 'options' => [ 'pid_file' => sys_get_temp_dir() . '/shlink-test-swoole.pid', - 'worker_num' => 1, - 'task_worker_num' => 1, 'enable_coroutine' => false, ], ], ], + 'mercure' => [ + 'public_hub_url' => null, + 'internal_hub_url' => null, + 'jwt_secret' => null, + ], + 'dependencies' => [ 'services' => [ 'shlink_test_api_client' => new Client([ diff --git a/docker/README.md b/docker/README.md index 3977fa37..3240f3d0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -73,18 +73,73 @@ It is possible to use a set of env vars to make this shlink instance interact wi Taking this into account, you could run shlink on a local docker service like this: ```bash -docker run --name shlink -p 8080:8080 -e SHORT_DOMAIN_HOST=doma.in -e SHORT_DOMAIN_SCHEMA=https -e DB_DRIVER=mysql -e DB_USER=root -e DB_PASSWORD=123abc -e DB_HOST=something.rds.amazonaws.com shlinkio/shlink:stable +docker run \ + --name shlink \ + -p 8080:8080 \ + -e SHORT_DOMAIN_HOST=doma.in \ + -e SHORT_DOMAIN_SCHEMA=https \ + -e DB_DRIVER=mysql \ + -e DB_USER=root \ + -e DB_PASSWORD=123abc \ + -e DB_HOST=something.rds.amazonaws.com \ + shlinkio/shlink:stable ``` You could even link to a local database running on a different container: ```bash -docker run --name shlink -p 8080:8080 [...] -e DB_HOST=some_mysql_container --link some_mysql_container shlinkio/shlink:stable +docker run \ + --name shlink \ + -p 8080:8080 \ + [...] \ + -e DB_HOST=some_mysql_container \ + --link some_mysql_container \ + shlinkio/shlink:stable ``` > If you have considered using SQLite but sharing the database file with a volume, read [this issue](https://github.com/shlinkio/shlink-docker-image/issues/40) first. -## Supported env vars +## Other integrations + +### Use an external redis server + +If you plan to run more than one Shlink instance, there are some resources that should be shared ([Multi instance considerations](#multi-instance-considerations)). + +One of those resources are the locks Shlink generates to prevent some operations to be run more than once in parallel (in the future, these redis servers could be used for other caching operations). + +In order to share those locks, you should use an external redis server (or a cluster of redis servers), by providing the `REDIS_SERVERS` env var. + +It can be either one server name or a comma-separated list of servers. + +> If more than one redis server is provided, Shlink will expect them to be configured as a [redis cluster](https://redis.io/topics/cluster-tutorial). + +### Integrate with a mercure hub server + +One way to get real time updates when certain events happen in Shlink is by integrating it with a [mercure hub](https://mercure.rocks/) server. + +If you do that, Shlink will publish updates and other clients can subscribe to those. + +There are three env vars you need to provide if you want to enable this: + +* `MERCURE_HUB_PUBLIC_URL`: **[Mandatory]**. The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. +* `MERCURE_HUB_INTERNAL_URL`: **[Optional]**. An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided, the `MERCURE_HUB_PUBLIC_URL` one will be used to publish updates. +* `MERCURE_JWT_SECRET`: **[Mandatory]**. The secret key that was provided to the mercure hub server, in order to be able to generate valid JWTs for publishing/subscribing to that server. + +So in order to run shlink with mercure integration, you would do it like this: + +```bash +docker run \ + --name shlink \ + -p 8080:8080 \ + -e SHORT_DOMAIN_HOST=doma.in \ + -e SHORT_DOMAIN_SCHEMA=https \ + -e "MERCURE_HUB_PUBLIC_URL=https://example.com" + -e "MERCURE_HUB_INTERNAL_URL=http://my-mercure-hub.prod.svc.cluster.local" + -e MERCURE_JWT_SECRET=super_secret_key + shlinkio/shlink:stable +``` + +## All supported env vars A few env vars have been already used in previous examples, but this image supports others that can be used to customize its behavior. @@ -114,12 +169,9 @@ This is the complete list of supported env vars: * `VISITS_WEBHOOKS`: A comma-separated list of URLs that will receive a `POST` request when a short URL receives a visit. * `DEFAULT_SHORT_CODES_LENGTH`: The length you want generated short codes to have. It defaults to 5 and has to be at least 4, so any value smaller than that will fall back to 4. * `REDIS_SERVERS`: A comma-separated list of redis servers where Shlink locks are stored (locks are used to prevent some operations to be run more than once in parallel). - - This is important when running more than one Shlink instance ([Multi instance considerations](#multi-instance-considerations)). If not provided, Shlink stores locks on every instance separately. - - If more than one server is provided, Shlink will expect them to be configured as a [redis cluster](https://redis.io/topics/cluster-tutorial). - - In the future, these redis servers could be used for other caching operations performed by shlink. +* `MERCURE_HUB_PUBLIC_URL`: The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. +* `MERCURE_HUB_INTERNAL_URL`: An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided but `MERCURE_HUB_PUBLIC_URL` was, the former one will be used to publish updates. +* `MERCURE_JWT_SECRET`: The secret key that was provided to the mercure hub server, in order to be able to generate valid JWTs for publishing/subscribing to that server. An example using all env vars could look like this: @@ -147,6 +199,9 @@ docker run \ -e TASK_WORKER_NUM=32 \ -e "VISITS_WEBHOOKS=http://my-api.com/api/v2.3/notify,https://third-party.io/foo" \ -e DEFAULT_SHORT_CODES_LENGTH=6 \ + -e "MERCURE_HUB_PUBLIC_URL=https://example.com" + -e "MERCURE_HUB_INTERNAL_URL=http://my-mercure-hub.prod.svc.cluster.local" + -e MERCURE_JWT_SECRET=super_secret_key shlinkio/shlink:stable ``` @@ -187,7 +242,10 @@ The whole configuration should have this format, but it can be split into multip "password": "123abc", "host": "something.rds.amazonaws.com", "port": "3306" - } + }, + "mercure_hub_public_url": "https://example.com", + "mercure_hub_internal_url": "http://my-mercure-hub.prod.svc.cluster.local", + "mercure_jwt_secret": "super_secret_key" } ``` From 934fa937b5ba92131a4261e595635ca03aeb698e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Apr 2020 20:41:23 +0200 Subject: [PATCH 13/87] Updated config parsers for docker image to accept new mercure env vars and configs --- docker/README.md | 20 +++++++++---------- docker/config/shlink_in_docker.local.php | 13 ++++++++++++ .../src/Config/SimplifiedConfigParser.php | 3 +++ .../Config/SimplifiedConfigParserTest.php | 9 +++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/docker/README.md b/docker/README.md index 3240f3d0..7cb4fb54 100644 --- a/docker/README.md +++ b/docker/README.md @@ -121,8 +121,8 @@ If you do that, Shlink will publish updates and other clients can subscribe to t There are three env vars you need to provide if you want to enable this: -* `MERCURE_HUB_PUBLIC_URL`: **[Mandatory]**. The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. -* `MERCURE_HUB_INTERNAL_URL`: **[Optional]**. An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided, the `MERCURE_HUB_PUBLIC_URL` one will be used to publish updates. +* `MERCURE_PUBLIC_HUB_URL`: **[Mandatory]**. The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. +* `MERCURE_INTERNAL_HUB_URL`: **[Optional]**. An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided, the `MERCURE_PUBLIC_HUB_URL` one will be used to publish updates. * `MERCURE_JWT_SECRET`: **[Mandatory]**. The secret key that was provided to the mercure hub server, in order to be able to generate valid JWTs for publishing/subscribing to that server. So in order to run shlink with mercure integration, you would do it like this: @@ -133,8 +133,8 @@ docker run \ -p 8080:8080 \ -e SHORT_DOMAIN_HOST=doma.in \ -e SHORT_DOMAIN_SCHEMA=https \ - -e "MERCURE_HUB_PUBLIC_URL=https://example.com" - -e "MERCURE_HUB_INTERNAL_URL=http://my-mercure-hub.prod.svc.cluster.local" + -e "MERCURE_PUBLIC_HUB_URL=https://example.com" + -e "MERCURE_INTERNAL_HUB_URL=http://my-mercure-hub.prod.svc.cluster.local" -e MERCURE_JWT_SECRET=super_secret_key shlinkio/shlink:stable ``` @@ -169,8 +169,8 @@ This is the complete list of supported env vars: * `VISITS_WEBHOOKS`: A comma-separated list of URLs that will receive a `POST` request when a short URL receives a visit. * `DEFAULT_SHORT_CODES_LENGTH`: The length you want generated short codes to have. It defaults to 5 and has to be at least 4, so any value smaller than that will fall back to 4. * `REDIS_SERVERS`: A comma-separated list of redis servers where Shlink locks are stored (locks are used to prevent some operations to be run more than once in parallel). -* `MERCURE_HUB_PUBLIC_URL`: The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. -* `MERCURE_HUB_INTERNAL_URL`: An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided but `MERCURE_HUB_PUBLIC_URL` was, the former one will be used to publish updates. +* `MERCURE_PUBLIC_HUB_URL`: The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. +* `MERCURE_INTERNAL_HUB_URL`: An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided but `MERCURE_PUBLIC_HUB_URL` was, the former one will be used to publish updates. * `MERCURE_JWT_SECRET`: The secret key that was provided to the mercure hub server, in order to be able to generate valid JWTs for publishing/subscribing to that server. An example using all env vars could look like this: @@ -199,8 +199,8 @@ docker run \ -e TASK_WORKER_NUM=32 \ -e "VISITS_WEBHOOKS=http://my-api.com/api/v2.3/notify,https://third-party.io/foo" \ -e DEFAULT_SHORT_CODES_LENGTH=6 \ - -e "MERCURE_HUB_PUBLIC_URL=https://example.com" - -e "MERCURE_HUB_INTERNAL_URL=http://my-mercure-hub.prod.svc.cluster.local" + -e "MERCURE_PUBLIC_HUB_URL=https://example.com" + -e "MERCURE_INTERNAL_HUB_URL=http://my-mercure-hub.prod.svc.cluster.local" -e MERCURE_JWT_SECRET=super_secret_key shlinkio/shlink:stable ``` @@ -243,8 +243,8 @@ The whole configuration should have this format, but it can be split into multip "host": "something.rds.amazonaws.com", "port": "3306" }, - "mercure_hub_public_url": "https://example.com", - "mercure_hub_internal_url": "http://my-mercure-hub.prod.svc.cluster.local", + "mercure_public_hub_url": "https://example.com", + "mercure_internal_hub_url": "http://my-mercure-hub.prod.svc.cluster.local", "mercure_jwt_secret": "super_secret_key" } ``` diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index 6cf86434..f1842de2 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -79,6 +79,17 @@ $helper = new class { $value = (int) env('DEFAULT_SHORT_CODES_LENGTH', DEFAULT_SHORT_CODES_LENGTH); return $value < MIN_SHORT_CODES_LENGTH ? MIN_SHORT_CODES_LENGTH : $value; } + + public function getMercureConfig(): array + { + $publicUrl = env('MERCURE_PUBLIC_HUB_URL'); + + return [ + 'public_hub_url' => $publicUrl, + 'internal_hub_url' => env('MERCURE_INTERNAL_HUB_URL', $publicUrl), + 'jwt_secret' => env('MERCURE_JWT_SECRET'), + ]; + } }; return [ @@ -147,4 +158,6 @@ return [ ], ], + 'mercure' => $helper->getMercureConfig(), + ]; diff --git a/module/Core/src/Config/SimplifiedConfigParser.php b/module/Core/src/Config/SimplifiedConfigParser.php index ee29d195..439a931a 100644 --- a/module/Core/src/Config/SimplifiedConfigParser.php +++ b/module/Core/src/Config/SimplifiedConfigParser.php @@ -33,6 +33,9 @@ class SimplifiedConfigParser 'task_worker_num' => ['mezzio-swoole', 'swoole-http-server', 'options', 'task_worker_num'], 'visits_webhooks' => ['url_shortener', 'visits_webhooks'], 'default_short_codes_length' => ['url_shortener', 'default_short_codes_length'], + 'mercure_public_hub_url' => ['mercure', 'public_hub_url'], + 'mercure_internal_hub_url' => ['mercure', 'internal_hub_url'], + 'mercure_jwt_secret' => ['mercure', 'jwt_secret'], ]; private const SIMPLIFIED_CONFIG_SIDE_EFFECTS = [ 'delete_short_url_threshold' => [ diff --git a/module/Core/test/Config/SimplifiedConfigParserTest.php b/module/Core/test/Config/SimplifiedConfigParserTest.php index 02f96423..6eee6737 100644 --- a/module/Core/test/Config/SimplifiedConfigParserTest.php +++ b/module/Core/test/Config/SimplifiedConfigParserTest.php @@ -60,6 +60,9 @@ class SimplifiedConfigParserTest extends TestCase 'https://third-party.io/foo', ], 'default_short_codes_length' => 8, + 'mercure_public_hub_url' => 'public_url', + 'mercure_internal_hub_url' => 'internal_url', + 'mercure_jwt_secret' => 'super_secret_value', ]; $expected = [ 'app_options' => [ @@ -127,6 +130,12 @@ class SimplifiedConfigParserTest extends TestCase ], ], ], + + 'mercure' => [ + 'public_hub_url' => 'public_url', + 'internal_hub_url' => 'internal_url', + 'jwt_secret' => 'super_secret_value', + ], ]; $result = ($this->postProcessor)(array_merge($config, $simplified)); From ba0678946f20ef1e68928fff005228dea7e8d4b3 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 13 Apr 2020 09:38:18 +0200 Subject: [PATCH 14/87] Updated installer to use a version supporting mercure options --- composer.json | 2 +- config/autoload/installer.global.php | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 324d83b2..7e0a8bfe 100644 --- a/composer.json +++ b/composer.json @@ -53,7 +53,7 @@ "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", - "shlinkio/shlink-installer": "^4.3.2", + "shlinkio/shlink-installer": "dev-master#c1412b9e9a150f443874f05452f7ce8e6f9e0339 as 4.4.0", "shlinkio/shlink-ip-geolocation": "^1.4", "symfony/console": "^5.0", "symfony/filesystem": "^5.0", diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php index c40d75d1..8f8562e6 100644 --- a/config/autoload/installer.global.php +++ b/config/autoload/installer.global.php @@ -31,6 +31,10 @@ return [ Option\WebWorkerNumConfigOption::class, Option\RedisServersConfigOption::class, Option\ShortCodeLengthOption::class, + Option\Mercure\EnableMercureConfigOption::class, + Option\Mercure\MercurePublicUrlConfigOption::class, + Option\Mercure\MercureInternalUrlConfigOption::class, + Option\Mercure\MercureJwtSecretConfigOption::class, ], 'installation_commands' => [ From a3a3ac1859e9ad35548f345323a2cf27e6cf2e86 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 13 Apr 2020 13:23:26 +0200 Subject: [PATCH 15/87] Added missing escaped characters --- docker/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index 7cb4fb54..bad7c010 100644 --- a/docker/README.md +++ b/docker/README.md @@ -199,9 +199,9 @@ docker run \ -e TASK_WORKER_NUM=32 \ -e "VISITS_WEBHOOKS=http://my-api.com/api/v2.3/notify,https://third-party.io/foo" \ -e DEFAULT_SHORT_CODES_LENGTH=6 \ - -e "MERCURE_PUBLIC_HUB_URL=https://example.com" - -e "MERCURE_INTERNAL_HUB_URL=http://my-mercure-hub.prod.svc.cluster.local" - -e MERCURE_JWT_SECRET=super_secret_key + -e "MERCURE_PUBLIC_HUB_URL=https://example.com" \ + -e "MERCURE_INTERNAL_HUB_URL=http://my-mercure-hub.prod.svc.cluster.local" \ + -e MERCURE_JWT_SECRET=super_secret_key \ shlinkio/shlink:stable ``` From ca2c32fa8c994e616f12b65247b211b4decad00e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 14 Apr 2020 20:24:36 +0200 Subject: [PATCH 16/87] Removed no-longer used dependencies --- composer.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/composer.json b/composer.json index 7e0a8bfe..3d710f83 100644 --- a/composer.json +++ b/composer.json @@ -17,13 +17,11 @@ "ext-pdo": "*", "akrabat/ip-address-middleware": "^1.0", "cakephp/chronos": "^1.2", - "cocur/slugify": "^3.0", "doctrine/cache": "^1.9", "doctrine/dbal": "^2.10", "doctrine/migrations": "^2.2", "doctrine/orm": "^2.7", "endroid/qr-code": "^3.6", - "firebase/php-jwt": "^4.0", "geoip2/geoip2": "^2.9", "guzzlehttp/guzzle": "^6.5.1", "laminas/laminas-config": "^3.3", From 3908f63b0d079ba4e743fa4c0c399f0fb425cdc4 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 14 Apr 2020 20:30:05 +0200 Subject: [PATCH 17/87] Updated to latest installer version --- composer.json | 2 +- config/autoload/installer.global.php | 40 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index 3d710f83..191a7d1b 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", - "shlinkio/shlink-installer": "dev-master#c1412b9e9a150f443874f05452f7ce8e6f9e0339 as 4.4.0", + "shlinkio/shlink-installer": "dev-master#487227bfc03233b44e8a113af0bc5bd0c6d67d5d as 5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", "symfony/console": "^5.0", "symfony/filesystem": "^5.0", diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php index 8f8562e6..6c1bea01 100644 --- a/config/autoload/installer.global.php +++ b/config/autoload/installer.global.php @@ -8,29 +8,29 @@ return [ 'installer' => [ 'enabled_options' => [ - Option\DatabaseDriverConfigOption::class, - Option\DatabaseNameConfigOption::class, - Option\DatabaseHostConfigOption::class, - Option\DatabasePortConfigOption::class, - Option\DatabaseUserConfigOption::class, - Option\DatabasePasswordConfigOption::class, - Option\DatabaseSqlitePathConfigOption::class, - Option\DatabaseMySqlOptionsConfigOption::class, - Option\ShortDomainHostConfigOption::class, - Option\ShortDomainSchemaConfigOption::class, - Option\ValidateUrlConfigOption::class, - Option\VisitsWebhooksConfigOption::class, - Option\BaseUrlRedirectConfigOption::class, - Option\InvalidShortUrlRedirectConfigOption::class, - Option\Regular404RedirectConfigOption::class, + Option\Database\DatabaseDriverConfigOption::class, + Option\Database\DatabaseNameConfigOption::class, + Option\Database\DatabaseHostConfigOption::class, + Option\Database\DatabasePortConfigOption::class, + Option\Database\DatabaseUserConfigOption::class, + Option\Database\DatabasePasswordConfigOption::class, + Option\Database\DatabaseSqlitePathConfigOption::class, + Option\Database\DatabaseMySqlOptionsConfigOption::class, + Option\UrlShortener\ShortDomainHostConfigOption::class, + Option\UrlShortener\ShortDomainSchemaConfigOption::class, + Option\UrlShortener\ValidateUrlConfigOption::class, + Option\Visit\VisitsWebhooksConfigOption::class, + Option\Redirect\BaseUrlRedirectConfigOption::class, + Option\Redirect\InvalidShortUrlRedirectConfigOption::class, + Option\Redirect\Regular404RedirectConfigOption::class, Option\DisableTrackParamConfigOption::class, - Option\CheckVisitsThresholdConfigOption::class, - Option\VisitsThresholdConfigOption::class, + Option\Visit\CheckVisitsThresholdConfigOption::class, + Option\Visit\VisitsThresholdConfigOption::class, Option\BasePathConfigOption::class, - Option\TaskWorkerNumConfigOption::class, - Option\WebWorkerNumConfigOption::class, + Option\Worker\TaskWorkerNumConfigOption::class, + Option\Worker\WebWorkerNumConfigOption::class, Option\RedisServersConfigOption::class, - Option\ShortCodeLengthOption::class, + Option\UrlShortener\ShortCodeLengthOption::class, Option\Mercure\EnableMercureConfigOption::class, Option\Mercure\MercurePublicUrlConfigOption::class, Option\Mercure\MercureInternalUrlConfigOption::class, From 18b12ab1e6062300893e462167404a9c01b0504a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 14 Apr 2020 20:57:25 +0200 Subject: [PATCH 18/87] Updated NotifyVisitToMercure to send both an update for all short URLs and one specific short URL --- .../EventDispatcher/NotifyVisitToMercure.php | 1 + .../src/Mercure/MercureUpdatesGenerator.php | 23 ++++++++++++++++--- .../MercureUpdatesGeneratorInterface.php | 2 ++ .../NotifyVisitToMercureTest.php | 16 +++++++++---- .../Mercure/MercureUpdatesGeneratorTest.php | 20 ++++++++++++---- 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/module/Core/src/EventDispatcher/NotifyVisitToMercure.php b/module/Core/src/EventDispatcher/NotifyVisitToMercure.php index 69527413..af6dd33f 100644 --- a/module/Core/src/EventDispatcher/NotifyVisitToMercure.php +++ b/module/Core/src/EventDispatcher/NotifyVisitToMercure.php @@ -44,6 +44,7 @@ class NotifyVisitToMercure } try { + ($this->publisher)($this->updatesGenerator->newShortUrlVisitUpdate($visit)); ($this->publisher)($this->updatesGenerator->newVisitUpdate($visit)); } catch (Throwable $e) { $this->logger->debug('Error while trying to notify mercure hub with new visit. {e}', [ diff --git a/module/Core/src/Mercure/MercureUpdatesGenerator.php b/module/Core/src/Mercure/MercureUpdatesGenerator.php index 47d4b7be..513b8e50 100644 --- a/module/Core/src/Mercure/MercureUpdatesGenerator.php +++ b/module/Core/src/Mercure/MercureUpdatesGenerator.php @@ -9,6 +9,7 @@ use Shlinkio\Shlink\Core\Transformer\ShortUrlDataTransformer; use Symfony\Component\Mercure\Update; use function json_encode; +use function sprintf; use const JSON_THROW_ON_ERROR; @@ -25,9 +26,25 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface public function newVisitUpdate(Visit $visit): Update { - return new Update(self::NEW_VISIT_TOPIC, json_encode([ + return new Update(self::NEW_VISIT_TOPIC, $this->serialize([ 'shortUrl' => $this->transformer->transform($visit->getShortUrl()), - 'visit' => $visit->jsonSerialize(), - ], JSON_THROW_ON_ERROR)); + 'visit' => $visit, + ])); + } + + public function newShortUrlVisitUpdate(Visit $visit): Update + { + $shortUrl = $visit->getShortUrl(); + $topic = sprintf('%s/%s', self::NEW_VISIT_TOPIC, $shortUrl->getShortCode()); + + return new Update($topic, $this->serialize([ + 'shortUrl' => $this->transformer->transform($visit->getShortUrl()), + 'visit' => $visit, + ])); + } + + private function serialize(array $data): string + { + return json_encode($data, JSON_THROW_ON_ERROR); } } diff --git a/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php b/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php index af539803..d433d9ad 100644 --- a/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php +++ b/module/Core/src/Mercure/MercureUpdatesGeneratorInterface.php @@ -10,4 +10,6 @@ use Symfony\Component\Mercure\Update; interface MercureUpdatesGeneratorInterface { public function newVisitUpdate(Visit $visit): Update; + + public function newShortUrlVisitUpdate(Visit $visit): Update; } diff --git a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php index 07f8bd1c..fce53344 100644 --- a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php +++ b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php @@ -43,7 +43,7 @@ class NotifyVisitToMercureTest extends TestCase } /** @test */ - public function notificationIsNotSentWhenVisitCannotBeFound(): void + public function notificationsAreNotSentWhenVisitCannotBeFound(): void { $visitId = '123'; $findVisit = $this->em->find(Visit::class, $visitId)->willReturn(null); @@ -52,6 +52,9 @@ class NotifyVisitToMercureTest extends TestCase ['visitId' => $visitId], ); $logDebug = $this->logger->debug(Argument::cetera()); + $buildNewShortUrlVisitUpdate = $this->updatesGenerator->newShortUrlVisitUpdate( + Argument::type(Visit::class), + )->willReturn(new Update('', '')); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate(Argument::type(Visit::class))->willReturn( new Update('', ''), ); @@ -62,12 +65,13 @@ class NotifyVisitToMercureTest extends TestCase $findVisit->shouldHaveBeenCalledOnce(); $logWarning->shouldHaveBeenCalledOnce(); $logDebug->shouldNotHaveBeenCalled(); + $buildNewShortUrlVisitUpdate->shouldNotHaveBeenCalled(); $buildNewVisitUpdate->shouldNotHaveBeenCalled(); $publish->shouldNotHaveBeenCalled(); } /** @test */ - public function notificationIsSentWhenVisitIsFound(): void + public function notificationsAreSentWhenVisitIsFound(): void { $visitId = '123'; $visit = new Visit(new ShortUrl(''), Visitor::emptyInstance()); @@ -76,6 +80,7 @@ class NotifyVisitToMercureTest extends TestCase $findVisit = $this->em->find(Visit::class, $visitId)->willReturn($visit); $logWarning = $this->logger->warning(Argument::cetera()); $logDebug = $this->logger->debug(Argument::cetera()); + $buildNewShortUrlVisitUpdate = $this->updatesGenerator->newShortUrlVisitUpdate($visit)->willReturn($update); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); $publish = $this->publisher->__invoke($update); @@ -84,8 +89,9 @@ class NotifyVisitToMercureTest extends TestCase $findVisit->shouldHaveBeenCalledOnce(); $logWarning->shouldNotHaveBeenCalled(); $logDebug->shouldNotHaveBeenCalled(); + $buildNewShortUrlVisitUpdate->shouldHaveBeenCalledOnce(); $buildNewVisitUpdate->shouldHaveBeenCalledOnce(); - $publish->shouldHaveBeenCalledOnce(); + $publish->shouldHaveBeenCalledTimes(2); } /** @test */ @@ -101,6 +107,7 @@ class NotifyVisitToMercureTest extends TestCase $logDebug = $this->logger->debug('Error while trying to notify mercure hub with new visit. {e}', [ 'e' => $e, ]); + $buildNewShortUrlVisitUpdate = $this->updatesGenerator->newShortUrlVisitUpdate($visit)->willReturn($update); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); $publish = $this->publisher->__invoke($update)->willThrow($e); @@ -109,7 +116,8 @@ class NotifyVisitToMercureTest extends TestCase $findVisit->shouldHaveBeenCalledOnce(); $logWarning->shouldNotHaveBeenCalled(); $logDebug->shouldHaveBeenCalledOnce(); - $buildNewVisitUpdate->shouldHaveBeenCalledOnce(); + $buildNewShortUrlVisitUpdate->shouldHaveBeenCalledOnce(); + $buildNewVisitUpdate->shouldNotHaveBeenCalled(); $publish->shouldHaveBeenCalledOnce(); } } diff --git a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php index 361d3b2f..f72cd82d 100644 --- a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php +++ b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php @@ -8,6 +8,7 @@ use PHPUnit\Framework\TestCase; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Mercure\MercureUpdatesGenerator; +use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\Visitor; use function Shlinkio\Shlink\Common\json_decode; @@ -21,15 +22,18 @@ class MercureUpdatesGeneratorTest extends TestCase $this->generator = new MercureUpdatesGenerator([]); } - /** @test */ - public function visitIsProperlySerializedIntoUpdate(): void + /** + * @test + * @dataProvider provideMethod + */ + public function visitIsProperlySerializedIntoUpdate(string $method, string $expectedTopic): void { - $shortUrl = new ShortUrl(''); + $shortUrl = new ShortUrl('', ShortUrlMeta::fromRawData(['customSlug' => 'foo'])); $visit = new Visit($shortUrl, Visitor::emptyInstance()); - $update = $this->generator->newVisitUpdate($visit); + $update = $this->generator->{$method}($visit); - $this->assertEquals(['https://shlink.io/new_visit'], $update->getTopics()); + $this->assertEquals([$expectedTopic], $update->getTopics()); $this->assertEquals([ 'shortUrl' => [ 'shortCode' => $shortUrl->getShortCode(), @@ -53,4 +57,10 @@ class MercureUpdatesGeneratorTest extends TestCase ], ], json_decode($update->getData())); } + + public function provideMethod(): iterable + { + yield 'newVisitUpdate' => ['newVisitUpdate', 'https://shlink.io/new_visit']; + yield 'newShortUrlVisitUpdate' => ['newShortUrlVisitUpdate', 'https://shlink.io/new_visit/foo']; + } } From 6ba6b951bf07837235292135c434fcf7c61dcee0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 16 Apr 2020 22:25:12 +0200 Subject: [PATCH 19/87] Changed mercure topics to be dash-cased --- CHANGELOG.md | 5 ++++- module/Core/src/Mercure/MercureUpdatesGenerator.php | 2 +- module/Core/test/Mercure/MercureUpdatesGeneratorTest.php | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 429c521a..57d41f0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this Thanks to that, Shlink will be able to publish events that can be consumed in real time. - For now, only one topic (event) is published, identified by `https://shlink.io/new_visit`, which includes a payload with the visit and the shortUrl, every time a new visit occurs. + For now, two topics (events) are published, when new visits occur. Both include a payload with the visit and the shortUrl: + + * A visit occurs on any short URL: `https://shlink.io/new-visit`. + * A visit occurs on short URLs with a specific short code: `https://shlink.io/new-visit/{shortCode}`. The updates are only published when serving Shlink with swoole. diff --git a/module/Core/src/Mercure/MercureUpdatesGenerator.php b/module/Core/src/Mercure/MercureUpdatesGenerator.php index 513b8e50..aad072f2 100644 --- a/module/Core/src/Mercure/MercureUpdatesGenerator.php +++ b/module/Core/src/Mercure/MercureUpdatesGenerator.php @@ -15,7 +15,7 @@ use const JSON_THROW_ON_ERROR; final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface { - private const NEW_VISIT_TOPIC = 'https://shlink.io/new_visit'; + private const NEW_VISIT_TOPIC = 'https://shlink.io/new-visit'; private ShortUrlDataTransformer $transformer; diff --git a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php index f72cd82d..992e25d6 100644 --- a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php +++ b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php @@ -60,7 +60,7 @@ class MercureUpdatesGeneratorTest extends TestCase public function provideMethod(): iterable { - yield 'newVisitUpdate' => ['newVisitUpdate', 'https://shlink.io/new_visit']; - yield 'newShortUrlVisitUpdate' => ['newShortUrlVisitUpdate', 'https://shlink.io/new_visit/foo']; + yield 'newVisitUpdate' => ['newVisitUpdate', 'https://shlink.io/new-visit']; + yield 'newShortUrlVisitUpdate' => ['newShortUrlVisitUpdate', 'https://shlink.io/new-visit/foo']; } } From 655fd58a9d00ae452461bc02f3261e37c850566c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 16 Apr 2020 22:44:08 +0200 Subject: [PATCH 20/87] Added async API spec file --- docs/async-api/async-api.json | 210 ++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 docs/async-api/async-api.json diff --git a/docs/async-api/async-api.json b/docs/async-api/async-api.json new file mode 100644 index 00000000..5279ce91 --- /dev/null +++ b/docs/async-api/async-api.json @@ -0,0 +1,210 @@ +{ + "asyncapi": "2.0.0", + "info": { + "title": "Shlink", + "version": "2.0.0", + "description": "Shlink, the self-hosted URL shortener", + "license": { + "name": "MIT", + "url": "https://github.com/shlinkio/shlink/blob/develop/LICENSE" + } + }, + "defaultContentType": "application/json", + "channels": { + "http://shlink.io/new-visit": { + "subscribe": { + "summary": "Receive information about any new visit occurring on any short URL.", + "operationId": "newVisit", + "message": { + "payload": { + "type": "object", + "additionalProperties": false, + "properties": { + "shortUrl": { + "$ref": "#/components/schemas/ShortUrl" + }, + "visit": { + "$ref": "#/components/schemas/Visit" + } + } + } + } + } + }, + "http://shlink.io/new-visit/{shortCode}": { + "parameters": { + "shortCode": { + "description": "The short code of the short URL", + "schema": { + "type": "string" + } + } + }, + "subscribe": { + "summary": "Receive information about any new visit occurring on a specific short URL.", + "operationId": "newShortUrlVisit", + "message": { + "payload": { + "type": "object", + "additionalProperties": false, + "properties": { + "shortUrl": { + "$ref": "#/components/schemas/ShortUrl" + }, + "visit": { + "$ref": "#/components/schemas/Visit" + } + } + } + } + } + } + }, + "components": { + "schemas": { + "ShortUrl": { + "type": "object", + "properties": { + "shortCode": { + "type": "string", + "description": "The short code for this short URL." + }, + "shortUrl": { + "type": "string", + "description": "The short URL." + }, + "longUrl": { + "type": "string", + "description": "The original long URL." + }, + "dateCreated": { + "type": "string", + "format": "date-time", + "description": "The date in which the short URL was created in ISO format." + }, + "visitsCount": { + "type": "integer", + "description": "The number of visits that this short URL has recieved." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "A list of tags applied to this short URL" + }, + "meta": { + "$ref": "#/components/schemas/ShortUrlMeta" + }, + "domain": { + "type": "string", + "description": "The domain in which the short URL was created. Null if it belongs to default domain." + } + }, + "example": { + "shortCode": "12C18", + "shortUrl": "https://doma.in/12C18", + "longUrl": "https://store.steampowered.com", + "dateCreated": "2016-08-21T20:34:16+02:00", + "visitsCount": 328, + "tags": [ + "games", + "tech" + ], + "meta": { + "validSince": "2017-01-21T00:00:00+02:00", + "validUntil": null, + "maxVisits": 100 + }, + "domain": "example.com" + } + }, + "ShortUrlMeta": { + "type": "object", + "required": [ + "validSince", + "validUntil", + "maxVisits" + ], + "properties": { + "validSince": { + "description": "The date (in ISO-8601 format) from which this short code will be valid", + "type": "string", + "nullable": true + }, + "validUntil": { + "description": "The date (in ISO-8601 format) until which this short code will be valid", + "type": "string", + "nullable": true + }, + "maxVisits": { + "description": "The maximum number of allowed visits for this short code", + "type": "number", + "nullable": true + } + } + }, + "Visit": { + "type": "object", + "properties": { + "referer": { + "type": "string", + "description": "The origin from which the visit was performed" + }, + "date": { + "type": "string", + "format": "date-time", + "description": "The date in which the visit was performed" + }, + "userAgent": { + "type": "string", + "description": "The user agent from which the visit was performed" + }, + "visitLocation": { + "$ref": "#/components/schemas/VisitLocation" + } + }, + "example": { + "referer": "https://t.co", + "date": "2015-08-20T05:05:03+04:00", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "visitLocation": { + "cityName": "Cupertino", + "countryCode": "US", + "countryName": "United States", + "latitude": 37.3042, + "longitude": -122.0946, + "regionName": "California", + "timezone": "America/Los_Angeles" + } + } + }, + "VisitLocation": { + "type": "object", + "properties": { + "cityName": { + "type": "string" + }, + "countryCode": { + "type": "string" + }, + "countryName": { + "type": "string" + }, + "latitude": { + "type": "number" + }, + "longitude": { + "type": "number" + }, + "regionName": { + "type": "string" + }, + "timezone": { + "type": "string" + } + } + } + } + } +} From 078c8ea01151b97eda1815709a59041e2be0c4a0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 18 Apr 2020 11:29:49 +0200 Subject: [PATCH 21/87] Changed default mercure token duration to 1 day --- config/autoload/mercure.global.php | 1 - module/Rest/src/Action/MercureInfoAction.php | 2 +- module/Rest/test/Action/MercureInfoActionTest.php | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php index 04a698c3..1a404dca 100644 --- a/config/autoload/mercure.global.php +++ b/config/autoload/mercure.global.php @@ -13,7 +13,6 @@ return [ 'public_hub_url' => null, 'internal_hub_url' => null, 'jwt_secret' => null, - 'jwt_days_duration' => 5, 'jwt_issuer' => 'Shlink', ], diff --git a/module/Rest/src/Action/MercureInfoAction.php b/module/Rest/src/Action/MercureInfoAction.php index 73526d9f..bad4cb9c 100644 --- a/module/Rest/src/Action/MercureInfoAction.php +++ b/module/Rest/src/Action/MercureInfoAction.php @@ -40,7 +40,7 @@ class MercureInfoAction extends AbstractRestAction throw MercureException::mercureNotConfigured(); } - $days = $this->mercureConfig['jwt_days_duration'] ?? 3; + $days = $this->mercureConfig['jwt_days_duration'] ?? 1; $expiresAt = Chronos::now()->addDays($days); try { diff --git a/module/Rest/test/Action/MercureInfoActionTest.php b/module/Rest/test/Action/MercureInfoActionTest.php index df4b7260..d40b3f70 100644 --- a/module/Rest/test/Action/MercureInfoActionTest.php +++ b/module/Rest/test/Action/MercureInfoActionTest.php @@ -92,7 +92,7 @@ class MercureInfoActionTest extends TestCase $this->assertArrayHasKey('token', $payload); $this->assertArrayHasKey('jwtExpiration', $payload); $this->assertEquals( - Chronos::now()->addDays($days ?? 3)->startOfDay(), + Chronos::now()->addDays($days ?? 1)->startOfDay(), Chronos::parse($payload['jwtExpiration'])->startOfDay(), ); $buildToken->shouldHaveBeenCalledOnce(); From 7c6827ea9fd9a3cd61bbc3f583cc32f7e94b3b02 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 18 Apr 2020 13:21:46 +0200 Subject: [PATCH 22/87] Added MYSQL_ATTR_USE_BUFFERED_QUERY driver option with value true for mysql/maria connections --- composer.json | 2 +- config/autoload/entity-manager.local.php.dist | 1 + config/test/test_config.global.php | 1 + docker/config/shlink_in_docker.local.php | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 191a7d1b..c324b39c 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", - "shlinkio/shlink-installer": "dev-master#487227bfc03233b44e8a113af0bc5bd0c6d67d5d as 5.0.0", + "shlinkio/shlink-installer": "dev-master#508652c895fb88512b8685198be1e2cfca59f4ce as 5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", "symfony/console": "^5.0", "symfony/filesystem": "^5.0", diff --git a/config/autoload/entity-manager.local.php.dist b/config/autoload/entity-manager.local.php.dist index 3c38fb82..d2fc3fd8 100644 --- a/config/autoload/entity-manager.local.php.dist +++ b/config/autoload/entity-manager.local.php.dist @@ -12,6 +12,7 @@ return [ 'host' => 'shlink_db', 'driverOptions' => [ PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', + PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, ], ], ], diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index 5d57e981..6aad5375 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -35,6 +35,7 @@ $buildDbConnection = function (): array { 'charset' => 'utf8', 'driverOptions' => [ PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', + PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, ], ], 'postgres' => [ diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index f1842de2..725a019b 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -41,6 +41,8 @@ $helper = new class { $driverOptions = ! contains(['maria', 'mysql'], $driver) ? [] : [ // 1002 -> PDO::MYSQL_ATTR_INIT_COMMAND 1002 => 'SET NAMES utf8', + // 1000 -> PDO::MYSQL_ATTR_USE_BUFFERED_QUERY + 1000 => true, ]; return [ 'driver' => self::DB_DRIVERS_MAP[$driver], From 66a7f279c255e21cfedd5551bd511e619846d37d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 18 Apr 2020 13:22:51 +0200 Subject: [PATCH 23/87] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57d41f0a..bad91e85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Fixed -* *Nothing* +* [#729](https://github.com/shlinkio/shlink/issues/729) Fixed weird error when fetching multiple visits result sets concurrently using mariadb or mysql. ## 2.1.3 - 2020-04-09 From e6d914cfe18ad1a94a7a9f3e1bbaa5fb7d0245c1 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 18 Apr 2020 14:01:24 +0200 Subject: [PATCH 24/87] Ensured mysql config is not loaded for sqlite test envs --- config/autoload/entity-manager.local.php.dist | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/config/autoload/entity-manager.local.php.dist b/config/autoload/entity-manager.local.php.dist index d2fc3fd8..1faed328 100644 --- a/config/autoload/entity-manager.local.php.dist +++ b/config/autoload/entity-manager.local.php.dist @@ -2,6 +2,14 @@ declare(strict_types=1); +use function Shlinkio\Shlink\Common\env; + +// When running tests, any mysql-specific option can interfere with other drivers +$driverOptions = env('APP_ENV') === 'test' ? [] : [ + PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', + PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, +]; + return [ 'entity_manager' => [ @@ -10,10 +18,7 @@ return [ 'password' => 'root', 'driver' => 'pdo_mysql', 'host' => 'shlink_db', - 'driverOptions' => [ - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', - PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, - ], + 'driverOptions' => $driverOptions, ], ], From 4917e53acd26ceda9c554e77529cd1c9221d0e5c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 25 Apr 2020 13:44:09 +0200 Subject: [PATCH 25/87] Configured an nginx container acting as a reverse proxy for the mercure container --- config/autoload/mercure.local.php.dist | 4 ++-- data/infra/mercure_proxy_vhost.conf | 17 +++++++++++++++++ docker-compose.yml | 16 +++++++++++++++- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 data/infra/mercure_proxy_vhost.conf diff --git a/config/autoload/mercure.local.php.dist b/config/autoload/mercure.local.php.dist index c760e771..b10ad86e 100644 --- a/config/autoload/mercure.local.php.dist +++ b/config/autoload/mercure.local.php.dist @@ -5,8 +5,8 @@ declare(strict_types=1); return [ 'mercure' => [ - 'public_hub_url' => 'http://localhost:3080', - 'internal_hub_url' => 'http://shlink_mercure', + 'public_hub_url' => 'http://localhost:8001', + 'internal_hub_url' => 'http://shlink_mercure_proxy', 'jwt_secret' => 'mercure_jwt_key', ], diff --git a/data/infra/mercure_proxy_vhost.conf b/data/infra/mercure_proxy_vhost.conf new file mode 100644 index 00000000..df986b37 --- /dev/null +++ b/data/infra/mercure_proxy_vhost.conf @@ -0,0 +1,17 @@ +server { + listen 80 default_server; + + error_log /home/shlink/www/data/infra/nginx/mercure_proxy.error.log; + + location / { + proxy_pass http://shlink_mercure; + proxy_read_timeout 24h; + proxy_http_version 1.1; + proxy_set_header Connection ""; + + ## Be sure to set USE_FORWARDED_HEADERS=1 to allow the hub to use those headers ## + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 51a8a83b..ec56b883 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: shlink_nginx: container_name: shlink_nginx - image: nginx:1.17.6-alpine + image: nginx:1.17.10-alpine ports: - "8000:80" volumes: @@ -28,6 +28,7 @@ services: - shlink_db_ms - shlink_redis - shlink_mercure + - shlink_mercure_proxy environment: LC_ALL: C @@ -49,6 +50,7 @@ services: - shlink_db_ms - shlink_redis - shlink_mercure + - shlink_mercure_proxy environment: LC_ALL: C @@ -105,6 +107,17 @@ services: ports: - "6380:6379" + shlink_mercure_proxy: + container_name: shlink_mercure_proxy + image: nginx:1.17.10-alpine + ports: + - "8001:80" + volumes: + - ./:/home/shlink/www + - ./data/infra/mercure_proxy_vhost.conf:/etc/nginx/conf.d/default.conf + links: + - shlink_mercure + shlink_mercure: container_name: shlink_mercure image: dunglas/mercure:v0.8 @@ -113,3 +126,4 @@ services: environment: CORS_ALLOWED_ORIGINS: "*" JWT_KEY: "mercure_jwt_key" + USE_FORWARDED_HEADERS: "1" From 4f988d223b45b5198cd8881d9503dc4b902f675d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 25 Apr 2020 19:13:47 +0200 Subject: [PATCH 26/87] Fixed error when cleaning metadata cache during installation with APCu enabled --- CHANGELOG.md | 1 + composer.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bad91e85..7e26b013 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Fixed * [#729](https://github.com/shlinkio/shlink/issues/729) Fixed weird error when fetching multiple visits result sets concurrently using mariadb or mysql. +* [#735](https://github.com/shlinkio/shlink/issues/735) Fixed error when cleaning metadata cache during installation when APCu is enabled. ## 2.1.3 - 2020-04-09 diff --git a/composer.json b/composer.json index c324b39c..1737f7a1 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", - "shlinkio/shlink-installer": "dev-master#508652c895fb88512b8685198be1e2cfca59f4ce as 5.0.0", + "shlinkio/shlink-installer": "dev-master#f51a2186cf474fb5773b0ef74b8533878de9dd1e as 5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", "symfony/console": "^5.0", "symfony/filesystem": "^5.0", From 15a8305209ba3cfcd5b6a5673e9b8c50bbfe1c47 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 25 Apr 2020 19:58:49 +0200 Subject: [PATCH 27/87] Fixed random 503 responses from the HealthAction when the database connection injected on it has expired --- module/Rest/config/dependencies.config.php | 3 +-- module/Rest/src/Action/HealthAction.php | 10 +++++----- module/Rest/test/Action/HealthActionTest.php | 6 +++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index 2434c88b..f6af6f85 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -4,7 +4,6 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Rest; -use Doctrine\DBAL\Connection; use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Laminas\ServiceManager\Factory\InvokableFactory; use Mezzio\Router\Middleware\ImplicitOptionsMiddleware; @@ -47,7 +46,7 @@ return [ ConfigAbstractFactory::class => [ ApiKeyService::class => ['em'], - Action\HealthAction::class => [Connection::class, AppOptions::class, 'Logger_Shlink'], + Action\HealthAction::class => ['em', AppOptions::class, 'Logger_Shlink'], Action\MercureInfoAction::class => [LcobucciJwtProvider::class, 'config.mercure', 'Logger_Shlink'], Action\ShortUrl\CreateShortUrlAction::class => [ Service\UrlShortener::class, diff --git a/module/Rest/src/Action/HealthAction.php b/module/Rest/src/Action/HealthAction.php index 4f3b9c64..ef1b6b88 100644 --- a/module/Rest/src/Action/HealthAction.php +++ b/module/Rest/src/Action/HealthAction.php @@ -4,7 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Rest\Action; -use Doctrine\DBAL\Connection; +use Doctrine\ORM\EntityManagerInterface; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; @@ -21,13 +21,13 @@ class HealthAction extends AbstractRestAction protected const ROUTE_PATH = '/health'; protected const ROUTE_ALLOWED_METHODS = [self::METHOD_GET]; + private EntityManagerInterface $em; private AppOptions $options; - private Connection $conn; - public function __construct(Connection $conn, AppOptions $options, ?LoggerInterface $logger = null) + public function __construct(EntityManagerInterface $em, AppOptions $options, ?LoggerInterface $logger = null) { parent::__construct($logger); - $this->conn = $conn; + $this->em = $em; $this->options = $options; } @@ -39,7 +39,7 @@ class HealthAction extends AbstractRestAction public function handle(ServerRequestInterface $request): ResponseInterface { try { - $connected = $this->conn->ping(); + $connected = $this->em->getConnection()->ping(); } catch (Throwable $e) { $connected = false; } diff --git a/module/Rest/test/Action/HealthActionTest.php b/module/Rest/test/Action/HealthActionTest.php index 813fa5cc..2ec68d25 100644 --- a/module/Rest/test/Action/HealthActionTest.php +++ b/module/Rest/test/Action/HealthActionTest.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Rest\Action; use Doctrine\DBAL\Connection; +use Doctrine\ORM\EntityManagerInterface; use Exception; use Laminas\Diactoros\Response\JsonResponse; use Laminas\Diactoros\ServerRequest; @@ -21,7 +22,10 @@ class HealthActionTest extends TestCase public function setUp(): void { $this->conn = $this->prophesize(Connection::class); - $this->action = new HealthAction($this->conn->reveal(), new AppOptions(['version' => '1.2.3'])); + $em = $this->prophesize(EntityManagerInterface::class); + $em->getConnection()->willReturn($this->conn->reveal()); + + $this->action = new HealthAction($em->reveal(), new AppOptions(['version' => '1.2.3'])); } /** @test */ From a6864bca7c14d36fe3bf76c3869ad8d1840d0dd5 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 25 Apr 2020 20:00:01 +0200 Subject: [PATCH 28/87] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e26b013..3ac1fb53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * [#729](https://github.com/shlinkio/shlink/issues/729) Fixed weird error when fetching multiple visits result sets concurrently using mariadb or mysql. * [#735](https://github.com/shlinkio/shlink/issues/735) Fixed error when cleaning metadata cache during installation when APCu is enabled. +* [#677](https://github.com/shlinkio/shlink/issues/677) Fixed `/health` endpoint returning `503` fail responses when the database connection has expired. ## 2.1.3 - 2020-04-09 From 5beaab85ac56e0b9c553655d6d0d1259de4362a4 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 11:17:07 +0200 Subject: [PATCH 29/87] Renamed GetVisitsAction to ShortUrlVisitsAction --- module/Rest/config/dependencies.config.php | 4 ++-- module/Rest/config/routes.config.php | 2 +- .../{GetVisitsAction.php => ShortUrlVisitsAction.php} | 2 +- ...tVisitsActionTest.php => ShortUrlVisitsActionTest.php} | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) rename module/Rest/src/Action/Visit/{GetVisitsAction.php => ShortUrlVisitsAction.php} (96%) rename module/Rest/test/Action/Visit/{GetVisitsActionTest.php => ShortUrlVisitsActionTest.php} (89%) diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index f6af6f85..1daa9300 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -28,7 +28,7 @@ return [ Action\ShortUrl\ResolveShortUrlAction::class => ConfigAbstractFactory::class, Action\ShortUrl\ListShortUrlsAction::class => ConfigAbstractFactory::class, Action\ShortUrl\EditShortUrlTagsAction::class => ConfigAbstractFactory::class, - Action\Visit\GetVisitsAction::class => ConfigAbstractFactory::class, + Action\Visit\ShortUrlVisitsAction::class => ConfigAbstractFactory::class, Action\Tag\ListTagsAction::class => ConfigAbstractFactory::class, Action\Tag\DeleteTagsAction::class => ConfigAbstractFactory::class, Action\Tag\CreateTagsAction::class => ConfigAbstractFactory::class, @@ -65,7 +65,7 @@ return [ Service\ShortUrl\ShortUrlResolver::class, 'config.url_shortener.domain', ], - Action\Visit\GetVisitsAction::class => [Service\VisitsTracker::class, 'Logger_Shlink'], + Action\Visit\ShortUrlVisitsAction::class => [Service\VisitsTracker::class, 'Logger_Shlink'], Action\ShortUrl\ListShortUrlsAction::class => [ Service\ShortUrlService::class, 'config.url_shortener.domain', diff --git a/module/Rest/config/routes.config.php b/module/Rest/config/routes.config.php index 3ced8357..7498d1b3 100644 --- a/module/Rest/config/routes.config.php +++ b/module/Rest/config/routes.config.php @@ -26,7 +26,7 @@ return [ Action\ShortUrl\EditShortUrlTagsAction::getRouteDef([$dropDomainMiddleware]), // Visits - Action\Visit\GetVisitsAction::getRouteDef([$dropDomainMiddleware]), + Action\Visit\ShortUrlVisitsAction::getRouteDef([$dropDomainMiddleware]), // Tags Action\Tag\ListTagsAction::getRouteDef(), diff --git a/module/Rest/src/Action/Visit/GetVisitsAction.php b/module/Rest/src/Action/Visit/ShortUrlVisitsAction.php similarity index 96% rename from module/Rest/src/Action/Visit/GetVisitsAction.php rename to module/Rest/src/Action/Visit/ShortUrlVisitsAction.php index bd6ae5a5..f7f0224a 100644 --- a/module/Rest/src/Action/Visit/GetVisitsAction.php +++ b/module/Rest/src/Action/Visit/ShortUrlVisitsAction.php @@ -14,7 +14,7 @@ use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Service\VisitsTrackerInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; -class GetVisitsAction extends AbstractRestAction +class ShortUrlVisitsAction extends AbstractRestAction { use PaginatorUtilsTrait; diff --git a/module/Rest/test/Action/Visit/GetVisitsActionTest.php b/module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php similarity index 89% rename from module/Rest/test/Action/Visit/GetVisitsActionTest.php rename to module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php index a1f1681a..07508acf 100644 --- a/module/Rest/test/Action/Visit/GetVisitsActionTest.php +++ b/module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php @@ -15,17 +15,17 @@ use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Service\VisitsTracker; -use Shlinkio\Shlink\Rest\Action\Visit\GetVisitsAction; +use Shlinkio\Shlink\Rest\Action\Visit\ShortUrlVisitsAction; -class GetVisitsActionTest extends TestCase +class ShortUrlVisitsActionTest extends TestCase { - private GetVisitsAction $action; + private ShortUrlVisitsAction $action; private ObjectProphecy $visitsTracker; public function setUp(): void { $this->visitsTracker = $this->prophesize(VisitsTracker::class); - $this->action = new GetVisitsAction($this->visitsTracker->reveal()); + $this->action = new ShortUrlVisitsAction($this->visitsTracker->reveal()); } /** @test */ From 1ef10f11cb4a33acd50f51b94e8e9ee532ce2166 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 11:40:02 +0200 Subject: [PATCH 30/87] Created new action to get default visit stats --- module/Core/config/dependencies.config.php | 2 ++ module/Core/src/Visit/Model/VisitsStats.php | 24 ++++++++++++++ module/Core/src/Visit/VisitsStatsHelper.php | 32 ++++++++++++++++++ .../src/Visit/VisitsStatsHelperInterface.php | 12 +++++++ module/Rest/config/dependencies.config.php | 3 ++ module/Rest/config/routes.config.php | 1 + .../src/Action/Visit/GlobalVisitsAction.php | 33 +++++++++++++++++++ .../Action/GlobalVisitsActionTest.php | 21 ++++++++++++ ...nTest.php => ShortUrlVisitsActionTest.php} | 2 +- 9 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 module/Core/src/Visit/Model/VisitsStats.php create mode 100644 module/Core/src/Visit/VisitsStatsHelper.php create mode 100644 module/Core/src/Visit/VisitsStatsHelperInterface.php create mode 100644 module/Rest/src/Action/Visit/GlobalVisitsAction.php create mode 100644 module/Rest/test-api/Action/GlobalVisitsActionTest.php rename module/Rest/test-api/Action/{GetVisitsActionTest.php => ShortUrlVisitsActionTest.php} (97%) diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 63e6cfed..67d18c40 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -27,6 +27,7 @@ return [ Service\VisitsTracker::class => ConfigAbstractFactory::class, Service\ShortUrlService::class => ConfigAbstractFactory::class, Visit\VisitLocator::class => ConfigAbstractFactory::class, + Visit\VisitsStatsHelper::class => ConfigAbstractFactory::class, Service\Tag\TagService::class => ConfigAbstractFactory::class, Service\ShortUrl\DeleteShortUrlService::class => ConfigAbstractFactory::class, Service\ShortUrl\ShortUrlResolver::class => ConfigAbstractFactory::class, @@ -56,6 +57,7 @@ return [ Service\VisitsTracker::class => ['em', EventDispatcherInterface::class], Service\ShortUrlService::class => ['em', Service\ShortUrl\ShortUrlResolver::class, Util\UrlValidator::class], Visit\VisitLocator::class => ['em'], + Visit\VisitsStatsHelper::class => ['em'], Service\Tag\TagService::class => ['em'], Service\ShortUrl\DeleteShortUrlService::class => [ 'em', diff --git a/module/Core/src/Visit/Model/VisitsStats.php b/module/Core/src/Visit/Model/VisitsStats.php new file mode 100644 index 00000000..ac5083c7 --- /dev/null +++ b/module/Core/src/Visit/Model/VisitsStats.php @@ -0,0 +1,24 @@ +visitsCount = $visitsCount; + } + + public function jsonSerialize(): array + { + return [ + 'visitsCount' => $this->visitsCount, + ]; + } +} diff --git a/module/Core/src/Visit/VisitsStatsHelper.php b/module/Core/src/Visit/VisitsStatsHelper.php new file mode 100644 index 00000000..de3219ff --- /dev/null +++ b/module/Core/src/Visit/VisitsStatsHelper.php @@ -0,0 +1,32 @@ +em = $em; + } + + public function getVisitsStats(): VisitsStats + { + return new VisitsStats($this->getVisitsCount()); + } + + private function getVisitsCount(): int + { + /** @var VisitRepository $visitsRepo */ + $visitsRepo = $this->em->getRepository(Visit::class); + return $visitsRepo->count([]); + } +} diff --git a/module/Core/src/Visit/VisitsStatsHelperInterface.php b/module/Core/src/Visit/VisitsStatsHelperInterface.php new file mode 100644 index 00000000..81423cb0 --- /dev/null +++ b/module/Core/src/Visit/VisitsStatsHelperInterface.php @@ -0,0 +1,12 @@ + ConfigAbstractFactory::class, Action\ShortUrl\EditShortUrlTagsAction::class => ConfigAbstractFactory::class, Action\Visit\ShortUrlVisitsAction::class => ConfigAbstractFactory::class, + Action\Visit\GlobalVisitsAction::class => ConfigAbstractFactory::class, Action\Tag\ListTagsAction::class => ConfigAbstractFactory::class, Action\Tag\DeleteTagsAction::class => ConfigAbstractFactory::class, Action\Tag\CreateTagsAction::class => ConfigAbstractFactory::class, @@ -66,6 +68,7 @@ return [ 'config.url_shortener.domain', ], Action\Visit\ShortUrlVisitsAction::class => [Service\VisitsTracker::class, 'Logger_Shlink'], + Action\Visit\GlobalVisitsAction::class => [Visit\VisitsStatsHelper::class, 'Logger_Shlink'], Action\ShortUrl\ListShortUrlsAction::class => [ Service\ShortUrlService::class, 'config.url_shortener.domain', diff --git a/module/Rest/config/routes.config.php b/module/Rest/config/routes.config.php index 7498d1b3..d2795971 100644 --- a/module/Rest/config/routes.config.php +++ b/module/Rest/config/routes.config.php @@ -27,6 +27,7 @@ return [ // Visits Action\Visit\ShortUrlVisitsAction::getRouteDef([$dropDomainMiddleware]), + Action\Visit\GlobalVisitsAction::getRouteDef(), // Tags Action\Tag\ListTagsAction::getRouteDef(), diff --git a/module/Rest/src/Action/Visit/GlobalVisitsAction.php b/module/Rest/src/Action/Visit/GlobalVisitsAction.php new file mode 100644 index 00000000..1946e222 --- /dev/null +++ b/module/Rest/src/Action/Visit/GlobalVisitsAction.php @@ -0,0 +1,33 @@ +statsHelper = $statsHelper; + } + + public function handle(ServerRequestInterface $request): ResponseInterface + { + return new JsonResponse([ + 'visits' => $this->statsHelper->getVisitsStats(), + ]); + } +} diff --git a/module/Rest/test-api/Action/GlobalVisitsActionTest.php b/module/Rest/test-api/Action/GlobalVisitsActionTest.php new file mode 100644 index 00000000..8e4f5e11 --- /dev/null +++ b/module/Rest/test-api/Action/GlobalVisitsActionTest.php @@ -0,0 +1,21 @@ +callApiWithKey(self::METHOD_GET, '/visits'); + $payload = $this->getJsonResponsePayload($resp); + + $this->assertArrayHasKey('visits', $payload); + $this->assertArrayHasKey('visitsCount', $payload['visits']); + $this->assertEquals(7, $payload['visits']['visitsCount']); + } +} diff --git a/module/Rest/test-api/Action/GetVisitsActionTest.php b/module/Rest/test-api/Action/ShortUrlVisitsActionTest.php similarity index 97% rename from module/Rest/test-api/Action/GetVisitsActionTest.php rename to module/Rest/test-api/Action/ShortUrlVisitsActionTest.php index cee466a3..ea39a267 100644 --- a/module/Rest/test-api/Action/GetVisitsActionTest.php +++ b/module/Rest/test-api/Action/ShortUrlVisitsActionTest.php @@ -11,7 +11,7 @@ use ShlinkioApiTest\Shlink\Rest\Utils\NotFoundUrlHelpersTrait; use function GuzzleHttp\Psr7\build_query; use function sprintf; -class GetVisitsActionTest extends ApiTestCase +class ShortUrlVisitsActionTest extends ApiTestCase { use NotFoundUrlHelpersTrait; From 3232ab401f3703b7a6323249f76ef4f1216ddd66 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 11:44:55 +0200 Subject: [PATCH 31/87] Documented new visits endpoint --- docs/swagger/definitions/VisitStats.json | 10 +++++ docs/swagger/paths/v2_visits.json | 54 ++++++++++++++++++++++++ docs/swagger/swagger.json | 3 ++ 3 files changed, 67 insertions(+) create mode 100644 docs/swagger/definitions/VisitStats.json create mode 100644 docs/swagger/paths/v2_visits.json diff --git a/docs/swagger/definitions/VisitStats.json b/docs/swagger/definitions/VisitStats.json new file mode 100644 index 00000000..5f439c9b --- /dev/null +++ b/docs/swagger/definitions/VisitStats.json @@ -0,0 +1,10 @@ +{ + "type": "object", + "required": ["visitsCount"], + "properties": { + "visitsCount": { + "type": "number", + "description": "The total amount of visits received." + } + } +} diff --git a/docs/swagger/paths/v2_visits.json b/docs/swagger/paths/v2_visits.json new file mode 100644 index 00000000..089223b3 --- /dev/null +++ b/docs/swagger/paths/v2_visits.json @@ -0,0 +1,54 @@ +{ + "get": { + "operationId": "getGlobalVisits", + "tags": [ + "Visits" + ], + "summary": "Get general visits stats", + "description": "Get general visits stats not linked to one specific short URL.", + "parameters": [ + { + "$ref": "../parameters/version.json" + } + ], + "security": [ + { + "ApiKey": [] + } + ], + "responses": { + "200": { + "description": "Visits stats.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "visits": { + "$ref": "../definitions/VisitStats.json" + } + } + } + } + }, + "examples": { + "application/json": { + "visits": { + "visitsCount": 1569874 + } + } + } + }, + "500": { + "description": "Unexpected error.", + "content": { + "application/problem+json": { + "schema": { + "$ref": "../definitions/Error.json" + } + } + } + } + } + } +} diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index c30bab97..e7663820 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -78,6 +78,9 @@ "$ref": "paths/v1_tags.json" }, + "/rest/v{version}/visits": { + "$ref": "paths/v2_visits.json" + }, "/rest/v{version}/short-urls/{shortCode}/visits": { "$ref": "paths/v1_short-urls_{shortCode}_visits.json" }, From b5947d1642f0aa32cf4ba6241a7e7deb244e7115 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 11:57:46 +0200 Subject: [PATCH 32/87] Created more unit tests --- .../Core/test/Visit/VisitsStatsHelperTest.php | 50 +++++++++++++++++++ module/Rest/src/Action/AbstractRestAction.php | 2 +- .../Action/Visit/GlobalVisitsActionTest.php | 39 +++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 module/Core/test/Visit/VisitsStatsHelperTest.php create mode 100644 module/Rest/test/Action/Visit/GlobalVisitsActionTest.php diff --git a/module/Core/test/Visit/VisitsStatsHelperTest.php b/module/Core/test/Visit/VisitsStatsHelperTest.php new file mode 100644 index 00000000..a4b692d5 --- /dev/null +++ b/module/Core/test/Visit/VisitsStatsHelperTest.php @@ -0,0 +1,50 @@ +em = $this->prophesize(EntityManagerInterface::class); + $this->helper = new VisitsStatsHelper($this->em->reveal()); + } + + /** + * @test + * @dataProvider provideCounts + */ + public function returnsExpectedVisitsStats(int $expectedCount): void + { + $repo = $this->prophesize(VisitRepository::class); + $count = $repo->count([])->willReturn($expectedCount); + $getRepo = $this->em->getRepository(Visit::class)->willReturn($repo->reveal()); + + $stats = $this->helper->getVisitsStats(); + + $this->assertEquals(new VisitsStats($expectedCount), $stats); + $count->shouldHaveBeenCalledOnce(); + $getRepo->shouldHaveBeenCalledOnce(); + } + + public function provideCounts(): iterable + { + return map(range(0, 50, 5), fn (int $value) => [$value]); + } +} diff --git a/module/Rest/src/Action/AbstractRestAction.php b/module/Rest/src/Action/AbstractRestAction.php index 826290b7..589e2d0e 100644 --- a/module/Rest/src/Action/AbstractRestAction.php +++ b/module/Rest/src/Action/AbstractRestAction.php @@ -21,7 +21,7 @@ abstract class AbstractRestAction implements RequestHandlerInterface, RequestMet public function __construct(?LoggerInterface $logger = null) { - $this->logger = $logger ?: new NullLogger(); + $this->logger = $logger ?? new NullLogger(); } public static function getRouteDef(array $prevMiddleware = [], array $postMiddleware = []): array diff --git a/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php b/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php new file mode 100644 index 00000000..7e1dec06 --- /dev/null +++ b/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php @@ -0,0 +1,39 @@ +helper = $this->prophesize(VisitsStatsHelperInterface::class); + $this->action = new GlobalVisitsAction($this->helper->reveal()); + } + + /** @test */ + public function statsAreReturnedFromHelper(): void + { + $stats = new VisitsStats(5); + $getStats = $this->helper->getVisitsStats()->willReturn($stats); + + /** @var JsonResponse $resp */ + $resp = $this->action->handle(ServerRequestFactory::fromGlobals()); + $payload = $resp->getPayload(); + + $this->assertEquals($payload, ['visits' => $stats]); + $getStats->shouldHaveBeenCalledOnce(); + } +} From d067f52ac2eaedd6ace9925757d195913c378113 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 11:58:59 +0200 Subject: [PATCH 33/87] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 584b8dc8..0ed6d0eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this Also, Shlink exposes a new endpoint `GET /rest/v2/mercure-info`, which returns the public URL of the mercure hub, and a valid JWT that can be used to subsribe to updates. +* [#673](https://github.com/shlinkio/shlink/issues/673) Added new `[GET /visits]` rest endpoint which returns basic visits stats. + #### Changed * *Nothing* From aece9e68ba9ca6ad8d4c5b3623f0fa4129e3e848 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 12:08:44 +0200 Subject: [PATCH 34/87] Removed logger dependency from rest actions --- module/Rest/config/dependencies.config.php | 36 +++++++------------ module/Rest/src/Action/AbstractRestAction.php | 9 ----- module/Rest/src/Action/HealthAction.php | 4 +-- module/Rest/src/Action/MercureInfoAction.php | 9 ++--- .../ShortUrl/AbstractCreateShortUrlAction.php | 9 ++--- .../Action/ShortUrl/DeleteShortUrlAction.php | 4 +-- .../Action/ShortUrl/EditShortUrlAction.php | 4 +-- .../ShortUrl/EditShortUrlTagsAction.php | 4 +-- .../Action/ShortUrl/ListShortUrlsAction.php | 9 ++--- .../Action/ShortUrl/ResolveShortUrlAction.php | 9 ++--- .../SingleStepCreateShortUrlAction.php | 6 ++-- .../Rest/src/Action/Tag/CreateTagsAction.php | 4 +-- .../Rest/src/Action/Tag/DeleteTagsAction.php | 4 +-- module/Rest/src/Action/Tag/ListTagsAction.php | 4 +-- .../Rest/src/Action/Tag/UpdateTagAction.php | 4 +-- .../src/Action/Visit/GlobalVisitsAction.php | 4 +-- .../src/Action/Visit/ShortUrlVisitsAction.php | 4 +-- 17 files changed, 33 insertions(+), 94 deletions(-) diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index d067d739..bd347897 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest; use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Laminas\ServiceManager\Factory\InvokableFactory; use Mezzio\Router\Middleware\ImplicitOptionsMiddleware; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider; use Shlinkio\Shlink\Core\Options\AppOptions; use Shlinkio\Shlink\Core\Service; @@ -48,37 +47,28 @@ return [ ConfigAbstractFactory::class => [ ApiKeyService::class => ['em'], - Action\HealthAction::class => ['em', AppOptions::class, 'Logger_Shlink'], - Action\MercureInfoAction::class => [LcobucciJwtProvider::class, 'config.mercure', 'Logger_Shlink'], - Action\ShortUrl\CreateShortUrlAction::class => [ - Service\UrlShortener::class, - 'config.url_shortener.domain', - 'Logger_Shlink', - ], + Action\HealthAction::class => ['em', AppOptions::class], + Action\MercureInfoAction::class => [LcobucciJwtProvider::class, 'config.mercure'], + Action\ShortUrl\CreateShortUrlAction::class => [Service\UrlShortener::class, 'config.url_shortener.domain'], Action\ShortUrl\SingleStepCreateShortUrlAction::class => [ Service\UrlShortener::class, ApiKeyService::class, 'config.url_shortener.domain', - 'Logger_Shlink', ], - Action\ShortUrl\EditShortUrlAction::class => [Service\ShortUrlService::class, 'Logger_Shlink'], - Action\ShortUrl\DeleteShortUrlAction::class => [Service\ShortUrl\DeleteShortUrlService::class, 'Logger_Shlink'], + Action\ShortUrl\EditShortUrlAction::class => [Service\ShortUrlService::class], + Action\ShortUrl\DeleteShortUrlAction::class => [Service\ShortUrl\DeleteShortUrlService::class], Action\ShortUrl\ResolveShortUrlAction::class => [ Service\ShortUrl\ShortUrlResolver::class, 'config.url_shortener.domain', ], - Action\Visit\ShortUrlVisitsAction::class => [Service\VisitsTracker::class, 'Logger_Shlink'], - Action\Visit\GlobalVisitsAction::class => [Visit\VisitsStatsHelper::class, 'Logger_Shlink'], - Action\ShortUrl\ListShortUrlsAction::class => [ - Service\ShortUrlService::class, - 'config.url_shortener.domain', - 'Logger_Shlink', - ], - Action\ShortUrl\EditShortUrlTagsAction::class => [Service\ShortUrlService::class, 'Logger_Shlink'], - Action\Tag\ListTagsAction::class => [Service\Tag\TagService::class, LoggerInterface::class], - Action\Tag\DeleteTagsAction::class => [Service\Tag\TagService::class, LoggerInterface::class], - Action\Tag\CreateTagsAction::class => [Service\Tag\TagService::class, LoggerInterface::class], - Action\Tag\UpdateTagAction::class => [Service\Tag\TagService::class, LoggerInterface::class], + Action\Visit\ShortUrlVisitsAction::class => [Service\VisitsTracker::class], + Action\Visit\GlobalVisitsAction::class => [Visit\VisitsStatsHelper::class], + Action\ShortUrl\ListShortUrlsAction::class => [Service\ShortUrlService::class, 'config.url_shortener.domain'], + Action\ShortUrl\EditShortUrlTagsAction::class => [Service\ShortUrlService::class], + Action\Tag\ListTagsAction::class => [Service\Tag\TagService::class], + Action\Tag\DeleteTagsAction::class => [Service\Tag\TagService::class], + Action\Tag\CreateTagsAction::class => [Service\Tag\TagService::class], + Action\Tag\UpdateTagAction::class => [Service\Tag\TagService::class], Middleware\ShortUrl\DropDefaultDomainFromRequestMiddleware::class => ['config.url_shortener.domain.hostname'], Middleware\ShortUrl\DefaultShortCodesLengthMiddleware::class => [ diff --git a/module/Rest/src/Action/AbstractRestAction.php b/module/Rest/src/Action/AbstractRestAction.php index 589e2d0e..da8b6d80 100644 --- a/module/Rest/src/Action/AbstractRestAction.php +++ b/module/Rest/src/Action/AbstractRestAction.php @@ -7,8 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action; use Fig\Http\Message\RequestMethodInterface; use Fig\Http\Message\StatusCodeInterface; use Psr\Http\Server\RequestHandlerInterface; -use Psr\Log\LoggerInterface; -use Psr\Log\NullLogger; use function array_merge; @@ -17,13 +15,6 @@ abstract class AbstractRestAction implements RequestHandlerInterface, RequestMet protected const ROUTE_PATH = ''; protected const ROUTE_ALLOWED_METHODS = []; - protected LoggerInterface $logger; - - public function __construct(?LoggerInterface $logger = null) - { - $this->logger = $logger ?? new NullLogger(); - } - public static function getRouteDef(array $prevMiddleware = [], array $postMiddleware = []): array { return [ diff --git a/module/Rest/src/Action/HealthAction.php b/module/Rest/src/Action/HealthAction.php index ef1b6b88..ef89da64 100644 --- a/module/Rest/src/Action/HealthAction.php +++ b/module/Rest/src/Action/HealthAction.php @@ -8,7 +8,6 @@ use Doctrine\ORM\EntityManagerInterface; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Options\AppOptions; use Throwable; @@ -24,9 +23,8 @@ class HealthAction extends AbstractRestAction private EntityManagerInterface $em; private AppOptions $options; - public function __construct(EntityManagerInterface $em, AppOptions $options, ?LoggerInterface $logger = null) + public function __construct(EntityManagerInterface $em, AppOptions $options) { - parent::__construct($logger); $this->em = $em; $this->options = $options; } diff --git a/module/Rest/src/Action/MercureInfoAction.php b/module/Rest/src/Action/MercureInfoAction.php index bad4cb9c..75893ab9 100644 --- a/module/Rest/src/Action/MercureInfoAction.php +++ b/module/Rest/src/Action/MercureInfoAction.php @@ -8,7 +8,6 @@ use Cake\Chronos\Chronos; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Common\Mercure\JwtProviderInterface; use Shlinkio\Shlink\Rest\Exception\MercureException; use Throwable; @@ -23,12 +22,8 @@ class MercureInfoAction extends AbstractRestAction private JwtProviderInterface $jwtProvider; private array $mercureConfig; - public function __construct( - JwtProviderInterface $jwtProvider, - array $mercureConfig, - ?LoggerInterface $logger = null - ) { - parent::__construct($logger); + public function __construct(JwtProviderInterface $jwtProvider, array $mercureConfig) + { $this->jwtProvider = $jwtProvider; $this->mercureConfig = $mercureConfig; } diff --git a/module/Rest/src/Action/ShortUrl/AbstractCreateShortUrlAction.php b/module/Rest/src/Action/ShortUrl/AbstractCreateShortUrlAction.php index 3335e1fa..feed626d 100644 --- a/module/Rest/src/Action/ShortUrl/AbstractCreateShortUrlAction.php +++ b/module/Rest/src/Action/ShortUrl/AbstractCreateShortUrlAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface as Response; use Psr\Http\Message\ServerRequestInterface as Request; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Exception\ValidationException; use Shlinkio\Shlink\Core\Model\CreateShortUrlData; use Shlinkio\Shlink\Core\Service\UrlShortenerInterface; @@ -19,12 +18,8 @@ abstract class AbstractCreateShortUrlAction extends AbstractRestAction private UrlShortenerInterface $urlShortener; private array $domainConfig; - public function __construct( - UrlShortenerInterface $urlShortener, - array $domainConfig, - ?LoggerInterface $logger = null - ) { - parent::__construct($logger); + public function __construct(UrlShortenerInterface $urlShortener, array $domainConfig) + { $this->urlShortener = $urlShortener; $this->domainConfig = $domainConfig; } diff --git a/module/Rest/src/Action/ShortUrl/DeleteShortUrlAction.php b/module/Rest/src/Action/ShortUrl/DeleteShortUrlAction.php index d86c60e9..bd5b487e 100644 --- a/module/Rest/src/Action/ShortUrl/DeleteShortUrlAction.php +++ b/module/Rest/src/Action/ShortUrl/DeleteShortUrlAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Response\EmptyResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Service\ShortUrl\DeleteShortUrlServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; @@ -19,9 +18,8 @@ class DeleteShortUrlAction extends AbstractRestAction private DeleteShortUrlServiceInterface $deleteShortUrlService; - public function __construct(DeleteShortUrlServiceInterface $deleteShortUrlService, ?LoggerInterface $logger = null) + public function __construct(DeleteShortUrlServiceInterface $deleteShortUrlService) { - parent::__construct($logger); $this->deleteShortUrlService = $deleteShortUrlService; } diff --git a/module/Rest/src/Action/ShortUrl/EditShortUrlAction.php b/module/Rest/src/Action/ShortUrl/EditShortUrlAction.php index da7012b6..30d95ae1 100644 --- a/module/Rest/src/Action/ShortUrl/EditShortUrlAction.php +++ b/module/Rest/src/Action/ShortUrl/EditShortUrlAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Response\EmptyResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Model\ShortUrlEdit; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; @@ -20,9 +19,8 @@ class EditShortUrlAction extends AbstractRestAction private ShortUrlServiceInterface $shortUrlService; - public function __construct(ShortUrlServiceInterface $shortUrlService, ?LoggerInterface $logger = null) + public function __construct(ShortUrlServiceInterface $shortUrlService) { - parent::__construct($logger); $this->shortUrlService = $shortUrlService; } diff --git a/module/Rest/src/Action/ShortUrl/EditShortUrlTagsAction.php b/module/Rest/src/Action/ShortUrl/EditShortUrlTagsAction.php index 0a48d986..def36d6c 100644 --- a/module/Rest/src/Action/ShortUrl/EditShortUrlTagsAction.php +++ b/module/Rest/src/Action/ShortUrl/EditShortUrlTagsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface as Response; use Psr\Http\Message\ServerRequestInterface as Request; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Exception\ValidationException; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; @@ -20,9 +19,8 @@ class EditShortUrlTagsAction extends AbstractRestAction private ShortUrlServiceInterface $shortUrlService; - public function __construct(ShortUrlServiceInterface $shortUrlService, ?LoggerInterface $logger = null) + public function __construct(ShortUrlServiceInterface $shortUrlService) { - parent::__construct($logger); $this->shortUrlService = $shortUrlService; } diff --git a/module/Rest/src/Action/ShortUrl/ListShortUrlsAction.php b/module/Rest/src/Action/ShortUrl/ListShortUrlsAction.php index 5801eeec..10a0effc 100644 --- a/module/Rest/src/Action/ShortUrl/ListShortUrlsAction.php +++ b/module/Rest/src/Action/ShortUrl/ListShortUrlsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface as Response; use Psr\Http\Message\ServerRequestInterface as Request; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Common\Paginator\Util\PaginatorUtilsTrait; use Shlinkio\Shlink\Core\Model\ShortUrlsParams; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; @@ -24,12 +23,8 @@ class ListShortUrlsAction extends AbstractRestAction private ShortUrlServiceInterface $shortUrlService; private array $domainConfig; - public function __construct( - ShortUrlServiceInterface $shortUrlService, - array $domainConfig, - ?LoggerInterface $logger = null - ) { - parent::__construct($logger); + public function __construct(ShortUrlServiceInterface $shortUrlService, array $domainConfig) + { $this->shortUrlService = $shortUrlService; $this->domainConfig = $domainConfig; } diff --git a/module/Rest/src/Action/ShortUrl/ResolveShortUrlAction.php b/module/Rest/src/Action/ShortUrl/ResolveShortUrlAction.php index 41cd2b2d..9c2cb3e4 100644 --- a/module/Rest/src/Action/ShortUrl/ResolveShortUrlAction.php +++ b/module/Rest/src/Action/ShortUrl/ResolveShortUrlAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface as Response; use Psr\Http\Message\ServerRequestInterface as Request; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortUrlResolverInterface; use Shlinkio\Shlink\Core\Transformer\ShortUrlDataTransformer; @@ -21,12 +20,8 @@ class ResolveShortUrlAction extends AbstractRestAction private ShortUrlResolverInterface $urlResolver; private array $domainConfig; - public function __construct( - ShortUrlResolverInterface $urlResolver, - array $domainConfig, - ?LoggerInterface $logger = null - ) { - parent::__construct($logger); + public function __construct(ShortUrlResolverInterface $urlResolver, array $domainConfig) + { $this->urlResolver = $urlResolver; $this->domainConfig = $domainConfig; } diff --git a/module/Rest/src/Action/ShortUrl/SingleStepCreateShortUrlAction.php b/module/Rest/src/Action/ShortUrl/SingleStepCreateShortUrlAction.php index e754e3ad..daeb3d04 100644 --- a/module/Rest/src/Action/ShortUrl/SingleStepCreateShortUrlAction.php +++ b/module/Rest/src/Action/ShortUrl/SingleStepCreateShortUrlAction.php @@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\Rest\Action\ShortUrl; use Laminas\Diactoros\Uri; use Psr\Http\Message\ServerRequestInterface as Request; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Exception\ValidationException; use Shlinkio\Shlink\Core\Model\CreateShortUrlData; use Shlinkio\Shlink\Core\Service\UrlShortenerInterface; @@ -22,10 +21,9 @@ class SingleStepCreateShortUrlAction extends AbstractCreateShortUrlAction public function __construct( UrlShortenerInterface $urlShortener, ApiKeyServiceInterface $apiKeyService, - array $domainConfig, - ?LoggerInterface $logger = null + array $domainConfig ) { - parent::__construct($urlShortener, $domainConfig, $logger); + parent::__construct($urlShortener, $domainConfig); $this->apiKeyService = $apiKeyService; } diff --git a/module/Rest/src/Action/Tag/CreateTagsAction.php b/module/Rest/src/Action/Tag/CreateTagsAction.php index eb4a279b..c481b463 100644 --- a/module/Rest/src/Action/Tag/CreateTagsAction.php +++ b/module/Rest/src/Action/Tag/CreateTagsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; @@ -18,9 +17,8 @@ class CreateTagsAction extends AbstractRestAction private TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService, ?LoggerInterface $logger = null) + public function __construct(TagServiceInterface $tagService) { - parent::__construct($logger); $this->tagService = $tagService; } diff --git a/module/Rest/src/Action/Tag/DeleteTagsAction.php b/module/Rest/src/Action/Tag/DeleteTagsAction.php index b8bedab9..5002eba0 100644 --- a/module/Rest/src/Action/Tag/DeleteTagsAction.php +++ b/module/Rest/src/Action/Tag/DeleteTagsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\EmptyResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; @@ -18,9 +17,8 @@ class DeleteTagsAction extends AbstractRestAction private TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService, ?LoggerInterface $logger = null) + public function __construct(TagServiceInterface $tagService) { - parent::__construct($logger); $this->tagService = $tagService; } diff --git a/module/Rest/src/Action/Tag/ListTagsAction.php b/module/Rest/src/Action/Tag/ListTagsAction.php index 7cc7e063..7211bce6 100644 --- a/module/Rest/src/Action/Tag/ListTagsAction.php +++ b/module/Rest/src/Action/Tag/ListTagsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; @@ -18,9 +17,8 @@ class ListTagsAction extends AbstractRestAction private TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService, ?LoggerInterface $logger = null) + public function __construct(TagServiceInterface $tagService) { - parent::__construct($logger); $this->tagService = $tagService; } diff --git a/module/Rest/src/Action/Tag/UpdateTagAction.php b/module/Rest/src/Action/Tag/UpdateTagAction.php index 6fb72e01..de5eb476 100644 --- a/module/Rest/src/Action/Tag/UpdateTagAction.php +++ b/module/Rest/src/Action/Tag/UpdateTagAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\EmptyResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Exception\ValidationException; use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; @@ -19,9 +18,8 @@ class UpdateTagAction extends AbstractRestAction private TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService, ?LoggerInterface $logger = null) + public function __construct(TagServiceInterface $tagService) { - parent::__construct($logger); $this->tagService = $tagService; } diff --git a/module/Rest/src/Action/Visit/GlobalVisitsAction.php b/module/Rest/src/Action/Visit/GlobalVisitsAction.php index 1946e222..a27412b2 100644 --- a/module/Rest/src/Action/Visit/GlobalVisitsAction.php +++ b/module/Rest/src/Action/Visit/GlobalVisitsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\Visit; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; @@ -18,9 +17,8 @@ class GlobalVisitsAction extends AbstractRestAction private VisitsStatsHelperInterface $statsHelper; - public function __construct(VisitsStatsHelperInterface $statsHelper, ?LoggerInterface $logger = null) + public function __construct(VisitsStatsHelperInterface $statsHelper) { - parent::__construct($logger); $this->statsHelper = $statsHelper; } diff --git a/module/Rest/src/Action/Visit/ShortUrlVisitsAction.php b/module/Rest/src/Action/Visit/ShortUrlVisitsAction.php index f7f0224a..92a7e873 100644 --- a/module/Rest/src/Action/Visit/ShortUrlVisitsAction.php +++ b/module/Rest/src/Action/Visit/ShortUrlVisitsAction.php @@ -7,7 +7,6 @@ namespace Shlinkio\Shlink\Rest\Action\Visit; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface as Response; use Psr\Http\Message\ServerRequestInterface as Request; -use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Common\Paginator\Util\PaginatorUtilsTrait; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\VisitsParams; @@ -23,9 +22,8 @@ class ShortUrlVisitsAction extends AbstractRestAction private VisitsTrackerInterface $visitsTracker; - public function __construct(VisitsTrackerInterface $visitsTracker, ?LoggerInterface $logger = null) + public function __construct(VisitsTrackerInterface $visitsTracker) { - parent::__construct($logger); $this->visitsTracker = $visitsTracker; } From 56932e4ea6904c5acfe3f77387ddef97e181ba6a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 1 May 2020 18:24:48 +0200 Subject: [PATCH 35/87] Disabled swoole coroutines --- config/autoload/swoole.global.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/autoload/swoole.global.php b/config/autoload/swoole.global.php index afc48a97..29c1ea37 100644 --- a/config/autoload/swoole.global.php +++ b/config/autoload/swoole.global.php @@ -5,7 +5,8 @@ declare(strict_types=1); return [ 'mezzio-swoole' => [ - 'enable_coroutine' => true, + // Setting this to true can have unexpected behaviors when running several concurrent slow DB queries + 'enable_coroutine' => false, 'swoole-http-server' => [ 'host' => '0.0.0.0', From 6c30fc73ee05727956405498df46acc95f5c35bc Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 2 May 2020 12:04:42 +0200 Subject: [PATCH 36/87] Added swoole reverse proxy container --- data/infra/swoole_proxy_vhost.conf | 14 ++++++++++++++ docker-compose.yml | 11 +++++++++++ 2 files changed, 25 insertions(+) create mode 100644 data/infra/swoole_proxy_vhost.conf diff --git a/data/infra/swoole_proxy_vhost.conf b/data/infra/swoole_proxy_vhost.conf new file mode 100644 index 00000000..af31b1ea --- /dev/null +++ b/data/infra/swoole_proxy_vhost.conf @@ -0,0 +1,14 @@ +server { + listen 80 default_server; + + error_log /home/shlink/www/data/infra/nginx/swoole_proxy.error.log; + + location / { + proxy_http_version 1.1; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://shlink_swoole:8080; + proxy_read_timeout 90s; + } +} diff --git a/docker-compose.yml b/docker-compose.yml index ec56b883..36153ad2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,17 @@ services: environment: LC_ALL: C + shlink_swoole_proxy: + container_name: shlink_swoole_proxy + image: nginx:1.17.10-alpine + ports: + - "8002:80" + volumes: + - ./:/home/shlink/www + - ./data/infra/swoole_proxy_vhost.conf:/etc/nginx/conf.d/default.conf + links: + - shlink_swoole + shlink_swoole: container_name: shlink_swoole build: From 80d41db90178f98a22fd0e31c7431eab9a9c0aae Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 2 May 2020 22:47:59 +0200 Subject: [PATCH 37/87] Improved performance on query that returns the list of visits for a short URL --- .../Core/src/Repository/VisitRepository.php | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 61b2afb8..e8a530f2 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\QueryBuilder; use Shlinkio\Shlink\Common\Util\DateRange; +use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; class VisitRepository extends EntityRepository implements VisitRepositoryInterface @@ -82,15 +83,11 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ?int $offset = null ): array { $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); - $qb->select('v') - ->orderBy('v.date', 'DESC'); - - if ($limit !== null) { - $qb->setMaxResults($limit); - } - if ($offset !== null) { - $qb->setFirstResult($offset); - } + $qb->select('v', 'vl') + ->leftJoin('v.visitLocation', 'vl') + ->orderBy('v.id', 'DESC') + ->setMaxResults($limit) + ->setFirstResult($offset); return $qb->getQuery()->getResult(); } @@ -108,20 +105,14 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ?string $domain, ?DateRange $dateRange ): QueryBuilder { + /** @var ShortUrlRepositoryInterface $shortUrlRepo */ + $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); + $shortUrl = $shortUrlRepo->findOne($shortCode, $domain) ?? -1; + $qb = $this->getEntityManager()->createQueryBuilder(); $qb->from(Visit::class, 'v') - ->join('v.shortUrl', 'su') - ->where($qb->expr()->eq('su.shortCode', ':shortCode')) - ->setParameter('shortCode', $shortCode); - - // Apply domain filtering - if ($domain !== null) { - $qb->join('su.domain', 'd') - ->andWhere($qb->expr()->eq('d.authority', ':domain')) - ->setParameter('domain', $domain); - } else { - $qb->andWhere($qb->expr()->isNull('su.domain')); - } + ->where($qb->expr()->eq('v.shortUrl', ':shortUrl')) + ->setParameter('shortUrl', $shortUrl); // Apply date range filtering if ($dateRange !== null && $dateRange->getStartDate() !== null) { From c4ae89a2794ea8d5802a0edfbb642ec989b3d845 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 10:22:00 +0200 Subject: [PATCH 38/87] Removed DISTINCT when counting visits for a short URL --- module/Core/src/Repository/VisitRepository.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index e8a530f2..e38980c4 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -95,7 +95,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa public function countVisitsByShortCode(string $shortCode, ?string $domain = null, ?DateRange $dateRange = null): int { $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); - $qb->select('COUNT(DISTINCT v.id)'); + $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); } From 0e4bccc4bbb0525bb3b3efba7b488f80160d0dd9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 10:44:01 +0200 Subject: [PATCH 39/87] Cached result of the count query on VisitsPaginatorAdapter --- .../Paginator/Adapter/VisitsPaginatorAdapter.php | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php index 247ea93e..6a42ecbc 100644 --- a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php @@ -15,6 +15,8 @@ class VisitsPaginatorAdapter implements AdapterInterface private ShortUrlIdentifier $identifier; private VisitsParams $params; + private ?int $count = null; + public function __construct( VisitRepositoryInterface $visitRepository, ShortUrlIdentifier $identifier, @@ -38,7 +40,17 @@ class VisitsPaginatorAdapter implements AdapterInterface public function count(): int { - return $this->visitRepository->countVisitsByShortCode( + // Since a new adapter instance is created every time visits are fetched, it is reasonably safe to internally + // cache the count value. + // The reason it is cached is because the Paginator is actually calling the method twice. + // An inconsistent value could be returned if between the first call and the second one, a new visit is created. + // However, it's almost instant, and then the adapter instance is discarded immediately after. + + if ($this->count !== null) { + return $this->count; + } + + return $this->count = $this->visitRepository->countVisitsByShortCode( $this->identifier->shortCode(), $this->identifier->domain(), $this->params->getDateRange(), From 8b0ce8e6f333ef8b1e585ca2cb8a047e7557a0a4 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 18:18:24 +0200 Subject: [PATCH 40/87] Improved performance when loading visits chuncks at high offsets --- .../Core/src/Repository/VisitRepository.php | 65 ++++++++++++++++--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index e38980c4..4cb7ff42 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -5,10 +5,16 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; +use Doctrine\ORM\Query\ResultSetMappingBuilder; use Doctrine\ORM\QueryBuilder; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; +use Shlinkio\Shlink\Core\Entity\VisitLocation; + +use function preg_replace; + +use const PHP_INT_MAX; class VisitRepository extends EntityRepository implements VisitRepositoryInterface { @@ -82,19 +88,60 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ?int $limit = null, ?int $offset = null ): array { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); - $qb->select('v', 'vl') - ->leftJoin('v.visitLocation', 'vl') + /** + * @var QueryBuilder $qb + * @var ShortUrl|int $shortUrl + */ + [$qb, $shortUrl] = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); + $qb->select('v.id') ->orderBy('v.id', 'DESC') - ->setMaxResults($limit) - ->setFirstResult($offset); + // Falling back to values that will behave as no limit/offset, but will workaround MS SQL not allowing + // order on sub-queries without offset + ->setMaxResults($limit ?? PHP_INT_MAX) + ->setFirstResult($offset ?? 0); - return $qb->getQuery()->getResult(); + // FIXME Crappy way to resolve the params into the query. Best option would be to inject the sub-query with + // placeholders and then pass params to the main query + $shortUrlId = $shortUrl instanceof ShortUrl ? $shortUrl->getId() : $shortUrl; + $subQuery = $qb->getQuery()->getSQL(); + $subQuery = preg_replace('/\?/', $shortUrlId, $subQuery, 1); + if ($dateRange !== null && $dateRange->getStartDate() !== null) { + $subQuery = preg_replace( + '/\?/', + '\'' . $dateRange->getStartDate()->toDateTimeString() . '\'', + $subQuery, + 1, + ); + } + if ($dateRange !== null && $dateRange->getEndDate() !== null) { + $subQuery = preg_replace('/\?/', '\'' . $dateRange->getEndDate()->toDateTimeString() . '\'', $subQuery, 1); + } + + // A native query builder needs to be used here because DQL and ORM query builders do not accept + // sub-queries at "from" and "join" level. + // If no sub-query is used, then the performance drops dramatically while the "offset" grows. + $nativeQb = $this->getEntityManager()->getConnection()->createQueryBuilder(); + $nativeQb->select('v.*', 'vl.*') + ->from('visits', 'v') + ->join('v', '(' . $subQuery . ')', 'o', $nativeQb->expr()->eq('o.id_0', 'v.id')) + ->leftJoin('v', 'visit_locations', 'vl', $nativeQb->expr()->eq('v.visit_location_id', 'vl.id')) + ->orderBy('v.id', 'DESC'); + + $rsm = new ResultSetMappingBuilder($this->getEntityManager()); + $rsm->addRootEntityFromClassMetadata(Visit::class, 'v'); + $rsm->addJoinedEntityFromClassMetadata(VisitLocation::class, 'vl', 'v', 'visitLocation', [ + 'id' => 'visit_location_id', + ]); + + $query = $this->getEntityManager()->createNativeQuery($nativeQb->getSQL(), $rsm); + + return $query->getResult(); } public function countVisitsByShortCode(string $shortCode, ?string $domain = null, ?DateRange $dateRange = null): int { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); + /** @var QueryBuilder $qb */ + [$qb] = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); @@ -104,7 +151,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa string $shortCode, ?string $domain, ?DateRange $dateRange - ): QueryBuilder { + ): array { /** @var ShortUrlRepositoryInterface $shortUrlRepo */ $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); $shortUrl = $shortUrlRepo->findOne($shortCode, $domain) ?? -1; @@ -124,6 +171,6 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ->setParameter('endDate', $dateRange->getEndDate()); } - return $qb; + return [$qb, $shortUrl]; } } From 74ad3553cbf1be4a79373ac6b690cdb633db5002 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 19:02:13 +0200 Subject: [PATCH 41/87] Hardcoded types on date fields when filtering visits lists --- module/Core/src/Repository/VisitRepository.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 4cb7ff42..3efd263f 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\Query\ResultSetMappingBuilder; use Doctrine\ORM\QueryBuilder; +use Shlinkio\Shlink\Common\Doctrine\Type\ChronosDateTimeType; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; @@ -123,7 +124,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa $nativeQb = $this->getEntityManager()->getConnection()->createQueryBuilder(); $nativeQb->select('v.*', 'vl.*') ->from('visits', 'v') - ->join('v', '(' . $subQuery . ')', 'o', $nativeQb->expr()->eq('o.id_0', 'v.id')) + ->join('v', '(' . $subQuery . ')', 'sq', $nativeQb->expr()->eq('sq.id_0', 'v.id')) ->leftJoin('v', 'visit_locations', 'vl', $nativeQb->expr()->eq('v.visit_location_id', 'vl.id')) ->orderBy('v.id', 'DESC'); @@ -164,11 +165,11 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa // Apply date range filtering if ($dateRange !== null && $dateRange->getStartDate() !== null) { $qb->andWhere($qb->expr()->gte('v.date', ':startDate')) - ->setParameter('startDate', $dateRange->getStartDate()); + ->setParameter('startDate', $dateRange->getStartDate(), ChronosDateTimeType::CHRONOS_DATETIME); } if ($dateRange !== null && $dateRange->getEndDate() !== null) { $qb->andWhere($qb->expr()->lte('v.date', ':endDate')) - ->setParameter('endDate', $dateRange->getEndDate()); + ->setParameter('endDate', $dateRange->getEndDate(), ChronosDateTimeType::CHRONOS_DATETIME); } return [$qb, $shortUrl]; From 867659ea25d6e9ed79ea368c3af2090f4204e412 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 19:15:26 +0200 Subject: [PATCH 42/87] Created index on visits.date column --- data/migrations/Version20200503170404.php | 27 +++++++++++++++++++ .../Shlinkio.Shlink.Core.Entity.Visit.php | 2 ++ 2 files changed, 29 insertions(+) create mode 100644 data/migrations/Version20200503170404.php diff --git a/data/migrations/Version20200503170404.php b/data/migrations/Version20200503170404.php new file mode 100644 index 00000000..a102c2c8 --- /dev/null +++ b/data/migrations/Version20200503170404.php @@ -0,0 +1,27 @@ +getTable('visits'); + $this->skipIf($visits->hasIndex(self::INDEX_NAME)); + $visits->addIndex(['date'], self::INDEX_NAME); + } + + public function down(Schema $schema): void + { + $visits = $schema->getTable('visits'); + $this->skipIf(! $visits->hasIndex(self::INDEX_NAME)); + $visits->dropIndex(self::INDEX_NAME); + } +} diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php index 803b9790..5143389b 100644 --- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php @@ -32,6 +32,8 @@ return static function (ClassMetadata $metadata, array $emConfig): void { ->columnName('`date`') ->build(); + $builder->addIndex(['date'], 'IDX_visits_date'); + $builder->createField('remoteAddr', Types::STRING) ->columnName('remote_addr') ->length(Visitor::REMOTE_ADDRESS_MAX_LENGTH) From d5288f756e1ca578a1abfb8b8dc0a54898692595 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 19:52:40 +0200 Subject: [PATCH 43/87] Fixed entity mapping for visits without a visit location --- module/Core/src/Repository/VisitRepository.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 3efd263f..547493c5 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -104,8 +104,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa // FIXME Crappy way to resolve the params into the query. Best option would be to inject the sub-query with // placeholders and then pass params to the main query $shortUrlId = $shortUrl instanceof ShortUrl ? $shortUrl->getId() : $shortUrl; - $subQuery = $qb->getQuery()->getSQL(); - $subQuery = preg_replace('/\?/', $shortUrlId, $subQuery, 1); + $subQuery = preg_replace('/\?/', $shortUrlId, $qb->getQuery()->getSQL(), 1); if ($dateRange !== null && $dateRange->getStartDate() !== null) { $subQuery = preg_replace( '/\?/', @@ -120,16 +119,16 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa // A native query builder needs to be used here because DQL and ORM query builders do not accept // sub-queries at "from" and "join" level. - // If no sub-query is used, then the performance drops dramatically while the "offset" grows. + // If no sub-query is used, then performance drops dramatically while the "offset" grows. $nativeQb = $this->getEntityManager()->getConnection()->createQueryBuilder(); - $nativeQb->select('v.*', 'vl.*') + $nativeQb->select('v.id AS visit_id', 'v.*', 'vl.*') ->from('visits', 'v') ->join('v', '(' . $subQuery . ')', 'sq', $nativeQb->expr()->eq('sq.id_0', 'v.id')) ->leftJoin('v', 'visit_locations', 'vl', $nativeQb->expr()->eq('v.visit_location_id', 'vl.id')) ->orderBy('v.id', 'DESC'); $rsm = new ResultSetMappingBuilder($this->getEntityManager()); - $rsm->addRootEntityFromClassMetadata(Visit::class, 'v'); + $rsm->addRootEntityFromClassMetadata(Visit::class, 'v', ['id' => 'visit_id']); $rsm->addJoinedEntityFromClassMetadata(VisitLocation::class, 'vl', 'v', 'visitLocation', [ 'id' => 'visit_location_id', ]); From 313b6a59b922f840bf4139d94179d7356cce2398 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 3 May 2020 20:02:50 +0200 Subject: [PATCH 44/87] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ed6d0eb..f50e81d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Changed -* *Nothing* +* [#692](https://github.com/shlinkio/shlink/issues/692) Drastically improved performance when loading visits. Specially noticeable when loading big result sets. #### Deprecated From e747a0b25037ea16546e1120d7608652c485dcd6 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 19:55:03 +0200 Subject: [PATCH 45/87] Updated how database tests are run in travis, so that all DB engines are covered --- .travis.yml | 5 +---- composer.json | 10 ++-------- config/test/test_config.global.php | 9 ++++++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8b232c76..d53d2ed0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,6 @@ php: - '7.4' services: - - mysql - - postgresql - docker cache: @@ -26,8 +24,7 @@ install: - composer install --no-interaction --prefer-dist before_script: - - mysql -e 'CREATE DATABASE shlink_test;' - - psql -c 'create database shlink_test;' -U postgres + - docker-compose up shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms -d - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) diff --git a/composer.json b/composer.json index 39c1e6db..111b60c4 100644 --- a/composer.json +++ b/composer.json @@ -109,7 +109,7 @@ ], "test:ci": [ "@test:unit:ci", - "@test:db:ci", + "@test:db", "@test:api:ci" ], "test:unit": "phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", @@ -121,11 +121,6 @@ "@test:db:postgres", "@test:db:ms" ], - "test:db:ci": [ - "@test:db:sqlite", - "@test:db:mysql", - "@test:db:postgres" - ], "test:db:sqlite": "APP_ENV=test phpdbg -qrr vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-db.cov --testdox -c phpunit-db.xml", "test:db:mysql": "DB_DRIVER=mysql composer test:db:sqlite", "test:db:maria": "DB_DRIVER=maria composer test:db:sqlite", @@ -152,8 +147,7 @@ "test:ci": "Runs all test suites, generating all needed reports and logs for CI envs", "test:unit": "Runs unit test suites", "test:unit:ci": "Runs unit test suites, generating all needed reports and logs for CI envs", - "test:db": "Runs database test suites on a SQLite, MySQL, MariaDB and PostgreSQL", - "test:db:ci": "Runs database test suites on a SQLite, MySQL and PostgreSQL", + "test:db": "Runs database test suites on a SQLite, MySQL, MariaDB, PostgreSQL and MsSQL", "test:db:sqlite": "Runs database test suites on a SQLite database", "test:db:mysql": "Runs database test suites on a MySQL database", "test:db:maria": "Runs database test suites on a MariaDB database", diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index 6aad5375..0086bcd0 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -20,6 +20,7 @@ $buildDbConnection = function (): array { $driver = env('DB_DRIVER', 'sqlite'); $isCi = env('TRAVIS', false); $getMysqlHost = fn (string $driver) => sprintf('shlink_db%s', $driver === 'mysql' ? '' : '_maria'); + $getCiMysqlPort = fn (string $driver) => $driver === 'mysql' ? '3307' : '3308'; $driverConfigMap = [ 'sqlite' => [ @@ -29,8 +30,9 @@ $buildDbConnection = function (): array { 'mysql' => [ 'driver' => 'pdo_mysql', 'host' => $isCi ? '127.0.0.1' : $getMysqlHost($driver), + 'port' => $isCi ? $getCiMysqlPort($driver) : '3306', 'user' => 'root', - 'password' => $isCi ? '' : 'root', + 'password' => 'root', 'dbname' => 'shlink_test', 'charset' => 'utf8', 'driverOptions' => [ @@ -41,8 +43,9 @@ $buildDbConnection = function (): array { 'postgres' => [ 'driver' => 'pdo_pgsql', 'host' => $isCi ? '127.0.0.1' : 'shlink_db_postgres', + 'port' => $isCi ? '5433' : '5432', 'user' => 'postgres', - 'password' => $isCi ? '' : 'root', + 'password' => 'root', 'dbname' => 'shlink_test', 'charset' => 'utf8', ], @@ -50,7 +53,7 @@ $buildDbConnection = function (): array { 'driver' => 'pdo_sqlsrv', 'host' => $isCi ? '127.0.0.1' : 'shlink_db_ms', 'user' => 'sa', - 'password' => $isCi ? '' : 'Passw0rd!', + 'password' => 'Passw0rd!', 'dbname' => 'shlink_test', ], ]; From 1e8c36b5f1679bfea2f778a1be515e3c2952e0ca Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 19:55:52 +0200 Subject: [PATCH 46/87] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f50e81d2..0358e791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Changed * [#692](https://github.com/shlinkio/shlink/issues/692) Drastically improved performance when loading visits. Specially noticeable when loading big result sets. +* [#657](https://github.com/shlinkio/shlink/issues/657) Updated how DB tests are run in travis by using docker containers which allow all engines to be covered. #### Deprecated From 9f13063b1f1da8056e0540eafeac2d5d34528436 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 20:02:48 +0200 Subject: [PATCH 47/87] Fixed docker-compose command run in travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d53d2ed0..426f55f6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ install: - composer install --no-interaction --prefer-dist before_script: - - docker-compose up shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms -d + - docker-compose up -d shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) From dfdae96da5bacf2bcff5c2834e536db1476518b0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 20:34:28 +0200 Subject: [PATCH 48/87] Added commands to initially create all testing database for all database engines in travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 426f55f6..cb180c03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,10 @@ install: before_script: - docker-compose up -d shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms + - docker-compose exec shlink_db mysql -u root -proot -e "CREATE DATABASE shlink_test" + - docker-compose exec -e PGPASSWORD=root shlink_db_postgres psql -U postgres -c 'create database shlink_test;' + - docker-compose exec shlink_db_maria mysql -u root -proot -e "CREATE DATABASE shlink_test" + - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) From f6d9a83202abbc5d3816fab8f0e19917cc506675 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 20:42:48 +0200 Subject: [PATCH 49/87] Moved initial ci databases to specific docker-compose file --- .travis.yml | 6 ++---- docker-compose.ci.yml | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 docker-compose.ci.yml diff --git a/.travis.yml b/.travis.yml index cb180c03..b9b8c6b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,10 +24,8 @@ install: - composer install --no-interaction --prefer-dist before_script: - - docker-compose up -d shlink_db shlink_db_postgres shlink_db_maria shlink_db_ms - - docker-compose exec shlink_db mysql -u root -proot -e "CREATE DATABASE shlink_test" - - docker-compose exec -e PGPASSWORD=root shlink_db_postgres psql -U postgres -c 'create database shlink_test;' - - docker-compose exec shlink_db_maria mysql -u root -proot -e "CREATE DATABASE shlink_test" + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria + - sleep 10 # Let MS database start-up before trying to interact with it - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml new file mode 100644 index 00000000..3783fef2 --- /dev/null +++ b/docker-compose.ci.yml @@ -0,0 +1,14 @@ +version: '3' + +services: + shlink_db: + environment: + MYSQL_DATABASE: shlink_test + + shlink_db_postgres: + environment: + POSTGRES_DB: shlink_test + + shlink_db_maria: + environment: + MYSQL_DATABASE: shlink_test From 13bb48d06886e2cba93766ce87974889c6af79b7 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:12:43 +0200 Subject: [PATCH 50/87] Installed pdo_sqlsrv extension in travis --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9b8c6b3..20ebd912 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,8 +15,9 @@ cache: - $HOME/.composer/cache/files before_install: + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - yes | pecl install swoole-4.4.15 + - yes | pecl install swoole-4.4.15 pdo_sqlsrv - phpenv config-rm xdebug.ini || return 0 install: @@ -24,8 +25,6 @@ install: - composer install --no-interaction --prefer-dist before_script: - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - - sleep 10 # Let MS database start-up before trying to interact with it - docker-compose exec shlink_db_ms /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P 'Passw0rd!' -Q "CREATE DATABASE shlink_test;" - mkdir build - export DOCKERFILE_CHANGED=$(git diff ${TRAVIS_COMMIT_RANGE:-origin/master} --name-only | grep Dockerfile) From 78359c28c7226181356b63d6722a9c04fd1df9ff Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:22:41 +0200 Subject: [PATCH 51/87] Added MS ODBC package installation to travis --- .travis.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 20ebd912..e59423a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,10 @@ cache: before_install: - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - yes | pecl install swoole-4.4.15 pdo_sqlsrv + - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - + - curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + - sudo ACCEPT_EULA=Y apt-get install msodbcsql17 + - yes | pecl install pdo_sqlsrv swoole-4.4.15 - phpenv config-rm xdebug.ini || return 0 install: From 522d8ed236d34687962b754cc438ae267a88110f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:28:53 +0200 Subject: [PATCH 52/87] Ensured some commands are run as sudo during travis CI --- .travis.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e59423a2..b8c131b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,14 @@ cache: - $HOME/.composer/cache/files before_install: - - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + - sudo su - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - - curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + - curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list + - exit + - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install msodbcsql17 + - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 - phpenv config-rm xdebug.ini || return 0 From 5f42266cf2b41b5e0aec9adffca699454e016261 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:39:48 +0200 Subject: [PATCH 53/87] Moved ms odbc commands to a script --- .travis.yml | 10 +++------- data/infra/ci/install-ms-odbc.sh | 9 +++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100755 data/infra/ci/install-ms-odbc.sh diff --git a/.travis.yml b/.travis.yml index b8c131b6..36bf55cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,15 +15,11 @@ cache: - $HOME/.composer/cache/files before_install: - - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - - sudo su - - curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - - - curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list - - exit - - sudo apt-get update - - sudo ACCEPT_EULA=Y apt-get install msodbcsql17 + - sudo ./data/infra/ci/install-ms-odbc.sh - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 + - echo 'extension = pdo_sqlsrv.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini + - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 install: diff --git a/data/infra/ci/install-ms-odbc.sh b/data/infra/ci/install-ms-odbc.sh new file mode 100755 index 00000000..8cd60580 --- /dev/null +++ b/data/infra/ci/install-ms-odbc.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -ex + +curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list > /etc/apt/sources.list.d/mssql-release.list +apt-get update +ACCEPT_EULA=Y apt-get install msodbcsql17 +apt-get install unixodbc-dev From cc41c51f77a35b801b2da7f3f4c81f5eb9f720bd Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 4 May 2020 21:55:18 +0200 Subject: [PATCH 54/87] Removed duplicated pdo_sqlsrv enabling on travis config --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 36bf55cf..a95bf67d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,6 @@ before_install: - sudo ./data/infra/ci/install-ms-odbc.sh - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - yes | pecl install pdo_sqlsrv swoole-4.4.15 - - echo 'extension = pdo_sqlsrv.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 From 0cab51b01b61754a0f1f16713f6159c63ed49d43 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 5 May 2020 12:51:47 +0200 Subject: [PATCH 55/87] Enforced mezzio-swoole 2.6.4 or greater --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 111b60c4..5ba58a8e 100644 --- a/composer.json +++ b/composer.json @@ -39,7 +39,7 @@ "mezzio/mezzio-helpers": "^5.3", "mezzio/mezzio-platesrenderer": "^2.1", "mezzio/mezzio-problem-details": "^1.1", - "mezzio/mezzio-swoole": "^2.6", + "mezzio/mezzio-swoole": "^2.6.4", "monolog/monolog": "^2.0", "nikolaposa/monolog-factory": "^3.0", "ocramius/proxy-manager": "^2.7.0", From 53a37feafe8582eee451324ddc95b16d24805c0f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 5 May 2020 12:54:08 +0200 Subject: [PATCH 56/87] Updated changelogs --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0358e791..67105b4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * [#729](https://github.com/shlinkio/shlink/issues/729) Fixed weird error when fetching multiple visits result sets concurrently using mariadb or mysql. * [#735](https://github.com/shlinkio/shlink/issues/735) Fixed error when cleaning metadata cache during installation when APCu is enabled. * [#677](https://github.com/shlinkio/shlink/issues/677) Fixed `/health` endpoint returning `503` fail responses when the database connection has expired. +* [#732](https://github.com/shlinkio/shlink/issues/732) Fixed wrong client IP in access logs when serving app with swoole behind load balancer. ## 2.1.4 - 2020-04-30 From d9ae83a92b9bbdcaf493c61da801bce33bd3bfe3 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 7 May 2020 10:16:20 +0200 Subject: [PATCH 57/87] Updated everything related with dependencies in docker images --- .travis.yml | 2 +- Dockerfile | 11 ++++------- data/infra/php.Dockerfile | 7 ++----- data/infra/swoole.Dockerfile | 9 +++------ docker-compose.yml | 8 ++++---- 5 files changed, 14 insertions(+), 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index a95bf67d..6ab12fa8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ cache: before_install: - sudo ./data/infra/ci/install-ms-odbc.sh - docker-compose -f docker-compose.yml -f docker-compose.ci.yml up -d shlink_db_ms shlink_db shlink_db_postgres shlink_db_maria - - yes | pecl install pdo_sqlsrv swoole-4.4.15 + - yes | pecl install pdo_sqlsrv swoole-4.4.18 - echo 'extension = apcu.so' >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - phpenv config-rm xdebug.ini || return 0 diff --git a/Dockerfile b/Dockerfile index 64cd7ebe..3f2c5856 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -FROM php:7.4.2-alpine3.11 as base +FROM php:7.4.5-alpine3.11 as base -ARG SHLINK_VERSION=2.0.5 +ARG SHLINK_VERSION=2.1.4 ENV SHLINK_VERSION ${SHLINK_VERSION} -ENV SWOOLE_VERSION 4.4.15 +ENV SWOOLE_VERSION 4.4.18 ENV LC_ALL "C" WORKDIR /etc/shlink @@ -25,15 +25,12 @@ RUN \ # Install swoole and sqlsrv driver RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.1.1-1_amd64.apk && \ apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted mssql-tools_17.5.1.1-1_amd64.apk && \ apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv && \ docker-php-ext-enable swoole pdo_sqlsrv && \ apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk && \ - rm mssql-tools_17.5.1.1-1_amd64.apk + rm msodbcsql17_17.5.1.1-1_amd64.apk # Install shlink diff --git a/data/infra/php.Dockerfile b/data/infra/php.Dockerfile index c5401651..33b654c5 100644 --- a/data/infra/php.Dockerfile +++ b/data/infra/php.Dockerfile @@ -1,4 +1,4 @@ -FROM php:7.4.2-fpm-alpine3.11 +FROM php:7.4.5-fpm-alpine3.11 MAINTAINER Alejandro Celaya ENV APCU_VERSION 5.1.18 @@ -67,15 +67,12 @@ RUN rm /tmp/xdebug.tar.gz # Install sqlsrv driver RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.1.1-1_amd64.apk && \ apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted mssql-tools_17.5.1.1-1_amd64.apk && \ apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ pecl install pdo_sqlsrv && \ docker-php-ext-enable pdo_sqlsrv && \ apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk && \ - rm mssql-tools_17.5.1.1-1_amd64.apk + rm msodbcsql17_17.5.1.1-1_amd64.apk # Install composer RUN php -r "readfile('https://getcomposer.org/installer');" | php diff --git a/data/infra/swoole.Dockerfile b/data/infra/swoole.Dockerfile index 3f7a1513..9d8d4240 100644 --- a/data/infra/swoole.Dockerfile +++ b/data/infra/swoole.Dockerfile @@ -1,10 +1,10 @@ -FROM php:7.4.2-alpine3.11 +FROM php:7.4.5-alpine3.11 MAINTAINER Alejandro Celaya ENV APCU_VERSION 5.1.18 ENV APCU_BC_VERSION 1.0.5 ENV INOTIFY_VERSION 2.0.0 -ENV SWOOLE_VERSION 4.4.15 +ENV SWOOLE_VERSION 4.4.18 RUN apk update @@ -68,15 +68,12 @@ RUN rm /tmp/inotify.tar.gz # Install swoole and mssql driver RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/mssql-tools_17.5.1.1-1_amd64.apk && \ apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted mssql-tools_17.5.1.1-1_amd64.apk && \ apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv && \ docker-php-ext-enable swoole pdo_sqlsrv && \ apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk && \ - rm mssql-tools_17.5.1.1-1_amd64.apk + rm msodbcsql17_17.5.1.1-1_amd64.apk # Install composer RUN php -r "readfile('https://getcomposer.org/installer');" | php diff --git a/docker-compose.yml b/docker-compose.yml index 36153ad2..d700f3b3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,7 +79,7 @@ services: shlink_db_postgres: container_name: shlink_db_postgres - image: postgres:10.7-alpine + image: postgres:12.2-alpine ports: - "5433:5432" volumes: @@ -92,7 +92,7 @@ services: shlink_db_maria: container_name: shlink_db_maria - image: mariadb:10.2 + image: mariadb:10.5 ports: - "3308:3306" volumes: @@ -114,7 +114,7 @@ services: shlink_redis: container_name: shlink_redis - image: redis:5.0-alpine + image: redis:6.0-alpine ports: - "6380:6379" @@ -131,7 +131,7 @@ services: shlink_mercure: container_name: shlink_mercure - image: dunglas/mercure:v0.8 + image: dunglas/mercure:v0.9 ports: - "3080:80" environment: From b75922f1d366f78c9f233519e67fffb6e7141979 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 7 May 2020 10:17:34 +0200 Subject: [PATCH 58/87] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0358e791..14814b99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * [#692](https://github.com/shlinkio/shlink/issues/692) Drastically improved performance when loading visits. Specially noticeable when loading big result sets. * [#657](https://github.com/shlinkio/shlink/issues/657) Updated how DB tests are run in travis by using docker containers which allow all engines to be covered. +* [#751](https://github.com/shlinkio/shlink/issues/751) Updated PHP and swoole versions used in docker image, and removed mssql-tools, as they are not needed. #### Deprecated From 5f0293bc21491090acfc07dac6a4b77fce06c8ff Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 7 May 2020 10:45:53 +0200 Subject: [PATCH 59/87] Ensured stable tag is not pushed when building docker image for alpha or beta versions --- docker/build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/build b/docker/build index 5eea7888..f7e4b923 100755 --- a/docker/build +++ b/docker/build @@ -7,7 +7,9 @@ echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin if [[ ! -z $TRAVIS_TAG ]]; then docker build --build-arg SHLINK_VERSION=${TRAVIS_TAG#?} -t shlinkio/shlink:${TRAVIS_TAG#?} -t shlinkio/shlink:stable . docker push shlinkio/shlink:${TRAVIS_TAG#?} - docker push shlinkio/shlink:stable + + # Push stable tag only if this is not an alpha or beta tag + [[ $TRAVIS_TAG != *"alpha"* && $TRAVIS_TAG != *"beta"* ]] && docker push shlinkio/shlink:stable # If build branch is develop, build latest (on master, when there's no tag, do not build anything) elif [[ "$TRAVIS_BRANCH" == 'develop' ]]; then docker build -t shlinkio/shlink:latest . From 7e0a14493e4b6b4db945adb45a82caf5fbb0b499 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 10:14:38 +0200 Subject: [PATCH 60/87] Documented updates on the tags endpoint to return more detailed information --- docs/swagger/definitions/TagInfo.json | 17 +++++++++++++++ docs/swagger/paths/v1_tags.json | 21 +++++++++++++++++++ .../Core/src/{Service => }/Tag/TagService.php | 0 .../{Service => }/Tag/TagServiceInterface.php | 0 4 files changed, 38 insertions(+) create mode 100644 docs/swagger/definitions/TagInfo.json rename module/Core/src/{Service => }/Tag/TagService.php (100%) rename module/Core/src/{Service => }/Tag/TagServiceInterface.php (100%) diff --git a/docs/swagger/definitions/TagInfo.json b/docs/swagger/definitions/TagInfo.json new file mode 100644 index 00000000..e881ce02 --- /dev/null +++ b/docs/swagger/definitions/TagInfo.json @@ -0,0 +1,17 @@ +{ + "type": "object", + "properties": { + "tag": { + "type": "string", + "description": "The unique tag name" + }, + "shortUrlsCount": { + "type": "number", + "description": "The amount of short URLs using this tag" + }, + "userAgent": { + "type": "number", + "description": "The combined amount of visits received by short URLs with this tag" + } + } +} diff --git a/docs/swagger/paths/v1_tags.json b/docs/swagger/paths/v1_tags.json index 5e7fd71c..83bc7d68 100644 --- a/docs/swagger/paths/v1_tags.json +++ b/docs/swagger/paths/v1_tags.json @@ -14,6 +14,19 @@ "parameters": [ { "$ref": "../parameters/version.json" + }, + { + "name": "withStats", + "description": "Whether you want to include also a list with general stats by tag or not.", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "true", + "false" + ] + } } ], "responses": { @@ -26,12 +39,20 @@ "properties": { "tags": { "type": "object", + "required": ["data"], "properties": { "data": { "type": "array", "items": { "type": "string" } + }, + "stats": { + "description": "The tag stats will be returned only if the withStats param was provided with value 'true'", + "type": "array", + "items": { + "$ref": "../definitions/TagInfo.json" + } } } } diff --git a/module/Core/src/Service/Tag/TagService.php b/module/Core/src/Tag/TagService.php similarity index 100% rename from module/Core/src/Service/Tag/TagService.php rename to module/Core/src/Tag/TagService.php diff --git a/module/Core/src/Service/Tag/TagServiceInterface.php b/module/Core/src/Tag/TagServiceInterface.php similarity index 100% rename from module/Core/src/Service/Tag/TagServiceInterface.php rename to module/Core/src/Tag/TagServiceInterface.php From 626c92460bcd1d157a70cfb5d6e3bceb18110f47 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 10:15:33 +0200 Subject: [PATCH 61/87] Enhanced list tags endpoint so that it can also return stats foir every tag --- module/CLI/config/dependencies.config.php | 8 ++-- .../CLI/src/Command/Tag/CreateTagCommand.php | 2 +- .../CLI/src/Command/Tag/DeleteTagsCommand.php | 6 +-- .../CLI/src/Command/Tag/ListTagsCommand.php | 2 +- .../CLI/src/Command/Tag/RenameTagCommand.php | 4 +- .../test/Command/Tag/CreateTagCommandTest.php | 2 +- .../Command/Tag/DeleteTagsCommandTest.php | 4 +- .../test/Command/Tag/ListTagsCommandTest.php | 4 +- .../test/Command/Tag/RenameTagCommandTest.php | 2 +- module/Core/config/dependencies.config.php | 4 +- .../Shlinkio.Shlink.Core.Entity.Tag.php | 6 +++ module/Core/src/Entity/Tag.php | 3 ++ module/Core/src/Repository/TagRepository.php | 23 ++++++++++ .../src/Repository/TagRepositoryInterface.php | 6 +++ module/Core/src/Tag/Model/TagInfo.php | 46 +++++++++++++++++++ module/Core/src/Tag/TagService.php | 15 +++++- module/Core/src/Tag/TagServiceInterface.php | 8 +++- .../Core/test/Service/Tag/TagServiceTest.php | 2 +- module/Rest/config/dependencies.config.php | 8 ++-- .../Rest/src/Action/Tag/CreateTagsAction.php | 2 +- .../Rest/src/Action/Tag/DeleteTagsAction.php | 4 +- module/Rest/src/Action/Tag/ListTagsAction.php | 29 ++++++++---- .../Rest/src/Action/Tag/UpdateTagAction.php | 4 +- .../test/Action/Tag/CreateTagsActionTest.php | 2 +- .../test/Action/Tag/DeleteTagsActionTest.php | 2 +- .../test/Action/Tag/ListTagsActionTest.php | 4 +- .../test/Action/Tag/UpdateTagActionTest.php | 4 +- 27 files changed, 159 insertions(+), 47 deletions(-) create mode 100644 module/Core/src/Tag/Model/TagInfo.php diff --git a/module/CLI/config/dependencies.config.php b/module/CLI/config/dependencies.config.php index 0f2e70a5..1231a9b3 100644 --- a/module/CLI/config/dependencies.config.php +++ b/module/CLI/config/dependencies.config.php @@ -78,10 +78,10 @@ return [ Command\Api\DisableKeyCommand::class => [ApiKeyService::class], Command\Api\ListKeysCommand::class => [ApiKeyService::class], - Command\Tag\ListTagsCommand::class => [Service\Tag\TagService::class], - Command\Tag\CreateTagCommand::class => [Service\Tag\TagService::class], - Command\Tag\RenameTagCommand::class => [Service\Tag\TagService::class], - Command\Tag\DeleteTagsCommand::class => [Service\Tag\TagService::class], + Command\Tag\ListTagsCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Command\Tag\CreateTagCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Command\Tag\RenameTagCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Command\Tag\DeleteTagsCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], Command\Db\CreateDatabaseCommand::class => [ LockFactory::class, diff --git a/module/CLI/src/Command/Tag/CreateTagCommand.php b/module/CLI/src/Command/Tag/CreateTagCommand.php index 5fe56d46..451eb81e 100644 --- a/module/CLI/src/Command/Tag/CreateTagCommand.php +++ b/module/CLI/src/Command/Tag/CreateTagCommand.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Tag; use Shlinkio\Shlink\CLI\Util\ExitCodes; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; diff --git a/module/CLI/src/Command/Tag/DeleteTagsCommand.php b/module/CLI/src/Command/Tag/DeleteTagsCommand.php index 1cebe895..f50f835a 100644 --- a/module/CLI/src/Command/Tag/DeleteTagsCommand.php +++ b/module/CLI/src/Command/Tag/DeleteTagsCommand.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Tag; use Shlinkio\Shlink\CLI\Util\ExitCodes; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -16,9 +16,9 @@ class DeleteTagsCommand extends Command { public const NAME = 'tag:delete'; - private TagServiceInterface $tagService; + private \Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService) + public function __construct(\Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService) { parent::__construct(); $this->tagService = $tagService; diff --git a/module/CLI/src/Command/Tag/ListTagsCommand.php b/module/CLI/src/Command/Tag/ListTagsCommand.php index 0b8f0aa3..5a8389f3 100644 --- a/module/CLI/src/Command/Tag/ListTagsCommand.php +++ b/module/CLI/src/Command/Tag/ListTagsCommand.php @@ -7,7 +7,7 @@ namespace Shlinkio\Shlink\CLI\Command\Tag; use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\CLI\Util\ShlinkTable; use Shlinkio\Shlink\Core\Entity\Tag; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; diff --git a/module/CLI/src/Command/Tag/RenameTagCommand.php b/module/CLI/src/Command/Tag/RenameTagCommand.php index f30bc757..2a4a1245 100644 --- a/module/CLI/src/Command/Tag/RenameTagCommand.php +++ b/module/CLI/src/Command/Tag/RenameTagCommand.php @@ -7,7 +7,7 @@ namespace Shlinkio\Shlink\CLI\Command\Tag; use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\Core\Exception\TagConflictException; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; @@ -20,7 +20,7 @@ class RenameTagCommand extends Command private TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService) + public function __construct(\Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService) { parent::__construct(); $this->tagService = $tagService; diff --git a/module/CLI/test/Command/Tag/CreateTagCommandTest.php b/module/CLI/test/Command/Tag/CreateTagCommandTest.php index bed087a5..e156cf28 100644 --- a/module/CLI/test/Command/Tag/CreateTagCommandTest.php +++ b/module/CLI/test/Command/Tag/CreateTagCommandTest.php @@ -8,7 +8,7 @@ use Doctrine\Common\Collections\ArrayCollection; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\CreateTagCommand; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\CommandTester; diff --git a/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php b/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php index 060e5aac..1ec75c8f 100644 --- a/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php @@ -7,7 +7,7 @@ namespace ShlinkioTest\Shlink\CLI\Command\Tag; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\DeleteTagsCommand; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\CommandTester; @@ -18,7 +18,7 @@ class DeleteTagsCommandTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(TagServiceInterface::class); + $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); $command = new DeleteTagsCommand($this->tagService->reveal()); $app = new Application(); diff --git a/module/CLI/test/Command/Tag/ListTagsCommandTest.php b/module/CLI/test/Command/Tag/ListTagsCommandTest.php index f171127c..b3914916 100644 --- a/module/CLI/test/Command/Tag/ListTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/ListTagsCommandTest.php @@ -8,7 +8,7 @@ use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\ListTagsCommand; use Shlinkio\Shlink\Core\Entity\Tag; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\CommandTester; @@ -19,7 +19,7 @@ class ListTagsCommandTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(TagServiceInterface::class); + $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); $command = new ListTagsCommand($this->tagService->reveal()); $app = new Application(); diff --git a/module/CLI/test/Command/Tag/RenameTagCommandTest.php b/module/CLI/test/Command/Tag/RenameTagCommandTest.php index 59f8d89c..ee499c48 100644 --- a/module/CLI/test/Command/Tag/RenameTagCommandTest.php +++ b/module/CLI/test/Command/Tag/RenameTagCommandTest.php @@ -9,7 +9,7 @@ use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\RenameTagCommand; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\CommandTester; diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 67d18c40..5db524b8 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -28,7 +28,7 @@ return [ Service\ShortUrlService::class => ConfigAbstractFactory::class, Visit\VisitLocator::class => ConfigAbstractFactory::class, Visit\VisitsStatsHelper::class => ConfigAbstractFactory::class, - Service\Tag\TagService::class => ConfigAbstractFactory::class, + Tag\TagService::class => ConfigAbstractFactory::class, Service\ShortUrl\DeleteShortUrlService::class => ConfigAbstractFactory::class, Service\ShortUrl\ShortUrlResolver::class => ConfigAbstractFactory::class, @@ -58,7 +58,7 @@ return [ Service\ShortUrlService::class => ['em', Service\ShortUrl\ShortUrlResolver::class, Util\UrlValidator::class], Visit\VisitLocator::class => ['em'], Visit\VisitsStatsHelper::class => ['em'], - Service\Tag\TagService::class => ['em'], + Tag\TagService::class => ['em'], Service\ShortUrl\DeleteShortUrlService::class => [ 'em', Options\DeleteShortUrlsOptions::class, diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php index 214396bd..c3104a9d 100644 --- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php @@ -24,4 +24,10 @@ return static function (ClassMetadata $metadata, array $emConfig): void { $builder->createField('name', Types::STRING) ->unique() ->build(); + + $builder->createManyToMany('shortUrls', Entity\ShortUrl::class) + ->setJoinTable(determineTableName('short_urls_in_tags', $emConfig)) + ->addInverseJoinColumn('short_url_id', 'id', true, false, 'CASCADE') + ->addJoinColumn('tag_id', 'id', true, false, 'CASCADE') + ->build(); }; diff --git a/module/Core/src/Entity/Tag.php b/module/Core/src/Entity/Tag.php index 7530b70a..54c05c56 100644 --- a/module/Core/src/Entity/Tag.php +++ b/module/Core/src/Entity/Tag.php @@ -4,16 +4,19 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Entity; +use Doctrine\Common\Collections; use JsonSerializable; use Shlinkio\Shlink\Common\Entity\AbstractEntity; class Tag extends AbstractEntity implements JsonSerializable { private string $name; + private Collections\Collection $shortUrls; public function __construct(string $name) { $this->name = $name; + $this->shortUrls = new Collections\ArrayCollection(); } public function rename(string $name): void diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index 92328630..6f4cbf8c 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -6,6 +6,8 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Shlinkio\Shlink\Core\Entity\Tag; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; +use function Functional\map; class TagRepository extends EntityRepository implements TagRepositoryInterface { @@ -21,4 +23,25 @@ class TagRepository extends EntityRepository implements TagRepositoryInterface return $qb->getQuery()->execute(); } + + /** + * @return TagInfo[] + */ + public function findTagsWithInfo(): array + { + $dql = <<getEntityManager()->createQuery($dql); + + return map( + $query->getResult(), + fn (array $row) => new TagInfo($row['tag'], (int) $row['shortUrlsCount'], (int) $row['visitsCount']), + ); + } } diff --git a/module/Core/src/Repository/TagRepositoryInterface.php b/module/Core/src/Repository/TagRepositoryInterface.php index e253f7a4..37179e21 100644 --- a/module/Core/src/Repository/TagRepositoryInterface.php +++ b/module/Core/src/Repository/TagRepositoryInterface.php @@ -5,8 +5,14 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; interface TagRepositoryInterface extends ObjectRepository { public function deleteByName(array $names): int; + + /** + * @return TagInfo[] + */ + public function findTagsWithInfo(): array; } diff --git a/module/Core/src/Tag/Model/TagInfo.php b/module/Core/src/Tag/Model/TagInfo.php new file mode 100644 index 00000000..dbc51316 --- /dev/null +++ b/module/Core/src/Tag/Model/TagInfo.php @@ -0,0 +1,46 @@ +tag = $tag; + $this->shortUrlsCount = $shortUrlsCount; + $this->visitsCount = $visitsCount; + } + + public function tag(): Tag + { + return $this->tag; + } + + public function shortUrlsCount(): int + { + return $this->shortUrlsCount; + } + + public function visitsCount(): int + { + return $this->visitsCount; + } + + public function jsonSerialize(): array + { + return [ + 'tag' => $this->tag, + 'shortUrlsCount' => $this->shortUrlsCount, + 'visitsCount' => $this->visitsCount, + ]; + } +} diff --git a/module/Core/src/Tag/TagService.php b/module/Core/src/Tag/TagService.php index b95ddf82..7137e885 100644 --- a/module/Core/src/Tag/TagService.php +++ b/module/Core/src/Tag/TagService.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Shlinkio\Shlink\Core\Service\Tag; +namespace Shlinkio\Shlink\Core\Tag; use Doctrine\Common\Collections\Collection; use Doctrine\ORM; @@ -10,6 +10,8 @@ use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\TagConflictException; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Repository\TagRepository; +use Shlinkio\Shlink\Core\Repository\TagRepositoryInterface; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Util\TagManagerTrait; class TagService implements TagServiceInterface @@ -25,7 +27,6 @@ class TagService implements TagServiceInterface /** * @return Tag[] - * @throws \UnexpectedValueException */ public function listTags(): array { @@ -34,6 +35,16 @@ class TagService implements TagServiceInterface return $tags; } + /** + * @return TagInfo[] + */ + public function tagsInfo(): array + { + /** @var TagRepositoryInterface $repo */ + $repo = $this->em->getRepository(Tag::class); + return $repo->findTagsWithInfo(); + } + /** * @param string[] $tagNames */ diff --git a/module/Core/src/Tag/TagServiceInterface.php b/module/Core/src/Tag/TagServiceInterface.php index 16da503c..ed643fc5 100644 --- a/module/Core/src/Tag/TagServiceInterface.php +++ b/module/Core/src/Tag/TagServiceInterface.php @@ -2,12 +2,13 @@ declare(strict_types=1); -namespace Shlinkio\Shlink\Core\Service\Tag; +namespace Shlinkio\Shlink\Core\Tag; use Doctrine\Common\Collections\Collection; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\TagConflictException; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; interface TagServiceInterface { @@ -16,6 +17,11 @@ interface TagServiceInterface */ public function listTags(): array; + /** + * @return TagInfo[] + */ + public function tagsInfo(): array; + /** * @param string[] $tagNames */ diff --git a/module/Core/test/Service/Tag/TagServiceTest.php b/module/Core/test/Service/Tag/TagServiceTest.php index b8c9d59b..bd9c447f 100644 --- a/module/Core/test/Service/Tag/TagServiceTest.php +++ b/module/Core/test/Service/Tag/TagServiceTest.php @@ -13,7 +13,7 @@ use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\TagConflictException; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Repository\TagRepository; -use Shlinkio\Shlink\Core\Service\Tag\TagService; +use Shlinkio\Shlink\Core\Tag\TagService; class TagServiceTest extends TestCase { diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index bd347897..c7623ff7 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -65,10 +65,10 @@ return [ Action\Visit\GlobalVisitsAction::class => [Visit\VisitsStatsHelper::class], Action\ShortUrl\ListShortUrlsAction::class => [Service\ShortUrlService::class, 'config.url_shortener.domain'], Action\ShortUrl\EditShortUrlTagsAction::class => [Service\ShortUrlService::class], - Action\Tag\ListTagsAction::class => [Service\Tag\TagService::class], - Action\Tag\DeleteTagsAction::class => [Service\Tag\TagService::class], - Action\Tag\CreateTagsAction::class => [Service\Tag\TagService::class], - Action\Tag\UpdateTagAction::class => [Service\Tag\TagService::class], + Action\Tag\ListTagsAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Action\Tag\DeleteTagsAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Action\Tag\CreateTagsAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Action\Tag\UpdateTagAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], Middleware\ShortUrl\DropDefaultDomainFromRequestMiddleware::class => ['config.url_shortener.domain.hostname'], Middleware\ShortUrl\DefaultShortCodesLengthMiddleware::class => [ diff --git a/module/Rest/src/Action/Tag/CreateTagsAction.php b/module/Rest/src/Action/Tag/CreateTagsAction.php index c481b463..08f617c2 100644 --- a/module/Rest/src/Action/Tag/CreateTagsAction.php +++ b/module/Rest/src/Action/Tag/CreateTagsAction.php @@ -7,7 +7,7 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; class CreateTagsAction extends AbstractRestAction diff --git a/module/Rest/src/Action/Tag/DeleteTagsAction.php b/module/Rest/src/Action/Tag/DeleteTagsAction.php index 5002eba0..62d13a16 100644 --- a/module/Rest/src/Action/Tag/DeleteTagsAction.php +++ b/module/Rest/src/Action/Tag/DeleteTagsAction.php @@ -7,7 +7,7 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\EmptyResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; class DeleteTagsAction extends AbstractRestAction @@ -17,7 +17,7 @@ class DeleteTagsAction extends AbstractRestAction private TagServiceInterface $tagService; - public function __construct(TagServiceInterface $tagService) + public function __construct(\Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService) { $this->tagService = $tagService; } diff --git a/module/Rest/src/Action/Tag/ListTagsAction.php b/module/Rest/src/Action/Tag/ListTagsAction.php index 7211bce6..0832f17c 100644 --- a/module/Rest/src/Action/Tag/ListTagsAction.php +++ b/module/Rest/src/Action/Tag/ListTagsAction.php @@ -7,9 +7,12 @@ namespace Shlinkio\Shlink\Rest\Action\Tag; use Laminas\Diactoros\Response\JsonResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; +use function Functional\map; + class ListTagsAction extends AbstractRestAction { protected const ROUTE_PATH = '/tags'; @@ -22,18 +25,26 @@ class ListTagsAction extends AbstractRestAction $this->tagService = $tagService; } - /** - * Process an incoming server request and return a response, optionally delegating - * to the next middleware component to create the response. - * - * - * @throws \InvalidArgumentException - */ public function handle(ServerRequestInterface $request): ResponseInterface { + $query = $request->getQueryParams(); + $withStats = ($query['withStats'] ?? null) === 'true'; + + if (! $withStats) { + return new JsonResponse([ + 'tags' => [ + 'data' => $this->tagService->listTags(), + ], + ]); + } + + $tagsInfo = $this->tagService->tagsInfo(); + $data = map($tagsInfo, fn (TagInfo $info) => (string) $info->tag()); + return new JsonResponse([ 'tags' => [ - 'data' => $this->tagService->listTags(), + 'data' => $data, + 'stats' => $tagsInfo, ], ]); } diff --git a/module/Rest/src/Action/Tag/UpdateTagAction.php b/module/Rest/src/Action/Tag/UpdateTagAction.php index de5eb476..34924f78 100644 --- a/module/Rest/src/Action/Tag/UpdateTagAction.php +++ b/module/Rest/src/Action/Tag/UpdateTagAction.php @@ -8,7 +8,7 @@ use Laminas\Diactoros\Response\EmptyResponse; use Psr\Http\Message\ResponseInterface; use Psr\Http\Message\ServerRequestInterface; use Shlinkio\Shlink\Core\Exception\ValidationException; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\AbstractRestAction; class UpdateTagAction extends AbstractRestAction @@ -16,7 +16,7 @@ class UpdateTagAction extends AbstractRestAction protected const ROUTE_PATH = '/tags'; protected const ROUTE_ALLOWED_METHODS = [self::METHOD_PUT]; - private TagServiceInterface $tagService; + private \Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService; public function __construct(TagServiceInterface $tagService) { diff --git a/module/Rest/test/Action/Tag/CreateTagsActionTest.php b/module/Rest/test/Action/Tag/CreateTagsActionTest.php index 357abc5d..33aa0ba7 100644 --- a/module/Rest/test/Action/Tag/CreateTagsActionTest.php +++ b/module/Rest/test/Action/Tag/CreateTagsActionTest.php @@ -8,7 +8,7 @@ use Doctrine\Common\Collections\ArrayCollection; use Laminas\Diactoros\ServerRequest; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\Tag\CreateTagsAction; class CreateTagsActionTest extends TestCase diff --git a/module/Rest/test/Action/Tag/DeleteTagsActionTest.php b/module/Rest/test/Action/Tag/DeleteTagsActionTest.php index 484bd549..819a608a 100644 --- a/module/Rest/test/Action/Tag/DeleteTagsActionTest.php +++ b/module/Rest/test/Action/Tag/DeleteTagsActionTest.php @@ -7,7 +7,7 @@ namespace ShlinkioTest\Shlink\Rest\Action\Tag; use Laminas\Diactoros\ServerRequest; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\Tag\DeleteTagsAction; class DeleteTagsActionTest extends TestCase diff --git a/module/Rest/test/Action/Tag/ListTagsActionTest.php b/module/Rest/test/Action/Tag/ListTagsActionTest.php index 7e9b061f..daacb657 100644 --- a/module/Rest/test/Action/Tag/ListTagsActionTest.php +++ b/module/Rest/test/Action/Tag/ListTagsActionTest.php @@ -8,7 +8,7 @@ use Laminas\Diactoros\ServerRequest; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\Core\Entity\Tag; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\Tag\ListTagsAction; use function Shlinkio\Shlink\Common\json_decode; @@ -20,7 +20,7 @@ class ListTagsActionTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(TagServiceInterface::class); + $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); $this->action = new ListTagsAction($this->tagService->reveal()); } diff --git a/module/Rest/test/Action/Tag/UpdateTagActionTest.php b/module/Rest/test/Action/Tag/UpdateTagActionTest.php index ab09b4ea..7d865642 100644 --- a/module/Rest/test/Action/Tag/UpdateTagActionTest.php +++ b/module/Rest/test/Action/Tag/UpdateTagActionTest.php @@ -9,7 +9,7 @@ use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\ValidationException; -use Shlinkio\Shlink\Core\Service\Tag\TagServiceInterface; +use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\Tag\UpdateTagAction; class UpdateTagActionTest extends TestCase @@ -19,7 +19,7 @@ class UpdateTagActionTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(TagServiceInterface::class); + $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); $this->action = new UpdateTagAction($this->tagService->reveal()); } From 9a78fd1a261441919daffabbe44c9d5f0905fe67 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 10:25:33 +0200 Subject: [PATCH 62/87] Fixed definition of inversed many to many entity relationship --- .../entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php index c3104a9d..97d15758 100644 --- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Tag.php @@ -25,9 +25,5 @@ return static function (ClassMetadata $metadata, array $emConfig): void { ->unique() ->build(); - $builder->createManyToMany('shortUrls', Entity\ShortUrl::class) - ->setJoinTable(determineTableName('short_urls_in_tags', $emConfig)) - ->addInverseJoinColumn('short_url_id', 'id', true, false, 'CASCADE') - ->addJoinColumn('tag_id', 'id', true, false, 'CASCADE') - ->build(); + $builder->addInverseManyToMany('shortUrls', Entity\ShortUrl::class, 'tags'); }; From 06c59fe2dddaa05e65e19cb18a1a35b8339e4e5b Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 10:29:24 +0200 Subject: [PATCH 63/87] Fixed invalid imports after class refactoring --- module/CLI/config/dependencies.config.php | 9 +++++---- module/CLI/src/Command/Tag/DeleteTagsCommand.php | 4 ++-- module/CLI/src/Command/Tag/RenameTagCommand.php | 2 +- module/CLI/test/Command/Tag/DeleteTagsCommandTest.php | 2 +- module/CLI/test/Command/Tag/ListTagsCommandTest.php | 2 +- module/Core/src/Repository/TagRepository.php | 1 + module/Rest/config/dependencies.config.php | 9 +++++---- module/Rest/src/Action/Tag/DeleteTagsAction.php | 2 +- module/Rest/src/Action/Tag/UpdateTagAction.php | 2 +- module/Rest/test/Action/Tag/ListTagsActionTest.php | 2 +- module/Rest/test/Action/Tag/UpdateTagActionTest.php | 2 +- 11 files changed, 20 insertions(+), 17 deletions(-) diff --git a/module/CLI/config/dependencies.config.php b/module/CLI/config/dependencies.config.php index 1231a9b3..516bbbd4 100644 --- a/module/CLI/config/dependencies.config.php +++ b/module/CLI/config/dependencies.config.php @@ -11,6 +11,7 @@ use Laminas\ServiceManager\Factory\InvokableFactory; use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater; use Shlinkio\Shlink\Common\Doctrine\NoDbNameConnectionFactory; use Shlinkio\Shlink\Core\Service; +use Shlinkio\Shlink\Core\Tag\TagService; use Shlinkio\Shlink\Core\Visit; use Shlinkio\Shlink\Installer\Factory\ProcessHelperFactory; use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater; @@ -78,10 +79,10 @@ return [ Command\Api\DisableKeyCommand::class => [ApiKeyService::class], Command\Api\ListKeysCommand::class => [ApiKeyService::class], - Command\Tag\ListTagsCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], - Command\Tag\CreateTagCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], - Command\Tag\RenameTagCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], - Command\Tag\DeleteTagsCommand::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Command\Tag\ListTagsCommand::class => [TagService::class], + Command\Tag\CreateTagCommand::class => [TagService::class], + Command\Tag\RenameTagCommand::class => [TagService::class], + Command\Tag\DeleteTagsCommand::class => [TagService::class], Command\Db\CreateDatabaseCommand::class => [ LockFactory::class, diff --git a/module/CLI/src/Command/Tag/DeleteTagsCommand.php b/module/CLI/src/Command/Tag/DeleteTagsCommand.php index f50f835a..2b3eae14 100644 --- a/module/CLI/src/Command/Tag/DeleteTagsCommand.php +++ b/module/CLI/src/Command/Tag/DeleteTagsCommand.php @@ -16,9 +16,9 @@ class DeleteTagsCommand extends Command { public const NAME = 'tag:delete'; - private \Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService; + private TagServiceInterface $tagService; - public function __construct(\Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService) + public function __construct(TagServiceInterface $tagService) { parent::__construct(); $this->tagService = $tagService; diff --git a/module/CLI/src/Command/Tag/RenameTagCommand.php b/module/CLI/src/Command/Tag/RenameTagCommand.php index 2a4a1245..fe42a832 100644 --- a/module/CLI/src/Command/Tag/RenameTagCommand.php +++ b/module/CLI/src/Command/Tag/RenameTagCommand.php @@ -20,7 +20,7 @@ class RenameTagCommand extends Command private TagServiceInterface $tagService; - public function __construct(\Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService) + public function __construct(TagServiceInterface $tagService) { parent::__construct(); $this->tagService = $tagService; diff --git a/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php b/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php index 1ec75c8f..27a95de8 100644 --- a/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php @@ -18,7 +18,7 @@ class DeleteTagsCommandTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); + $this->tagService = $this->prophesize(TagServiceInterface::class); $command = new DeleteTagsCommand($this->tagService->reveal()); $app = new Application(); diff --git a/module/CLI/test/Command/Tag/ListTagsCommandTest.php b/module/CLI/test/Command/Tag/ListTagsCommandTest.php index b3914916..4318e906 100644 --- a/module/CLI/test/Command/Tag/ListTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/ListTagsCommandTest.php @@ -19,7 +19,7 @@ class ListTagsCommandTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); + $this->tagService = $this->prophesize(TagServiceInterface::class); $command = new ListTagsCommand($this->tagService->reveal()); $app = new Application(); diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index 6f4cbf8c..25c05596 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; + use function Functional\map; class TagRepository extends EntityRepository implements TagRepositoryInterface diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index c7623ff7..a10fd254 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -10,6 +10,7 @@ use Mezzio\Router\Middleware\ImplicitOptionsMiddleware; use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider; use Shlinkio\Shlink\Core\Options\AppOptions; use Shlinkio\Shlink\Core\Service; +use Shlinkio\Shlink\Core\Tag\TagService; use Shlinkio\Shlink\Core\Visit; use Shlinkio\Shlink\Rest\Service\ApiKeyService; @@ -65,10 +66,10 @@ return [ Action\Visit\GlobalVisitsAction::class => [Visit\VisitsStatsHelper::class], Action\ShortUrl\ListShortUrlsAction::class => [Service\ShortUrlService::class, 'config.url_shortener.domain'], Action\ShortUrl\EditShortUrlTagsAction::class => [Service\ShortUrlService::class], - Action\Tag\ListTagsAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], - Action\Tag\DeleteTagsAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], - Action\Tag\CreateTagsAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], - Action\Tag\UpdateTagAction::class => [\Shlinkio\Shlink\Core\Tag\TagService::class], + Action\Tag\ListTagsAction::class => [TagService::class], + Action\Tag\DeleteTagsAction::class => [TagService::class], + Action\Tag\CreateTagsAction::class => [TagService::class], + Action\Tag\UpdateTagAction::class => [TagService::class], Middleware\ShortUrl\DropDefaultDomainFromRequestMiddleware::class => ['config.url_shortener.domain.hostname'], Middleware\ShortUrl\DefaultShortCodesLengthMiddleware::class => [ diff --git a/module/Rest/src/Action/Tag/DeleteTagsAction.php b/module/Rest/src/Action/Tag/DeleteTagsAction.php index 62d13a16..f38c443a 100644 --- a/module/Rest/src/Action/Tag/DeleteTagsAction.php +++ b/module/Rest/src/Action/Tag/DeleteTagsAction.php @@ -17,7 +17,7 @@ class DeleteTagsAction extends AbstractRestAction private TagServiceInterface $tagService; - public function __construct(\Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService) + public function __construct(TagServiceInterface $tagService) { $this->tagService = $tagService; } diff --git a/module/Rest/src/Action/Tag/UpdateTagAction.php b/module/Rest/src/Action/Tag/UpdateTagAction.php index 34924f78..fbf93f50 100644 --- a/module/Rest/src/Action/Tag/UpdateTagAction.php +++ b/module/Rest/src/Action/Tag/UpdateTagAction.php @@ -16,7 +16,7 @@ class UpdateTagAction extends AbstractRestAction protected const ROUTE_PATH = '/tags'; protected const ROUTE_ALLOWED_METHODS = [self::METHOD_PUT]; - private \Shlinkio\Shlink\Core\Tag\TagServiceInterface $tagService; + private TagServiceInterface $tagService; public function __construct(TagServiceInterface $tagService) { diff --git a/module/Rest/test/Action/Tag/ListTagsActionTest.php b/module/Rest/test/Action/Tag/ListTagsActionTest.php index daacb657..813d62f4 100644 --- a/module/Rest/test/Action/Tag/ListTagsActionTest.php +++ b/module/Rest/test/Action/Tag/ListTagsActionTest.php @@ -20,7 +20,7 @@ class ListTagsActionTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); + $this->tagService = $this->prophesize(TagServiceInterface::class); $this->action = new ListTagsAction($this->tagService->reveal()); } diff --git a/module/Rest/test/Action/Tag/UpdateTagActionTest.php b/module/Rest/test/Action/Tag/UpdateTagActionTest.php index 7d865642..11b2c1c4 100644 --- a/module/Rest/test/Action/Tag/UpdateTagActionTest.php +++ b/module/Rest/test/Action/Tag/UpdateTagActionTest.php @@ -19,7 +19,7 @@ class UpdateTagActionTest extends TestCase public function setUp(): void { - $this->tagService = $this->prophesize(\Shlinkio\Shlink\Core\Tag\TagServiceInterface::class); + $this->tagService = $this->prophesize(TagServiceInterface::class); $this->action = new UpdateTagAction($this->tagService->reveal()); } From bdd14427d921c9969e9b8ffabc51f4339476cedd Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 11:09:28 +0200 Subject: [PATCH 64/87] Added tests for TagRepository::findTagsWithInfo --- module/Core/src/Repository/TagRepository.php | 2 +- .../test-db/Repository/TagRepositoryTest.php | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index 25c05596..05b2481c 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -35,7 +35,7 @@ class TagRepository extends EntityRepository implements TagRepositoryInterface FROM Shlinkio\Shlink\Core\Entity\Tag t LEFT JOIN t.shortUrls s LEFT JOIN s.visits v - GROUP BY tag + GROUP BY t ORDER BY t.name ASC DQL; $query = $this->getEntityManager()->createQuery($dql); diff --git a/module/Core/test-db/Repository/TagRepositoryTest.php b/module/Core/test-db/Repository/TagRepositoryTest.php index 94e38f53..8e1a11ef 100644 --- a/module/Core/test-db/Repository/TagRepositoryTest.php +++ b/module/Core/test-db/Repository/TagRepositoryTest.php @@ -4,13 +4,21 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\Repository; +use Doctrine\Common\Collections\ArrayCollection; +use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Tag; +use Shlinkio\Shlink\Core\Entity\Visit; +use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\TestUtils\DbTest\DatabaseTestCase; +use function array_chunk; + class TagRepositoryTest extends DatabaseTestCase { protected const ENTITIES_TO_EMPTY = [ + Visit::class, + ShortUrl::class, Tag::class, ]; @@ -40,4 +48,53 @@ class TagRepositoryTest extends DatabaseTestCase $this->assertEquals(2, $this->repo->deleteByName($toDelete)); } + + /** @test */ + public function properTagsInfoIsReturned(): void + { + $names = ['foo', 'bar', 'baz', 'another']; + $tags = []; + foreach ($names as $name) { + $tag = new Tag($name); + $tags[] = $tag; + $this->getEntityManager()->persist($tag); + } + + [$firstUrlTags] = array_chunk($tags, 3); + $secondUrlTags = [$tags[0]]; + + $shortUrl = new ShortUrl(''); + $shortUrl->setTags(new ArrayCollection($firstUrlTags)); + $this->getEntityManager()->persist($shortUrl); + $this->getEntityManager()->persist(new Visit($shortUrl, Visitor::emptyInstance())); + $this->getEntityManager()->persist(new Visit($shortUrl, Visitor::emptyInstance())); + $this->getEntityManager()->persist(new Visit($shortUrl, Visitor::emptyInstance())); + + $shortUrl2 = new ShortUrl(''); + $shortUrl2->setTags(new ArrayCollection($secondUrlTags)); + $this->getEntityManager()->persist($shortUrl2); + $this->getEntityManager()->persist(new Visit($shortUrl2, Visitor::emptyInstance())); + + $this->getEntityManager()->flush(); + + $result = $this->repo->findTagsWithInfo(); + + $this->assertCount(4, $result); + $this->assertEquals( + ['tag' => $tags[3], 'shortUrlsCount' => 0, 'visitsCount' => 0], + $result[0]->jsonSerialize(), + ); + $this->assertEquals( + ['tag' => $tags[1], 'shortUrlsCount' => 1, 'visitsCount' => 3], + $result[1]->jsonSerialize(), + ); + $this->assertEquals( + ['tag' => $tags[2], 'shortUrlsCount' => 1, 'visitsCount' => 3], + $result[2]->jsonSerialize(), + ); + $this->assertEquals( + ['tag' => $tags[0], 'shortUrlsCount' => 2, 'visitsCount' => 4], + $result[3]->jsonSerialize(), + ); + } } From 2e269bcacd36994be5756858239e55690d3a82ca Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 11:14:39 +0200 Subject: [PATCH 65/87] Updated TagServiceTest --- .../Core/test/Service/Tag/TagServiceTest.php | 53 +++++++++---------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/module/Core/test/Service/Tag/TagServiceTest.php b/module/Core/test/Service/Tag/TagServiceTest.php index bd9c447f..c031e51f 100644 --- a/module/Core/test/Service/Tag/TagServiceTest.php +++ b/module/Core/test/Service/Tag/TagServiceTest.php @@ -5,7 +5,6 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\Service\Tag; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\EntityRepository; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\Prophecy\ObjectProphecy; @@ -13,16 +12,21 @@ use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\TagConflictException; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Repository\TagRepository; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\TagService; class TagServiceTest extends TestCase { private TagService $service; private ObjectProphecy $em; + private ObjectProphecy $repo; public function setUp(): void { $this->em = $this->prophesize(EntityManagerInterface::class); + $this->repo = $this->prophesize(TagRepository::class); + $this->em->getRepository(Tag::class)->willReturn($this->repo->reveal())->shouldBeCalled(); + $this->service = new TagService($this->em->reveal()); } @@ -31,36 +35,41 @@ class TagServiceTest extends TestCase { $expected = [new Tag('foo'), new Tag('bar')]; - $repo = $this->prophesize(EntityRepository::class); - $find = $repo->findBy(Argument::cetera())->willReturn($expected); - $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + $find = $this->repo->findBy(Argument::cetera())->willReturn($expected); $result = $this->service->listTags(); $this->assertEquals($expected, $result); $find->shouldHaveBeenCalled(); - $getRepo->shouldHaveBeenCalled(); + } + + /** @test */ + public function tagsInfoDelegatesOnRepository(): void + { + $expected = [new TagInfo(new Tag('foo'), 1, 1), new TagInfo(new Tag('bar'), 3, 10)]; + + $find = $this->repo->findTagsWithInfo()->willReturn($expected); + + $result = $this->service->tagsInfo(); + + $this->assertEquals($expected, $result); + $find->shouldHaveBeenCalled(); } /** @test */ public function deleteTagsDelegatesOnRepository(): void { - $repo = $this->prophesize(TagRepository::class); - $delete = $repo->deleteByName(['foo', 'bar'])->willReturn(4); - $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + $delete = $this->repo->deleteByName(['foo', 'bar'])->willReturn(4); $this->service->deleteTags(['foo', 'bar']); $delete->shouldHaveBeenCalled(); - $getRepo->shouldHaveBeenCalled(); } /** @test */ public function createTagsPersistsEntities(): void { - $repo = $this->prophesize(TagRepository::class); - $find = $repo->findOneBy(Argument::cetera())->willReturn(new Tag('foo')); - $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + $find = $this->repo->findOneBy(Argument::cetera())->willReturn(new Tag('foo')); $persist = $this->em->persist(Argument::type(Tag::class))->willReturn(null); $flush = $this->em->flush()->willReturn(null); @@ -68,7 +77,6 @@ class TagServiceTest extends TestCase $this->assertCount(2, $result); $find->shouldHaveBeenCalled(); - $getRepo->shouldHaveBeenCalled(); $persist->shouldHaveBeenCalledTimes(2); $flush->shouldHaveBeenCalled(); } @@ -76,12 +84,9 @@ class TagServiceTest extends TestCase /** @test */ public function renameInvalidTagThrowsException(): void { - $repo = $this->prophesize(TagRepository::class); - $find = $repo->findOneBy(Argument::cetera())->willReturn(null); - $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + $find = $this->repo->findOneBy(Argument::cetera())->willReturn(null); $find->shouldBeCalled(); - $getRepo->shouldBeCalled(); $this->expectException(TagNotFoundException::class); $this->service->renameTag('foo', 'bar'); @@ -95,10 +100,8 @@ class TagServiceTest extends TestCase { $expected = new Tag('foo'); - $repo = $this->prophesize(TagRepository::class); - $find = $repo->findOneBy(Argument::cetera())->willReturn($expected); - $countTags = $repo->count(Argument::cetera())->willReturn($count); - $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + $find = $this->repo->findOneBy(Argument::cetera())->willReturn($expected); + $countTags = $this->repo->count(Argument::cetera())->willReturn($count); $flush = $this->em->flush()->willReturn(null); $tag = $this->service->renameTag($oldName, $newName); @@ -106,7 +109,6 @@ class TagServiceTest extends TestCase $this->assertSame($expected, $tag); $this->assertEquals($newName, (string) $tag); $find->shouldHaveBeenCalled(); - $getRepo->shouldHaveBeenCalled(); $flush->shouldHaveBeenCalled(); $countTags->shouldHaveBeenCalledTimes($count > 0 ? 0 : 1); } @@ -120,14 +122,11 @@ class TagServiceTest extends TestCase /** @test */ public function renameTagToAnExistingNameThrowsException(): void { - $repo = $this->prophesize(TagRepository::class); - $find = $repo->findOneBy(Argument::cetera())->willReturn(new Tag('foo')); - $countTags = $repo->count(Argument::cetera())->willReturn(1); - $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + $find = $this->repo->findOneBy(Argument::cetera())->willReturn(new Tag('foo')); + $countTags = $this->repo->count(Argument::cetera())->willReturn(1); $flush = $this->em->flush(Argument::any())->willReturn(null); $find->shouldBeCalled(); - $getRepo->shouldBeCalled(); $countTags->shouldBeCalled(); $flush->shouldNotBeCalled(); $this->expectException(TagConflictException::class); From 91aaffc6db942e5a20b29f5845cdd68bdfafaa73 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 11:32:06 +0200 Subject: [PATCH 66/87] Updated ListTagsActionTest --- module/Core/src/Tag/Model/TagInfo.php | 10 ---- .../test/Action/Tag/ListTagsActionTest.php | 53 +++++++++++++++---- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/module/Core/src/Tag/Model/TagInfo.php b/module/Core/src/Tag/Model/TagInfo.php index dbc51316..0237f062 100644 --- a/module/Core/src/Tag/Model/TagInfo.php +++ b/module/Core/src/Tag/Model/TagInfo.php @@ -25,16 +25,6 @@ final class TagInfo implements JsonSerializable return $this->tag; } - public function shortUrlsCount(): int - { - return $this->shortUrlsCount; - } - - public function visitsCount(): int - { - return $this->visitsCount; - } - public function jsonSerialize(): array { return [ diff --git a/module/Rest/test/Action/Tag/ListTagsActionTest.php b/module/Rest/test/Action/Tag/ListTagsActionTest.php index 813d62f4..461ddd3f 100644 --- a/module/Rest/test/Action/Tag/ListTagsActionTest.php +++ b/module/Rest/test/Action/Tag/ListTagsActionTest.php @@ -4,15 +4,15 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Rest\Action\Tag; -use Laminas\Diactoros\ServerRequest; +use Laminas\Diactoros\Response\JsonResponse; +use Laminas\Diactoros\ServerRequestFactory; use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\Core\Entity\Tag; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Shlinkio\Shlink\Rest\Action\Tag\ListTagsAction; -use function Shlinkio\Shlink\Common\json_decode; - class ListTagsActionTest extends TestCase { private ListTagsAction $action; @@ -24,18 +24,53 @@ class ListTagsActionTest extends TestCase $this->action = new ListTagsAction($this->tagService->reveal()); } - /** @test */ - public function returnsDataFromService(): void + /** + * @test + * @dataProvider provideNoStatsQueries + */ + public function returnsBaseDataWhenStatsAreNotRequested(array $query): void { - $listTags = $this->tagService->listTags()->willReturn([new Tag('foo'), new Tag('bar')]); + $tags = [new Tag('foo'), new Tag('bar')]; + $listTags = $this->tagService->listTags()->willReturn($tags); - $resp = $this->action->handle(new ServerRequest()); + /** @var JsonResponse $resp */ + $resp = $this->action->handle(ServerRequestFactory::fromGlobals()->withQueryParams($query)); + $payload = $resp->getPayload(); + + $this->assertEquals([ + 'tags' => [ + 'data' => $tags, + ], + ], $payload); + $listTags->shouldHaveBeenCalled(); + } + + public function provideNoStatsQueries(): iterable + { + yield 'no query' => [[]]; + yield 'withStats is false' => [['withStats' => 'withStats']]; + yield 'withStats is something else' => [['withStats' => 'foo']]; + } + + /** @test */ + public function returnsStatsWhenRequested(): void + { + $stats = [ + new TagInfo(new Tag('foo'), 1, 1), + new TagInfo(new Tag('bar'), 3, 10), + ]; + $tagsInfo = $this->tagService->tagsInfo()->willReturn($stats); + + /** @var JsonResponse $resp */ + $resp = $this->action->handle(ServerRequestFactory::fromGlobals()->withQueryParams(['withStats' => 'true'])); + $payload = $resp->getPayload(); $this->assertEquals([ 'tags' => [ 'data' => ['foo', 'bar'], + 'stats' => $stats, ], - ], json_decode((string) $resp->getBody())); - $listTags->shouldHaveBeenCalled(); + ], $payload); + $tagsInfo->shouldHaveBeenCalled(); } } From 00cac4ba720c641f0f817b4fbf4228d9b74c61e8 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 11:48:31 +0200 Subject: [PATCH 67/87] Created rest test for list tags action --- .../Shlinkio.Shlink.Core.Entity.ShortUrl.php | 1 + .../test-api/Action/ListTagsActionTest.php | 50 +++++++++++++++++++ module/Rest/test-api/Fixtures/TagsFixture.php | 1 + phpstan.neon | 1 + 4 files changed, 53 insertions(+) create mode 100644 module/Rest/test-api/Action/ListTagsActionTest.php diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php index a4aef29f..871ac113 100644 --- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php @@ -60,6 +60,7 @@ return static function (ClassMetadata $metadata, array $emConfig): void { ->setJoinTable(determineTableName('short_urls_in_tags', $emConfig)) ->addInverseJoinColumn('tag_id', 'id', true, false, 'CASCADE') ->addJoinColumn('short_url_id', 'id', true, false, 'CASCADE') + ->setOrderBy(['name' => 'ASC']) ->build(); $builder->createManyToOne('domain', Entity\Domain::class) diff --git a/module/Rest/test-api/Action/ListTagsActionTest.php b/module/Rest/test-api/Action/ListTagsActionTest.php new file mode 100644 index 00000000..0690d4f2 --- /dev/null +++ b/module/Rest/test-api/Action/ListTagsActionTest.php @@ -0,0 +1,50 @@ +callApiWithKey(self::METHOD_GET, '/tags', [RequestOptions::QUERY => $query]); + $payload = $this->getJsonResponsePayload($resp); + + $this->assertEquals(['tags' => $expectedTags], $payload); + } + + public function provideQueries(): iterable + { + yield 'stats not requested' => [[], [ + 'data' => ['bar', 'baz', 'foo'], + ]]; + yield 'stats requested' => [['withStats' => 'true'], [ + 'data' => ['bar', 'baz', 'foo'], + 'stats' => [ + [ + 'tag' => 'bar', + 'shortUrlsCount' => 1, + 'visitsCount' => 2, + ], + [ + 'tag' => 'baz', + 'shortUrlsCount' => 0, + 'visitsCount' => 0, + ], + [ + 'tag' => 'foo', + 'shortUrlsCount' => 2, + 'visitsCount' => 5, + ], + ], + ]]; + } +} diff --git a/module/Rest/test-api/Fixtures/TagsFixture.php b/module/Rest/test-api/Fixtures/TagsFixture.php index 5bd10ca7..5d3333cc 100644 --- a/module/Rest/test-api/Fixtures/TagsFixture.php +++ b/module/Rest/test-api/Fixtures/TagsFixture.php @@ -24,6 +24,7 @@ class TagsFixture extends AbstractFixture implements DependentFixtureInterface $manager->persist($fooTag); $barTag = new Tag('bar'); $manager->persist($barTag); + $manager->persist(new Tag('baz')); /** @var ShortUrl $abcShortUrl */ $abcShortUrl = $this->getReference('abc123_short_url'); diff --git a/phpstan.neon b/phpstan.neon index d983a985..e065acef 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -4,3 +4,4 @@ parameters: ignoreErrors: - '#AbstractQuery::setParameters()#' - '#mustRun()#' + - '#AssociationBuilder::setOrderBy#' From 252cc7f49d88373cdb01748c32d8901a23ff8151 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 11:53:26 +0200 Subject: [PATCH 68/87] Updated changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9fa8531..49fc0a29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this Also, Shlink exposes a new endpoint `GET /rest/v2/mercure-info`, which returns the public URL of the mercure hub, and a valid JWT that can be used to subsribe to updates. * [#673](https://github.com/shlinkio/shlink/issues/673) Added new `[GET /visits]` rest endpoint which returns basic visits stats. +* [#672](https://github.com/shlinkio/shlink/issues/672) Enhanced `[GET /tags]` rest endpoint so that it is possible to get basic stats info for every tag. + + Now, if the `withStats=true` query param is provided, the response payload will include a new `stats` property which is a list with the amount of short URLs and visits for every tag. #### Changed From c336bb19010cfdfeb01d81a8d835d92b946a17de Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 12:39:02 +0200 Subject: [PATCH 69/87] Updated ListTagsCommand so that it displays extended information --- .../CLI/src/Command/Tag/ListTagsCommand.php | 13 +++++++---- .../test/Command/Tag/ListTagsCommandTest.php | 23 +++++++++++-------- module/Core/src/Tag/Model/TagInfo.php | 10 ++++++++ 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/module/CLI/src/Command/Tag/ListTagsCommand.php b/module/CLI/src/Command/Tag/ListTagsCommand.php index 5a8389f3..11e22a4f 100644 --- a/module/CLI/src/Command/Tag/ListTagsCommand.php +++ b/module/CLI/src/Command/Tag/ListTagsCommand.php @@ -6,7 +6,7 @@ namespace Shlinkio\Shlink\CLI\Command\Tag; use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\CLI\Util\ShlinkTable; -use Shlinkio\Shlink\Core\Entity\Tag; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputInterface; @@ -35,17 +35,20 @@ class ListTagsCommand extends Command protected function execute(InputInterface $input, OutputInterface $output): ?int { - ShlinkTable::fromOutput($output)->render(['Name'], $this->getTagsRows()); + ShlinkTable::fromOutput($output)->render(['Name', 'URLs amount', 'Visits amount'], $this->getTagsRows()); return ExitCodes::EXIT_SUCCESS; } private function getTagsRows(): array { - $tags = $this->tagService->listTags(); + $tags = $this->tagService->tagsInfo(); if (empty($tags)) { - return [['No tags yet']]; + return [['No tags found', '-', '-']]; } - return map($tags, fn (Tag $tag) => [(string) $tag]); + return map( + $tags, + fn (TagInfo $tagInfo) => [(string) $tagInfo->tag(), $tagInfo->shortUrlsCount(), $tagInfo->visitsCount()], + ); } } diff --git a/module/CLI/test/Command/Tag/ListTagsCommandTest.php b/module/CLI/test/Command/Tag/ListTagsCommandTest.php index 4318e906..b6087307 100644 --- a/module/CLI/test/Command/Tag/ListTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/ListTagsCommandTest.php @@ -8,6 +8,7 @@ use PHPUnit\Framework\TestCase; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\ListTagsCommand; use Shlinkio\Shlink\Core\Entity\Tag; +use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; use Symfony\Component\Console\Application; use Symfony\Component\Console\Tester\CommandTester; @@ -31,28 +32,32 @@ class ListTagsCommandTest extends TestCase /** @test */ public function noTagsPrintsEmptyMessage(): void { - $listTags = $this->tagService->listTags()->willReturn([]); + $tagsInfo = $this->tagService->tagsInfo()->willReturn([]); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); - $this->assertStringContainsString('No tags yet', $output); - $listTags->shouldHaveBeenCalled(); + $this->assertStringContainsString('No tags found', $output); + $tagsInfo->shouldHaveBeenCalled(); } /** @test */ public function listOfTagsIsPrinted(): void { - $listTags = $this->tagService->listTags()->willReturn([ - new Tag('foo'), - new Tag('bar'), + $tagsInfo = $this->tagService->tagsInfo()->willReturn([ + new TagInfo(new Tag('foo'), 10, 2), + new TagInfo(new Tag('bar'), 7, 32), ]); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); - $this->assertStringContainsString('foo', $output); - $this->assertStringContainsString('bar', $output); - $listTags->shouldHaveBeenCalled(); + $this->assertStringContainsString('| foo', $output); + $this->assertStringContainsString('| bar', $output); + $this->assertStringContainsString('| 10 ', $output); + $this->assertStringContainsString('| 2 ', $output); + $this->assertStringContainsString('| 7 ', $output); + $this->assertStringContainsString('| 32 ', $output); + $tagsInfo->shouldHaveBeenCalled(); } } diff --git a/module/Core/src/Tag/Model/TagInfo.php b/module/Core/src/Tag/Model/TagInfo.php index 0237f062..dbc51316 100644 --- a/module/Core/src/Tag/Model/TagInfo.php +++ b/module/Core/src/Tag/Model/TagInfo.php @@ -25,6 +25,16 @@ final class TagInfo implements JsonSerializable return $this->tag; } + public function shortUrlsCount(): int + { + return $this->shortUrlsCount; + } + + public function visitsCount(): int + { + return $this->visitsCount; + } + public function jsonSerialize(): array { return [ From 7da00fbc8cc2a8115b4f457924463f7a85e8650e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 12:58:49 +0200 Subject: [PATCH 70/87] Updated Visit entity so that the address can be optionally obfuscated --- module/Core/src/Entity/Visit.php | 8 +++---- .../Repository/VisitRepositoryTest.php | 8 ++++++- module/Core/test/Entity/VisitTest.php | 24 ++++++++++++++++++- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index e8cbb119..fb5e3bfb 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -21,19 +21,19 @@ class Visit extends AbstractEntity implements JsonSerializable private ShortUrl $shortUrl; private ?VisitLocation $visitLocation = null; - public function __construct(ShortUrl $shortUrl, Visitor $visitor, ?Chronos $date = null) + public function __construct(ShortUrl $shortUrl, Visitor $visitor, bool $obfuscate = true, ?Chronos $date = null) { $this->shortUrl = $shortUrl; $this->date = $date ?? Chronos::now(); $this->userAgent = $visitor->getUserAgent(); $this->referer = $visitor->getReferer(); - $this->remoteAddr = $this->obfuscateAddress($visitor->getRemoteAddress()); + $this->remoteAddr = $this->processAddress($obfuscate, $visitor->getRemoteAddress()); } - private function obfuscateAddress(?string $address): ?string + private function processAddress(bool $obfuscate, ?string $address): ?string { // Localhost addresses do not need to be obfuscated - if ($address === null || $address === IpAddress::LOCALHOST) { + if (! $obfuscate || $address === null || $address === IpAddress::LOCALHOST) { return $address; } diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 034b15f9..13fc8581 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -139,13 +139,19 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->persist($shortUrlWithDomain); for ($i = 0; $i < 6; $i++) { - $visit = new Visit($shortUrl, Visitor::emptyInstance(), Chronos::parse(sprintf('2016-01-0%s', $i + 1))); + $visit = new Visit( + $shortUrl, + Visitor::emptyInstance(), + true, + Chronos::parse(sprintf('2016-01-0%s', $i + 1)), + ); $this->getEntityManager()->persist($visit); } for ($i = 0; $i < 3; $i++) { $visit = new Visit( $shortUrlWithDomain, Visitor::emptyInstance(), + true, Chronos::parse(sprintf('2016-01-0%s', $i + 1)), ); $this->getEntityManager()->persist($visit); diff --git a/module/Core/test/Entity/VisitTest.php b/module/Core/test/Entity/VisitTest.php index 9af71a09..a82e1939 100644 --- a/module/Core/test/Entity/VisitTest.php +++ b/module/Core/test/Entity/VisitTest.php @@ -6,6 +6,7 @@ namespace ShlinkioTest\Shlink\Core\Entity; use Cake\Chronos\Chronos; use PHPUnit\Framework\TestCase; +use Shlinkio\Shlink\Common\Util\IpAddress; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Model\Visitor; @@ -18,7 +19,7 @@ class VisitTest extends TestCase */ public function isProperlyJsonSerialized(?Chronos $date): void { - $visit = new Visit(new ShortUrl(''), new Visitor('Chrome', 'some site', '1.2.3.4'), $date); + $visit = new Visit(new ShortUrl(''), new Visitor('Chrome', 'some site', '1.2.3.4'), true, $date); $this->assertEquals([ 'referer' => 'some site', @@ -33,4 +34,25 @@ class VisitTest extends TestCase yield 'null date' => [null]; yield 'not null date' => [Chronos::now()->subDays(10)]; } + + /** + * @test + * @dataProvider provideAddresses + */ + public function addressIsObfuscatedWhenRequested(bool $obfuscate, ?string $address, ?string $expectedAddress): void + { + $visit = new Visit(new ShortUrl(''), new Visitor('Chrome', 'some site', $address), $obfuscate); + + $this->assertEquals($expectedAddress, $visit->getRemoteAddr()); + } + + public function provideAddresses(): iterable + { + yield 'obfuscated null address' => [true, null, null]; + yield 'non-obfuscated null address' => [false, null, null]; + yield 'obfuscated localhost' => [true, IpAddress::LOCALHOST, IpAddress::LOCALHOST]; + yield 'non-obfuscated localhost' => [false, IpAddress::LOCALHOST, IpAddress::LOCALHOST]; + yield 'obfuscated regular address' => [true, '1.2.3.4', '1.2.3.0']; + yield 'non-obfuscated regular address' => [false, '1.2.3.4', '1.2.3.4']; + } } From eac468514ba3e4a6d6909b328170514af630879a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 13:10:58 +0200 Subject: [PATCH 71/87] Allow to determine if remote addresses should be obfuscated at configuration level --- config/autoload/url-shortener.global.php | 1 + module/Core/config/dependencies.config.php | 6 +++++- module/Core/src/Service/VisitsTracker.php | 11 ++++++++--- module/Core/test/Service/VisitsTrackerTest.php | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/config/autoload/url-shortener.global.php b/config/autoload/url-shortener.global.php index 165e0258..8439cc8a 100644 --- a/config/autoload/url-shortener.global.php +++ b/config/autoload/url-shortener.global.php @@ -12,6 +12,7 @@ return [ 'hostname' => '', ], 'validate_url' => false, + 'obfuscate_remote_addr' => true, 'visits_webhooks' => [], 'default_short_codes_length' => DEFAULT_SHORT_CODES_LENGTH, ], diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 5db524b8..90a4ffaf 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -54,7 +54,11 @@ return [ Options\UrlShortenerOptions::class => ['config.url_shortener'], Service\UrlShortener::class => [Util\UrlValidator::class, 'em', Resolver\PersistenceDomainResolver::class], - Service\VisitsTracker::class => ['em', EventDispatcherInterface::class], + Service\VisitsTracker::class => [ + 'em', + EventDispatcherInterface::class, + 'config.url_shortener.obfuscate_remote_addr', + ], Service\ShortUrlService::class => ['em', Service\ShortUrl\ShortUrlResolver::class, Util\UrlValidator::class], Visit\VisitLocator::class => ['em'], Visit\VisitsStatsHelper::class => ['em'], diff --git a/module/Core/src/Service/VisitsTracker.php b/module/Core/src/Service/VisitsTracker.php index f477681a..a60513e4 100644 --- a/module/Core/src/Service/VisitsTracker.php +++ b/module/Core/src/Service/VisitsTracker.php @@ -22,11 +22,16 @@ class VisitsTracker implements VisitsTrackerInterface { private ORM\EntityManagerInterface $em; private EventDispatcherInterface $eventDispatcher; + private bool $obfuscateRemoteAddr; - public function __construct(ORM\EntityManagerInterface $em, EventDispatcherInterface $eventDispatcher) - { + public function __construct( + ORM\EntityManagerInterface $em, + EventDispatcherInterface $eventDispatcher, + bool $obfuscateRemoteAddr + ) { $this->em = $em; $this->eventDispatcher = $eventDispatcher; + $this->obfuscateRemoteAddr = $obfuscateRemoteAddr; } /** @@ -34,7 +39,7 @@ class VisitsTracker implements VisitsTrackerInterface */ public function track(ShortUrl $shortUrl, Visitor $visitor): void { - $visit = new Visit($shortUrl, $visitor); + $visit = new Visit($shortUrl, $visitor, $this->obfuscateRemoteAddr); $this->em->persist($visit); $this->em->flush(); diff --git a/module/Core/test/Service/VisitsTrackerTest.php b/module/Core/test/Service/VisitsTrackerTest.php index 9028d2c7..6ae5acf6 100644 --- a/module/Core/test/Service/VisitsTrackerTest.php +++ b/module/Core/test/Service/VisitsTrackerTest.php @@ -37,7 +37,7 @@ class VisitsTrackerTest extends TestCase $this->em = $this->prophesize(EntityManager::class); $this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class); - $this->visitsTracker = new VisitsTracker($this->em->reveal(), $this->eventDispatcher->reveal()); + $this->visitsTracker = new VisitsTracker($this->em->reveal(), $this->eventDispatcher->reveal(), true); } /** @test */ From ba13d99a71a78b9de839f0f0abed3cc1b19a686f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 13:19:40 +0200 Subject: [PATCH 72/87] Allowed remote addr obfuscation to be configured on docker image by using the OBFUSCATE_REMOTE_ADDR env var --- docker/README.md | 8 +++++--- docker/config/shlink_in_docker.local.php | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/README.md b/docker/README.md index aa9ae16b..a2283710 100644 --- a/docker/README.md +++ b/docker/README.md @@ -168,12 +168,12 @@ This is the complete list of supported env vars: * `TASK_WORKER_NUM`: The amount of concurrent background tasks this shlink instance will be able to execute. Defaults to 16. * `VISITS_WEBHOOKS`: A comma-separated list of URLs that will receive a `POST` request when a short URL receives a visit. * `DEFAULT_SHORT_CODES_LENGTH`: The length you want generated short codes to have. It defaults to 5 and has to be at least 4, so any value smaller than that will fall back to 4. +* `GEOLITE_LICENSE_KEY`: The license key used to download new GeoLite2 database files. This is not mandatory, as a default license key is provided, but it is **strongly recommended** that you provide your own. Go to [https://shlink.io/documentation/geolite-license-key](https://shlink.io/documentation/geolite-license-key) to know how to generate it. * `REDIS_SERVERS`: A comma-separated list of redis servers where Shlink locks are stored (locks are used to prevent some operations to be run more than once in parallel). * `MERCURE_PUBLIC_HUB_URL`: The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. * `MERCURE_INTERNAL_HUB_URL`: An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided but `MERCURE_PUBLIC_HUB_URL` was, the former one will be used to publish updates. * `MERCURE_JWT_SECRET`: The secret key that was provided to the mercure hub server, in order to be able to generate valid JWTs for publishing/subscribing to that server. - -* `GEOLITE_LICENSE_KEY`: The license key used to download new GeoLite2 database files. This is not mandatory, as a default license key is provided, but it is **strongly recommended** that you provide your own. Go to [https://shlink.io/documentation/geolite-license-key](https://shlink.io/documentation/geolite-license-key) to know how to generate it. +* `OBFUSCATE_REMOTE_ADDR`: Tells if IP addresses from visitors should be obfuscated before storing them in the database. Default value is `true`. **Careful!** Setting this to `false` will make your Shlink instance no longer be in compliance with the GDPR and other similar laws. An example using all env vars could look like this: @@ -205,6 +205,7 @@ docker run \ -e "MERCURE_PUBLIC_HUB_URL=https://example.com" \ -e "MERCURE_INTERNAL_HUB_URL=http://my-mercure-hub.prod.svc.cluster.local" \ -e MERCURE_JWT_SECRET=super_secret_key \ + -e OBFUSCATE_REMOTE_ADDR=false \ shlinkio/shlink:stable ``` @@ -249,7 +250,8 @@ The whole configuration should have this format, but it can be split into multip "geolite_license_key": "kjh23ljkbndskj345", "mercure_public_hub_url": "https://example.com", "mercure_internal_hub_url": "http://my-mercure-hub.prod.svc.cluster.local", - "mercure_jwt_secret": "super_secret_key" + "mercure_jwt_secret": "super_secret_key", + "obfuscate_remote_addr": false } ``` diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index 5662aaee..d85d0e79 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -117,6 +117,7 @@ return [ 'hostname' => env('SHORT_DOMAIN_HOST', ''), ], 'validate_url' => (bool) env('VALIDATE_URLS', false), + 'obfuscate_remote_addr' => (bool) env('OBFUSCATE_REMOTE_ADDR', true), 'visits_webhooks' => $helper->getVisitsWebhooks(), 'default_short_codes_length' => $helper->getDefaultShortCodesLength(), ], From bfdd6e0c508590b5743be0da36b4395f9b989283 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 13:21:49 +0200 Subject: [PATCH 73/87] Ensured SimplifiedConfigParser properly handles obfuscate_remote_addr option --- module/Core/src/Config/SimplifiedConfigParser.php | 1 + module/Core/test/Config/SimplifiedConfigParserTest.php | 2 ++ 2 files changed, 3 insertions(+) diff --git a/module/Core/src/Config/SimplifiedConfigParser.php b/module/Core/src/Config/SimplifiedConfigParser.php index fe4a4192..d3c22118 100644 --- a/module/Core/src/Config/SimplifiedConfigParser.php +++ b/module/Core/src/Config/SimplifiedConfigParser.php @@ -37,6 +37,7 @@ class SimplifiedConfigParser 'mercure_public_hub_url' => ['mercure', 'public_hub_url'], 'mercure_internal_hub_url' => ['mercure', 'internal_hub_url'], 'mercure_jwt_secret' => ['mercure', 'jwt_secret'], + 'obfuscate_remote_addr' => ['url_shortener', 'obfuscate_remote_addr'], ]; private const SIMPLIFIED_CONFIG_SIDE_EFFECTS = [ 'delete_short_url_threshold' => [ diff --git a/module/Core/test/Config/SimplifiedConfigParserTest.php b/module/Core/test/Config/SimplifiedConfigParserTest.php index 6bc6f1f7..94eb4116 100644 --- a/module/Core/test/Config/SimplifiedConfigParserTest.php +++ b/module/Core/test/Config/SimplifiedConfigParserTest.php @@ -64,6 +64,7 @@ class SimplifiedConfigParserTest extends TestCase 'mercure_public_hub_url' => 'public_url', 'mercure_internal_hub_url' => 'internal_url', 'mercure_jwt_secret' => 'super_secret_value', + 'obfuscate_remote_addr' => false, ]; $expected = [ 'app_options' => [ @@ -92,6 +93,7 @@ class SimplifiedConfigParserTest extends TestCase 'https://third-party.io/foo', ], 'default_short_codes_length' => 8, + 'obfuscate_remote_addr' => false, ], 'delete_short_urls' => [ From 8f06e4b20fb32085804da8c1c943823d2fb27528 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 15:43:09 +0200 Subject: [PATCH 74/87] Replaced references to obfuscate by anonymize --- config/autoload/url-shortener.global.php | 2 +- docker/README.md | 6 +++--- docker/config/shlink_in_docker.local.php | 2 +- module/Core/config/dependencies.config.php | 2 +- .../src/Config/SimplifiedConfigParser.php | 2 +- module/Core/src/Entity/Visit.php | 10 +++++----- module/Core/src/Service/VisitsTracker.php | 8 ++++---- .../Config/SimplifiedConfigParserTest.php | 4 ++-- module/Core/test/Entity/VisitTest.php | 16 +++++++-------- .../Core/test/Service/VisitsTrackerTest.php | 20 ------------------- 10 files changed, 26 insertions(+), 46 deletions(-) diff --git a/config/autoload/url-shortener.global.php b/config/autoload/url-shortener.global.php index 8439cc8a..5ad66bea 100644 --- a/config/autoload/url-shortener.global.php +++ b/config/autoload/url-shortener.global.php @@ -12,7 +12,7 @@ return [ 'hostname' => '', ], 'validate_url' => false, - 'obfuscate_remote_addr' => true, + 'anonymize_remote_addr' => true, 'visits_webhooks' => [], 'default_short_codes_length' => DEFAULT_SHORT_CODES_LENGTH, ], diff --git a/docker/README.md b/docker/README.md index a2283710..e17e570e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -173,7 +173,7 @@ This is the complete list of supported env vars: * `MERCURE_PUBLIC_HUB_URL`: The public URL of a mercure hub server to which Shlink will sent updates. This URL will also be served to consumers that want to subscribe to those updates. * `MERCURE_INTERNAL_HUB_URL`: An internal URL for a mercure hub. Will be used only when publishing updates to mercure, and does not need to be public. If this is not provided but `MERCURE_PUBLIC_HUB_URL` was, the former one will be used to publish updates. * `MERCURE_JWT_SECRET`: The secret key that was provided to the mercure hub server, in order to be able to generate valid JWTs for publishing/subscribing to that server. -* `OBFUSCATE_REMOTE_ADDR`: Tells if IP addresses from visitors should be obfuscated before storing them in the database. Default value is `true`. **Careful!** Setting this to `false` will make your Shlink instance no longer be in compliance with the GDPR and other similar laws. +* `ANONYMIZE_REMOTE_ADDR`: Tells if IP addresses from visitors should be obfuscated before storing them in the database. Default value is `true`. **Careful!** Setting this to `false` will make your Shlink instance no longer be in compliance with the GDPR and other similar data protection regulations. An example using all env vars could look like this: @@ -205,7 +205,7 @@ docker run \ -e "MERCURE_PUBLIC_HUB_URL=https://example.com" \ -e "MERCURE_INTERNAL_HUB_URL=http://my-mercure-hub.prod.svc.cluster.local" \ -e MERCURE_JWT_SECRET=super_secret_key \ - -e OBFUSCATE_REMOTE_ADDR=false \ + -e ANONYMIZE_REMOTE_ADDR=false \ shlinkio/shlink:stable ``` @@ -251,7 +251,7 @@ The whole configuration should have this format, but it can be split into multip "mercure_public_hub_url": "https://example.com", "mercure_internal_hub_url": "http://my-mercure-hub.prod.svc.cluster.local", "mercure_jwt_secret": "super_secret_key", - "obfuscate_remote_addr": false + "anonymize_remote_addr": false } ``` diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index d85d0e79..b870ccd7 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -117,7 +117,7 @@ return [ 'hostname' => env('SHORT_DOMAIN_HOST', ''), ], 'validate_url' => (bool) env('VALIDATE_URLS', false), - 'obfuscate_remote_addr' => (bool) env('OBFUSCATE_REMOTE_ADDR', true), + 'anonymize_remote_addr' => (bool) env('ANONYMIZE_REMOTE_ADDR', true), 'visits_webhooks' => $helper->getVisitsWebhooks(), 'default_short_codes_length' => $helper->getDefaultShortCodesLength(), ], diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 90a4ffaf..debf021f 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -57,7 +57,7 @@ return [ Service\VisitsTracker::class => [ 'em', EventDispatcherInterface::class, - 'config.url_shortener.obfuscate_remote_addr', + 'config.url_shortener.anonymize_remote_addr', ], Service\ShortUrlService::class => ['em', Service\ShortUrl\ShortUrlResolver::class, Util\UrlValidator::class], Visit\VisitLocator::class => ['em'], diff --git a/module/Core/src/Config/SimplifiedConfigParser.php b/module/Core/src/Config/SimplifiedConfigParser.php index d3c22118..81f05d14 100644 --- a/module/Core/src/Config/SimplifiedConfigParser.php +++ b/module/Core/src/Config/SimplifiedConfigParser.php @@ -37,7 +37,7 @@ class SimplifiedConfigParser 'mercure_public_hub_url' => ['mercure', 'public_hub_url'], 'mercure_internal_hub_url' => ['mercure', 'internal_hub_url'], 'mercure_jwt_secret' => ['mercure', 'jwt_secret'], - 'obfuscate_remote_addr' => ['url_shortener', 'obfuscate_remote_addr'], + 'anonymize_remote_addr' => ['url_shortener', 'anonymize_remote_addr'], ]; private const SIMPLIFIED_CONFIG_SIDE_EFFECTS = [ 'delete_short_url_threshold' => [ diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index fb5e3bfb..6a4f44e5 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -21,19 +21,19 @@ class Visit extends AbstractEntity implements JsonSerializable private ShortUrl $shortUrl; private ?VisitLocation $visitLocation = null; - public function __construct(ShortUrl $shortUrl, Visitor $visitor, bool $obfuscate = true, ?Chronos $date = null) + public function __construct(ShortUrl $shortUrl, Visitor $visitor, bool $anonymize = true, ?Chronos $date = null) { $this->shortUrl = $shortUrl; $this->date = $date ?? Chronos::now(); $this->userAgent = $visitor->getUserAgent(); $this->referer = $visitor->getReferer(); - $this->remoteAddr = $this->processAddress($obfuscate, $visitor->getRemoteAddress()); + $this->remoteAddr = $this->processAddress($anonymize, $visitor->getRemoteAddress()); } - private function processAddress(bool $obfuscate, ?string $address): ?string + private function processAddress(bool $anonymize, ?string $address): ?string { - // Localhost addresses do not need to be obfuscated - if (! $obfuscate || $address === null || $address === IpAddress::LOCALHOST) { + // Localhost addresses do not need to be anonymized + if (! $anonymize || $address === null || $address === IpAddress::LOCALHOST) { return $address; } diff --git a/module/Core/src/Service/VisitsTracker.php b/module/Core/src/Service/VisitsTracker.php index a60513e4..39f74cae 100644 --- a/module/Core/src/Service/VisitsTracker.php +++ b/module/Core/src/Service/VisitsTracker.php @@ -22,16 +22,16 @@ class VisitsTracker implements VisitsTrackerInterface { private ORM\EntityManagerInterface $em; private EventDispatcherInterface $eventDispatcher; - private bool $obfuscateRemoteAddr; + private bool $anonymizeRemoteAddr; public function __construct( ORM\EntityManagerInterface $em, EventDispatcherInterface $eventDispatcher, - bool $obfuscateRemoteAddr + bool $anonymizeRemoteAddr ) { $this->em = $em; $this->eventDispatcher = $eventDispatcher; - $this->obfuscateRemoteAddr = $obfuscateRemoteAddr; + $this->anonymizeRemoteAddr = $anonymizeRemoteAddr; } /** @@ -39,7 +39,7 @@ class VisitsTracker implements VisitsTrackerInterface */ public function track(ShortUrl $shortUrl, Visitor $visitor): void { - $visit = new Visit($shortUrl, $visitor, $this->obfuscateRemoteAddr); + $visit = new Visit($shortUrl, $visitor, $this->anonymizeRemoteAddr); $this->em->persist($visit); $this->em->flush(); diff --git a/module/Core/test/Config/SimplifiedConfigParserTest.php b/module/Core/test/Config/SimplifiedConfigParserTest.php index 94eb4116..3700b042 100644 --- a/module/Core/test/Config/SimplifiedConfigParserTest.php +++ b/module/Core/test/Config/SimplifiedConfigParserTest.php @@ -64,7 +64,7 @@ class SimplifiedConfigParserTest extends TestCase 'mercure_public_hub_url' => 'public_url', 'mercure_internal_hub_url' => 'internal_url', 'mercure_jwt_secret' => 'super_secret_value', - 'obfuscate_remote_addr' => false, + 'anonymize_remote_addr' => false, ]; $expected = [ 'app_options' => [ @@ -93,7 +93,7 @@ class SimplifiedConfigParserTest extends TestCase 'https://third-party.io/foo', ], 'default_short_codes_length' => 8, - 'obfuscate_remote_addr' => false, + 'anonymize_remote_addr' => false, ], 'delete_short_urls' => [ diff --git a/module/Core/test/Entity/VisitTest.php b/module/Core/test/Entity/VisitTest.php index a82e1939..73e41f12 100644 --- a/module/Core/test/Entity/VisitTest.php +++ b/module/Core/test/Entity/VisitTest.php @@ -39,20 +39,20 @@ class VisitTest extends TestCase * @test * @dataProvider provideAddresses */ - public function addressIsObfuscatedWhenRequested(bool $obfuscate, ?string $address, ?string $expectedAddress): void + public function addressIsAnonymizedWhenRequested(bool $anonymize, ?string $address, ?string $expectedAddress): void { - $visit = new Visit(new ShortUrl(''), new Visitor('Chrome', 'some site', $address), $obfuscate); + $visit = new Visit(new ShortUrl(''), new Visitor('Chrome', 'some site', $address), $anonymize); $this->assertEquals($expectedAddress, $visit->getRemoteAddr()); } public function provideAddresses(): iterable { - yield 'obfuscated null address' => [true, null, null]; - yield 'non-obfuscated null address' => [false, null, null]; - yield 'obfuscated localhost' => [true, IpAddress::LOCALHOST, IpAddress::LOCALHOST]; - yield 'non-obfuscated localhost' => [false, IpAddress::LOCALHOST, IpAddress::LOCALHOST]; - yield 'obfuscated regular address' => [true, '1.2.3.4', '1.2.3.0']; - yield 'non-obfuscated regular address' => [false, '1.2.3.4', '1.2.3.4']; + yield 'anonymized null address' => [true, null, null]; + yield 'non-anonymized null address' => [false, null, null]; + yield 'anonymized localhost' => [true, IpAddress::LOCALHOST, IpAddress::LOCALHOST]; + yield 'non-anonymized localhost' => [false, IpAddress::LOCALHOST, IpAddress::LOCALHOST]; + yield 'anonymized regular address' => [true, '1.2.3.4', '1.2.3.0']; + yield 'non-anonymized regular address' => [false, '1.2.3.4', '1.2.3.4']; } } diff --git a/module/Core/test/Service/VisitsTrackerTest.php b/module/Core/test/Service/VisitsTrackerTest.php index 6ae5acf6..0722352f 100644 --- a/module/Core/test/Service/VisitsTrackerTest.php +++ b/module/Core/test/Service/VisitsTrackerTest.php @@ -6,7 +6,6 @@ namespace ShlinkioTest\Shlink\Core\Service; use Doctrine\ORM\EntityManager; use Laminas\Stdlib\ArrayUtils; -use PHPUnit\Framework\Assert; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\Prophecy\ObjectProphecy; @@ -53,25 +52,6 @@ class VisitsTrackerTest extends TestCase $this->eventDispatcher->dispatch(Argument::type(ShortUrlVisited::class))->shouldHaveBeenCalled(); } - /** @test */ - public function trackedIpAddressGetsObfuscated(): void - { - $shortCode = '123ABC'; - - $this->em->persist(Argument::any())->will(function ($args) { - /** @var Visit $visit */ - $visit = $args[0]; - Assert::assertEquals('4.3.2.0', $visit->getRemoteAddr()); - $visit->setId('1'); - return $visit; - })->shouldBeCalledOnce(); - $this->em->flush()->shouldBeCalledOnce(); - - $this->visitsTracker->track(new ShortUrl($shortCode), new Visitor('', '', '4.3.2.1')); - - $this->eventDispatcher->dispatch(Argument::type(ShortUrlVisited::class))->shouldHaveBeenCalled(); - } - /** @test */ public function infoReturnsVisitsForCertainShortCode(): void { From f4bf3551f6a13cec6a47a4aeaa77e5762209684f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 15:50:16 +0200 Subject: [PATCH 75/87] Updated shlink-installer to a version supporting IP anonymization param --- composer.json | 2 +- config/autoload/installer.global.php | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 5ba58a8e..380cd237 100644 --- a/composer.json +++ b/composer.json @@ -51,7 +51,7 @@ "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", - "shlinkio/shlink-installer": "dev-master#dae6644587d0c1c59ca773722531551b9f436786 as 5.0.0", + "shlinkio/shlink-installer": "dev-master#50be18de1e505d2609d96c6cc86571b1b1ca7b57 as 5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", "symfony/console": "^5.0", "symfony/filesystem": "^5.0", diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php index d46aa8d7..db1914db 100644 --- a/config/autoload/installer.global.php +++ b/config/autoload/installer.global.php @@ -36,6 +36,7 @@ return [ Option\Mercure\MercureInternalUrlConfigOption::class, Option\Mercure\MercureJwtSecretConfigOption::class, Option\UrlShortener\GeoLiteLicenseKeyConfigOption::class, + Option\UrlShortener\IpAnonymizationConfigOption::class, ], 'installation_commands' => [ From e8ab664561c53e8feef7ee95aa650f791fff9c43 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 15:52:35 +0200 Subject: [PATCH 76/87] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49fc0a29..c4b6fd82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this Now, if the `withStats=true` query param is provided, the response payload will include a new `stats` property which is a list with the amount of short URLs and visits for every tag. +* [#640](https://github.com/shlinkio/shlink/issues/640) Allowed to optionally disable visitors' IP address anonymization. This will make Shlink no longer be GDPR-compliant, but it's OK if you only plan to share your URLs in countries without this regulation. + #### Changed * [#692](https://github.com/shlinkio/shlink/issues/692) Drastically improved performance when loading visits. Specially noticeable when loading big result sets. From 5be882a31b923b6e0e3e60d590bcfaab3b40a775 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 19:41:21 +0200 Subject: [PATCH 77/87] Improved parameter definition in some private queries in VisitRepository --- .../Core/src/Repository/VisitRepository.php | 69 ++++++++----------- .../Repository/VisitRepositoryInterface.php | 12 ++++ 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 547493c5..e9d93d10 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -7,14 +7,11 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\Query\ResultSetMappingBuilder; use Doctrine\ORM\QueryBuilder; -use Shlinkio\Shlink\Common\Doctrine\Type\ChronosDateTimeType; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; -use function preg_replace; - use const PHP_INT_MAX; class VisitRepository extends EntityRepository implements VisitRepositoryInterface @@ -29,7 +26,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ->from(Visit::class, 'v') ->where($qb->expr()->isNull('v.visitLocation')); - return $this->findVisitsForQuery($qb, $blockSize); + return $this->visitsIterableForQuery($qb, $blockSize); } /** @@ -45,7 +42,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ->andWhere($qb->expr()->eq('vl.isEmpty', ':isEmpty')) ->setParameter('isEmpty', true); - return $this->findVisitsForQuery($qb, $blockSize); + return $this->visitsIterableForQuery($qb, $blockSize); } public function findAllVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable @@ -54,10 +51,10 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa $qb->select('v') ->from(Visit::class, 'v'); - return $this->findVisitsForQuery($qb, $blockSize); + return $this->visitsIterableForQuery($qb, $blockSize); } - private function findVisitsForQuery(QueryBuilder $qb, int $blockSize): iterable + private function visitsIterableForQuery(QueryBuilder $qb, int $blockSize): iterable { $originalQueryBuilder = $qb->setMaxResults($blockSize) ->orderBy('v.id', 'ASC'); @@ -89,33 +86,14 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ?int $limit = null, ?int $offset = null ): array { - /** - * @var QueryBuilder $qb - * @var ShortUrl|int $shortUrl - */ - [$qb, $shortUrl] = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); + $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); $qb->select('v.id') ->orderBy('v.id', 'DESC') // Falling back to values that will behave as no limit/offset, but will workaround MS SQL not allowing // order on sub-queries without offset ->setMaxResults($limit ?? PHP_INT_MAX) ->setFirstResult($offset ?? 0); - - // FIXME Crappy way to resolve the params into the query. Best option would be to inject the sub-query with - // placeholders and then pass params to the main query - $shortUrlId = $shortUrl instanceof ShortUrl ? $shortUrl->getId() : $shortUrl; - $subQuery = preg_replace('/\?/', $shortUrlId, $qb->getQuery()->getSQL(), 1); - if ($dateRange !== null && $dateRange->getStartDate() !== null) { - $subQuery = preg_replace( - '/\?/', - '\'' . $dateRange->getStartDate()->toDateTimeString() . '\'', - $subQuery, - 1, - ); - } - if ($dateRange !== null && $dateRange->getEndDate() !== null) { - $subQuery = preg_replace('/\?/', '\'' . $dateRange->getEndDate()->toDateTimeString() . '\'', $subQuery, 1); - } + $subQuery = $qb->getQuery()->getSQL(); // A native query builder needs to be used here because DQL and ORM query builders do not accept // sub-queries at "from" and "join" level. @@ -140,8 +118,7 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa public function countVisitsByShortCode(string $shortCode, ?string $domain = null, ?DateRange $dateRange = null): int { - /** @var QueryBuilder $qb */ - [$qb] = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); + $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); @@ -151,26 +128,40 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa string $shortCode, ?string $domain, ?DateRange $dateRange - ): array { + ): QueryBuilder { /** @var ShortUrlRepositoryInterface $shortUrlRepo */ $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); - $shortUrl = $shortUrlRepo->findOne($shortCode, $domain) ?? -1; + $shortUrl = $shortUrlRepo->findOne($shortCode, $domain); + $shortUrlId = $shortUrl !== null ? $shortUrl->getId() : -1; + // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later + // Since they are not strictly provided by the caller, it's reasonably safe $qb = $this->getEntityManager()->createQueryBuilder(); $qb->from(Visit::class, 'v') - ->where($qb->expr()->eq('v.shortUrl', ':shortUrl')) - ->setParameter('shortUrl', $shortUrl); + ->where($qb->expr()->eq('v.shortUrl', $shortUrlId)); // Apply date range filtering if ($dateRange !== null && $dateRange->getStartDate() !== null) { - $qb->andWhere($qb->expr()->gte('v.date', ':startDate')) - ->setParameter('startDate', $dateRange->getStartDate(), ChronosDateTimeType::CHRONOS_DATETIME); + $qb->andWhere($qb->expr()->gte('v.date', '\'' . $dateRange->getStartDate()->toDateTimeString() . '\'')); } if ($dateRange !== null && $dateRange->getEndDate() !== null) { - $qb->andWhere($qb->expr()->lte('v.date', ':endDate')) - ->setParameter('endDate', $dateRange->getEndDate(), ChronosDateTimeType::CHRONOS_DATETIME); + $qb->andWhere($qb->expr()->lte('v.date', '\'' . $dateRange->getEndDate()->toDateTimeString() . '\'')); } - return [$qb, $shortUrl]; + return $qb; + } + + public function findVisitsByTag( + string $tag, + ?DateRange $dateRange = null, + ?int $limit = null, + ?int $offset = null + ): array { + return []; + } + + public function countVisitsByTag(string $tag, ?DateRange $dateRange = null): int + { + return 0; } } diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index f9cbc8d9..5a540171 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -43,4 +43,16 @@ interface VisitRepositoryInterface extends ObjectRepository ?string $domain = null, ?DateRange $dateRange = null ): int; + + /** + * @return Visit[] + */ + public function findVisitsByTag( + string $tag, + ?DateRange $dateRange = null, + ?int $limit = null, + ?int $offset = null + ): array; + + public function countVisitsByTag(string $tag, ?DateRange $dateRange = null): int; } From baf77b6ffbc9cd859956b5c930c86e71b6f46f68 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 19:55:05 +0200 Subject: [PATCH 78/87] Implemented methods to get paginated list of visits by tag, reusing methods used for short code filtering --- .../Core/src/Repository/VisitRepository.php | 135 +++++++++++------- 1 file changed, 86 insertions(+), 49 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index e9d93d10..e779a8bb 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -12,6 +12,8 @@ use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; +use function array_column; + use const PHP_INT_MAX; class VisitRepository extends EntityRepository implements VisitRepositoryInterface @@ -87,6 +89,90 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ?int $offset = null ): array { $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); + return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); + } + + public function countVisitsByShortCode(string $shortCode, ?string $domain = null, ?DateRange $dateRange = null): int + { + $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); + $qb->select('COUNT(v.id)'); + + return (int) $qb->getQuery()->getSingleScalarResult(); + } + + private function createVisitsByShortCodeQueryBuilder( + string $shortCode, + ?string $domain, + ?DateRange $dateRange + ): QueryBuilder { + /** @var ShortUrlRepositoryInterface $shortUrlRepo */ + $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); + $shortUrl = $shortUrlRepo->findOne($shortCode, $domain); + $shortUrlId = $shortUrl !== null ? $shortUrl->getId() : -1; + + // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later + // Since they are not strictly provided by the caller, it's reasonably safe + $qb = $this->getEntityManager()->createQueryBuilder(); + $qb->from(Visit::class, 'v') + ->where($qb->expr()->eq('v.shortUrl', $shortUrlId)); + + // Apply date range filtering + $this->applyDatesInline($qb, $dateRange); + + return $qb; + } + + public function findVisitsByTag( + string $tag, + ?DateRange $dateRange = null, + ?int $limit = null, + ?int $offset = null + ): array { + $qb = $this->createVisitsByTagQueryBuilder($tag, $dateRange); + return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); + } + + public function countVisitsByTag(string $tag, ?DateRange $dateRange = null): int + { + $qb = $this->createVisitsByTagQueryBuilder($tag, $dateRange); + $qb->select('COUNT(v.id)'); + + return (int) $qb->getQuery()->getSingleScalarResult(); + } + + private function createVisitsByTagQueryBuilder(string $tag, ?DateRange $dateRange = null): QueryBuilder + { + $qb = $this->getEntityManager()->createQueryBuilder(); + $qb->select('s.id') + ->from(ShortUrl::class, 's') + ->join('s.tags', 't') + ->where($qb->expr()->eq('t.name', ':tag')) + ->setParameter('tag', $tag); + + // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later + // Since they are not strictly provided by the caller, it's reasonably safe + $qb2 = $this->getEntityManager()->createQueryBuilder(); + $qb2->from(Visit::class, 'v') + ->where($qb2->expr()->in('v.shortUrl', array_column($qb->getQuery()->getArrayResult(), 'id'))); + + // Apply date range filtering + $this->applyDatesInline($qb2, $dateRange); + + return $qb2; + } + + private function applyDatesInline(QueryBuilder $qb, ?DateRange $dateRange): void + { + if ($dateRange !== null && $dateRange->getStartDate() !== null) { + $qb->andWhere($qb->expr()->gte('v.date', '\'' . $dateRange->getStartDate()->toDateTimeString() . '\'')); + } + if ($dateRange !== null && $dateRange->getEndDate() !== null) { + $qb->andWhere($qb->expr()->lte('v.date', '\'' . $dateRange->getEndDate()->toDateTimeString() . '\'')); + } + } + + private function resolveVisitsWithNativeQuery(QueryBuilder $qb, ?int $limit, ?int $offset): array + { $qb->select('v.id') ->orderBy('v.id', 'DESC') // Falling back to values that will behave as no limit/offset, but will workaround MS SQL not allowing @@ -115,53 +201,4 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa return $query->getResult(); } - - public function countVisitsByShortCode(string $shortCode, ?string $domain = null, ?DateRange $dateRange = null): int - { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange); - $qb->select('COUNT(v.id)'); - - return (int) $qb->getQuery()->getSingleScalarResult(); - } - - private function createVisitsByShortCodeQueryBuilder( - string $shortCode, - ?string $domain, - ?DateRange $dateRange - ): QueryBuilder { - /** @var ShortUrlRepositoryInterface $shortUrlRepo */ - $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); - $shortUrl = $shortUrlRepo->findOne($shortCode, $domain); - $shortUrlId = $shortUrl !== null ? $shortUrl->getId() : -1; - - // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later - // Since they are not strictly provided by the caller, it's reasonably safe - $qb = $this->getEntityManager()->createQueryBuilder(); - $qb->from(Visit::class, 'v') - ->where($qb->expr()->eq('v.shortUrl', $shortUrlId)); - - // Apply date range filtering - if ($dateRange !== null && $dateRange->getStartDate() !== null) { - $qb->andWhere($qb->expr()->gte('v.date', '\'' . $dateRange->getStartDate()->toDateTimeString() . '\'')); - } - if ($dateRange !== null && $dateRange->getEndDate() !== null) { - $qb->andWhere($qb->expr()->lte('v.date', '\'' . $dateRange->getEndDate()->toDateTimeString() . '\'')); - } - - return $qb; - } - - public function findVisitsByTag( - string $tag, - ?DateRange $dateRange = null, - ?int $limit = null, - ?int $offset = null - ): array { - return []; - } - - public function countVisitsByTag(string $tag, ?DateRange $dateRange = null): int - { - return 0; - } } From dd4b4277c92f07651b666e91b084236bf4236bb7 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 8 May 2020 20:11:37 +0200 Subject: [PATCH 79/87] Added test for VisitRepository tag methods --- .../Core/src/Repository/VisitRepository.php | 5 +- .../Repository/VisitRepositoryTest.php | 99 +++++++++++++++---- 2 files changed, 85 insertions(+), 19 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index e779a8bb..b3761c9f 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -149,11 +149,14 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa ->where($qb->expr()->eq('t.name', ':tag')) ->setParameter('tag', $tag); + $shortUrlIds = array_column($qb->getQuery()->getArrayResult(), 'id'); + $shortUrlIds[] = '-1'; // Add an invalid ID, in case the list is empty + // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later // Since they are not strictly provided by the caller, it's reasonably safe $qb2 = $this->getEntityManager()->createQueryBuilder(); $qb2->from(Visit::class, 'v') - ->where($qb2->expr()->in('v.shortUrl', array_column($qb->getQuery()->getArrayResult(), 'id'))); + ->where($qb2->expr()->in('v.shortUrl', $shortUrlIds)); // Apply date range filtering $this->applyDatesInline($qb2, $dateRange); diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 13fc8581..529a5ae0 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -5,9 +5,11 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\Repository; use Cake\Chronos\Chronos; +use Doctrine\Common\Collections\ArrayCollection; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; @@ -27,6 +29,7 @@ class VisitRepositoryTest extends DatabaseTestCase Visit::class, ShortUrl::class, Domain::class, + Tag::class, ]; private VisitRepository $repo; @@ -125,18 +128,69 @@ class VisitRepositoryTest extends DatabaseTestCase ))); } - private function createShortUrlsAndVisits(): array + /** @test */ + public function findVisitsByTagReturnsProperData(): void + { + $foo = new Tag('foo'); + $this->getEntityManager()->persist($foo); + + /** @var ShortUrl $shortUrl */ + [,, $shortUrl] = $this->createShortUrlsAndVisits(false); + /** @var ShortUrl $shortUrl2 */ + [,, $shortUrl2] = $this->createShortUrlsAndVisits(false); + /** @var ShortUrl $shortUrl3 */ + [,, $shortUrl3] = $this->createShortUrlsAndVisits(false); + + $shortUrl->setTags(new ArrayCollection([$foo])); + $shortUrl2->setTags(new ArrayCollection([$foo])); + $shortUrl3->setTags(new ArrayCollection([$foo])); + + $this->getEntityManager()->flush(); + + $this->assertCount(0, $this->repo->findVisitsByTag('invalid')); + $this->assertCount(18, $this->repo->findVisitsByTag((string) $foo)); + $this->assertCount(6, $this->repo->findVisitsByTag((string) $foo, new DateRange( + Chronos::parse('2016-01-02'), + Chronos::parse('2016-01-03'), + ))); + $this->assertCount(12, $this->repo->findVisitsByTag((string) $foo, new DateRange( + Chronos::parse('2016-01-03'), + ))); + } + + /** @test */ + public function countVisitsByTagReturnsProperData(): void + { + $foo = new Tag('foo'); + $this->getEntityManager()->persist($foo); + + /** @var ShortUrl $shortUrl */ + [,, $shortUrl] = $this->createShortUrlsAndVisits(false); + /** @var ShortUrl $shortUrl2 */ + [,, $shortUrl2] = $this->createShortUrlsAndVisits(false); + + $shortUrl->setTags(new ArrayCollection([$foo])); + $shortUrl2->setTags(new ArrayCollection([$foo])); + + $this->getEntityManager()->flush(); + + $this->assertEquals(0, $this->repo->countVisitsByTag('invalid')); + $this->assertEquals(12, $this->repo->countVisitsByTag((string) $foo)); + $this->assertEquals(4, $this->repo->countVisitsByTag((string) $foo, new DateRange( + Chronos::parse('2016-01-02'), + Chronos::parse('2016-01-03'), + ))); + $this->assertEquals(8, $this->repo->countVisitsByTag((string) $foo, new DateRange( + Chronos::parse('2016-01-03'), + ))); + } + + private function createShortUrlsAndVisits(bool $withDomain = true): array { $shortUrl = new ShortUrl(''); $domain = 'example.com'; $shortCode = $shortUrl->getShortCode(); - $shortUrlWithDomain = new ShortUrl('', ShortUrlMeta::fromRawData([ - 'customSlug' => $shortCode, - 'domain' => $domain, - ])); - $this->getEntityManager()->persist($shortUrl); - $this->getEntityManager()->persist($shortUrlWithDomain); for ($i = 0; $i < 6; $i++) { $visit = new Visit( @@ -147,17 +201,26 @@ class VisitRepositoryTest extends DatabaseTestCase ); $this->getEntityManager()->persist($visit); } - for ($i = 0; $i < 3; $i++) { - $visit = new Visit( - $shortUrlWithDomain, - Visitor::emptyInstance(), - true, - Chronos::parse(sprintf('2016-01-0%s', $i + 1)), - ); - $this->getEntityManager()->persist($visit); - } - $this->getEntityManager()->flush(); - return [$shortCode, $domain]; + if ($withDomain) { + $shortUrlWithDomain = new ShortUrl('', ShortUrlMeta::fromRawData([ + 'customSlug' => $shortCode, + 'domain' => $domain, + ])); + $this->getEntityManager()->persist($shortUrlWithDomain); + + for ($i = 0; $i < 3; $i++) { + $visit = new Visit( + $shortUrlWithDomain, + Visitor::emptyInstance(), + true, + Chronos::parse(sprintf('2016-01-0%s', $i + 1)), + ); + $this->getEntityManager()->persist($visit); + } + $this->getEntityManager()->flush(); + } + + return [$shortCode, $domain, $shortUrl]; } } From f0acce1be05e55f9fabe75e23667d2040521e100 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 09:34:59 +0200 Subject: [PATCH 80/87] Updated to latest common --- composer.json | 2 +- module/Core/src/Entity/Visit.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 380cd237..650e00b5 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.5", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "dev-master#e659cf9d9b5b3b131419e2f55f2e595f562baafc as 3.1.0", + "shlinkio/shlink-common": "dev-master#26109f1e3f1d83e0fc8056d16848ffaca74a8806 as 3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", "shlinkio/shlink-installer": "dev-master#50be18de1e505d2609d96c6cc86571b1b1ca7b57 as 5.0.0", diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index 6a4f44e5..7e6ed060 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -38,7 +38,7 @@ class Visit extends AbstractEntity implements JsonSerializable } try { - return (string) IpAddress::fromString($address)->getObfuscatedCopy(); + return (string) IpAddress::fromString($address)->getAnonymizedCopy(); } catch (InvalidArgumentException $e) { return null; } From 3218f8c2835049964ecf60831c6997246f65d6e7 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 09:53:45 +0200 Subject: [PATCH 81/87] Added Created endpoint to serve visits by tag --- docs/swagger/paths/v2_tags_{tag}_visits.json | 154 ++++++++++++++++++ docs/swagger/swagger.json | 3 + .../Adapter/VisitsForTagPaginatorAdapter.php | 50 ++++++ module/Core/src/Service/VisitsTracker.php | 35 +++- .../src/Service/VisitsTrackerInterface.php | 12 +- module/Rest/config/dependencies.config.php | 2 + module/Rest/config/routes.config.php | 1 + .../Rest/src/Action/Visit/TagVisitsAction.php | 38 +++++ 8 files changed, 283 insertions(+), 12 deletions(-) create mode 100644 docs/swagger/paths/v2_tags_{tag}_visits.json create mode 100644 module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php create mode 100644 module/Rest/src/Action/Visit/TagVisitsAction.php diff --git a/docs/swagger/paths/v2_tags_{tag}_visits.json b/docs/swagger/paths/v2_tags_{tag}_visits.json new file mode 100644 index 00000000..d9d9dda7 --- /dev/null +++ b/docs/swagger/paths/v2_tags_{tag}_visits.json @@ -0,0 +1,154 @@ +{ + "get": { + "operationId": "getTagVisits", + "tags": [ + "Visits" + ], + "summary": "List visits for tag", + "description": "Get the list of visits on any short URL which is tagged with provided tag.", + "parameters": [ + { + "$ref": "../parameters/version.json" + }, + { + "name": "tag", + "in": "path", + "description": "The tag from which we want to get the visits.", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "startDate", + "in": "query", + "description": "The date (in ISO-8601 format) from which we want to get visits.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "endDate", + "in": "query", + "description": "The date (in ISO-8601 format) until which we want to get visits.", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "page", + "in": "query", + "description": "The page to display. Defaults to 1", + "required": false, + "schema": { + "type": "number" + } + }, + { + "name": "itemsPerPage", + "in": "query", + "description": "The amount of items to return on every page. Defaults to all the items", + "required": false, + "schema": { + "type": "number" + } + } + ], + "security": [ + { + "ApiKey": [] + } + ], + "responses": { + "200": { + "description": "List of visits.", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "visits": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "../definitions/Visit.json" + } + }, + "pagination": { + "$ref": "../definitions/Pagination.json" + } + } + } + } + } + } + }, + "examples": { + "application/json": { + "visits": { + "data": [ + { + "referer": "https://twitter.com", + "date": "2015-08-20T05:05:03+04:00", + "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0", + "visitLocation": null + }, + { + "referer": "https://t.co", + "date": "2015-08-20T05:05:03+04:00", + "userAgent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", + "visitLocation": { + "cityName": "Cupertino", + "countryCode": "US", + "countryName": "United States", + "latitude": 37.3042, + "longitude": -122.0946, + "regionName": "California", + "timezone": "America/Los_Angeles" + } + }, + { + "referer": null, + "date": "2015-08-20T05:05:03+04:00", + "userAgent": "some_web_crawler/1.4", + "visitLocation": null + } + ], + "pagination": { + "currentPage": 5, + "pagesCount": 12, + "itemsPerPage": 10, + "itemsInCurrentPage": 10, + "totalItems": 115 + } + } + } + } + }, + "404": { + "description": "The tag does not exist.", + "content": { + "application/problem+json": { + "schema": { + "$ref": "../definitions/Error.json" + } + } + } + }, + "500": { + "description": "Unexpected error.", + "content": { + "application/problem+json": { + "schema": { + "$ref": "../definitions/Error.json" + } + } + } + } + } + } +} diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index e7663820..8dc21997 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -84,6 +84,9 @@ "/rest/v{version}/short-urls/{shortCode}/visits": { "$ref": "paths/v1_short-urls_{shortCode}_visits.json" }, + "/rest/v{version}/tags/{tag}/visits": { + "$ref": "paths/v2_tags_{tag}_visits.json" + }, "/rest/v{version}/mercure-info": { "$ref": "paths/v2_mercure-info.json" diff --git a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php new file mode 100644 index 00000000..d456ad8c --- /dev/null +++ b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php @@ -0,0 +1,50 @@ +visitRepository = $visitRepository; + $this->params = $params; + $this->tag = $tag; + } + + public function getItems($offset, $itemCountPerPage): array // phpcs:ignore + { + return $this->visitRepository->findVisitsByTag( + $this->tag, + $this->params->getDateRange(), + $itemCountPerPage, + $offset, + ); + } + + public function count(): int + { + // Since a new adapter instance is created every time visits are fetched, it is reasonably safe to internally + // cache the count value. + // The reason it is cached is because the Paginator is actually calling the method twice. + // An inconsistent value could be returned if between the first call and the second one, a new visit is created. + // However, it's almost instant, and then the adapter instance is discarded immediately after. + + if ($this->count !== null) { + return $this->count; + } + + return $this->count = $this->visitRepository->countVisitsByTag($this->tag, $this->params->getDateRange()); + } +} diff --git a/module/Core/src/Service/VisitsTracker.php b/module/Core/src/Service/VisitsTracker.php index 39f74cae..e777af76 100644 --- a/module/Core/src/Service/VisitsTracker.php +++ b/module/Core/src/Service/VisitsTracker.php @@ -8,15 +8,19 @@ use Doctrine\ORM; use Laminas\Paginator\Paginator; use Psr\EventDispatcher\EventDispatcherInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\EventDispatcher\ShortUrlVisited; use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException; +use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Model\VisitsParams; +use Shlinkio\Shlink\Core\Paginator\Adapter\VisitsForTagPaginatorAdapter; use Shlinkio\Shlink\Core\Paginator\Adapter\VisitsPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; -use Shlinkio\Shlink\Core\Repository\VisitRepository; +use Shlinkio\Shlink\Core\Repository\TagRepository; +use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; class VisitsTracker implements VisitsTrackerInterface { @@ -34,9 +38,6 @@ class VisitsTracker implements VisitsTrackerInterface $this->anonymizeRemoteAddr = $anonymizeRemoteAddr; } - /** - * Tracks a new visit to provided short code from provided visitor - */ public function track(ShortUrl $shortUrl, Visitor $visitor): void { $visit = new Visit($shortUrl, $visitor, $this->anonymizeRemoteAddr); @@ -48,8 +49,6 @@ class VisitsTracker implements VisitsTrackerInterface } /** - * Returns the visits on certain short code - * * @return Visit[]|Paginator * @throws ShortUrlNotFoundException */ @@ -61,7 +60,7 @@ class VisitsTracker implements VisitsTrackerInterface throw ShortUrlNotFoundException::fromNotFound($identifier); } - /** @var VisitRepository $repo */ + /** @var VisitRepositoryInterface $repo */ $repo = $this->em->getRepository(Visit::class); $paginator = new Paginator(new VisitsPaginatorAdapter($repo, $identifier, $params)); $paginator->setItemCountPerPage($params->getItemsPerPage()) @@ -69,4 +68,26 @@ class VisitsTracker implements VisitsTrackerInterface return $paginator; } + + /** + * @return Visit[]|Paginator + * @throws TagNotFoundException + */ + public function visitsForTag(string $tag, VisitsParams $params): Paginator + { + /** @var TagRepository $tagRepo */ + $tagRepo = $this->em->getRepository(Tag::class); + $count = $tagRepo->count(['name' => $tag]); + if ($count === 0) { + throw TagNotFoundException::fromTag($tag); + } + + /** @var VisitRepositoryInterface $repo */ + $repo = $this->em->getRepository(Visit::class); + $paginator = new Paginator(new VisitsForTagPaginatorAdapter($repo, $tag, $params)); + $paginator->setItemCountPerPage($params->getItemsPerPage()) + ->setCurrentPageNumber($params->getPage()); + + return $paginator; + } } diff --git a/module/Core/src/Service/VisitsTrackerInterface.php b/module/Core/src/Service/VisitsTrackerInterface.php index 1ec4e110..2c2759c2 100644 --- a/module/Core/src/Service/VisitsTrackerInterface.php +++ b/module/Core/src/Service/VisitsTrackerInterface.php @@ -8,22 +8,24 @@ use Laminas\Paginator\Paginator; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException; +use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Model\VisitsParams; interface VisitsTrackerInterface { - /** - * Tracks a new visit to provided short code from provided visitor - */ public function track(ShortUrl $shortUrl, Visitor $visitor): void; /** - * Returns the visits on certain short code - * * @return Visit[]|Paginator * @throws ShortUrlNotFoundException */ public function info(ShortUrlIdentifier $identifier, VisitsParams $params): Paginator; + + /** + * @return Visit[]|Paginator + * @throws TagNotFoundException + */ + public function visitsForTag(string $tag, VisitsParams $params): Paginator; } diff --git a/module/Rest/config/dependencies.config.php b/module/Rest/config/dependencies.config.php index a10fd254..258404ef 100644 --- a/module/Rest/config/dependencies.config.php +++ b/module/Rest/config/dependencies.config.php @@ -30,6 +30,7 @@ return [ Action\ShortUrl\ListShortUrlsAction::class => ConfigAbstractFactory::class, Action\ShortUrl\EditShortUrlTagsAction::class => ConfigAbstractFactory::class, Action\Visit\ShortUrlVisitsAction::class => ConfigAbstractFactory::class, + Action\Visit\TagVisitsAction::class => ConfigAbstractFactory::class, Action\Visit\GlobalVisitsAction::class => ConfigAbstractFactory::class, Action\Tag\ListTagsAction::class => ConfigAbstractFactory::class, Action\Tag\DeleteTagsAction::class => ConfigAbstractFactory::class, @@ -63,6 +64,7 @@ return [ 'config.url_shortener.domain', ], Action\Visit\ShortUrlVisitsAction::class => [Service\VisitsTracker::class], + Action\Visit\TagVisitsAction::class => [Service\VisitsTracker::class], Action\Visit\GlobalVisitsAction::class => [Visit\VisitsStatsHelper::class], Action\ShortUrl\ListShortUrlsAction::class => [Service\ShortUrlService::class, 'config.url_shortener.domain'], Action\ShortUrl\EditShortUrlTagsAction::class => [Service\ShortUrlService::class], diff --git a/module/Rest/config/routes.config.php b/module/Rest/config/routes.config.php index d2795971..0bde3da0 100644 --- a/module/Rest/config/routes.config.php +++ b/module/Rest/config/routes.config.php @@ -27,6 +27,7 @@ return [ // Visits Action\Visit\ShortUrlVisitsAction::getRouteDef([$dropDomainMiddleware]), + Action\Visit\TagVisitsAction::getRouteDef(), Action\Visit\GlobalVisitsAction::getRouteDef(), // Tags diff --git a/module/Rest/src/Action/Visit/TagVisitsAction.php b/module/Rest/src/Action/Visit/TagVisitsAction.php new file mode 100644 index 00000000..1107ca5c --- /dev/null +++ b/module/Rest/src/Action/Visit/TagVisitsAction.php @@ -0,0 +1,38 @@ +visitsTracker = $visitsTracker; + } + + public function handle(Request $request): Response + { + $tag = $request->getAttribute('tag', ''); + $visits = $this->visitsTracker->visitsForTag($tag, VisitsParams::fromRawData($request->getQueryParams())); + + return new JsonResponse([ + 'visits' => $this->serializePaginator($visits), + ]); + } +} From e1e3c7f0614e456f76ab75910569386273fcb2b0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 10:10:48 +0200 Subject: [PATCH 82/87] Created paginator adapter tests --- ...AbstractCacheableCountPaginatorAdapter.php | 29 ++++++++++ .../Adapter/VisitsForTagPaginatorAdapter.php | 19 +------ .../Adapter/VisitsPaginatorAdapter.php | 19 +------ .../VisitsForTagPaginatorAdapterTest.php | 52 +++++++++++++++++ .../Adapter/VisitsPaginatorAdapterTest.php | 57 +++++++++++++++++++ 5 files changed, 144 insertions(+), 32 deletions(-) create mode 100644 module/Core/src/Paginator/Adapter/AbstractCacheableCountPaginatorAdapter.php create mode 100644 module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php create mode 100644 module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php diff --git a/module/Core/src/Paginator/Adapter/AbstractCacheableCountPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/AbstractCacheableCountPaginatorAdapter.php new file mode 100644 index 00000000..cc2a8287 --- /dev/null +++ b/module/Core/src/Paginator/Adapter/AbstractCacheableCountPaginatorAdapter.php @@ -0,0 +1,29 @@ +count !== null) { + return $this->count; + } + + return $this->count = $this->doCount(); + } + + abstract protected function doCount(): int; +} diff --git a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php index d456ad8c..e80fbcdd 100644 --- a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php @@ -4,18 +4,15 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Paginator\Adapter; -use Laminas\Paginator\Adapter\AdapterInterface; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; -class VisitsForTagPaginatorAdapter implements AdapterInterface +class VisitsForTagPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter { private VisitRepositoryInterface $visitRepository; private string $tag; private VisitsParams $params; - private ?int $count = null; - public function __construct(VisitRepositoryInterface $visitRepository, string $tag, VisitsParams $params) { $this->visitRepository = $visitRepository; @@ -33,18 +30,8 @@ class VisitsForTagPaginatorAdapter implements AdapterInterface ); } - public function count(): int + protected function doCount(): int { - // Since a new adapter instance is created every time visits are fetched, it is reasonably safe to internally - // cache the count value. - // The reason it is cached is because the Paginator is actually calling the method twice. - // An inconsistent value could be returned if between the first call and the second one, a new visit is created. - // However, it's almost instant, and then the adapter instance is discarded immediately after. - - if ($this->count !== null) { - return $this->count; - } - - return $this->count = $this->visitRepository->countVisitsByTag($this->tag, $this->params->getDateRange()); + return $this->visitRepository->countVisitsByTag($this->tag, $this->params->getDateRange()); } } diff --git a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php index 6a42ecbc..404ae309 100644 --- a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php @@ -4,19 +4,16 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Paginator\Adapter; -use Laminas\Paginator\Adapter\AdapterInterface; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; -class VisitsPaginatorAdapter implements AdapterInterface +class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter { private VisitRepositoryInterface $visitRepository; private ShortUrlIdentifier $identifier; private VisitsParams $params; - private ?int $count = null; - public function __construct( VisitRepositoryInterface $visitRepository, ShortUrlIdentifier $identifier, @@ -38,19 +35,9 @@ class VisitsPaginatorAdapter implements AdapterInterface ); } - public function count(): int + protected function doCount(): int { - // Since a new adapter instance is created every time visits are fetched, it is reasonably safe to internally - // cache the count value. - // The reason it is cached is because the Paginator is actually calling the method twice. - // An inconsistent value could be returned if between the first call and the second one, a new visit is created. - // However, it's almost instant, and then the adapter instance is discarded immediately after. - - if ($this->count !== null) { - return $this->count; - } - - return $this->count = $this->visitRepository->countVisitsByShortCode( + return $this->visitRepository->countVisitsByShortCode( $this->identifier->shortCode(), $this->identifier->domain(), $this->params->getDateRange(), diff --git a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php new file mode 100644 index 00000000..e4418c5b --- /dev/null +++ b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php @@ -0,0 +1,52 @@ +repo = $this->prophesize(VisitRepositoryInterface::class); + $this->adapter = new VisitsForTagPaginatorAdapter($this->repo->reveal(), 'foo', VisitsParams::fromRawData([])); + } + + /** @test */ + public function repoIsCalledEveryTimeItemsAreFetched(): void + { + $count = 3; + $limit = 1; + $offset = 5; + $findVisits = $this->repo->findVisitsByTag('foo', new DateRange(), $limit, $offset)->willReturn([]); + + for ($i = 0; $i < $count; $i++) { + $this->adapter->getItems($offset, $limit); + } + + $findVisits->shouldHaveBeenCalledTimes($count); + } + + /** @test */ + public function repoIsCalledOnlyOnceForCount(): void + { + $count = 3; + $countVisits = $this->repo->countVisitsByTag('foo', new DateRange())->willReturn(3); + + for ($i = 0; $i < $count; $i++) { + $this->adapter->count(); + } + + $countVisits->shouldHaveBeenCalledOnce(); + } +} diff --git a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php new file mode 100644 index 00000000..744582b7 --- /dev/null +++ b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php @@ -0,0 +1,57 @@ +repo = $this->prophesize(VisitRepositoryInterface::class); + $this->adapter = new VisitsPaginatorAdapter( + $this->repo->reveal(), + new ShortUrlIdentifier(''), + VisitsParams::fromRawData([]), + ); + } + + /** @test */ + public function repoIsCalledEveryTimeItemsAreFetched(): void + { + $count = 3; + $limit = 1; + $offset = 5; + $findVisits = $this->repo->findVisitsByShortCode('', null, new DateRange(), $limit, $offset)->willReturn([]); + + for ($i = 0; $i < $count; $i++) { + $this->adapter->getItems($offset, $limit); + } + + $findVisits->shouldHaveBeenCalledTimes($count); + } + + /** @test */ + public function repoIsCalledOnlyOnceForCount(): void + { + $count = 3; + $countVisits = $this->repo->countVisitsByShortCode('', null, new DateRange())->willReturn(3); + + for ($i = 0; $i < $count; $i++) { + $this->adapter->count(); + } + + $countVisits->shouldHaveBeenCalledOnce(); + } +} From 9b9de8e290a6140c2c8b251e39e27b0e150f11f3 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 10:14:26 +0200 Subject: [PATCH 83/87] Updated VisitsTrackerTest --- .../Core/test/Service/VisitsTrackerTest.php | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/module/Core/test/Service/VisitsTrackerTest.php b/module/Core/test/Service/VisitsTrackerTest.php index 0722352f..5893b952 100644 --- a/module/Core/test/Service/VisitsTrackerTest.php +++ b/module/Core/test/Service/VisitsTrackerTest.php @@ -12,13 +12,16 @@ use Prophecy\Prophecy\ObjectProphecy; use Psr\EventDispatcher\EventDispatcherInterface; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\EventDispatcher\ShortUrlVisited; use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException; +use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; +use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\Service\VisitsTracker; @@ -85,4 +88,40 @@ class VisitsTrackerTest extends TestCase $this->visitsTracker->info(new ShortUrlIdentifier($shortCode), new VisitsParams()); } + + /** @test */ + public function throwsExceptionWhenRequestingVisitsForInvalidTag(): void + { + $tag = 'foo'; + $repo = $this->prophesize(TagRepository::class); + $count = $repo->count(['name' => $tag])->willReturn(0); + $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + + $this->expectException(TagNotFoundException::class); + $count->shouldBeCalledOnce(); + $getRepo->shouldBeCalledOnce(); + + $this->visitsTracker->visitsForTag($tag, new VisitsParams()); + } + + /** @test */ + public function visitsForTagAreReturnedAsExpected(): void + { + $tag = 'foo'; + $repo = $this->prophesize(TagRepository::class); + $count = $repo->count(['name' => $tag])->willReturn(1); + $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); + + $list = map(range(0, 1), fn () => new Visit(new ShortUrl(''), Visitor::emptyInstance())); + $repo2 = $this->prophesize(VisitRepository::class); + $repo2->findVisitsByTag($tag, Argument::type(DateRange::class), 1, 0)->willReturn($list); + $repo2->countVisitsByTag($tag, Argument::type(DateRange::class))->willReturn(1); + $this->em->getRepository(Visit::class)->willReturn($repo2->reveal())->shouldBeCalledOnce(); + + $paginator = $this->visitsTracker->visitsForTag($tag, new VisitsParams()); + + $this->assertEquals($list, ArrayUtils::iteratorToArray($paginator->getCurrentItems())); + $count->shouldHaveBeenCalledOnce(); + $getRepo->shouldHaveBeenCalledOnce(); + } } From 7f39e6d7681e3c8b67e7d72c3e5e560c2820812c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 10:22:07 +0200 Subject: [PATCH 84/87] Created TagVisitsActionTest --- .../test/Action/Visit/TagVisitsActionTest.php | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 module/Rest/test/Action/Visit/TagVisitsActionTest.php diff --git a/module/Rest/test/Action/Visit/TagVisitsActionTest.php b/module/Rest/test/Action/Visit/TagVisitsActionTest.php new file mode 100644 index 00000000..863bc725 --- /dev/null +++ b/module/Rest/test/Action/Visit/TagVisitsActionTest.php @@ -0,0 +1,41 @@ +visitsTracker = $this->prophesize(VisitsTracker::class); + $this->action = new TagVisitsAction($this->visitsTracker->reveal()); + } + + /** @test */ + public function providingCorrectShortCodeReturnsVisits(): void + { + $tag = 'foo'; + $getVisits = $this->visitsTracker->visitsForTag($tag, Argument::type(VisitsParams::class))->willReturn( + new Paginator(new ArrayAdapter([])), + ); + + $response = $this->action->handle((new ServerRequest())->withAttribute('tag', $tag)); + + $this->assertEquals(200, $response->getStatusCode()); + $getVisits->shouldHaveBeenCalledOnce(); + } +} From 4d346d1feac64d1e442477640ec7b8e0af902591 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 10:31:39 +0200 Subject: [PATCH 85/87] Created API test for tags visits endpoint --- .../test-api/Action/TagVisitsActionTest.php | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 module/Rest/test-api/Action/TagVisitsActionTest.php diff --git a/module/Rest/test-api/Action/TagVisitsActionTest.php b/module/Rest/test-api/Action/TagVisitsActionTest.php new file mode 100644 index 00000000..94e592f6 --- /dev/null +++ b/module/Rest/test-api/Action/TagVisitsActionTest.php @@ -0,0 +1,46 @@ +callApiWithKey(self::METHOD_GET, sprintf('/tags/%s/visits', $tag)); + $payload = $this->getJsonResponsePayload($resp); + + $this->assertArrayHasKey('visits', $payload); + $this->assertArrayHasKey('data', $payload['visits']); + $this->assertCount($expectedVisitsAmount, $payload['visits']['data']); + } + + public function provideTags(): iterable + { + yield 'foo' => ['foo', 5]; + yield 'bar' => ['bar', 2]; + yield 'baz' => ['baz', 0]; + } + + /** @test */ + public function notFoundErrorIsReturnedForInvalidTags(): void + { + $resp = $this->callApiWithKey(self::METHOD_GET, '/tags/invalid_tag/visits'); + $payload = $this->getJsonResponsePayload($resp); + + $this->assertEquals(self::STATUS_NOT_FOUND, $resp->getStatusCode()); + $this->assertEquals(self::STATUS_NOT_FOUND, $payload['status']); + $this->assertEquals('TAG_NOT_FOUND', $payload['type']); + $this->assertEquals('Tag with name "invalid_tag" could not be found', $payload['detail']); + $this->assertEquals('Tag not found', $payload['title']); + } +} From 4c5cd88041120b3d556406fe7eaeab3f2943c856 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 10:36:49 +0200 Subject: [PATCH 86/87] Updated changelog --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4b6fd82..f2491f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). -## [Unreleased] +## 2.2.0 - 2020-05-09 #### Added @@ -19,13 +19,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this The updates are only published when serving Shlink with swoole. - Also, Shlink exposes a new endpoint `GET /rest/v2/mercure-info`, which returns the public URL of the mercure hub, and a valid JWT that can be used to subsribe to updates. + Also, Shlink exposes a new endpoint `GET /rest/v2/mercure-info`, which returns the public URL of the mercure hub, and a valid JWT that can be used to subscribe to updates. * [#673](https://github.com/shlinkio/shlink/issues/673) Added new `[GET /visits]` rest endpoint which returns basic visits stats. +* [#674](https://github.com/shlinkio/shlink/issues/674) Added new `[GET /tags/{tag}/visits]` rest endpoint which returns visits by tag. + + It works in the same way as the `[GET /short-urls/{shortCode}/visits]` one, returning the same response payload, and supporting the same query params, but the response is the list of visits in all short URLs which have provided tag. + * [#672](https://github.com/shlinkio/shlink/issues/672) Enhanced `[GET /tags]` rest endpoint so that it is possible to get basic stats info for every tag. Now, if the `withStats=true` query param is provided, the response payload will include a new `stats` property which is a list with the amount of short URLs and visits for every tag. + Also, the `tag:list` CLI command has been changed and it always behaves like this. + * [#640](https://github.com/shlinkio/shlink/issues/640) Allowed to optionally disable visitors' IP address anonymization. This will make Shlink no longer be GDPR-compliant, but it's OK if you only plan to share your URLs in countries without this regulation. #### Changed From cf605407ad34e3a2b80f1061e9419ec65221946b Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 9 May 2020 10:56:07 +0200 Subject: [PATCH 87/87] =?UTF-8?q?Used=20definitive=20dependency=20versions?= =?UTF-8?q?=20for=20shlink-common=20and=20shl=C3=B1ink-installer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- composer.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/composer.json b/composer.json index 650e00b5..5d531a37 100644 --- a/composer.json +++ b/composer.json @@ -48,10 +48,10 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.5", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "dev-master#26109f1e3f1d83e0fc8056d16848ffaca74a8806 as 3.1.0", + "shlinkio/shlink-common": "^3.1.0", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^1.4", - "shlinkio/shlink-installer": "dev-master#50be18de1e505d2609d96c6cc86571b1b1ca7b57 as 5.0.0", + "shlinkio/shlink-installer": "^5.0.0", "shlinkio/shlink-ip-geolocation": "^1.4", "symfony/console": "^5.0", "symfony/filesystem": "^5.0",