From 1498b72966f976a0ba2c72000c6fba7236cf26cd Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 26 Feb 2021 20:01:16 +0100 Subject: [PATCH 01/86] Updated to happyr/doctrine-specification 2, with some fixes --- composer.json | 2 +- config/autoload/entity-manager.global.php | 2 +- module/Core/src/Domain/Repository/DomainRepository.php | 2 +- .../src/Domain/Repository/DomainRepositoryInterface.php | 2 +- module/Core/src/Repository/ShortUrlRepository.php | 2 +- .../Core/src/Repository/ShortUrlRepositoryInterface.php | 2 +- module/Core/src/Repository/TagRepository.php | 2 +- module/Core/src/Repository/TagRepositoryInterface.php | 2 +- module/Core/src/Repository/VisitRepository.php | 2 +- module/Core/src/Repository/VisitRepositoryInterface.php | 2 +- module/Core/src/ShortUrl/Spec/BelongsToApiKey.php | 6 +++--- module/Core/src/ShortUrl/Spec/BelongsToApiKeyInlined.php | 8 ++------ module/Core/src/ShortUrl/Spec/BelongsToDomain.php | 6 +++--- module/Core/src/ShortUrl/Spec/BelongsToDomainInlined.php | 8 ++------ module/Core/src/Spec/InDateRange.php | 2 +- module/Core/src/Tag/Spec/CountTagsWithName.php | 2 +- module/Core/src/Visit/Spec/CountOfOrphanVisits.php | 2 +- module/Core/src/Visit/Spec/CountOfShortUrlVisits.php | 2 +- module/Rest/src/ApiKey/Role.php | 5 +++-- .../Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php | 2 +- 20 files changed, 28 insertions(+), 35 deletions(-) diff --git a/composer.json b/composer.json index a22c26ae..d544ac41 100644 --- a/composer.json +++ b/composer.json @@ -24,7 +24,7 @@ "endroid/qr-code": "dev-master#0f1613a as 3.10", "geoip2/geoip2": "^2.9", "guzzlehttp/guzzle": "^7.0", - "happyr/doctrine-specification": "2.0.x-dev#cb116d3 as 2.0", + "happyr/doctrine-specification": "^2.0", "laminas/laminas-config": "^3.3", "laminas/laminas-config-aggregator": "^1.1", "laminas/laminas-diactoros": "^2.1.3", diff --git a/config/autoload/entity-manager.global.php b/config/autoload/entity-manager.global.php index 639df7ec..c3d2ab83 100644 --- a/config/autoload/entity-manager.global.php +++ b/config/autoload/entity-manager.global.php @@ -4,7 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Common; -use Happyr\DoctrineSpecification\EntitySpecificationRepository; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; return [ diff --git a/module/Core/src/Domain/Repository/DomainRepository.php b/module/Core/src/Domain/Repository/DomainRepository.php index f2152fbe..2e4f3bb2 100644 --- a/module/Core/src/Domain/Repository/DomainRepository.php +++ b/module/Core/src/Domain/Repository/DomainRepository.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Domain\Repository; use Doctrine\ORM\Query\Expr\Join; -use Happyr\DoctrineSpecification\EntitySpecificationRepository; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Rest\Entity\ApiKey; diff --git a/module/Core/src/Domain/Repository/DomainRepositoryInterface.php b/module/Core/src/Domain/Repository/DomainRepositoryInterface.php index 13917dc6..1d201520 100644 --- a/module/Core/src/Domain/Repository/DomainRepositoryInterface.php +++ b/module/Core/src/Domain/Repository/DomainRepositoryInterface.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Domain\Repository; use Doctrine\Persistence\ObjectRepository; -use Happyr\DoctrineSpecification\EntitySpecificationRepositoryInterface; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Rest\Entity\ApiKey; diff --git a/module/Core/src/Repository/ShortUrlRepository.php b/module/Core/src/Repository/ShortUrlRepository.php index f7a089b7..24b20a38 100644 --- a/module/Core/src/Repository/ShortUrlRepository.php +++ b/module/Core/src/Repository/ShortUrlRepository.php @@ -6,7 +6,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\Query\Expr\Join; use Doctrine\ORM\QueryBuilder; -use Happyr\DoctrineSpecification\EntitySpecificationRepository; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Doctrine\Type\ChronosDateTimeType; use Shlinkio\Shlink\Common\Util\DateRange; diff --git a/module/Core/src/Repository/ShortUrlRepositoryInterface.php b/module/Core/src/Repository/ShortUrlRepositoryInterface.php index e5662e20..ca04ffda 100644 --- a/module/Core/src/Repository/ShortUrlRepositoryInterface.php +++ b/module/Core/src/Repository/ShortUrlRepositoryInterface.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; -use Happyr\DoctrineSpecification\EntitySpecificationRepositoryInterface; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index dd15c292..d5a33c8b 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -4,7 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; -use Happyr\DoctrineSpecification\EntitySpecificationRepository; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Happyr\DoctrineSpecification\Spec; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Entity\Tag; diff --git a/module/Core/src/Repository/TagRepositoryInterface.php b/module/Core/src/Repository/TagRepositoryInterface.php index 86898ed1..97c308df 100644 --- a/module/Core/src/Repository/TagRepositoryInterface.php +++ b/module/Core/src/Repository/TagRepositoryInterface.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; -use Happyr\DoctrineSpecification\EntitySpecificationRepositoryInterface; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Rest\Entity\ApiKey; diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index b869093e..cd51f60d 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -6,7 +6,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\Query\ResultSetMappingBuilder; use Doctrine\ORM\QueryBuilder; -use Happyr\DoctrineSpecification\EntitySpecificationRepository; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index 3ecf0bca..96fb21ee 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -5,7 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; -use Happyr\DoctrineSpecification\EntitySpecificationRepositoryInterface; +use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Visit; diff --git a/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php b/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php index 9e094b90..ac20d0db 100644 --- a/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php +++ b/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php @@ -4,20 +4,20 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\ShortUrl\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Filter\Filter; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Shlinkio\Shlink\Rest\Entity\ApiKey; class BelongsToApiKey extends BaseSpecification { private ApiKey $apiKey; - private string $dqlAlias; + private ?string $dqlAlias; public function __construct(ApiKey $apiKey, ?string $dqlAlias = null) { $this->apiKey = $apiKey; - $this->dqlAlias = $dqlAlias ?? 's'; + $this->dqlAlias = $dqlAlias; parent::__construct($this->dqlAlias); } diff --git a/module/Core/src/ShortUrl/Spec/BelongsToApiKeyInlined.php b/module/Core/src/ShortUrl/Spec/BelongsToApiKeyInlined.php index 197031f3..579407cd 100644 --- a/module/Core/src/ShortUrl/Spec/BelongsToApiKeyInlined.php +++ b/module/Core/src/ShortUrl/Spec/BelongsToApiKeyInlined.php @@ -5,10 +5,10 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\ShortUrl\Spec; use Doctrine\ORM\QueryBuilder; -use Happyr\DoctrineSpecification\Specification\Specification; +use Happyr\DoctrineSpecification\Filter\Filter; use Shlinkio\Shlink\Rest\Entity\ApiKey; -class BelongsToApiKeyInlined implements Specification +class BelongsToApiKeyInlined implements Filter { private ApiKey $apiKey; @@ -22,8 +22,4 @@ class BelongsToApiKeyInlined implements Specification // Parameters in this query need to be inlined, not bound, as we need to use it as sub-query later return (string) $qb->expr()->eq('s.authorApiKey', '\'' . $this->apiKey->getId() . '\''); } - - public function modify(QueryBuilder $qb, string $dqlAlias): void - { - } } diff --git a/module/Core/src/ShortUrl/Spec/BelongsToDomain.php b/module/Core/src/ShortUrl/Spec/BelongsToDomain.php index 81b4388a..59898476 100644 --- a/module/Core/src/ShortUrl/Spec/BelongsToDomain.php +++ b/module/Core/src/ShortUrl/Spec/BelongsToDomain.php @@ -4,19 +4,19 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\ShortUrl\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Filter\Filter; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; class BelongsToDomain extends BaseSpecification { private string $domainId; - private string $dqlAlias; + private ?string $dqlAlias; public function __construct(string $domainId, ?string $dqlAlias = null) { $this->domainId = $domainId; - $this->dqlAlias = $dqlAlias ?? 's'; + $this->dqlAlias = $dqlAlias; parent::__construct($this->dqlAlias); } diff --git a/module/Core/src/ShortUrl/Spec/BelongsToDomainInlined.php b/module/Core/src/ShortUrl/Spec/BelongsToDomainInlined.php index a8ef527e..cb69a359 100644 --- a/module/Core/src/ShortUrl/Spec/BelongsToDomainInlined.php +++ b/module/Core/src/ShortUrl/Spec/BelongsToDomainInlined.php @@ -5,9 +5,9 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\ShortUrl\Spec; use Doctrine\ORM\QueryBuilder; -use Happyr\DoctrineSpecification\Specification\Specification; +use Happyr\DoctrineSpecification\Filter\Filter; -class BelongsToDomainInlined implements Specification +class BelongsToDomainInlined implements Filter { private string $domainId; @@ -21,8 +21,4 @@ class BelongsToDomainInlined implements Specification // Parameters in this query need to be inlined, not bound, as we need to use it as sub-query later return (string) $qb->expr()->eq('s.domain', '\'' . $this->domainId . '\''); } - - public function modify(QueryBuilder $qb, string $dqlAlias): void - { - } } diff --git a/module/Core/src/Spec/InDateRange.php b/module/Core/src/Spec/InDateRange.php index 44944aed..953ed9f2 100644 --- a/module/Core/src/Spec/InDateRange.php +++ b/module/Core/src/Spec/InDateRange.php @@ -4,8 +4,8 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; diff --git a/module/Core/src/Tag/Spec/CountTagsWithName.php b/module/Core/src/Tag/Spec/CountTagsWithName.php index a3f90a78..8dd3e44d 100644 --- a/module/Core/src/Tag/Spec/CountTagsWithName.php +++ b/module/Core/src/Tag/Spec/CountTagsWithName.php @@ -4,8 +4,8 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Tag\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Happyr\DoctrineSpecification\Specification\Specification; class CountTagsWithName extends BaseSpecification diff --git a/module/Core/src/Visit/Spec/CountOfOrphanVisits.php b/module/Core/src/Visit/Spec/CountOfOrphanVisits.php index fb8ee3bd..97712944 100644 --- a/module/Core/src/Visit/Spec/CountOfOrphanVisits.php +++ b/module/Core/src/Visit/Spec/CountOfOrphanVisits.php @@ -4,8 +4,8 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Visit\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Spec\InDateRange; diff --git a/module/Core/src/Visit/Spec/CountOfShortUrlVisits.php b/module/Core/src/Visit/Spec/CountOfShortUrlVisits.php index 6a125ee9..ea4a4800 100644 --- a/module/Core/src/Visit/Spec/CountOfShortUrlVisits.php +++ b/module/Core/src/Visit/Spec/CountOfShortUrlVisits.php @@ -4,8 +4,8 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Visit\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Rest\ApiKey\Spec\WithApiKeySpecsEnsuringJoin; use Shlinkio\Shlink\Rest\Entity\ApiKey; diff --git a/module/Rest/src/ApiKey/Role.php b/module/Rest/src/ApiKey/Role.php index ff3211ba..93ac717a 100644 --- a/module/Rest/src/ApiKey/Role.php +++ b/module/Rest/src/ApiKey/Role.php @@ -24,12 +24,13 @@ class Role public static function toSpec(ApiKeyRole $role, bool $inlined): Specification { if ($role->name() === self::AUTHORED_SHORT_URLS) { - return $inlined ? new BelongsToApiKeyInlined($role->apiKey()) : new BelongsToApiKey($role->apiKey()); + $apiKey = $role->apiKey(); + return $inlined ? Spec::andX(new BelongsToApiKeyInlined($apiKey)) : new BelongsToApiKey($apiKey); } if ($role->name() === self::DOMAIN_SPECIFIC) { $domainId = self::domainIdFromMeta($role->meta()); - return $inlined ? new BelongsToDomainInlined($domainId) : new BelongsToDomain($domainId); + return $inlined ? Spec::andX(new BelongsToDomainInlined($domainId)) : new BelongsToDomain($domainId); } return Spec::andX(); diff --git a/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php b/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php index 64359d15..e05dca41 100644 --- a/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php +++ b/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php @@ -4,8 +4,8 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Rest\ApiKey\Spec; -use Happyr\DoctrineSpecification\BaseSpecification; use Happyr\DoctrineSpecification\Spec; +use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Rest\Entity\ApiKey; From e093480a5b9ce8c300062a9688e1d02f0d798f54 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 26 Feb 2021 20:24:57 +0100 Subject: [PATCH 02/86] Fixed API tests --- module/Core/src/Repository/TagRepository.php | 7 ++++--- module/Core/src/Repository/TagRepositoryInterface.php | 2 +- module/Core/src/ShortUrl/Spec/BelongsToApiKey.php | 2 +- module/Core/src/ShortUrl/Spec/BelongsToDomain.php | 2 +- module/Core/src/Tag/TagService.php | 2 +- module/Core/test/Service/Tag/TagServiceTest.php | 2 +- module/Rest/src/ApiKey/Role.php | 8 +++++--- .../Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php | 2 +- module/Rest/src/Entity/ApiKey.php | 4 ++-- 9 files changed, 17 insertions(+), 14 deletions(-) diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index d5a33c8b..d21122d0 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\Core\Repository; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Happyr\DoctrineSpecification\Spec; -use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\Spec\CountTagsWithName; @@ -33,7 +32,7 @@ class TagRepository extends EntitySpecificationRepository implements TagReposito /** * @return TagInfo[] */ - public function findTagsWithInfo(?Specification $spec = null): array + public function findTagsWithInfo(?ApiKey $apiKey = null): array { $qb = $this->createQueryBuilder('t'); $qb->select('t AS tag', 'COUNT(DISTINCT s.id) AS shortUrlsCount', 'COUNT(DISTINCT v.id) AS visitsCount') @@ -42,7 +41,9 @@ class TagRepository extends EntitySpecificationRepository implements TagReposito ->groupBy('t') ->orderBy('t.name', 'ASC'); - $this->applySpecification($qb, $spec, 't'); + if ($apiKey !== null) { + $this->applySpecification($qb, $apiKey->spec(false, 'shortUrls'), 't'); + } $query = $qb->getQuery(); diff --git a/module/Core/src/Repository/TagRepositoryInterface.php b/module/Core/src/Repository/TagRepositoryInterface.php index 97c308df..9833eea1 100644 --- a/module/Core/src/Repository/TagRepositoryInterface.php +++ b/module/Core/src/Repository/TagRepositoryInterface.php @@ -17,7 +17,7 @@ interface TagRepositoryInterface extends ObjectRepository, EntitySpecificationRe /** * @return TagInfo[] */ - public function findTagsWithInfo(?Specification $spec = null): array; + public function findTagsWithInfo(?ApiKey $apiKey = null): array; public function tagExists(string $tag, ?ApiKey $apiKey = null): bool; } diff --git a/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php b/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php index ac20d0db..4aa3579f 100644 --- a/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php +++ b/module/Core/src/ShortUrl/Spec/BelongsToApiKey.php @@ -18,7 +18,7 @@ class BelongsToApiKey extends BaseSpecification { $this->apiKey = $apiKey; $this->dqlAlias = $dqlAlias; - parent::__construct($this->dqlAlias); + parent::__construct(); } protected function getSpec(): Filter diff --git a/module/Core/src/ShortUrl/Spec/BelongsToDomain.php b/module/Core/src/ShortUrl/Spec/BelongsToDomain.php index 59898476..7745ff27 100644 --- a/module/Core/src/ShortUrl/Spec/BelongsToDomain.php +++ b/module/Core/src/ShortUrl/Spec/BelongsToDomain.php @@ -17,7 +17,7 @@ class BelongsToDomain extends BaseSpecification { $this->domainId = $domainId; $this->dqlAlias = $dqlAlias; - parent::__construct($this->dqlAlias); + parent::__construct(); } protected function getSpec(): Filter diff --git a/module/Core/src/Tag/TagService.php b/module/Core/src/Tag/TagService.php index ae46a312..4619bd9d 100644 --- a/module/Core/src/Tag/TagService.php +++ b/module/Core/src/Tag/TagService.php @@ -52,7 +52,7 @@ class TagService implements TagServiceInterface { /** @var TagRepositoryInterface $repo */ $repo = $this->em->getRepository(Tag::class); - return $repo->findTagsWithInfo($apiKey !== null ? $apiKey->spec() : null); + return $repo->findTagsWithInfo($apiKey); } /** diff --git a/module/Core/test/Service/Tag/TagServiceTest.php b/module/Core/test/Service/Tag/TagServiceTest.php index 5f518184..9b484791 100644 --- a/module/Core/test/Service/Tag/TagServiceTest.php +++ b/module/Core/test/Service/Tag/TagServiceTest.php @@ -60,7 +60,7 @@ class TagServiceTest extends TestCase { $expected = [new TagInfo(new Tag('foo'), 1, 1), new TagInfo(new Tag('bar'), 3, 10)]; - $find = $this->repo->findTagsWithInfo($apiKey === null ? null : $apiKey->spec())->willReturn($expected); + $find = $this->repo->findTagsWithInfo($apiKey)->willReturn($expected); $result = $this->service->tagsInfo($apiKey); diff --git a/module/Rest/src/ApiKey/Role.php b/module/Rest/src/ApiKey/Role.php index 93ac717a..c3677029 100644 --- a/module/Rest/src/ApiKey/Role.php +++ b/module/Rest/src/ApiKey/Role.php @@ -21,16 +21,18 @@ class Role self::DOMAIN_SPECIFIC => 'Domain only', ]; - public static function toSpec(ApiKeyRole $role, bool $inlined): Specification + public static function toSpec(ApiKeyRole $role, bool $inlined, ?string $context = null): Specification { if ($role->name() === self::AUTHORED_SHORT_URLS) { $apiKey = $role->apiKey(); - return $inlined ? Spec::andX(new BelongsToApiKeyInlined($apiKey)) : new BelongsToApiKey($apiKey); + return $inlined ? Spec::andX(new BelongsToApiKeyInlined($apiKey)) : new BelongsToApiKey($apiKey, $context); } if ($role->name() === self::DOMAIN_SPECIFIC) { $domainId = self::domainIdFromMeta($role->meta()); - return $inlined ? Spec::andX(new BelongsToDomainInlined($domainId)) : new BelongsToDomain($domainId); + return $inlined + ? Spec::andX(new BelongsToDomainInlined($domainId)) + : new BelongsToDomain($domainId, $context); } return Spec::andX(); diff --git a/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php b/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php index e05dca41..a1f9b361 100644 --- a/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php +++ b/module/Rest/src/ApiKey/Spec/WithApiKeySpecsEnsuringJoin.php @@ -25,7 +25,7 @@ class WithApiKeySpecsEnsuringJoin extends BaseSpecification { return $this->apiKey === null || $this->apiKey->isAdmin() ? Spec::andX() : Spec::andX( Spec::join($this->fieldToJoin, 's'), - $this->apiKey->spec(), + $this->apiKey->spec(false, $this->fieldToJoin), ); } } diff --git a/module/Rest/src/Entity/ApiKey.php b/module/Rest/src/Entity/ApiKey.php index 62729031..d153b7f1 100644 --- a/module/Rest/src/Entity/ApiKey.php +++ b/module/Rest/src/Entity/ApiKey.php @@ -92,9 +92,9 @@ class ApiKey extends AbstractEntity return $this->key; } - public function spec(bool $inlined = false): Specification + public function spec(bool $inlined = false, ?string $context = null): Specification { - $specs = $this->roles->map(fn (ApiKeyRole $role) => Role::toSpec($role, $inlined))->getValues(); + $specs = $this->roles->map(fn (ApiKeyRole $role) => Role::toSpec($role, $inlined, $context))->getValues(); return Spec::andX(...$specs); } From c622804950bf5b9f3ad9805fe1202250bf8cb40f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 26 Feb 2021 20:27:41 +0100 Subject: [PATCH 03/86] Fixed unit tests --- module/Rest/test/ApiKey/RoleTest.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/Rest/test/ApiKey/RoleTest.php b/module/Rest/test/ApiKey/RoleTest.php index 4cb9ba1b..60a55ca5 100644 --- a/module/Rest/test/ApiKey/RoleTest.php +++ b/module/Rest/test/ApiKey/RoleTest.php @@ -35,7 +35,7 @@ class RoleTest extends TestCase yield 'inline author role' => [ new ApiKeyRole(Role::AUTHORED_SHORT_URLS, [], $apiKey), true, - new BelongsToApiKeyInlined($apiKey), + Spec::andX(new BelongsToApiKeyInlined($apiKey)), ]; yield 'not inline author role' => [ new ApiKeyRole(Role::AUTHORED_SHORT_URLS, [], $apiKey), @@ -45,7 +45,7 @@ class RoleTest extends TestCase yield 'inline domain role' => [ new ApiKeyRole(Role::DOMAIN_SPECIFIC, ['domain_id' => '123'], $apiKey), true, - new BelongsToDomainInlined('123'), + Spec::andX(new BelongsToDomainInlined('123')), ]; yield 'not inline domain role' => [ new ApiKeyRole(Role::DOMAIN_SPECIFIC, ['domain_id' => '456'], $apiKey), From 0fc123b249019beff20333384deac0768bddc313 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 26 Feb 2021 20:28:41 +0100 Subject: [PATCH 04/86] Fixed coding styles --- module/Core/src/Repository/TagRepositoryInterface.php | 1 - 1 file changed, 1 deletion(-) diff --git a/module/Core/src/Repository/TagRepositoryInterface.php b/module/Core/src/Repository/TagRepositoryInterface.php index 9833eea1..924706ff 100644 --- a/module/Core/src/Repository/TagRepositoryInterface.php +++ b/module/Core/src/Repository/TagRepositoryInterface.php @@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; -use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Rest\Entity\ApiKey; From 1b168ac3d2a394198189da335db7458acfbc90a6 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 26 Feb 2021 20:34:04 +0100 Subject: [PATCH 05/86] Updated changelog --- CHANGELOG.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2913aae2..9c41f6d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). +## [Unreleased] +### Added +* *Nothing* + +### Changed +* [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. + +### Deprecated +* *Nothing* + +### Removed +* *Nothing* + +### Fixed +* *Nothing* + + ## [2.6.1] - 2021-02-22 ### Added * *Nothing* From b244c5686294255c5bc958c6414a8f5706165059 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 28 Feb 2021 16:41:52 +0100 Subject: [PATCH 06/86] Updated to endroid/qr-code 4 --- composer.json | 4 ++-- module/Core/src/Action/QrCodeAction.php | 13 +++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index d544ac41..548d5e0c 100644 --- a/composer.json +++ b/composer.json @@ -21,7 +21,7 @@ "doctrine/cache": "^1.9", "doctrine/migrations": "^3.0.2", "doctrine/orm": "2.8.1 || ^2.8.3", - "endroid/qr-code": "dev-master#0f1613a as 3.10", + "endroid/qr-code": "^4.0", "geoip2/geoip2": "^2.9", "guzzlehttp/guzzle": "^7.0", "happyr/doctrine-specification": "^2.0", @@ -46,7 +46,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.7", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "^3.5", + "shlinkio/shlink-common": "^3.6", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", "shlinkio/shlink-importer": "^2.2", diff --git a/module/Core/src/Action/QrCodeAction.php b/module/Core/src/Action/QrCodeAction.php index 3209d651..1b2b5012 100644 --- a/module/Core/src/Action/QrCodeAction.php +++ b/module/Core/src/Action/QrCodeAction.php @@ -4,7 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Action; -use Endroid\QrCode\QrCode; +use Endroid\QrCode\Builder\Builder; use Endroid\QrCode\Writer\SvgWriter; use Psr\Http\Message\ResponseInterface as Response; use Psr\Http\Message\ServerRequestInterface as Request; @@ -50,16 +50,17 @@ class QrCodeAction implements MiddlewareInterface } $query = $request->getQueryParams(); - $qrCode = new QrCode($this->stringifier->stringify($shortUrl)); - $qrCode->setSize($this->resolveSize($request, $query)); - $qrCode->setMargin($this->resolveMargin($query)); + $qrCode = Builder::create() + ->data($this->stringifier->stringify($shortUrl)) + ->size($this->resolveSize($request, $query)) + ->margin($this->resolveMargin($query)); $format = $query['format'] ?? 'png'; if ($format === 'svg') { - $qrCode->setWriter(new SvgWriter()); + $qrCode->writer(new SvgWriter()); } - return new QrCodeResponse($qrCode); + return new QrCodeResponse($qrCode->build()); } private function resolveSize(Request $request, array $query): int From a2ca1618eabd6d783f013f8d12fcf6dd25e2a179 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 28 Feb 2021 16:42:25 +0100 Subject: [PATCH 07/86] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9c41f6d6..e8dce72c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. +* [#1039](https://github.com/shlinkio/shlink/issues/1039) Updated to `endroid/qr-code` 4.0. ### Deprecated * *Nothing* From 8128e85b6be75fce501e4fb7219b3bd968d627f0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 1 Mar 2021 17:04:16 +0100 Subject: [PATCH 08/86] Ensured latest docker image is built with SHLINK_VERSION=latest --- CHANGELOG.md | 2 +- Dockerfile | 2 +- docker/build | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e8dce72c..21dde341 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * *Nothing* ### Fixed -* *Nothing* +* [#1041](https://github.com/shlinkio/shlink/issues/1041) Added missing `--build-arg SHLINK_VERSION=latest` when building latest docker image. ## [2.6.1] - 2021-02-22 diff --git a/Dockerfile b/Dockerfile index fd703ebc..eb458bf1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM php:8.0.2-alpine3.13 as base -ARG SHLINK_VERSION=2.5.2 +ARG SHLINK_VERSION=2.6.1 ENV SHLINK_VERSION ${SHLINK_VERSION} ENV SWOOLE_VERSION 4.6.3 ENV PDO_SQLSRV_VERSION 5.9.0 diff --git a/docker/build b/docker/build index fdd58106..8e8db6c3 100755 --- a/docker/build +++ b/docker/build @@ -20,6 +20,7 @@ if [[ "$GITHUB_REF" != *"develop"* ]]; then # If build branch is develop, build latest elif [[ "$GITHUB_REF" == *"develop"* ]]; then docker buildx build --push \ + --build-arg SHLINK_VERSION=latest \ --platform ${PLATFORMS} \ -t ${DOCKER_IMAGE}:latest . fi From 65f2ab6720de09eb4c65bf1e2aa6a4e6096234c3 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 1 Mar 2021 21:17:32 +0100 Subject: [PATCH 09/86] Changed approach to ensure the default value for the version while building the docker image is latest --- CHANGELOG.md | 2 +- Dockerfile | 2 +- docker/build | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21dde341..2ee9cc37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * *Nothing* ### Fixed -* [#1041](https://github.com/shlinkio/shlink/issues/1041) Added missing `--build-arg SHLINK_VERSION=latest` when building latest docker image. +* [#1041](https://github.com/shlinkio/shlink/issues/1041) Ensured the default value for the version while building the docker image is `latest`. ## [2.6.1] - 2021-02-22 diff --git a/Dockerfile b/Dockerfile index eb458bf1..c058ef61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM php:8.0.2-alpine3.13 as base -ARG SHLINK_VERSION=2.6.1 +ARG SHLINK_VERSION=latest ENV SHLINK_VERSION ${SHLINK_VERSION} ENV SWOOLE_VERSION 4.6.3 ENV PDO_SQLSRV_VERSION 5.9.0 diff --git a/docker/build b/docker/build index 8e8db6c3..fdd58106 100755 --- a/docker/build +++ b/docker/build @@ -20,7 +20,6 @@ if [[ "$GITHUB_REF" != *"develop"* ]]; then # If build branch is develop, build latest elif [[ "$GITHUB_REF" == *"develop"* ]]; then docker buildx build --push \ - --build-arg SHLINK_VERSION=latest \ --platform ${PLATFORMS} \ -t ${DOCKER_IMAGE}:latest . fi From 1ade4e99177ec2e804743310445edaad086a2510 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 7 Mar 2021 09:22:04 +0100 Subject: [PATCH 10/86] Updated CONTRIBUTING doc mentioning API tests are run using Postgres --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 234bab5e..f9121b26 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,7 +100,7 @@ In order to ensure stability and no regressions are introduced while developing These are the best tests to catch regressions, and to verify everything behaves as expected. - They use MySQL as the database engine, and include some fixtures that ensure the same data exists at the beginning of the execution. + They use Postgres as the database engine, and include some fixtures that ensure the same data exists at the beginning of the execution. * **CLI tests**: *TBD. Once included, its purpose will be the same as API tests, but running through the command line* From b93b14986ecaa2b0f211c54da75378ce82f42662 Mon Sep 17 00:00:00 2001 From: KetchupBomb Date: Sat, 6 Mar 2021 17:27:34 +0000 Subject: [PATCH 11/86] Feature/name api keys --- data/migrations/Version20210306165711.php | 45 ++++++++++++ .../src/Command/Api/GenerateKeyCommand.php | 11 +++ .../CLI/src/Command/Api/ListKeysCommand.php | 3 + .../Command/Api/GenerateKeyCommandTest.php | 29 ++++++-- .../test/Command/Api/ListKeysCommandTest.php | 68 ++++++++++++------- .../Shlinkio.Shlink.Rest.Entity.ApiKey.php | 5 ++ module/Rest/src/Entity/ApiKey.php | 13 +++- module/Rest/src/Service/ApiKeyService.php | 9 ++- .../src/Service/ApiKeyServiceInterface.php | 6 +- .../Rest/test/Service/ApiKeyServiceTest.php | 14 ++-- 10 files changed, 163 insertions(+), 40 deletions(-) create mode 100644 data/migrations/Version20210306165711.php diff --git a/data/migrations/Version20210306165711.php b/data/migrations/Version20210306165711.php new file mode 100644 index 00000000..5a7b9fe7 --- /dev/null +++ b/data/migrations/Version20210306165711.php @@ -0,0 +1,45 @@ +getTable(self::TABLE); + $this->skipIf($apiKeys->hasColumn(self::COLUMN)); + + $apiKeys->addColumn( + self::COLUMN, + Types::STRING, + [ + 'notnull' => false, + ], + ); + } + + public function down(Schema $schema): void + { + $apiKeys = $schema->getTable(self::TABLE); + $this->skipIf(! $apiKeys->hasColumn(self::COLUMN)); + + $apiKeys->dropColumn(self::COLUMN); + } + + /** + * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 + */ + public function isTransactional(): bool + { + return false; + } +} diff --git a/module/CLI/src/Command/Api/GenerateKeyCommand.php b/module/CLI/src/Command/Api/GenerateKeyCommand.php index 119fa020..31df82a1 100644 --- a/module/CLI/src/Command/Api/GenerateKeyCommand.php +++ b/module/CLI/src/Command/Api/GenerateKeyCommand.php @@ -42,6 +42,10 @@ class GenerateKeyCommand extends BaseCommand %command.full_name% + You can optionally set its name for tracking purposes with --name or -m: + + %command.full_name% --name Alice + You can optionally set its expiration date with --expiration-date or -e: %command.full_name% --expiration-date 2020-01-01 @@ -56,6 +60,12 @@ class GenerateKeyCommand extends BaseCommand $this ->setName(self::NAME) ->setDescription('Generates a new valid API key.') + ->addOption( + 'name', + 'm', + InputOption::VALUE_REQUIRED, + 'The name by which this API key will be known.', + ) ->addOptionWithDeprecatedFallback( 'expiration-date', 'e', @@ -82,6 +92,7 @@ class GenerateKeyCommand extends BaseCommand $expirationDate = $this->getOptionWithDeprecatedFallback($input, 'expiration-date'); $apiKey = $this->apiKeyService->create( isset($expirationDate) ? Chronos::parse($expirationDate) : null, + $input->getOption('name'), ...$this->roleResolver->determineRoles($input), ); diff --git a/module/CLI/src/Command/Api/ListKeysCommand.php b/module/CLI/src/Command/Api/ListKeysCommand.php index 9243779b..ac5fc73d 100644 --- a/module/CLI/src/Command/Api/ListKeysCommand.php +++ b/module/CLI/src/Command/Api/ListKeysCommand.php @@ -58,6 +58,7 @@ class ListKeysCommand extends BaseCommand // Set columns for this row $rowData = [sprintf($messagePattern, $apiKey)]; + $rowData[] = $apiKey->name() ?? '-'; if (! $enabledOnly) { $rowData[] = sprintf($messagePattern, $this->getEnabledSymbol($apiKey)); } @@ -74,10 +75,12 @@ class ListKeysCommand extends BaseCommand ShlinkTable::fromOutput($output)->render(array_filter([ 'Key', + 'Name', ! $enabledOnly ? 'Is enabled' : null, 'Expiration date', 'Roles', ]), $rows); + return ExitCodes::EXIT_SUCCESS; } diff --git a/module/CLI/test/Command/Api/GenerateKeyCommandTest.php b/module/CLI/test/Command/Api/GenerateKeyCommandTest.php index 00548f17..ee822bee 100644 --- a/module/CLI/test/Command/Api/GenerateKeyCommandTest.php +++ b/module/CLI/test/Command/Api/GenerateKeyCommandTest.php @@ -40,22 +40,43 @@ class GenerateKeyCommandTest extends TestCase /** @test */ public function noExpirationDateIsDefinedIfNotProvided(): void { - $create = $this->apiKeyService->create(null)->willReturn(new ApiKey()); + $this->apiKeyService->create( + null, // Expiration date + null, // Name + )->shouldBeCalledOnce() + ->willReturn(new ApiKey()); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); self::assertStringContainsString('Generated API key: ', $output); - $create->shouldHaveBeenCalledOnce(); } /** @test */ public function expirationDateIsDefinedIfProvided(): void { - $this->apiKeyService->create(Argument::type(Chronos::class))->shouldBeCalledOnce() - ->willReturn(new ApiKey()); + $this->apiKeyService->create( + Argument::type(Chronos::class), // Expiration date + null, // Name + )->shouldBeCalledOnce() + ->willReturn(new ApiKey()); + $this->commandTester->execute([ '--expiration-date' => '2016-01-01', ]); } + + /** @test */ + public function nameIsDefinedIfProvided(): void + { + $this->apiKeyService->create( + null, // Expiration date + Argument::type('string'), // Name + )->shouldBeCalledOnce() + ->willReturn(new ApiKey()); + + $this->commandTester->execute([ + '--name' => 'Alice', + ]); + } } diff --git a/module/CLI/test/Command/Api/ListKeysCommandTest.php b/module/CLI/test/Command/Api/ListKeysCommandTest.php index e0cada5d..54ce19bc 100644 --- a/module/CLI/test/Command/Api/ListKeysCommandTest.php +++ b/module/CLI/test/Command/Api/ListKeysCommandTest.php @@ -52,13 +52,13 @@ class ListKeysCommandTest extends TestCase [ApiKey::withKey('foo'), ApiKey::withKey('bar'), ApiKey::withKey('baz')], false, <<disable(), ApiKey::withKey('bar')], true, << [ + [ + ApiKey::withKey('abc', null, 'Alice'), + ApiKey::withKey('def', null, 'Alice and Bob'), + ApiKey::withKey('ghi', null, ''), + ApiKey::withKey('jkl', null, null), + ], + true, + <<unique() ->build(); + $builder->createField('name', Types::STRING) + ->columnName('`name`') + ->nullable() + ->build(); + $builder->createField('expirationDate', ChronosDateTimeType::CHRONOS_DATETIME) ->columnName('expiration_date') ->nullable() diff --git a/module/Rest/src/Entity/ApiKey.php b/module/Rest/src/Entity/ApiKey.php index d153b7f1..b475a69d 100644 --- a/module/Rest/src/Entity/ApiKey.php +++ b/module/Rest/src/Entity/ApiKey.php @@ -22,14 +22,16 @@ class ApiKey extends AbstractEntity private bool $enabled; /** @var Collection|ApiKeyRole[] */ private Collection $roles; + private ?string $name; /** * @throws Exception */ - public function __construct(?Chronos $expirationDate = null) + public function __construct(?Chronos $expirationDate = null, ?string $name = null) { $this->key = Uuid::uuid4()->toString(); $this->expirationDate = $expirationDate; + $this->name = $name; $this->enabled = true; $this->roles = new ArrayCollection(); } @@ -45,9 +47,9 @@ class ApiKey extends AbstractEntity return $apiKey; } - public static function withKey(string $key, ?Chronos $expirationDate = null): self + public static function withKey(string $key, ?Chronos $expirationDate = null, ?string $name = null): self { - $apiKey = new self($expirationDate); + $apiKey = new self($expirationDate, $name); $apiKey->key = $key; return $apiKey; @@ -63,6 +65,11 @@ class ApiKey extends AbstractEntity return $this->expirationDate !== null && $this->expirationDate->lt(Chronos::now()); } + public function name(): ?string + { + return $this->name; + } + public function isEnabled(): bool { return $this->enabled; diff --git a/module/Rest/src/Service/ApiKeyService.php b/module/Rest/src/Service/ApiKeyService.php index 917cf048..ef60ee08 100644 --- a/module/Rest/src/Service/ApiKeyService.php +++ b/module/Rest/src/Service/ApiKeyService.php @@ -21,9 +21,12 @@ class ApiKeyService implements ApiKeyServiceInterface $this->em = $em; } - public function create(?Chronos $expirationDate = null, RoleDefinition ...$roleDefinitions): ApiKey - { - $key = new ApiKey($expirationDate); + public function create( + ?Chronos $expirationDate = null, + ?string $name = null, + RoleDefinition ...$roleDefinitions + ): ApiKey { + $key = new ApiKey($expirationDate, $name); foreach ($roleDefinitions as $definition) { $key->registerRole($definition); } diff --git a/module/Rest/src/Service/ApiKeyServiceInterface.php b/module/Rest/src/Service/ApiKeyServiceInterface.php index 562f106b..982bdf4f 100644 --- a/module/Rest/src/Service/ApiKeyServiceInterface.php +++ b/module/Rest/src/Service/ApiKeyServiceInterface.php @@ -11,7 +11,11 @@ use Shlinkio\Shlink\Rest\Entity\ApiKey; interface ApiKeyServiceInterface { - public function create(?Chronos $expirationDate = null, RoleDefinition ...$roleDefinitions): ApiKey; + public function create( + ?Chronos $expirationDate = null, + ?string $name = null, + RoleDefinition ...$roleDefinitions + ): ApiKey; public function check(string $key): ApiKeyCheckResult; diff --git a/module/Rest/test/Service/ApiKeyServiceTest.php b/module/Rest/test/Service/ApiKeyServiceTest.php index 6879d492..af50be23 100644 --- a/module/Rest/test/Service/ApiKeyServiceTest.php +++ b/module/Rest/test/Service/ApiKeyServiceTest.php @@ -35,14 +35,15 @@ class ApiKeyServiceTest extends TestCase * @dataProvider provideCreationDate * @param RoleDefinition[] $roles */ - public function apiKeyIsProperlyCreated(?Chronos $date, array $roles): void + public function apiKeyIsProperlyCreated(?Chronos $date, ?string $name, array $roles): void { $this->em->flush()->shouldBeCalledOnce(); $this->em->persist(Argument::type(ApiKey::class))->shouldBeCalledOnce(); - $key = $this->service->create($date, ...$roles); + $key = $this->service->create($date, $name, ...$roles); self::assertEquals($date, $key->getExpirationDate()); + self::assertEquals($name, $key->name()); foreach ($roles as $roleDefinition) { self::assertTrue($key->hasRole($roleDefinition->roleName())); } @@ -50,12 +51,15 @@ class ApiKeyServiceTest extends TestCase public function provideCreationDate(): iterable { - yield 'no expiration date' => [null, []]; - yield 'expiration date' => [Chronos::parse('2030-01-01'), []]; - yield 'roles' => [null, [ + yield 'no expiration date or name' => [null, null, []]; + yield 'expiration date' => [Chronos::parse('2030-01-01'), null, []]; + yield 'roles' => [null, null, [ RoleDefinition::forDomain((new Domain(''))->setId('123')), RoleDefinition::forAuthoredShortUrls(), ]]; + yield 'single name' => [null, 'Alice', []]; + yield 'multi-word name' => [null, 'Alice and Bob', []]; + yield 'empty name' => [null, '', []]; } /** From a03c4519c92b20e4d30017ae22f752d0c0023859 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 8 Mar 2021 07:00:25 +0100 Subject: [PATCH 12/86] Updated CONTRIBUTING doc --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f9121b26..eec7b0fb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -118,7 +118,7 @@ Depending on the kind of contribution, maybe not all kinds of tests are needed, For example, `test:db:postgres`. -* Run `./indocker composer test:api` to run API E2E tests. For these, the MySQL database engine is used. +* Run `./indocker composer test:api` to run API E2E tests. For these, the Postgres database engine is used. * Run `./indocker composer infect:test` ti run both unit and database tests (over sqlite) and then apply mutations to them with [infection](https://infection.github.io/). * Run `./indocker composer ci` to run all previous commands together. This command is run during the project's continuous integration. * Run `./indocker composer ci:parallel` to do the same as in previous case, but parallelizing non-conflicting tasks as much as possible. From 744b368cc1936511aaa7d8872f59518a0cf757bc Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Mon, 8 Mar 2021 19:50:43 +0100 Subject: [PATCH 13/86] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ee9cc37..2cb649b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this ## [Unreleased] ### Added -* *Nothing* +* [#1044](https://github.com/shlinkio/shlink/issues/1044) Added ability to set names on API keys, which help identifying them when the list grows. ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. From 9feb72235ae13de98e80b2060f5ca20bc4efd84e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 13 Feb 2021 11:40:19 +0100 Subject: [PATCH 14/86] Added config to log in filesystem while running API tests --- bin/test/run-api-tests.sh | 2 ++ config/test/test_config.global.php | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/bin/test/run-api-tests.sh b/bin/test/run-api-tests.sh index 07b36881..dbd87a84 100755 --- a/bin/test/run-api-tests.sh +++ b/bin/test/run-api-tests.sh @@ -3,6 +3,8 @@ export APP_ENV=test export DB_DRIVER=postgres export TEST_ENV=api +rm -rf data/log/api-tests + # Try to stop server just in case it hanged in last execution vendor/bin/laminas mezzio:swoole:stop diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index 3608257e..7ea5644a 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -9,6 +9,8 @@ use Laminas\ConfigAggregator\ConfigAggregator; use Laminas\Diactoros\Response\EmptyResponse; use Laminas\ServiceManager\Factory\InvokableFactory; use Laminas\Stdlib\Glob; +use Monolog\Handler\RotatingFileHandler; +use Monolog\Logger; use PDO; use PHPUnit\Runner\Version; use SebastianBergmann\CodeCoverage\CodeCoverage; @@ -163,4 +165,28 @@ return [ ], ], + 'logger' => [ + 'Shlink' => [ + 'handlers' => [ + 'shlink_handler' => [ + 'params' => [ + 'level' => Logger::DEBUG, + 'filename' => 'data/log/api-tests/shlink_log.log', + ], + ], + ], + ], + 'Access' => [ + 'handlers' => [ + 'access_handler' => [ + 'name' => RotatingFileHandler::class, + 'params' => [ + 'level' => Logger::DEBUG, + 'filename' => 'data/log/api-tests/access_log.log', + ], + ], + ], + ], + ], + ]; From 4439685403cdf55f1486b4d06026ed2b48afa02d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 08:49:38 +0100 Subject: [PATCH 15/86] Fixed logs generated by shlink during API tests --- composer.json | 2 +- config/test/test_config.global.php | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index 53acdf0a..89764118 100644 --- a/composer.json +++ b/composer.json @@ -33,7 +33,7 @@ "laminas/laminas-stdlib": "^3.2", "lcobucci/jwt": "^4.0", "league/uri": "^6.2", - "lstrojny/functional-php": "^1.15", + "lstrojny/functional-php": "^1.17", "mezzio/mezzio": "^3.3", "mezzio/mezzio-fastroute": "^3.1", "mezzio/mezzio-problem-details": "^1.3", diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index 7ea5644a..d12902a5 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -9,7 +9,7 @@ use Laminas\ConfigAggregator\ConfigAggregator; use Laminas\Diactoros\Response\EmptyResponse; use Laminas\ServiceManager\Factory\InvokableFactory; use Laminas\Stdlib\Glob; -use Monolog\Handler\RotatingFileHandler; +use Monolog\Handler\StreamHandler; use Monolog\Logger; use PDO; use PHPUnit\Runner\Version; @@ -169,9 +169,10 @@ return [ 'Shlink' => [ 'handlers' => [ 'shlink_handler' => [ + 'name' => StreamHandler::class, 'params' => [ 'level' => Logger::DEBUG, - 'filename' => 'data/log/api-tests/shlink_log.log', + 'stream' => 'data/log/api-tests/shlink.log', ], ], ], @@ -179,10 +180,10 @@ return [ 'Access' => [ 'handlers' => [ 'access_handler' => [ - 'name' => RotatingFileHandler::class, + 'name' => StreamHandler::class, 'params' => [ 'level' => Logger::DEBUG, - 'filename' => 'data/log/api-tests/access_log.log', + 'stream' => 'data/log/api-tests/access.log', ], ], ], From d104265f04dd14b6b14a5888313f08cf825fe9bb Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 08:54:05 +0100 Subject: [PATCH 16/86] Updated CONTRIBUTING file, explaining how the logs are dumped during API tests --- CONTRIBUTING.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eec7b0fb..837f7593 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,12 +96,14 @@ In order to ensure stability and no regressions are introduced while developing The project provides some tooling to run them against any of the supported database engines. -* **API tests**: These are E2E tests that spin up an instance of the app and test it from the outside, by interacting with the REST API. +* **API tests**: These are E2E tests that spin up an instance of the app with swoole, and test it from the outside by interacting with the REST API. These are the best tests to catch regressions, and to verify everything behaves as expected. They use Postgres as the database engine, and include some fixtures that ensure the same data exists at the beginning of the execution. + Since the app instance is run on a process different from the one running the tests, when a test fails it might not be obvious why. To help debugging that, the app will dump all its logs inside `data/log/api-tests`, where you will find the `shlink.log` and `access.log` files. + * **CLI tests**: *TBD. Once included, its purpose will be the same as API tests, but running through the command line* Depending on the kind of contribution, maybe not all kinds of tests are needed, but the more you provide, the better. From 562110fac4505cea84fa22d14dc424233c76752a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 08:55:39 +0100 Subject: [PATCH 17/86] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17d1799f..97cf925c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. * [#1039](https://github.com/shlinkio/shlink/issues/1039) Updated to `endroid/qr-code` 4.0. +* [#1008](https://github.com/shlinkio/shlink/issues/1008) Ensured all logs are sent to the filesystem while running API tests, which helps debugging the reason for tests to fail. ### Deprecated * *Nothing* From d7523bcb57d7a1752abd7a6442e19841959127c9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 09:01:11 +0100 Subject: [PATCH 18/86] Reduced duplication by creating a function that builds test logger config --- config/test/test_config.global.php | 36 ++++++++++++------------------ 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index d12902a5..c6d90e39 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -82,6 +82,18 @@ $buildDbConnection = function (): array { return $driverConfigMap[$driver] ?? []; }; +$buildTestLoggerConfig = fn (string $handlerName, string $filename) => [ + 'handlers' => [ + $handlerName => [ + 'name' => StreamHandler::class, + 'params' => [ + 'level' => Logger::DEBUG, + 'stream' => sprintf('data/log/api-tests/%s', $filename), + ], + ], + ], +]; + return [ 'debug' => true, @@ -166,28 +178,8 @@ return [ ], 'logger' => [ - 'Shlink' => [ - 'handlers' => [ - 'shlink_handler' => [ - 'name' => StreamHandler::class, - 'params' => [ - 'level' => Logger::DEBUG, - 'stream' => 'data/log/api-tests/shlink.log', - ], - ], - ], - ], - 'Access' => [ - 'handlers' => [ - 'access_handler' => [ - 'name' => StreamHandler::class, - 'params' => [ - 'level' => Logger::DEBUG, - 'stream' => 'data/log/api-tests/access.log', - ], - ], - ], - ], + 'Shlink' => $buildTestLoggerConfig('shlink_handler', 'shlink.log'), + 'Access' => $buildTestLoggerConfig('access_handler', 'access.log'), ], ]; From 9b553895389e7520c6778a1a3826c499c070f5a9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 08:28:21 +0100 Subject: [PATCH 19/86] First steps to create ApiKeyMeta --- .../test/Command/Api/ListKeysCommandTest.php | 24 +++++++++---------- module/Rest/src/ApiKey/Model/ApiKeyMeta.php | 18 ++++++++++++++ module/Rest/src/Entity/ApiKey.php | 5 ++++ 3 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 module/Rest/src/ApiKey/Model/ApiKeyMeta.php diff --git a/module/CLI/test/Command/Api/ListKeysCommandTest.php b/module/CLI/test/Command/Api/ListKeysCommandTest.php index 54ce19bc..0e806e87 100644 --- a/module/CLI/test/Command/Api/ListKeysCommandTest.php +++ b/module/CLI/test/Command/Api/ListKeysCommandTest.php @@ -105,21 +105,21 @@ class ListKeysCommandTest extends TestCase ]; yield 'with names' => [ [ - ApiKey::withKey('abc', null, 'Alice'), - ApiKey::withKey('def', null, 'Alice and Bob'), - ApiKey::withKey('ghi', null, ''), - ApiKey::withKey('jkl', null, null), + $apiKey1 = ApiKey::withName('Alice'), + $apiKey2 = ApiKey::withName('Alice and Bob'), + $apiKey3 = ApiKey::withName(''), + $apiKey4 = new ApiKey(), ], true, <<expirationDate; From 0a5c265b1289df778bd79a707152fc317cc1abc2 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 09:59:35 +0100 Subject: [PATCH 20/86] Extracted ApiKey metadata to the ApiKeyMeta object --- .../Command/Api/GenerateKeyCommandTest.php | 22 ++---- .../test/Command/Api/ListKeysCommandTest.php | 77 ++++++++++--------- module/Core/functions/functions.php | 1 + .../Repository/DomainRepositoryTest.php | 9 ++- .../Repository/ShortUrlRepositoryTest.php | 9 ++- .../test-db/Repository/TagRepositoryTest.php | 5 +- .../Repository/VisitRepositoryTest.php | 7 +- module/Core/test/Domain/DomainServiceTest.php | 7 +- .../Adapter/ShortUrlRepositoryAdapterTest.php | 2 +- .../VisitsForTagPaginatorAdapterTest.php | 2 +- .../Adapter/VisitsPaginatorAdapterTest.php | 2 +- .../Core/test/Service/ShortUrlServiceTest.php | 2 +- .../Core/test/Service/Tag/TagServiceTest.php | 8 +- module/Core/test/Util/ApiKeyHelpersTrait.php | 2 +- .../Core/test/Visit/VisitsStatsHelperTest.php | 2 +- module/Rest/src/ApiKey/Model/ApiKeyMeta.php | 48 +++++++++++- module/Rest/src/Entity/ApiKey.php | 26 +++---- module/Rest/src/Service/ApiKeyService.php | 21 ++++- .../Rest/test-api/Fixtures/ApiKeyFixture.php | 8 +- .../Action/Domain/ListDomainsActionTest.php | 2 +- .../ShortUrl/CreateShortUrlActionTest.php | 4 +- .../ShortUrl/DeleteShortUrlActionTest.php | 2 +- .../ShortUrl/EditShortUrlActionTest.php | 2 +- .../ShortUrl/EditShortUrlTagsActionTest.php | 2 +- .../ShortUrl/ListShortUrlsActionTest.php | 2 +- .../ShortUrl/ResolveShortUrlActionTest.php | 2 +- .../SingleStepCreateShortUrlActionTest.php | 2 +- .../test/Action/Tag/DeleteTagsActionTest.php | 2 +- .../test/Action/Tag/ListTagsActionTest.php | 2 +- .../test/Action/Tag/UpdateTagActionTest.php | 2 +- .../Action/Visit/GlobalVisitsActionTest.php | 2 +- .../Action/Visit/ShortUrlVisitsActionTest.php | 2 +- .../test/Action/Visit/TagVisitsActionTest.php | 2 +- module/Rest/test/ApiKey/RoleTest.php | 2 +- .../AuthenticationMiddlewareTest.php | 2 +- .../Rest/test/Service/ApiKeyServiceTest.php | 13 ++-- 36 files changed, 186 insertions(+), 121 deletions(-) diff --git a/module/CLI/test/Command/Api/GenerateKeyCommandTest.php b/module/CLI/test/Command/Api/GenerateKeyCommandTest.php index ee822bee..5c0c3c8a 100644 --- a/module/CLI/test/Command/Api/GenerateKeyCommandTest.php +++ b/module/CLI/test/Command/Api/GenerateKeyCommandTest.php @@ -40,11 +40,7 @@ class GenerateKeyCommandTest extends TestCase /** @test */ public function noExpirationDateIsDefinedIfNotProvided(): void { - $this->apiKeyService->create( - null, // Expiration date - null, // Name - )->shouldBeCalledOnce() - ->willReturn(new ApiKey()); + $this->apiKeyService->create(null, null)->shouldBeCalledOnce()->willReturn(ApiKey::create()); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); @@ -55,11 +51,9 @@ class GenerateKeyCommandTest extends TestCase /** @test */ public function expirationDateIsDefinedIfProvided(): void { - $this->apiKeyService->create( - Argument::type(Chronos::class), // Expiration date - null, // Name - )->shouldBeCalledOnce() - ->willReturn(new ApiKey()); + $this->apiKeyService->create(Argument::type(Chronos::class), null)->shouldBeCalledOnce()->willReturn( + ApiKey::create(), + ); $this->commandTester->execute([ '--expiration-date' => '2016-01-01', @@ -69,11 +63,9 @@ class GenerateKeyCommandTest extends TestCase /** @test */ public function nameIsDefinedIfProvided(): void { - $this->apiKeyService->create( - null, // Expiration date - Argument::type('string'), // Name - )->shouldBeCalledOnce() - ->willReturn(new ApiKey()); + $this->apiKeyService->create(null, Argument::type('string'))->shouldBeCalledOnce()->willReturn( + ApiKey::create(), + ); $this->commandTester->execute([ '--name' => 'Alice', diff --git a/module/CLI/test/Command/Api/ListKeysCommandTest.php b/module/CLI/test/Command/Api/ListKeysCommandTest.php index 0e806e87..fa6816d2 100644 --- a/module/CLI/test/Command/Api/ListKeysCommandTest.php +++ b/module/CLI/test/Command/Api/ListKeysCommandTest.php @@ -9,6 +9,7 @@ use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Api\ListKeysCommand; use Shlinkio\Shlink\Core\Entity\Domain; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface; @@ -49,66 +50,66 @@ class ListKeysCommandTest extends TestCase public function provideKeysAndOutputs(): iterable { yield 'all keys' => [ - [ApiKey::withKey('foo'), ApiKey::withKey('bar'), ApiKey::withKey('baz')], + [$apiKey1 = ApiKey::create(), $apiKey2 = ApiKey::create(), $apiKey3 = ApiKey::create()], false, << [ - [ApiKey::withKey('foo')->disable(), ApiKey::withKey('bar')], + [$apiKey1 = ApiKey::create()->disable(), $apiKey2 = ApiKey::create()], true, << [ [ - ApiKey::withKey('foo'), - $this->apiKeyWithRoles('bar', [RoleDefinition::forAuthoredShortUrls()]), - $this->apiKeyWithRoles('baz', [RoleDefinition::forDomain((new Domain('example.com'))->setId('1'))]), - ApiKey::withKey('foo2'), - $this->apiKeyWithRoles('baz2', [ + $apiKey1 = ApiKey::create(), + $apiKey2 = $this->apiKeyWithRoles([RoleDefinition::forAuthoredShortUrls()]), + $apiKey3 = $this->apiKeyWithRoles([RoleDefinition::forDomain((new Domain('example.com'))->setId('1'))]), + $apiKey4 = ApiKey::create(), + $apiKey5 = $this->apiKeyWithRoles([ RoleDefinition::forAuthoredShortUrls(), RoleDefinition::forDomain((new Domain('example.com'))->setId('1')), ]), - ApiKey::withKey('foo3'), + $apiKey6 = ApiKey::create(), ], true, << [ [ - $apiKey1 = ApiKey::withName('Alice'), - $apiKey2 = ApiKey::withName('Alice and Bob'), - $apiKey3 = ApiKey::withName(''), - $apiKey4 = new ApiKey(), + $apiKey1 = ApiKey::fromMeta(ApiKeyMeta::withName('Alice')), + $apiKey2 = ApiKey::fromMeta(ApiKeyMeta::withName('Alice and Bob')), + $apiKey3 = ApiKey::fromMeta(ApiKeyMeta::withName('')), + $apiKey4 = ApiKey::create(), ], true, <<registerRole($role); } diff --git a/module/Core/functions/functions.php b/module/Core/functions/functions.php index 00954049..62df2070 100644 --- a/module/Core/functions/functions.php +++ b/module/Core/functions/functions.php @@ -50,6 +50,7 @@ function parseDateRangeFromQuery(array $query, string $startDateName, string $en $startDate = parseDateFromQuery($query, $startDateName); $endDate = parseDateFromQuery($query, $endDateName); + // TODO Use match expression when migrating to PHP8 if ($startDate === null && $endDate === null) { return DateRange::emptyInstance(); } diff --git a/module/Core/test-db/Domain/Repository/DomainRepositoryTest.php b/module/Core/test-db/Domain/Repository/DomainRepositoryTest.php index 49265eb0..aaa63d9f 100644 --- a/module/Core/test-db/Domain/Repository/DomainRepositoryTest.php +++ b/module/Core/test-db/Domain/Repository/DomainRepositoryTest.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\ShortUrl\Resolver\ShortUrlRelationResolverInterface; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\TestUtils\DbTest\DatabaseTestCase; @@ -53,9 +54,9 @@ class DomainRepositoryTest extends DatabaseTestCase /** @test */ public function findDomainsReturnsJustThoseMatchingProvidedApiKey(): void { - $authorApiKey = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $authorApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($authorApiKey); - $authorAndDomainApiKey = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $authorAndDomainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($authorAndDomainApiKey); $fooDomain = new Domain('foo.com'); @@ -74,10 +75,10 @@ class DomainRepositoryTest extends DatabaseTestCase $authorAndDomainApiKey->registerRole(RoleDefinition::forDomain($fooDomain)); - $fooDomainApiKey = ApiKey::withRoles(RoleDefinition::forDomain($fooDomain)); + $fooDomainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forDomain($fooDomain))); $this->getEntityManager()->persist($fooDomainApiKey); - $barDomainApiKey = ApiKey::withRoles(RoleDefinition::forDomain($barDomain)); + $barDomainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forDomain($barDomain))); $this->getEntityManager()->persist($fooDomainApiKey); $this->getEntityManager()->flush(); diff --git a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php index 48381857..35a6b709 100644 --- a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php +++ b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php @@ -17,6 +17,7 @@ use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\ShortUrlRepository; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\TestUtils\DbTest\DatabaseTestCase; @@ -335,13 +336,13 @@ class ShortUrlRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - $apiKey = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $apiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($apiKey); - $otherApiKey = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $otherApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($otherApiKey); - $wrongDomainApiKey = ApiKey::withRoles(RoleDefinition::forDomain($wrongDomain)); + $wrongDomainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forDomain($wrongDomain))); $this->getEntityManager()->persist($wrongDomainApiKey); - $rightDomainApiKey = ApiKey::withRoles(RoleDefinition::forDomain($rightDomain)); + $rightDomainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forDomain($rightDomain))); $this->getEntityManager()->persist($rightDomainApiKey); $shortUrl = ShortUrl::fromMeta(ShortUrlMeta::fromRawData([ diff --git a/module/Core/test-db/Repository/TagRepositoryTest.php b/module/Core/test-db/Repository/TagRepositoryTest.php index 34a06a40..eea2ed8c 100644 --- a/module/Core/test-db/Repository/TagRepositoryTest.php +++ b/module/Core/test-db/Repository/TagRepositoryTest.php @@ -12,6 +12,7 @@ use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\TestUtils\DbTest\DatabaseTestCase; @@ -100,9 +101,9 @@ class TagRepositoryTest extends DatabaseTestCase $this->getEntityManager()->persist($domain); $this->getEntityManager()->flush(); - $authorApiKey = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $authorApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($authorApiKey); - $domainApiKey = ApiKey::withRoles(RoleDefinition::forDomain($domain)); + $domainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forDomain($domain))); $this->getEntityManager()->persist($domainApiKey); $names = ['foo', 'bar', 'baz', 'another']; diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index b6c23699..4e634493 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -16,6 +16,7 @@ use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; use Shlinkio\Shlink\IpGeolocation\Model\Location; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\TestUtils\DbTest\DatabaseTestCase; @@ -176,7 +177,7 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - $apiKey1 = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $apiKey1 = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($apiKey1); $shortUrl = ShortUrl::fromMeta( ShortUrlMeta::fromRawData(['apiKey' => $apiKey1, 'domain' => $domain->getAuthority(), 'longUrl' => '']), @@ -185,7 +186,7 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->persist($shortUrl); $this->createVisitsForShortUrl($shortUrl, 4); - $apiKey2 = ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()); + $apiKey2 = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())); $this->getEntityManager()->persist($apiKey2); $shortUrl2 = ShortUrl::fromMeta(ShortUrlMeta::fromRawData(['apiKey' => $apiKey2, 'longUrl' => ''])); $this->getEntityManager()->persist($shortUrl2); @@ -198,7 +199,7 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->persist($shortUrl3); $this->createVisitsForShortUrl($shortUrl3, 7); - $domainApiKey = ApiKey::withRoles(RoleDefinition::forDomain($domain)); + $domainApiKey = ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forDomain($domain))); $this->getEntityManager()->persist($domainApiKey); // Visits not linked to any short URL diff --git a/module/Core/test/Domain/DomainServiceTest.php b/module/Core/test/Domain/DomainServiceTest.php index 46e39c5a..0306f387 100644 --- a/module/Core/test/Domain/DomainServiceTest.php +++ b/module/Core/test/Domain/DomainServiceTest.php @@ -14,6 +14,7 @@ use Shlinkio\Shlink\Core\Domain\Model\DomainItem; use Shlinkio\Shlink\Core\Domain\Repository\DomainRepositoryInterface; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Exception\DomainNotFoundException; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -50,8 +51,10 @@ class DomainServiceTest extends TestCase public function provideExcludedDomains(): iterable { $default = new DomainItem('default.com', true); - $adminApiKey = new ApiKey(); - $domainSpecificApiKey = ApiKey::withRoles(RoleDefinition::forDomain((new Domain(''))->setId('123'))); + $adminApiKey = ApiKey::create(); + $domainSpecificApiKey = ApiKey::fromMeta( + ApiKeyMeta::withRoles(RoleDefinition::forDomain((new Domain(''))->setId('123'))), + ); yield 'empty list without API key' => [[], [$default], null]; yield 'one item without API key' => [ diff --git a/module/Core/test/Paginator/Adapter/ShortUrlRepositoryAdapterTest.php b/module/Core/test/Paginator/Adapter/ShortUrlRepositoryAdapterTest.php index 93aba122..5420e4b6 100644 --- a/module/Core/test/Paginator/Adapter/ShortUrlRepositoryAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/ShortUrlRepositoryAdapterTest.php @@ -66,7 +66,7 @@ class ShortUrlRepositoryAdapterTest extends TestCase 'startDate' => $startDate, 'endDate' => $endDate, ]); - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $adapter = new ShortUrlRepositoryAdapter($this->repo->reveal(), $params, $apiKey); $dateRange = $params->dateRange(); diff --git a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php index 8dc88495..6c95a60f 100644 --- a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php @@ -44,7 +44,7 @@ class VisitsForTagPaginatorAdapterTest extends TestCase public function repoIsCalledOnlyOnceForCount(): void { $count = 3; - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $adapter = $this->createAdapter($apiKey); $countVisits = $this->repo->countVisitsByTag('foo', new DateRange(), $apiKey->spec())->willReturn(3); diff --git a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php index 436b4b7d..955e5ac5 100644 --- a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php @@ -47,7 +47,7 @@ class VisitsPaginatorAdapterTest extends TestCase public function repoIsCalledOnlyOnceForCount(): void { $count = 3; - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $adapter = $this->createAdapter($apiKey); $countVisits = $this->repo->countVisitsByShortCode('', null, new DateRange(), $apiKey->spec())->willReturn(3); diff --git a/module/Core/test/Service/ShortUrlServiceTest.php b/module/Core/test/Service/ShortUrlServiceTest.php index 024957b0..67420edc 100644 --- a/module/Core/test/Service/ShortUrlServiceTest.php +++ b/module/Core/test/Service/ShortUrlServiceTest.php @@ -124,7 +124,7 @@ class ShortUrlServiceTest extends TestCase 'maxVisits' => 10, 'longUrl' => 'modifiedLongUrl', ], - ), new ApiKey()]; + ), ApiKey::create()]; yield 'long URL with validation' => [1, ShortUrlEdit::fromRawData( [ 'longUrl' => 'modifiedLongUrl', diff --git a/module/Core/test/Service/Tag/TagServiceTest.php b/module/Core/test/Service/Tag/TagServiceTest.php index 9b484791..33ae7be0 100644 --- a/module/Core/test/Service/Tag/TagServiceTest.php +++ b/module/Core/test/Service/Tag/TagServiceTest.php @@ -17,6 +17,7 @@ use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\Model\TagRenaming; use Shlinkio\Shlink\Core\Tag\TagService; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use ShlinkioTest\Shlink\Core\Util\ApiKeyHelpersTrait; @@ -90,7 +91,10 @@ class TagServiceTest extends TestCase $this->expectExceptionMessage('You are not allowed to delete tags'); $delete->shouldNotBeCalled(); - $this->service->deleteTags(['foo', 'bar'], ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls())); + $this->service->deleteTags( + ['foo', 'bar'], + ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())), + ); } /** @test */ @@ -178,7 +182,7 @@ class TagServiceTest extends TestCase $this->service->renameTag( TagRenaming::fromNames('foo', 'bar'), - ApiKey::withRoles(RoleDefinition::forAuthoredShortUrls()), + ApiKey::fromMeta(ApiKeyMeta::withRoles(RoleDefinition::forAuthoredShortUrls())), ); } } diff --git a/module/Core/test/Util/ApiKeyHelpersTrait.php b/module/Core/test/Util/ApiKeyHelpersTrait.php index 0b21ed5f..6624c8dd 100644 --- a/module/Core/test/Util/ApiKeyHelpersTrait.php +++ b/module/Core/test/Util/ApiKeyHelpersTrait.php @@ -11,6 +11,6 @@ trait ApiKeyHelpersTrait public function provideAdminApiKeys(): iterable { yield 'no API key' => [null]; - yield 'admin API key' => [new ApiKey()]; + yield 'admin API key' => [ApiKey::create()]; } } diff --git a/module/Core/test/Visit/VisitsStatsHelperTest.php b/module/Core/test/Visit/VisitsStatsHelperTest.php index de2a3534..8e90a447 100644 --- a/module/Core/test/Visit/VisitsStatsHelperTest.php +++ b/module/Core/test/Visit/VisitsStatsHelperTest.php @@ -113,7 +113,7 @@ class VisitsStatsHelperTest extends TestCase public function throwsExceptionWhenRequestingVisitsForInvalidTag(): void { $tag = 'foo'; - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $repo = $this->prophesize(TagRepository::class); $tagExists = $repo->tagExists($tag, $apiKey)->willReturn(false); $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); diff --git a/module/Rest/src/ApiKey/Model/ApiKeyMeta.php b/module/Rest/src/ApiKey/Model/ApiKeyMeta.php index b79b9adf..aa3c117a 100644 --- a/module/Rest/src/ApiKey/Model/ApiKeyMeta.php +++ b/module/Rest/src/ApiKey/Model/ApiKeyMeta.php @@ -4,15 +4,57 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Rest\ApiKey\Model; +use Cake\Chronos\Chronos; + final class ApiKeyMeta { - public static function withKey(string $key): self - { + private ?string $name = null; + private ?Chronos $expirationDate = null; + /** @var RoleDefinition[] */ + private array $roleDefinitions; + private function __construct(?string $name, ?Chronos $expirationDate, array $roleDefinitions) + { + $this->name = $name; + $this->expirationDate = $expirationDate; + $this->roleDefinitions = $roleDefinitions; } - public static function withName(string $key): self + public static function withName(string $name): self { + return new self($name, null, []); + } + public static function withExpirationDate(Chronos $expirationDate): self + { + return new self(null, $expirationDate, []); + } + + public static function withNameAndExpirationDate(string $name, Chronos $expirationDate): self + { + return new self($name, $expirationDate, []); + } + + public static function withRoles(RoleDefinition ...$roleDefinitions): self + { + return new self(null, null, $roleDefinitions); + } + + public function name(): ?string + { + return $this->name; + } + + public function expirationDate(): ?Chronos + { + return $this->expirationDate; + } + + /** + * @return RoleDefinition[] + */ + public function roleDefinitions(): array + { + return $this->roleDefinitions; } } diff --git a/module/Rest/src/Entity/ApiKey.php b/module/Rest/src/Entity/ApiKey.php index e15c59f8..0317390e 100644 --- a/module/Rest/src/Entity/ApiKey.php +++ b/module/Rest/src/Entity/ApiKey.php @@ -12,6 +12,7 @@ use Happyr\DoctrineSpecification\Spec; use Happyr\DoctrineSpecification\Specification\Specification; use Ramsey\Uuid\Uuid; use Shlinkio\Shlink\Common\Entity\AbstractEntity; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\ApiKey\Role; @@ -27,7 +28,7 @@ class ApiKey extends AbstractEntity /** * @throws Exception */ - public function __construct(?Chronos $expirationDate = null, ?string $name = null) + private function __construct(?string $name = null, ?Chronos $expirationDate = null) { $this->key = Uuid::uuid4()->toString(); $this->expirationDate = $expirationDate; @@ -36,30 +37,21 @@ class ApiKey extends AbstractEntity $this->roles = new ArrayCollection(); } - public static function withRoles(RoleDefinition ...$roleDefinitions): self + public static function create(): ApiKey { - $apiKey = new self(); + return new self(); + } - foreach ($roleDefinitions as $roleDefinition) { + public static function fromMeta(ApiKeyMeta $meta): self + { + $apiKey = new self($meta->name(), $meta->expirationDate()); + foreach ($meta->roleDefinitions() as $roleDefinition) { $apiKey->registerRole($roleDefinition); } return $apiKey; } - public static function withKey(string $key, ?Chronos $expirationDate = null, ?string $name = null): self - { - $apiKey = new self($expirationDate, $name); - $apiKey->key = $key; - - return $apiKey; - } - - public static function withName(string $name): self - { - return new self(null, $name); - } - public function getExpirationDate(): ?Chronos { return $this->expirationDate; diff --git a/module/Rest/src/Service/ApiKeyService.php b/module/Rest/src/Service/ApiKeyService.php index ef60ee08..e81c446f 100644 --- a/module/Rest/src/Service/ApiKeyService.php +++ b/module/Rest/src/Service/ApiKeyService.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Rest\Service; use Cake\Chronos\Chronos; use Doctrine\ORM\EntityManagerInterface; use Shlinkio\Shlink\Common\Exception\InvalidArgumentException; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -26,7 +27,7 @@ class ApiKeyService implements ApiKeyServiceInterface ?string $name = null, RoleDefinition ...$roleDefinitions ): ApiKey { - $key = new ApiKey($expirationDate, $name); + $key = $this->buildApiKeyWithParams($expirationDate, $name); foreach ($roleDefinitions as $definition) { $key->registerRole($definition); } @@ -37,6 +38,24 @@ class ApiKeyService implements ApiKeyServiceInterface return $key; } + private function buildApiKeyWithParams(?Chronos $expirationDate, ?string $name): ApiKey + { + // TODO Use match expression when migrating to PHP8 + if ($expirationDate === null && $name === null) { + return ApiKey::create(); + } + + if ($expirationDate !== null && $name !== null) { + return ApiKey::fromMeta(ApiKeyMeta::withNameAndExpirationDate($name, $expirationDate)); + } + + if ($name === null) { + return ApiKey::fromMeta(ApiKeyMeta::withExpirationDate($expirationDate)); + } + + return ApiKey::fromMeta(ApiKeyMeta::withName($name)); + } + public function check(string $key): ApiKeyCheckResult { $apiKey = $this->getByKey($key); diff --git a/module/Rest/test-api/Fixtures/ApiKeyFixture.php b/module/Rest/test-api/Fixtures/ApiKeyFixture.php index c6383968..ef6d1781 100644 --- a/module/Rest/test-api/Fixtures/ApiKeyFixture.php +++ b/module/Rest/test-api/Fixtures/ApiKeyFixture.php @@ -8,7 +8,9 @@ use Cake\Chronos\Chronos; use Doctrine\Common\DataFixtures\AbstractFixture; use Doctrine\Common\DataFixtures\DependentFixtureInterface; use Doctrine\Persistence\ObjectManager; +use ReflectionObject; use Shlinkio\Shlink\Core\Entity\Domain; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -41,7 +43,11 @@ class ApiKeyFixture extends AbstractFixture implements DependentFixtureInterface private function buildApiKey(string $key, bool $enabled, ?Chronos $expiresAt = null): ApiKey { - $apiKey = ApiKey::withKey($key, $expiresAt); + $apiKey = $expiresAt !== null ? ApiKey::fromMeta(ApiKeyMeta::withExpirationDate($expiresAt)) : ApiKey::create(); + $ref = new ReflectionObject($apiKey); + $keyProp = $ref->getProperty('key'); + $keyProp->setAccessible(true); + $keyProp->setValue($apiKey, $key); if (! $enabled) { $apiKey->disable(); diff --git a/module/Rest/test/Action/Domain/ListDomainsActionTest.php b/module/Rest/test/Action/Domain/ListDomainsActionTest.php index d6dcc4a3..cbe43895 100644 --- a/module/Rest/test/Action/Domain/ListDomainsActionTest.php +++ b/module/Rest/test/Action/Domain/ListDomainsActionTest.php @@ -30,7 +30,7 @@ class ListDomainsActionTest extends TestCase /** @test */ public function domainsAreProperlyListed(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $domains = [ new DomainItem('bar.com', true), new DomainItem('baz.com', false), diff --git a/module/Rest/test/Action/ShortUrl/CreateShortUrlActionTest.php b/module/Rest/test/Action/ShortUrl/CreateShortUrlActionTest.php index f8e95659..ffcd6c62 100644 --- a/module/Rest/test/Action/ShortUrl/CreateShortUrlActionTest.php +++ b/module/Rest/test/Action/ShortUrl/CreateShortUrlActionTest.php @@ -40,7 +40,7 @@ class CreateShortUrlActionTest extends TestCase /** @test */ public function properShortcodeConversionReturnsData(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $shortUrl = ShortUrl::createEmpty(); $expectedMeta = $body = [ 'longUrl' => 'http://www.domain.com/foo/bar', @@ -80,7 +80,7 @@ class CreateShortUrlActionTest extends TestCase $request = (new ServerRequest())->withParsedBody([ 'longUrl' => 'http://www.domain.com/foo/bar', 'domain' => $domain, - ])->withAttribute(ApiKey::class, new ApiKey()); + ])->withAttribute(ApiKey::class, ApiKey::create()); $this->expectException(ValidationException::class); $urlToShortCode->shouldNotBeCalled(); diff --git a/module/Rest/test/Action/ShortUrl/DeleteShortUrlActionTest.php b/module/Rest/test/Action/ShortUrl/DeleteShortUrlActionTest.php index 9be06756..9705cd59 100644 --- a/module/Rest/test/Action/ShortUrl/DeleteShortUrlActionTest.php +++ b/module/Rest/test/Action/ShortUrl/DeleteShortUrlActionTest.php @@ -29,7 +29,7 @@ class DeleteShortUrlActionTest extends TestCase /** @test */ public function emptyResponseIsReturnedIfProperlyDeleted(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $deleteByShortCode = $this->service->deleteByShortCode(Argument::any(), false, $apiKey)->will( function (): void { }, diff --git a/module/Rest/test/Action/ShortUrl/EditShortUrlActionTest.php b/module/Rest/test/Action/ShortUrl/EditShortUrlActionTest.php index eee75dbf..e1f434df 100644 --- a/module/Rest/test/Action/ShortUrl/EditShortUrlActionTest.php +++ b/module/Rest/test/Action/ShortUrl/EditShortUrlActionTest.php @@ -48,7 +48,7 @@ class EditShortUrlActionTest extends TestCase public function correctShortCodeReturnsSuccess(): void { $request = (new ServerRequest())->withAttribute('shortCode', 'abc123') - ->withAttribute(ApiKey::class, new ApiKey()) + ->withAttribute(ApiKey::class, ApiKey::create()) ->withParsedBody([ 'maxVisits' => 5, ]); diff --git a/module/Rest/test/Action/ShortUrl/EditShortUrlTagsActionTest.php b/module/Rest/test/Action/ShortUrl/EditShortUrlTagsActionTest.php index a345046a..59c55d84 100644 --- a/module/Rest/test/Action/ShortUrl/EditShortUrlTagsActionTest.php +++ b/module/Rest/test/Action/ShortUrl/EditShortUrlTagsActionTest.php @@ -58,6 +58,6 @@ class EditShortUrlTagsActionTest extends TestCase private function createRequestWithAPiKey(): ServerRequestInterface { - return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, new ApiKey()); + return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, ApiKey::create()); } } diff --git a/module/Rest/test/Action/ShortUrl/ListShortUrlsActionTest.php b/module/Rest/test/Action/ShortUrl/ListShortUrlsActionTest.php index 2683b514..712d605d 100644 --- a/module/Rest/test/Action/ShortUrl/ListShortUrlsActionTest.php +++ b/module/Rest/test/Action/ShortUrl/ListShortUrlsActionTest.php @@ -51,7 +51,7 @@ class ListShortUrlsActionTest extends TestCase ?string $startDate = null, ?string $endDate = null ): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $request = (new ServerRequest())->withQueryParams($query)->withAttribute(ApiKey::class, $apiKey); $listShortUrls = $this->service->listShortUrls(ShortUrlsParams::fromRawData([ 'page' => $expectedPage, diff --git a/module/Rest/test/Action/ShortUrl/ResolveShortUrlActionTest.php b/module/Rest/test/Action/ShortUrl/ResolveShortUrlActionTest.php index 748ab642..6f8ddbb9 100644 --- a/module/Rest/test/Action/ShortUrl/ResolveShortUrlActionTest.php +++ b/module/Rest/test/Action/ShortUrl/ResolveShortUrlActionTest.php @@ -37,7 +37,7 @@ class ResolveShortUrlActionTest extends TestCase public function correctShortCodeReturnsSuccess(): void { $shortCode = 'abc123'; - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $this->urlResolver->resolveShortUrl(new ShortUrlIdentifier($shortCode), $apiKey)->willReturn( ShortUrl::withLongUrl('http://domain.com/foo/bar'), )->shouldBeCalledOnce(); diff --git a/module/Rest/test/Action/ShortUrl/SingleStepCreateShortUrlActionTest.php b/module/Rest/test/Action/ShortUrl/SingleStepCreateShortUrlActionTest.php index f78a9de5..8bb1482a 100644 --- a/module/Rest/test/Action/ShortUrl/SingleStepCreateShortUrlActionTest.php +++ b/module/Rest/test/Action/ShortUrl/SingleStepCreateShortUrlActionTest.php @@ -39,7 +39,7 @@ class SingleStepCreateShortUrlActionTest extends TestCase /** @test */ public function properDataIsPassedWhenGeneratingShortCode(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $request = (new ServerRequest())->withQueryParams([ 'longUrl' => 'http://foobar.com', diff --git a/module/Rest/test/Action/Tag/DeleteTagsActionTest.php b/module/Rest/test/Action/Tag/DeleteTagsActionTest.php index 957c01a5..4812649d 100644 --- a/module/Rest/test/Action/Tag/DeleteTagsActionTest.php +++ b/module/Rest/test/Action/Tag/DeleteTagsActionTest.php @@ -34,7 +34,7 @@ class DeleteTagsActionTest extends TestCase { $request = (new ServerRequest()) ->withQueryParams(['tags' => $tags]) - ->withAttribute(ApiKey::class, new ApiKey()); + ->withAttribute(ApiKey::class, ApiKey::create()); $deleteTags = $this->tagService->deleteTags($tags ?: [], Argument::type(ApiKey::class)); $response = $this->action->handle($request); diff --git a/module/Rest/test/Action/Tag/ListTagsActionTest.php b/module/Rest/test/Action/Tag/ListTagsActionTest.php index 9bdad15b..8b7378fd 100644 --- a/module/Rest/test/Action/Tag/ListTagsActionTest.php +++ b/module/Rest/test/Action/Tag/ListTagsActionTest.php @@ -83,6 +83,6 @@ class ListTagsActionTest extends TestCase private function requestWithApiKey(): ServerRequestInterface { - return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, new ApiKey()); + return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, ApiKey::create()); } } diff --git a/module/Rest/test/Action/Tag/UpdateTagActionTest.php b/module/Rest/test/Action/Tag/UpdateTagActionTest.php index 681e68f6..d7b398db 100644 --- a/module/Rest/test/Action/Tag/UpdateTagActionTest.php +++ b/module/Rest/test/Action/Tag/UpdateTagActionTest.php @@ -70,6 +70,6 @@ class UpdateTagActionTest extends TestCase private function requestWithApiKey(): ServerRequestInterface { - return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, new ApiKey()); + return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, ApiKey::create()); } } diff --git a/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php b/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php index d53cb20d..829b820b 100644 --- a/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php +++ b/module/Rest/test/Action/Visit/GlobalVisitsActionTest.php @@ -30,7 +30,7 @@ class GlobalVisitsActionTest extends TestCase /** @test */ public function statsAreReturnedFromHelper(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $stats = new VisitsStats(5, 3); $getStats = $this->helper->getVisitsStats($apiKey)->willReturn($stats); diff --git a/module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php b/module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php index 6b149877..d0c67e7c 100644 --- a/module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php +++ b/module/Rest/test/Action/Visit/ShortUrlVisitsActionTest.php @@ -73,6 +73,6 @@ class ShortUrlVisitsActionTest extends TestCase private function requestWithApiKey(): ServerRequestInterface { - return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, new ApiKey()); + return ServerRequestFactory::fromGlobals()->withAttribute(ApiKey::class, ApiKey::create()); } } diff --git a/module/Rest/test/Action/Visit/TagVisitsActionTest.php b/module/Rest/test/Action/Visit/TagVisitsActionTest.php index da046f26..be3ce914 100644 --- a/module/Rest/test/Action/Visit/TagVisitsActionTest.php +++ b/module/Rest/test/Action/Visit/TagVisitsActionTest.php @@ -33,7 +33,7 @@ class TagVisitsActionTest extends TestCase public function providingCorrectShortCodeReturnsVisits(): void { $tag = 'foo'; - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $getVisits = $this->visitsHelper->visitsForTag($tag, Argument::type(VisitsParams::class), $apiKey)->willReturn( new Paginator(new ArrayAdapter([])), ); diff --git a/module/Rest/test/ApiKey/RoleTest.php b/module/Rest/test/ApiKey/RoleTest.php index 60a55ca5..278d37ff 100644 --- a/module/Rest/test/ApiKey/RoleTest.php +++ b/module/Rest/test/ApiKey/RoleTest.php @@ -28,7 +28,7 @@ class RoleTest extends TestCase public function provideRoles(): iterable { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); yield 'inline invalid role' => [new ApiKeyRole('invalid', [], $apiKey), true, Spec::andX()]; yield 'not inline invalid role' => [new ApiKeyRole('invalid', [], $apiKey), false, Spec::andX()]; diff --git a/module/Rest/test/Middleware/AuthenticationMiddlewareTest.php b/module/Rest/test/Middleware/AuthenticationMiddlewareTest.php index 2edbe5e6..68503b58 100644 --- a/module/Rest/test/Middleware/AuthenticationMiddlewareTest.php +++ b/module/Rest/test/Middleware/AuthenticationMiddlewareTest.php @@ -138,7 +138,7 @@ class AuthenticationMiddlewareTest extends TestCase /** @test */ public function validApiKeyFallsBackToNextMiddleware(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $key = $apiKey->toString(); $request = ServerRequestFactory::fromGlobals() ->withAttribute( diff --git a/module/Rest/test/Service/ApiKeyServiceTest.php b/module/Rest/test/Service/ApiKeyServiceTest.php index af50be23..addebbcd 100644 --- a/module/Rest/test/Service/ApiKeyServiceTest.php +++ b/module/Rest/test/Service/ApiKeyServiceTest.php @@ -13,6 +13,7 @@ use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\Common\Exception\InvalidArgumentException; use Shlinkio\Shlink\Core\Entity\Domain; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\Rest\Service\ApiKeyService; @@ -82,14 +83,14 @@ class ApiKeyServiceTest extends TestCase public function provideInvalidApiKeys(): iterable { yield 'non-existent api key' => [null]; - yield 'disabled api key' => [(new ApiKey())->disable()]; - yield 'expired api key' => [new ApiKey(Chronos::now()->subDay())]; + yield 'disabled api key' => [ApiKey::create()->disable()]; + yield 'expired api key' => [ApiKey::fromMeta(ApiKeyMeta::withExpirationDate(Chronos::now()->subDay()))]; } /** @test */ public function checkReturnsTrueWhenConditionsAreFavorable(): void { - $apiKey = new ApiKey(); + $apiKey = ApiKey::create(); $repo = $this->prophesize(EntityRepository::class); $repo->findOneBy(['key' => '12345'])->willReturn($apiKey) @@ -118,7 +119,7 @@ class ApiKeyServiceTest extends TestCase /** @test */ public function disableReturnsDisabledApiKeyWhenFound(): void { - $key = new ApiKey(); + $key = ApiKey::create(); $repo = $this->prophesize(EntityRepository::class); $repo->findOneBy(['key' => '12345'])->willReturn($key) ->shouldBeCalledOnce(); @@ -135,7 +136,7 @@ class ApiKeyServiceTest extends TestCase /** @test */ public function listFindsAllApiKeys(): void { - $expectedApiKeys = [new ApiKey(), new ApiKey(), new ApiKey()]; + $expectedApiKeys = [ApiKey::create(), ApiKey::create(), ApiKey::create()]; $repo = $this->prophesize(EntityRepository::class); $repo->findBy([])->willReturn($expectedApiKeys) @@ -150,7 +151,7 @@ class ApiKeyServiceTest extends TestCase /** @test */ public function listEnabledFindsOnlyEnabledApiKeys(): void { - $expectedApiKeys = [new ApiKey(), new ApiKey(), new ApiKey()]; + $expectedApiKeys = [ApiKey::create(), ApiKey::create(), ApiKey::create()]; $repo = $this->prophesize(EntityRepository::class); $repo->findBy(['enabled' => true])->willReturn($expectedApiKeys) From 4ce44034cb1c86ebff6fd7a5c8cab1752b936f6e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 10:20:05 +0100 Subject: [PATCH 21/86] Ensured API key name appears in the proper color in the console, for disabled or expired API keys --- module/CLI/src/Command/Api/ListKeysCommand.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/module/CLI/src/Command/Api/ListKeysCommand.php b/module/CLI/src/Command/Api/ListKeysCommand.php index ac5fc73d..e8326826 100644 --- a/module/CLI/src/Command/Api/ListKeysCommand.php +++ b/module/CLI/src/Command/Api/ListKeysCommand.php @@ -57,8 +57,7 @@ class ListKeysCommand extends BaseCommand $messagePattern = $this->determineMessagePattern($apiKey); // Set columns for this row - $rowData = [sprintf($messagePattern, $apiKey)]; - $rowData[] = $apiKey->name() ?? '-'; + $rowData = [sprintf($messagePattern, $apiKey), sprintf($messagePattern, $apiKey->name() ?? '-')]; if (! $enabledOnly) { $rowData[] = sprintf($messagePattern, $this->getEnabledSymbol($apiKey)); } From 554209d644efe997592d518c59ed5e8c7d702301 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 14 Mar 2021 12:28:30 +0100 Subject: [PATCH 22/86] Updated to latest migrations patch and removed workaround --- composer.json | 2 +- data/migrations/Version20160819142757.php | 8 -------- data/migrations/Version20160820191203.php | 8 -------- data/migrations/Version20171021093246.php | 8 -------- data/migrations/Version20171022064541.php | 8 -------- data/migrations/Version20180801183328.php | 8 -------- data/migrations/Version20180913205455.php | 8 -------- data/migrations/Version20180915110857.php | 8 -------- data/migrations/Version20181020060559.php | 8 -------- data/migrations/Version20181020065148.php | 8 -------- data/migrations/Version20181110175521.php | 8 -------- data/migrations/Version20190824075137.php | 8 -------- data/migrations/Version20190930165521.php | 8 -------- data/migrations/Version20191001201532.php | 8 -------- data/migrations/Version20191020074522.php | 8 -------- data/migrations/Version20200105165647.php | 8 -------- data/migrations/Version20200106215144.php | 8 -------- data/migrations/Version20200110182849.php | 8 -------- data/migrations/Version20200323190014.php | 8 -------- data/migrations/Version20200503170404.php | 8 -------- data/migrations/Version20201023090929.php | 8 -------- data/migrations/Version20201102113208.php | 8 -------- data/migrations/Version20210102174433.php | 8 -------- data/migrations/Version20210118153932.php | 8 -------- data/migrations/Version20210202181026.php | 8 -------- data/migrations/Version20210207100807.php | 8 -------- data/migrations/Version20210306165711.php | 8 -------- 27 files changed, 1 insertion(+), 209 deletions(-) diff --git a/composer.json b/composer.json index 89764118..8cf48639 100644 --- a/composer.json +++ b/composer.json @@ -19,7 +19,7 @@ "cakephp/chronos": "^2.0", "cocur/slugify": "^4.0", "doctrine/cache": "^1.9", - "doctrine/migrations": "^3.0.2", + "doctrine/migrations": "^3.1.1", "doctrine/orm": "2.8.1 || ^2.8.3", "endroid/qr-code": "^4.0", "geoip2/geoip2": "^2.9", diff --git a/data/migrations/Version20160819142757.php b/data/migrations/Version20160819142757.php index 966a53a0..2901836e 100644 --- a/data/migrations/Version20160819142757.php +++ b/data/migrations/Version20160819142757.php @@ -41,12 +41,4 @@ class Version20160819142757 extends AbstractMigration { $db = $this->connection->getDatabasePlatform()->getName(); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20160820191203.php b/data/migrations/Version20160820191203.php index ae2b7bcf..d0a4d673 100644 --- a/data/migrations/Version20160820191203.php +++ b/data/migrations/Version20160820191203.php @@ -73,12 +73,4 @@ class Version20160820191203 extends AbstractMigration $schema->dropTable('short_urls_in_tags'); $schema->dropTable('tags'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20171021093246.php b/data/migrations/Version20171021093246.php index 54dd54cc..83f08e41 100644 --- a/data/migrations/Version20171021093246.php +++ b/data/migrations/Version20171021093246.php @@ -45,12 +45,4 @@ class Version20171021093246 extends AbstractMigration $shortUrls->dropColumn('valid_since'); $shortUrls->dropColumn('valid_until'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20171022064541.php b/data/migrations/Version20171022064541.php index 86efc008..7ff39666 100644 --- a/data/migrations/Version20171022064541.php +++ b/data/migrations/Version20171022064541.php @@ -42,12 +42,4 @@ class Version20171022064541 extends AbstractMigration $shortUrls->dropColumn('max_visits'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20180801183328.php b/data/migrations/Version20180801183328.php index 232b37b0..24bcd825 100644 --- a/data/migrations/Version20180801183328.php +++ b/data/migrations/Version20180801183328.php @@ -39,12 +39,4 @@ final class Version20180801183328 extends AbstractMigration { $schema->getTable('short_urls')->getColumn('short_code')->setLength($size); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20180913205455.php b/data/migrations/Version20180913205455.php index 187b860e..ee6cd861 100644 --- a/data/migrations/Version20180913205455.php +++ b/data/migrations/Version20180913205455.php @@ -66,12 +66,4 @@ final class Version20180913205455 extends AbstractMigration { // Nothing to rollback } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20180915110857.php b/data/migrations/Version20180915110857.php index 51a1ed72..73a36597 100644 --- a/data/migrations/Version20180915110857.php +++ b/data/migrations/Version20180915110857.php @@ -47,12 +47,4 @@ final class Version20180915110857 extends AbstractMigration { // Nothing to run } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20181020060559.php b/data/migrations/Version20181020060559.php index 804027d2..78cd8614 100644 --- a/data/migrations/Version20181020060559.php +++ b/data/migrations/Version20181020060559.php @@ -65,12 +65,4 @@ final class Version20181020060559 extends AbstractMigration { // No down } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20181020065148.php b/data/migrations/Version20181020065148.php index 4e33f8e9..62b14ccf 100644 --- a/data/migrations/Version20181020065148.php +++ b/data/migrations/Version20181020065148.php @@ -38,12 +38,4 @@ final class Version20181020065148 extends AbstractMigration { // No down } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20181110175521.php b/data/migrations/Version20181110175521.php index 7142522a..aae8d419 100644 --- a/data/migrations/Version20181110175521.php +++ b/data/migrations/Version20181110175521.php @@ -34,12 +34,4 @@ final class Version20181110175521 extends AbstractMigration { return $schema->getTable('visits')->getColumn('user_agent'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20190824075137.php b/data/migrations/Version20190824075137.php index 893ea4f5..208d85a3 100644 --- a/data/migrations/Version20190824075137.php +++ b/data/migrations/Version20190824075137.php @@ -34,12 +34,4 @@ final class Version20190824075137 extends AbstractMigration { return $schema->getTable('visits')->getColumn('referer'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20190930165521.php b/data/migrations/Version20190930165521.php index de11c3b5..2e4e8f50 100644 --- a/data/migrations/Version20190930165521.php +++ b/data/migrations/Version20190930165521.php @@ -52,12 +52,4 @@ final class Version20190930165521 extends AbstractMigration $schema->getTable('short_urls')->dropColumn('domain_id'); $schema->dropTable('domains'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20191001201532.php b/data/migrations/Version20191001201532.php index 045c65cc..d067101c 100644 --- a/data/migrations/Version20191001201532.php +++ b/data/migrations/Version20191001201532.php @@ -46,12 +46,4 @@ final class Version20191001201532 extends AbstractMigration $shortUrls->dropIndex('unique_short_code_plus_domain'); $shortUrls->addUniqueIndex(['short_code']); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20191020074522.php b/data/migrations/Version20191020074522.php index 638f60db..baf1ce7a 100644 --- a/data/migrations/Version20191020074522.php +++ b/data/migrations/Version20191020074522.php @@ -34,12 +34,4 @@ final class Version20191020074522 extends AbstractMigration { return $schema->getTable('short_urls')->getColumn('original_url'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20200105165647.php b/data/migrations/Version20200105165647.php index 3672322c..e0e31f55 100644 --- a/data/migrations/Version20200105165647.php +++ b/data/migrations/Version20200105165647.php @@ -93,12 +93,4 @@ final class Version20200105165647 extends AbstractMigration $visitLocations->dropColumn($colName); } } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20200106215144.php b/data/migrations/Version20200106215144.php index 6ed83522..5682baaf 100644 --- a/data/migrations/Version20200106215144.php +++ b/data/migrations/Version20200106215144.php @@ -44,12 +44,4 @@ final class Version20200106215144 extends AbstractMigration ]); } } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20200110182849.php b/data/migrations/Version20200110182849.php index 2d8e18c8..16b858f9 100644 --- a/data/migrations/Version20200110182849.php +++ b/data/migrations/Version20200110182849.php @@ -50,12 +50,4 @@ final class Version20200110182849 extends AbstractMigration { // No need (and no way) to undo this migration } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20200323190014.php b/data/migrations/Version20200323190014.php index 7adb31b3..47cf402a 100644 --- a/data/migrations/Version20200323190014.php +++ b/data/migrations/Version20200323190014.php @@ -42,12 +42,4 @@ final class Version20200323190014 extends AbstractMigration $visitLocations->dropColumn('is_empty'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20200503170404.php b/data/migrations/Version20200503170404.php index 4779b93b..a102c2c8 100644 --- a/data/migrations/Version20200503170404.php +++ b/data/migrations/Version20200503170404.php @@ -24,12 +24,4 @@ final class Version20200503170404 extends AbstractMigration $this->skipIf(! $visits->hasIndex(self::INDEX_NAME)); $visits->dropIndex(self::INDEX_NAME); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20201023090929.php b/data/migrations/Version20201023090929.php index 8c1e8622..05d16c22 100644 --- a/data/migrations/Version20201023090929.php +++ b/data/migrations/Version20201023090929.php @@ -41,12 +41,4 @@ final class Version20201023090929 extends AbstractMigration $shortUrls->dropColumn('import_original_short_code'); $shortUrls->dropIndex('unique_imports'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20201102113208.php b/data/migrations/Version20201102113208.php index 88be074c..1e1237a4 100644 --- a/data/migrations/Version20201102113208.php +++ b/data/migrations/Version20201102113208.php @@ -86,12 +86,4 @@ final class Version20201102113208 extends AbstractMigration $shortUrls->removeForeignKey('FK_' . self::API_KEY_COLUMN); $shortUrls->dropColumn(self::API_KEY_COLUMN); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20210102174433.php b/data/migrations/Version20210102174433.php index 01616ffe..835fcbda 100644 --- a/data/migrations/Version20210102174433.php +++ b/data/migrations/Version20210102174433.php @@ -49,12 +49,4 @@ final class Version20210102174433 extends AbstractMigration $schema->getTable(self::TABLE_NAME)->dropIndex('UQ_role_plus_api_key'); $schema->dropTable(self::TABLE_NAME); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20210118153932.php b/data/migrations/Version20210118153932.php index e9f29759..e17ff533 100644 --- a/data/migrations/Version20210118153932.php +++ b/data/migrations/Version20210118153932.php @@ -23,12 +23,4 @@ final class Version20210118153932 extends AbstractMigration public function down(Schema $schema): void { } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20210202181026.php b/data/migrations/Version20210202181026.php index c964559c..ccf69572 100644 --- a/data/migrations/Version20210202181026.php +++ b/data/migrations/Version20210202181026.php @@ -33,12 +33,4 @@ final class Version20210202181026 extends AbstractMigration $shortUrls->dropColumn(self::TITLE); $shortUrls->dropColumn('title_was_auto_resolved'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20210207100807.php b/data/migrations/Version20210207100807.php index 24e73d34..4c4509c4 100644 --- a/data/migrations/Version20210207100807.php +++ b/data/migrations/Version20210207100807.php @@ -40,12 +40,4 @@ final class Version20210207100807 extends AbstractMigration $visits->dropColumn('visited_url'); $visits->dropColumn('type'); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } diff --git a/data/migrations/Version20210306165711.php b/data/migrations/Version20210306165711.php index 5a7b9fe7..5b4bd166 100644 --- a/data/migrations/Version20210306165711.php +++ b/data/migrations/Version20210306165711.php @@ -34,12 +34,4 @@ final class Version20210306165711 extends AbstractMigration $apiKeys->dropColumn(self::COLUMN); } - - /** - * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104 - */ - public function isTransactional(): bool - { - return false; - } } From 312f20d2f15ba383e2d823ca84522eb2593914a4 Mon Sep 17 00:00:00 2001 From: RaINi_ Date: Mon, 29 Mar 2021 12:46:45 +0200 Subject: [PATCH 23/86] Create volume for /etc/shlink/data Makes it so shlink can be used as a docker service without losing ur data every time --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index c058ef61..d9bf96e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,6 +69,8 @@ EXPOSE 8080 # Expose params config dir, since the user is expected to provide custom config from there VOLUME /etc/shlink/config/params +# Expose data to have it persistent when using shlink as a docker service or similar +VOLUME /etc/shlink/data # Copy config specific for the image COPY docker/docker-entrypoint.sh docker-entrypoint.sh From 0f0c4dc549c3a005713db73af4d33c1c0d8e46bb Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 30 Mar 2021 18:18:38 +0200 Subject: [PATCH 24/86] Fixed comment --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d9bf96e8..49bb28a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,7 @@ EXPOSE 8080 # Expose params config dir, since the user is expected to provide custom config from there VOLUME /etc/shlink/config/params -# Expose data to have it persistent when using shlink as a docker service or similar +# Expose data dir to allow persistent runtime data and SQLite db VOLUME /etc/shlink/data # Copy config specific for the image From d72b9cf646190f39fcaef7ef9c6bfe3638b7373a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 2 Apr 2021 09:46:02 +0200 Subject: [PATCH 25/86] Updated to symfony/mercure 0.5 --- composer.json | 4 ++-- config/autoload/mercure.global.php | 8 ++++---- module/Core/config/event_dispatcher.config.php | 4 ++-- .../src/EventDispatcher/NotifyVisitToMercure.php | 10 +++++----- .../EventDispatcher/NotifyVisitToMercureTest.php | 16 ++++++++-------- phpstan.neon | 1 + 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/composer.json b/composer.json index 8cf48639..c3d8aba4 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.7", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "^3.6", + "shlinkio/shlink-common": "dev-main#3777189 as 3.7", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", "shlinkio/shlink-importer": "^2.2", @@ -55,7 +55,7 @@ "symfony/console": "^5.1", "symfony/filesystem": "^5.1", "symfony/lock": "^5.1", - "symfony/mercure": "^0.4.1", + "symfony/mercure": "^0.5.1", "symfony/process": "^5.1", "symfony/string": "^5.1" }, diff --git a/config/autoload/mercure.global.php b/config/autoload/mercure.global.php index 1a404dca..72fafe58 100644 --- a/config/autoload/mercure.global.php +++ b/config/autoload/mercure.global.php @@ -4,8 +4,8 @@ declare(strict_types=1); use Laminas\ServiceManager\Proxy\LazyServiceFactory; use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider; -use Symfony\Component\Mercure\Publisher; -use Symfony\Component\Mercure\PublisherInterface; +use Symfony\Component\Mercure\Hub; +use Symfony\Component\Mercure\HubInterface; return [ @@ -21,14 +21,14 @@ return [ LcobucciJwtProvider::class => [ LazyServiceFactory::class, ], - Publisher::class => [ + Hub::class => [ LazyServiceFactory::class, ], ], 'lazy_services' => [ 'class_map' => [ LcobucciJwtProvider::class => LcobucciJwtProvider::class, - Publisher::class => PublisherInterface::class, + Hub::class => HubInterface::class, ], ], ], diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index 5c2c88e0..5e1a6c9f 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -8,7 +8,7 @@ use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Psr\EventDispatcher\EventDispatcherInterface; use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; -use Symfony\Component\Mercure\Publisher; +use Symfony\Component\Mercure\Hub; return [ @@ -57,7 +57,7 @@ return [ Options\AppOptions::class, ], EventDispatcher\NotifyVisitToMercure::class => [ - Publisher::class, + Hub::class, Mercure\MercureUpdatesGenerator::class, 'em', 'Logger_Shlink', diff --git a/module/Core/src/EventDispatcher/NotifyVisitToMercure.php b/module/Core/src/EventDispatcher/NotifyVisitToMercure.php index 0cf438ed..33adf965 100644 --- a/module/Core/src/EventDispatcher/NotifyVisitToMercure.php +++ b/module/Core/src/EventDispatcher/NotifyVisitToMercure.php @@ -9,7 +9,7 @@ use Psr\Log\LoggerInterface; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated; use Shlinkio\Shlink\Core\Mercure\MercureUpdatesGeneratorInterface; -use Symfony\Component\Mercure\PublisherInterface; +use Symfony\Component\Mercure\HubInterface; use Symfony\Component\Mercure\Update; use Throwable; @@ -17,18 +17,18 @@ use function Functional\each; class NotifyVisitToMercure { - private PublisherInterface $publisher; + private HubInterface $hub; private MercureUpdatesGeneratorInterface $updatesGenerator; private EntityManagerInterface $em; private LoggerInterface $logger; public function __construct( - PublisherInterface $publisher, + HubInterface $hub, MercureUpdatesGeneratorInterface $updatesGenerator, EntityManagerInterface $em, LoggerInterface $logger ) { - $this->publisher = $publisher; + $this->hub = $hub; $this->em = $em; $this->logger = $logger; $this->updatesGenerator = $updatesGenerator; @@ -48,7 +48,7 @@ class NotifyVisitToMercure } try { - each($this->determineUpdatesForVisit($visit), fn (Update $update) => ($this->publisher)($update)); + each($this->determineUpdatesForVisit($visit), fn (Update $update) => $this->hub->publish($update)); } catch (Throwable $e) { $this->logger->debug('Error while trying to notify mercure hub with new visit. {e}', [ 'e' => $e, diff --git a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php index f323a155..0b863b69 100644 --- a/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php +++ b/module/Core/test/EventDispatcher/NotifyVisitToMercureTest.php @@ -17,7 +17,7 @@ use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated; use Shlinkio\Shlink\Core\EventDispatcher\NotifyVisitToMercure; use Shlinkio\Shlink\Core\Mercure\MercureUpdatesGeneratorInterface; use Shlinkio\Shlink\Core\Model\Visitor; -use Symfony\Component\Mercure\PublisherInterface; +use Symfony\Component\Mercure\HubInterface; use Symfony\Component\Mercure\Update; class NotifyVisitToMercureTest extends TestCase @@ -25,20 +25,20 @@ class NotifyVisitToMercureTest extends TestCase use ProphecyTrait; private NotifyVisitToMercure $listener; - private ObjectProphecy $publisher; + private ObjectProphecy $hub; private ObjectProphecy $updatesGenerator; private ObjectProphecy $em; private ObjectProphecy $logger; public function setUp(): void { - $this->publisher = $this->prophesize(PublisherInterface::class); + $this->hub = $this->prophesize(HubInterface::class); $this->updatesGenerator = $this->prophesize(MercureUpdatesGeneratorInterface::class); $this->em = $this->prophesize(EntityManagerInterface::class); $this->logger = $this->prophesize(LoggerInterface::class); $this->listener = new NotifyVisitToMercure( - $this->publisher->reveal(), + $this->hub->reveal(), $this->updatesGenerator->reveal(), $this->em->reveal(), $this->logger->reveal(), @@ -60,7 +60,7 @@ class NotifyVisitToMercureTest extends TestCase ); $buildNewOrphanVisitUpdate = $this->updatesGenerator->newOrphanVisitUpdate(Argument::type(Visit::class)); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate(Argument::type(Visit::class)); - $publish = $this->publisher->__invoke(Argument::type(Update::class)); + $publish = $this->hub->publish(Argument::type(Update::class)); ($this->listener)(new VisitLocated($visitId)); @@ -86,7 +86,7 @@ class NotifyVisitToMercureTest extends TestCase $buildNewShortUrlVisitUpdate = $this->updatesGenerator->newShortUrlVisitUpdate($visit)->willReturn($update); $buildNewOrphanVisitUpdate = $this->updatesGenerator->newOrphanVisitUpdate($visit)->willReturn($update); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); - $publish = $this->publisher->__invoke($update); + $publish = $this->hub->publish($update); ($this->listener)(new VisitLocated($visitId)); @@ -115,7 +115,7 @@ class NotifyVisitToMercureTest extends TestCase $buildNewShortUrlVisitUpdate = $this->updatesGenerator->newShortUrlVisitUpdate($visit)->willReturn($update); $buildNewOrphanVisitUpdate = $this->updatesGenerator->newOrphanVisitUpdate($visit)->willReturn($update); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); - $publish = $this->publisher->__invoke($update)->willThrow($e); + $publish = $this->hub->publish($update)->willThrow($e); ($this->listener)(new VisitLocated($visitId)); @@ -143,7 +143,7 @@ class NotifyVisitToMercureTest extends TestCase $buildNewShortUrlVisitUpdate = $this->updatesGenerator->newShortUrlVisitUpdate($visit)->willReturn($update); $buildNewOrphanVisitUpdate = $this->updatesGenerator->newOrphanVisitUpdate($visit)->willReturn($update); $buildNewVisitUpdate = $this->updatesGenerator->newVisitUpdate($visit)->willReturn($update); - $publish = $this->publisher->__invoke($update); + $publish = $this->hub->publish($update); ($this->listener)(new VisitLocated($visitId)); diff --git a/phpstan.neon b/phpstan.neon index 80f1b083..fbddc81c 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -3,3 +3,4 @@ parameters: checkGenericClassInNonGenericObjectType: false ignoreErrors: - '#If condition is always false#' + - '#setOrderBy\(\) expects array\, array\ given#' From 3a6a1f25a7b36af8923e7ea055828134065d7e48 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Tue, 6 Apr 2021 17:34:31 +0200 Subject: [PATCH 26/86] Updated to latest doctrine/orm without security issues --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index c3d8aba4..bff515f0 100644 --- a/composer.json +++ b/composer.json @@ -20,7 +20,7 @@ "cocur/slugify": "^4.0", "doctrine/cache": "^1.9", "doctrine/migrations": "^3.1.1", - "doctrine/orm": "2.8.1 || ^2.8.3", + "doctrine/orm": "^2.8.4", "endroid/qr-code": "^4.0", "geoip2/geoip2": "^2.9", "guzzlehttp/guzzle": "^7.0", From 0621ae7735915da7a88765d36435f7a89c6bc27e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Wed, 7 Apr 2021 11:33:23 +0200 Subject: [PATCH 27/86] Ensured visits tracking is run transactionally together with the event dispatched afterwards --- module/Core/src/Visit/VisitsTracker.php | 8 +++++--- module/Core/test/Visit/VisitsTrackerTest.php | 13 +++++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/module/Core/src/Visit/VisitsTracker.php b/module/Core/src/Visit/VisitsTracker.php index 306da7a9..f8c82b49 100644 --- a/module/Core/src/Visit/VisitsTracker.php +++ b/module/Core/src/Visit/VisitsTracker.php @@ -65,9 +65,11 @@ class VisitsTracker implements VisitsTrackerInterface private function trackVisit(Visit $visit, Visitor $visitor): void { - $this->em->persist($visit); - $this->em->flush(); + $this->em->transactional(function () use ($visit, $visitor): void { + $this->em->persist($visit); + $this->em->flush(); - $this->eventDispatcher->dispatch(new UrlVisited($visit->getId(), $visitor->getRemoteAddress())); + $this->eventDispatcher->dispatch(new UrlVisited($visit->getId(), $visitor->getRemoteAddress())); + }); } } diff --git a/module/Core/test/Visit/VisitsTrackerTest.php b/module/Core/test/Visit/VisitsTrackerTest.php index 118ebc06..bba4e919 100644 --- a/module/Core/test/Visit/VisitsTrackerTest.php +++ b/module/Core/test/Visit/VisitsTrackerTest.php @@ -29,6 +29,11 @@ class VisitsTrackerTest extends TestCase public function setUp(): void { $this->em = $this->prophesize(EntityManager::class); + $this->em->transactional(Argument::any())->will(function (array $args) { + [$callback] = $args; + return $callback(); + }); + $this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class); $this->options = new UrlShortenerOptions(); @@ -41,11 +46,14 @@ class VisitsTrackerTest extends TestCase */ public function trackPersistsVisitAndDispatchesEvent(string $method, array $args): void { - $this->em->persist(Argument::that(fn (Visit $visit) => $visit->setId('1')))->shouldBeCalledOnce(); - $this->em->flush()->shouldBeCalledOnce(); + $persist = $this->em->persist(Argument::that(fn (Visit $visit) => $visit->setId('1')))->will(function (): void { + }); $this->visitsTracker->{$method}(...$args); + $persist->shouldHaveBeenCalledOnce(); + $this->em->transactional(Argument::cetera())->shouldHaveBeenCalledOnce(); + $this->em->flush()->shouldHaveBeenCalledOnce(); $this->eventDispatcher->dispatch(Argument::type(UrlVisited::class))->shouldHaveBeenCalled(); } @@ -68,6 +76,7 @@ class VisitsTrackerTest extends TestCase $this->visitsTracker->{$method}(Visitor::emptyInstance()); $this->eventDispatcher->dispatch(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->em->transactional(Argument::cetera())->shouldNotHaveBeenCalled(); $this->em->persist(Argument::cetera())->shouldNotHaveBeenCalled(); $this->em->flush()->shouldNotHaveBeenCalled(); } From b4d137375a9b2e9201d2cd9c8eaf345383faa54f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Wed, 7 Apr 2021 11:35:02 +0200 Subject: [PATCH 28/86] Flipped events triggered when locating a visit, so that geolocation is done synchronously --- .../GeolocationDbUpdateFailedException.php | 5 +++++ .../Core/config/event_dispatcher.config.php | 20 +++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/module/CLI/src/Exception/GeolocationDbUpdateFailedException.php b/module/CLI/src/Exception/GeolocationDbUpdateFailedException.php index f663fd8f..07d66855 100644 --- a/module/CLI/src/Exception/GeolocationDbUpdateFailedException.php +++ b/module/CLI/src/Exception/GeolocationDbUpdateFailedException.php @@ -13,6 +13,11 @@ class GeolocationDbUpdateFailedException extends RuntimeException implements Exc { private bool $olderDbExists; + private function __construct(string $message, int $code = 0, ?Throwable $previous = null) + { + parent::__construct($message, $code, $previous); + } + public static function withOlderDb(?Throwable $prev = null): self { $e = new self( diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index 5e1a6c9f..37b1d59d 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -6,7 +6,7 @@ namespace Shlinkio\Shlink\Core; use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Psr\EventDispatcher\EventDispatcherInterface; -use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater; +use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; use Symfony\Component\Mercure\Hub; @@ -14,14 +14,15 @@ return [ 'events' => [ 'regular' => [ - EventDispatcher\Event\VisitLocated::class => [ - EventDispatcher\NotifyVisitToMercure::class, - EventDispatcher\NotifyVisitToWebHooks::class, + EventDispatcher\Event\UrlVisited::class => [ + EventDispatcher\LocateVisit::class, ], ], 'async' => [ - EventDispatcher\Event\UrlVisited::class => [ - EventDispatcher\LocateVisit::class, + EventDispatcher\Event\VisitLocated::class => [ + EventDispatcher\NotifyVisitToMercure::class, + EventDispatcher\NotifyVisitToWebHooks::class, +// EventDispatcher\UpdateGeoliteDb::class, ], ], ], @@ -34,7 +35,10 @@ return [ ], 'delegators' => [ - EventDispatcher\LocateVisit::class => [ + EventDispatcher\NotifyVisitToMercure::class => [ + EventDispatcher\CloseDbConnectionEventListenerDelegator::class, + ], + EventDispatcher\NotifyVisitToWebHooks::class => [ EventDispatcher\CloseDbConnectionEventListenerDelegator::class, ], ], @@ -45,7 +49,7 @@ return [ IpLocationResolverInterface::class, 'em', 'Logger_Shlink', - GeolocationDbUpdater::class, + DbUpdater::class, EventDispatcherInterface::class, ], EventDispatcher\NotifyVisitToWebHooks::class => [ From 77d06b4b03db980517b28e14b74c105a52d3fdf0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Wed, 7 Apr 2021 11:48:01 +0200 Subject: [PATCH 29/86] Renamed argument to have a more clear intention --- module/CLI/src/Util/GeolocationDbUpdater.php | 16 +++--- .../Util/GeolocationDbUpdaterInterface.php | 2 +- .../Core/src/EventDispatcher/LocateVisit.php | 57 ++++++++++--------- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/module/CLI/src/Util/GeolocationDbUpdater.php b/module/CLI/src/Util/GeolocationDbUpdater.php index b8f5b756..ebdeb573 100644 --- a/module/CLI/src/Util/GeolocationDbUpdater.php +++ b/module/CLI/src/Util/GeolocationDbUpdater.php @@ -32,13 +32,13 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface /** * @throws GeolocationDbUpdateFailedException */ - public function checkDbUpdate(?callable $mustBeUpdated = null, ?callable $handleProgress = null): void + public function checkDbUpdate(?callable $beforeDownload = null, ?callable $handleProgress = null): void { $lock = $this->locker->createLock(self::LOCK_NAME); $lock->acquire(true); // Block until lock is released try { - $this->downloadIfNeeded($mustBeUpdated, $handleProgress); + $this->downloadIfNeeded($beforeDownload, $handleProgress); } finally { $lock->release(); } @@ -47,26 +47,26 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface /** * @throws GeolocationDbUpdateFailedException */ - private function downloadIfNeeded(?callable $mustBeUpdated, ?callable $handleProgress): void + private function downloadIfNeeded(?callable $beforeDownload, ?callable $handleProgress): void { if (! $this->dbUpdater->databaseFileExists()) { - $this->downloadNewDb(false, $mustBeUpdated, $handleProgress); + $this->downloadNewDb(false, $beforeDownload, $handleProgress); return; } $meta = $this->geoLiteDbReader->metadata(); if ($this->buildIsTooOld($meta)) { - $this->downloadNewDb(true, $mustBeUpdated, $handleProgress); + $this->downloadNewDb(true, $beforeDownload, $handleProgress); } } /** * @throws GeolocationDbUpdateFailedException */ - private function downloadNewDb(bool $olderDbExists, ?callable $mustBeUpdated, ?callable $handleProgress): void + private function downloadNewDb(bool $olderDbExists, ?callable $beforeDownload, ?callable $handleProgress): void { - if ($mustBeUpdated !== null) { - $mustBeUpdated($olderDbExists); + if ($beforeDownload !== null) { + $beforeDownload($olderDbExists); } try { diff --git a/module/CLI/src/Util/GeolocationDbUpdaterInterface.php b/module/CLI/src/Util/GeolocationDbUpdaterInterface.php index 1eda5123..714f6a11 100644 --- a/module/CLI/src/Util/GeolocationDbUpdaterInterface.php +++ b/module/CLI/src/Util/GeolocationDbUpdaterInterface.php @@ -11,5 +11,5 @@ interface GeolocationDbUpdaterInterface /** * @throws GeolocationDbUpdateFailedException */ - public function checkDbUpdate(?callable $mustBeUpdated = null, ?callable $handleProgress = null): void; + public function checkDbUpdate(?callable $beforeDownload = null, ?callable $handleProgress = null): void; } diff --git a/module/Core/src/EventDispatcher/LocateVisit.php b/module/Core/src/EventDispatcher/LocateVisit.php index 32da6060..d02f8be7 100644 --- a/module/Core/src/EventDispatcher/LocateVisit.php +++ b/module/Core/src/EventDispatcher/LocateVisit.php @@ -7,31 +7,28 @@ namespace Shlinkio\Shlink\Core\EventDispatcher; use Doctrine\ORM\EntityManagerInterface; use Psr\EventDispatcher\EventDispatcherInterface; use Psr\Log\LoggerInterface; -use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException; -use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; use Shlinkio\Shlink\Core\EventDispatcher\Event\UrlVisited; use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated; use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; +use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; -use function sprintf; - class LocateVisit { private IpLocationResolverInterface $ipLocationResolver; private EntityManagerInterface $em; private LoggerInterface $logger; - private GeolocationDbUpdaterInterface $dbUpdater; + private DbUpdaterInterface $dbUpdater; private EventDispatcherInterface $eventDispatcher; public function __construct( IpLocationResolverInterface $ipLocationResolver, EntityManagerInterface $em, LoggerInterface $logger, - GeolocationDbUpdaterInterface $dbUpdater, + DbUpdaterInterface $dbUpdater, EventDispatcherInterface $eventDispatcher ) { $this->ipLocationResolver = $ipLocationResolver; @@ -54,33 +51,37 @@ class LocateVisit return; } - if ($this->downloadOrUpdateGeoLiteDb($visitId)) { - $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit); + if (! $this->dbUpdater->databaseFileExists()) { + $this->logger->warning('Tried to locate visit with id "{visitId}", but a GeoLite2 db was not found.', [ + 'visitId' => $visitId, + ]); + return; } + $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit); $this->eventDispatcher->dispatch(new VisitLocated($visitId)); } - private function downloadOrUpdateGeoLiteDb(string $visitId): bool - { - try { - $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void { - $this->logger->notice(sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading')); - }); - } catch (GeolocationDbUpdateFailedException $e) { - if (! $e->olderDbExists()) { - $this->logger->error( - 'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}', - ['e' => $e, 'visitId' => $visitId], - ); - return false; - } - - $this->logger->warning('GeoLite2 database update failed. Proceeding with old version. {e}', ['e' => $e]); - } - - return true; - } +// private function downloadOrUpdateGeoLiteDb(string $visitId): bool +// { +// try { +// $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void { +// $this->logger->notice(sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading')); +// }); +// } catch (GeolocationDbUpdateFailedException $e) { +// if (! $e->olderDbExists()) { +// $this->logger->error( +// 'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}', +// ['e' => $e, 'visitId' => $visitId], +// ); +// return false; +// } +// +// $this->logger->warning('GeoLite2 database update failed. Proceeding with old version. {e}', ['e' => $e]); +// } +// +// return true; +// } private function locateVisit(string $visitId, ?string $originalIpAddress, Visit $visit): void { From 5de706e0fe9f58c178b3bfe993b44acec801bc92 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Wed, 7 Apr 2021 11:52:50 +0200 Subject: [PATCH 30/86] Fixed LocateVisitTest --- .../test/EventDispatcher/LocateVisitTest.php | 156 ++++++++++-------- 1 file changed, 91 insertions(+), 65 deletions(-) diff --git a/module/Core/test/EventDispatcher/LocateVisitTest.php b/module/Core/test/EventDispatcher/LocateVisitTest.php index 081f0f86..a01ec996 100644 --- a/module/Core/test/EventDispatcher/LocateVisitTest.php +++ b/module/Core/test/EventDispatcher/LocateVisitTest.php @@ -11,8 +11,6 @@ use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Psr\EventDispatcher\EventDispatcherInterface; use Psr\Log\LoggerInterface; -use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException; -use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface; use Shlinkio\Shlink\Common\Util\IpAddress; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; @@ -22,6 +20,7 @@ use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated; use Shlinkio\Shlink\Core\EventDispatcher\LocateVisit; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; +use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; @@ -41,9 +40,11 @@ class LocateVisitTest extends TestCase $this->ipLocationResolver = $this->prophesize(IpLocationResolverInterface::class); $this->em = $this->prophesize(EntityManagerInterface::class); $this->logger = $this->prophesize(LoggerInterface::class); - $this->dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class); $this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class); + $this->dbUpdater = $this->prophesize(DbUpdaterInterface::class); + $this->dbUpdater->databaseFileExists()->willReturn(true); + $this->locateVisit = new LocateVisit( $this->ipLocationResolver->reveal(), $this->em->reveal(), @@ -73,6 +74,31 @@ class LocateVisitTest extends TestCase $dispatch->shouldNotHaveBeenCalled(); } + /** @test */ + public function nonExistingGeoLiteDbLogsWarning(): void + { + $event = new UrlVisited('123'); + $findVisit = $this->em->find(Visit::class, '123')->willReturn( + Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', '1.2.3.4', '')), + ); + $dbExists = $this->dbUpdater->databaseFileExists()->willReturn(false); + $logWarning = $this->logger->warning( + 'Tried to locate visit with id "{visitId}", but a GeoLite2 db was not found.', + ['visitId' => 123], + ); + $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { + }); + + ($this->locateVisit)($event); + + $findVisit->shouldHaveBeenCalledOnce(); + $dbExists->shouldHaveBeenCalledOnce(); + $this->em->flush()->shouldNotHaveBeenCalled(); + $this->ipLocationResolver->resolveIpLocation(Argument::cetera())->shouldNotHaveBeenCalled(); + $logWarning->shouldHaveBeenCalled(); + $dispatch->shouldNotHaveBeenCalled(); + } + /** @test */ public function invalidAddressLogsWarning(): void { @@ -174,66 +200,66 @@ class LocateVisitTest extends TestCase yield 'regular not found' => [Visit::forRegularNotFound(new Visitor('', '', '1.2.3.4', '')), '1.2.3.4']; } - /** @test */ - public function errorWhenUpdatingGeoLiteWithExistingCopyLogsWarning(): void - { - $e = GeolocationDbUpdateFailedException::withOlderDb(); - $ipAddr = '1.2.3.0'; - $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $ipAddr, '')); - $location = new Location('', '', '', '', 0.0, 0.0, ''); - $event = new UrlVisited('123'); - - $findVisit = $this->em->find(Visit::class, '123')->willReturn($visit); - $flush = $this->em->flush()->will(function (): void { - }); - $resolveIp = $this->ipLocationResolver->resolveIpLocation($ipAddr)->willReturn($location); - $checkUpdateDb = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); - $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { - }); - - ($this->locateVisit)($event); - - self::assertEquals($visit->getVisitLocation(), new VisitLocation($location)); - $findVisit->shouldHaveBeenCalledOnce(); - $flush->shouldHaveBeenCalledOnce(); - $resolveIp->shouldHaveBeenCalledOnce(); - $checkUpdateDb->shouldHaveBeenCalledOnce(); - $this->logger->warning( - 'GeoLite2 database update failed. Proceeding with old version. {e}', - ['e' => $e], - )->shouldHaveBeenCalledOnce(); - $dispatch->shouldHaveBeenCalledOnce(); - } - - /** @test */ - public function errorWhenDownloadingGeoLiteCancelsLocation(): void - { - $e = GeolocationDbUpdateFailedException::withoutOlderDb(); - $ipAddr = '1.2.3.0'; - $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $ipAddr, '')); - $location = new Location('', '', '', '', 0.0, 0.0, ''); - $event = new UrlVisited('123'); - - $findVisit = $this->em->find(Visit::class, '123')->willReturn($visit); - $flush = $this->em->flush()->will(function (): void { - }); - $resolveIp = $this->ipLocationResolver->resolveIpLocation($ipAddr)->willReturn($location); - $checkUpdateDb = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); - $logError = $this->logger->error( - 'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}', - ['e' => $e, 'visitId' => 123], - ); - $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { - }); - - ($this->locateVisit)($event); - - self::assertNull($visit->getVisitLocation()); - $findVisit->shouldHaveBeenCalledOnce(); - $flush->shouldNotHaveBeenCalled(); - $resolveIp->shouldNotHaveBeenCalled(); - $checkUpdateDb->shouldHaveBeenCalledOnce(); - $logError->shouldHaveBeenCalledOnce(); - $dispatch->shouldHaveBeenCalledOnce(); - } +// /** @test */ +// public function errorWhenUpdatingGeoLiteWithExistingCopyLogsWarning(): void +// { +// $e = GeolocationDbUpdateFailedException::withOlderDb(); +// $ipAddr = '1.2.3.0'; +// $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $ipAddr, '')); +// $location = new Location('', '', '', '', 0.0, 0.0, ''); +// $event = new UrlVisited('123'); +// +// $findVisit = $this->em->find(Visit::class, '123')->willReturn($visit); +// $flush = $this->em->flush()->will(function (): void { +// }); +// $resolveIp = $this->ipLocationResolver->resolveIpLocation($ipAddr)->willReturn($location); +// $checkUpdateDb = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); +// $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { +// }); +// +// ($this->locateVisit)($event); +// +// self::assertEquals($visit->getVisitLocation(), new VisitLocation($location)); +// $findVisit->shouldHaveBeenCalledOnce(); +// $flush->shouldHaveBeenCalledOnce(); +// $resolveIp->shouldHaveBeenCalledOnce(); +// $checkUpdateDb->shouldHaveBeenCalledOnce(); +// $this->logger->warning( +// 'GeoLite2 database update failed. Proceeding with old version. {e}', +// ['e' => $e], +// )->shouldHaveBeenCalledOnce(); +// $dispatch->shouldHaveBeenCalledOnce(); +// } +// +// /** @test */ +// public function errorWhenDownloadingGeoLiteCancelsLocation(): void +// { +// $e = GeolocationDbUpdateFailedException::withoutOlderDb(); +// $ipAddr = '1.2.3.0'; +// $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $ipAddr, '')); +// $location = new Location('', '', '', '', 0.0, 0.0, ''); +// $event = new UrlVisited('123'); +// +// $findVisit = $this->em->find(Visit::class, '123')->willReturn($visit); +// $flush = $this->em->flush()->will(function (): void { +// }); +// $resolveIp = $this->ipLocationResolver->resolveIpLocation($ipAddr)->willReturn($location); +// $checkUpdateDb = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); +// $logError = $this->logger->error( +// 'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}', +// ['e' => $e, 'visitId' => 123], +// ); +// $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { +// }); +// +// ($this->locateVisit)($event); +// +// self::assertNull($visit->getVisitLocation()); +// $findVisit->shouldHaveBeenCalledOnce(); +// $flush->shouldNotHaveBeenCalled(); +// $resolveIp->shouldNotHaveBeenCalled(); +// $checkUpdateDb->shouldHaveBeenCalledOnce(); +// $logError->shouldHaveBeenCalledOnce(); +// $dispatch->shouldHaveBeenCalledOnce(); +// } } From c4718e75232980629b40fb23b3ff8eb8ef9b7a16 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Wed, 7 Apr 2021 12:53:53 +0200 Subject: [PATCH 31/86] Extended error handling on LocateVisit handler --- module/CLI/src/Util/GeolocationDbUpdater.php | 45 +++++++++++-------- .../Core/config/event_dispatcher.config.php | 2 +- .../Core/src/EventDispatcher/LocateVisit.php | 27 +++-------- .../test/EventDispatcher/LocateVisitTest.php | 29 +++++++++++- 4 files changed, 62 insertions(+), 41 deletions(-) diff --git a/module/CLI/src/Util/GeolocationDbUpdater.php b/module/CLI/src/Util/GeolocationDbUpdater.php index ebdeb573..6e7c2da2 100644 --- a/module/CLI/src/Util/GeolocationDbUpdater.php +++ b/module/CLI/src/Util/GeolocationDbUpdater.php @@ -60,24 +60,6 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface } } - /** - * @throws GeolocationDbUpdateFailedException - */ - private function downloadNewDb(bool $olderDbExists, ?callable $beforeDownload, ?callable $handleProgress): void - { - if ($beforeDownload !== null) { - $beforeDownload($olderDbExists); - } - - try { - $this->dbUpdater->downloadFreshCopy($handleProgress); - } catch (RuntimeException $e) { - throw $olderDbExists - ? GeolocationDbUpdateFailedException::withOlderDb($e) - : GeolocationDbUpdateFailedException::withoutOlderDb($e); - } - } - private function buildIsTooOld(Metadata $meta): bool { $buildTimestamp = $this->resolveBuildTimestamp($meta); @@ -105,4 +87,31 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface throw GeolocationDbUpdateFailedException::withInvalidEpochInOldDb($buildEpoch); } + + /** + * @throws GeolocationDbUpdateFailedException + */ + private function downloadNewDb(bool $olderDbExists, ?callable $beforeDownload, ?callable $handleProgress): void + { + if ($beforeDownload !== null) { + $beforeDownload($olderDbExists); + } + + try { + $this->dbUpdater->downloadFreshCopy($this->wrapHandleProgressCallback($handleProgress, $olderDbExists)); + } catch (RuntimeException $e) { + throw $olderDbExists + ? GeolocationDbUpdateFailedException::withOlderDb($e) + : GeolocationDbUpdateFailedException::withoutOlderDb($e); + } + } + + private function wrapHandleProgressCallback(?callable $handleProgress, bool $olderDbExists): ?callable + { + if ($handleProgress === null) { + return null; + } + + return fn (int $total, int $downloaded) => $handleProgress($total, $downloaded, $olderDbExists); + } } diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index 37b1d59d..610412fe 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -22,7 +22,7 @@ return [ EventDispatcher\Event\VisitLocated::class => [ EventDispatcher\NotifyVisitToMercure::class, EventDispatcher\NotifyVisitToWebHooks::class, -// EventDispatcher\UpdateGeoliteDb::class, +// EventDispatcher\UpdateGeoLiteDb::class, ], ], ], diff --git a/module/Core/src/EventDispatcher/LocateVisit.php b/module/Core/src/EventDispatcher/LocateVisit.php index d02f8be7..4d98f454 100644 --- a/module/Core/src/EventDispatcher/LocateVisit.php +++ b/module/Core/src/EventDispatcher/LocateVisit.php @@ -15,6 +15,7 @@ use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; +use Throwable; class LocateVisit { @@ -62,27 +63,6 @@ class LocateVisit $this->eventDispatcher->dispatch(new VisitLocated($visitId)); } -// private function downloadOrUpdateGeoLiteDb(string $visitId): bool -// { -// try { -// $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void { -// $this->logger->notice(sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading')); -// }); -// } catch (GeolocationDbUpdateFailedException $e) { -// if (! $e->olderDbExists()) { -// $this->logger->error( -// 'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}', -// ['e' => $e, 'visitId' => $visitId], -// ); -// return false; -// } -// -// $this->logger->warning('GeoLite2 database update failed. Proceeding with old version. {e}', ['e' => $e]); -// } -// -// return true; -// } - private function locateVisit(string $visitId, ?string $originalIpAddress, Visit $visit): void { $isLocatable = $originalIpAddress !== null || $visit->isLocatable(); @@ -98,6 +78,11 @@ class LocateVisit 'Tried to locate visit with id "{visitId}", but its address seems to be wrong. {e}', ['e' => $e, 'visitId' => $visitId], ); + } catch (Throwable $e) { + $this->logger->error( + 'An unexpected error occurred while trying to locate visit with id "{visitId}". {e}', + ['e' => $e, 'visitId' => $visitId], + ); } } } diff --git a/module/Core/test/EventDispatcher/LocateVisitTest.php b/module/Core/test/EventDispatcher/LocateVisitTest.php index a01ec996..b56cd97f 100644 --- a/module/Core/test/EventDispatcher/LocateVisitTest.php +++ b/module/Core/test/EventDispatcher/LocateVisitTest.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\EventDispatcher; use Doctrine\ORM\EntityManagerInterface; +use OutOfRangeException; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; @@ -110,7 +111,7 @@ class LocateVisitTest extends TestCase WrongIpException::class, ); $logWarning = $this->logger->warning( - Argument::containingString('Tried to locate visit with id "{visitId}", but its address seems to be wrong.'), + 'Tried to locate visit with id "{visitId}", but its address seems to be wrong. {e}', Argument::type('array'), ); $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { @@ -125,6 +126,32 @@ class LocateVisitTest extends TestCase $dispatch->shouldHaveBeenCalledOnce(); } + /** @test */ + public function unhandledExceptionLogsError(): void + { + $event = new UrlVisited('123'); + $findVisit = $this->em->find(Visit::class, '123')->willReturn( + Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', '1.2.3.4', '')), + ); + $resolveLocation = $this->ipLocationResolver->resolveIpLocation(Argument::cetera())->willThrow( + OutOfRangeException::class, + ); + $logError = $this->logger->error( + 'An unexpected error occurred while trying to locate visit with id "{visitId}". {e}', + Argument::type('array'), + ); + $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { + }); + + ($this->locateVisit)($event); + + $findVisit->shouldHaveBeenCalledOnce(); + $resolveLocation->shouldHaveBeenCalledOnce(); + $logError->shouldHaveBeenCalled(); + $this->em->flush()->shouldNotHaveBeenCalled(); + $dispatch->shouldHaveBeenCalledOnce(); + } + /** * @test * @dataProvider provideNonLocatableVisits From 74ea5969be76d39fc2a92ac70bf98f40bd12336f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Wed, 7 Apr 2021 16:29:29 +0200 Subject: [PATCH 32/86] Created new listener to update the GeoLite db after a visit occurs --- .../src/Command/Visit/LocateVisitsCommand.php | 2 +- .../Command/Visit/LocateVisitsCommandTest.php | 2 +- .../Core/config/event_dispatcher.config.php | 5 +- .../src/EventDispatcher/UpdateGeoLiteDb.php | 43 +++++++ .../test/EventDispatcher/LocateVisitTest.php | 63 ---------- .../EventDispatcher/UpdateGeoLiteDbTest.php | 114 ++++++++++++++++++ 6 files changed, 163 insertions(+), 66 deletions(-) create mode 100644 module/Core/src/EventDispatcher/UpdateGeoLiteDb.php create mode 100644 module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php diff --git a/module/CLI/src/Command/Visit/LocateVisitsCommand.php b/module/CLI/src/Command/Visit/LocateVisitsCommand.php index 67678d4d..a71ee410 100644 --- a/module/CLI/src/Command/Visit/LocateVisitsCommand.php +++ b/module/CLI/src/Command/Visit/LocateVisitsCommand.php @@ -181,7 +181,7 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat try { $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void { $this->io->writeln( - sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading'), + sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), ); $this->progressBar = new ProgressBar($this->io); }, function (int $total, int $downloaded): void { diff --git a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php index d5ee2982..be6846a8 100644 --- a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php @@ -227,7 +227,7 @@ class LocateVisitsCommandTest extends TestCase $output = $this->commandTester->getDisplay(); self::assertStringContainsString( - sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading'), + sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), $output, ); self::assertStringContainsString($expectedMessage, $output); diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index 610412fe..bddd59f5 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core; use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; use Psr\EventDispatcher\EventDispatcherInterface; +use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater; use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; use Symfony\Component\Mercure\Hub; @@ -22,7 +23,7 @@ return [ EventDispatcher\Event\VisitLocated::class => [ EventDispatcher\NotifyVisitToMercure::class, EventDispatcher\NotifyVisitToWebHooks::class, -// EventDispatcher\UpdateGeoLiteDb::class, + EventDispatcher\UpdateGeoLiteDb::class, ], ], ], @@ -32,6 +33,7 @@ return [ EventDispatcher\LocateVisit::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToMercure::class => ConfigAbstractFactory::class, + EventDispatcher\UpdateGeoLiteDb::class => ConfigAbstractFactory::class, ], 'delegators' => [ @@ -66,6 +68,7 @@ return [ 'em', 'Logger_Shlink', ], + EventDispatcher\UpdateGeoLiteDb::class => [GeolocationDbUpdater::class, 'Logger_Shlink'], ], ]; diff --git a/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php b/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php new file mode 100644 index 00000000..7b985026 --- /dev/null +++ b/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php @@ -0,0 +1,43 @@ +dbUpdater = $dbUpdater; + $this->logger = $logger; + } + + public function __invoke(): void + { + $beforeDownload = fn (bool $olderDbExists) => $this->logger->notice( + sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), + ); + $handleProgress = function (int $total, int $downloaded, bool $olderDbExists): void { + if ($total > $downloaded) { + return; + } + + $this->logger->notice(sprintf('Finished %s GeoLite2 db file', $olderDbExists ? 'updating' : 'downloading')); + }; + + try { + $this->dbUpdater->checkDbUpdate($beforeDownload, $handleProgress); + } catch (Throwable $e) { + $this->logger->error('GeoLite2 database download failed. {e}', ['e' => $e]); + } + } +} diff --git a/module/Core/test/EventDispatcher/LocateVisitTest.php b/module/Core/test/EventDispatcher/LocateVisitTest.php index b56cd97f..68c7496f 100644 --- a/module/Core/test/EventDispatcher/LocateVisitTest.php +++ b/module/Core/test/EventDispatcher/LocateVisitTest.php @@ -226,67 +226,4 @@ class LocateVisitTest extends TestCase yield 'invalid short url' => [Visit::forInvalidShortUrl(new Visitor('', '', '1.2.3.4', '')), '1.2.3.4']; yield 'regular not found' => [Visit::forRegularNotFound(new Visitor('', '', '1.2.3.4', '')), '1.2.3.4']; } - -// /** @test */ -// public function errorWhenUpdatingGeoLiteWithExistingCopyLogsWarning(): void -// { -// $e = GeolocationDbUpdateFailedException::withOlderDb(); -// $ipAddr = '1.2.3.0'; -// $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $ipAddr, '')); -// $location = new Location('', '', '', '', 0.0, 0.0, ''); -// $event = new UrlVisited('123'); -// -// $findVisit = $this->em->find(Visit::class, '123')->willReturn($visit); -// $flush = $this->em->flush()->will(function (): void { -// }); -// $resolveIp = $this->ipLocationResolver->resolveIpLocation($ipAddr)->willReturn($location); -// $checkUpdateDb = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); -// $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { -// }); -// -// ($this->locateVisit)($event); -// -// self::assertEquals($visit->getVisitLocation(), new VisitLocation($location)); -// $findVisit->shouldHaveBeenCalledOnce(); -// $flush->shouldHaveBeenCalledOnce(); -// $resolveIp->shouldHaveBeenCalledOnce(); -// $checkUpdateDb->shouldHaveBeenCalledOnce(); -// $this->logger->warning( -// 'GeoLite2 database update failed. Proceeding with old version. {e}', -// ['e' => $e], -// )->shouldHaveBeenCalledOnce(); -// $dispatch->shouldHaveBeenCalledOnce(); -// } -// -// /** @test */ -// public function errorWhenDownloadingGeoLiteCancelsLocation(): void -// { -// $e = GeolocationDbUpdateFailedException::withoutOlderDb(); -// $ipAddr = '1.2.3.0'; -// $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $ipAddr, '')); -// $location = new Location('', '', '', '', 0.0, 0.0, ''); -// $event = new UrlVisited('123'); -// -// $findVisit = $this->em->find(Visit::class, '123')->willReturn($visit); -// $flush = $this->em->flush()->will(function (): void { -// }); -// $resolveIp = $this->ipLocationResolver->resolveIpLocation($ipAddr)->willReturn($location); -// $checkUpdateDb = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); -// $logError = $this->logger->error( -// 'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}', -// ['e' => $e, 'visitId' => 123], -// ); -// $dispatch = $this->eventDispatcher->dispatch(new VisitLocated('123'))->will(function (): void { -// }); -// -// ($this->locateVisit)($event); -// -// self::assertNull($visit->getVisitLocation()); -// $findVisit->shouldHaveBeenCalledOnce(); -// $flush->shouldNotHaveBeenCalled(); -// $resolveIp->shouldNotHaveBeenCalled(); -// $checkUpdateDb->shouldHaveBeenCalledOnce(); -// $logError->shouldHaveBeenCalledOnce(); -// $dispatch->shouldHaveBeenCalledOnce(); -// } } diff --git a/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php b/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php new file mode 100644 index 00000000..a5edcbcb --- /dev/null +++ b/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php @@ -0,0 +1,114 @@ +dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class); + $this->logger = $this->prophesize(LoggerInterface::class); + + $this->listener = new UpdateGeoLiteDb($this->dbUpdater->reveal(), $this->logger->reveal()); + } + + /** @test */ + public function exceptionWhileUpdatingDbLogsError(): void + { + $e = new RuntimeException(); + + $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->willThrow($e); + $logError = $this->logger->error('GeoLite2 database download failed. {e}', ['e' => $e]); + + ($this->listener)(); + + $checkDbUpdate->shouldHaveBeenCalledOnce(); + $logError->shouldHaveBeenCalledOnce(); + $this->logger->notice(Argument::cetera())->shouldNotHaveBeenCalled(); + } + + /** + * @test + * @dataProvider provideFlags + */ + public function noticeMessageIsPrintedWhenFirstCallbackIsInvoked(bool $oldDbExists, string $expectedMessage): void + { + $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( + function (array $args) use ($oldDbExists): void { + [$firstCallback] = $args; + $firstCallback($oldDbExists); + }, + ); + $logNotice = $this->logger->notice($expectedMessage); + + ($this->listener)(); + + $checkDbUpdate->shouldHaveBeenCalledOnce(); + $logNotice->shouldHaveBeenCalledOnce(); + $this->logger->error(Argument::cetera())->shouldNotHaveBeenCalled(); + } + + public function provideFlags(): iterable + { + yield 'existing old db' => [true, 'Updating GeoLite2 db file...']; + yield 'not existing old db' => [false, 'Downloading GeoLite2 db file...']; + } + + /** + * @test + * @dataProvider provideDownloaded + */ + public function noticeMessageIsPrintedWhenSecondCallbackIsInvoked( + int $total, + int $downloaded, + bool $oldDbExists, + ?string $expectedMessage + ): void { + $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( + function (array $args) use ($total, $downloaded, $oldDbExists): void { + [, $secondCallback] = $args; + $secondCallback($total, $downloaded, $oldDbExists); + }, + ); + $logNotice = $this->logger->notice($expectedMessage ?? Argument::cetera()); + + ($this->listener)(); + + if ($expectedMessage !== null) { + $logNotice->shouldHaveBeenCalledOnce(); + } else { + $logNotice->shouldNotHaveBeenCalled(); + } + $checkDbUpdate->shouldHaveBeenCalledOnce(); + $this->logger->error(Argument::cetera())->shouldNotHaveBeenCalled(); + } + + public function provideDownloaded(): iterable + { + yield [100, 0, true, null]; + yield [100, 0, false, null]; + yield [100, 99, true, null]; + yield [100, 99, false, null]; + yield [100, 100, true, 'Finished updating GeoLite2 db file']; + yield [100, 100, false, 'Finished downloading GeoLite2 db file']; + yield [100, 101, true, 'Finished updating GeoLite2 db file']; + yield [100, 101, false, 'Finished downloading GeoLite2 db file']; + } +} From f7b6f4ba19c8b7dcdc20a0f404aa8b541dc8643f Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 13:12:37 +0200 Subject: [PATCH 33/86] Created new command containing the logic to download the GeoLite2 db file --- module/CLI/config/cli.config.php | 1 + module/CLI/config/dependencies.config.php | 3 +- .../Visit/DownloadGeoLiteDbCommand.php | 80 +++++++++++++++++ .../src/Command/Visit/LocateVisitsCommand.php | 51 +++-------- module/CLI/test/CliTestUtilsTrait.php | 32 +++++++ .../Command/Visit/LocateVisitsCommandTest.php | 90 +++++++++++-------- .../test/Factory/ApplicationFactoryTest.php | 21 +---- 7 files changed, 182 insertions(+), 96 deletions(-) create mode 100644 module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php create mode 100644 module/CLI/test/CliTestUtilsTrait.php diff --git a/module/CLI/config/cli.config.php b/module/CLI/config/cli.config.php index 6e32428a..6043833b 100644 --- a/module/CLI/config/cli.config.php +++ b/module/CLI/config/cli.config.php @@ -15,6 +15,7 @@ return [ Command\ShortUrl\DeleteShortUrlCommand::NAME => Command\ShortUrl\DeleteShortUrlCommand::class, Command\Visit\LocateVisitsCommand::NAME => Command\Visit\LocateVisitsCommand::class, + Command\Visit\DownloadGeoLiteDbCommand::NAME => Command\Visit\DownloadGeoLiteDbCommand::class, Command\Api\GenerateKeyCommand::NAME => Command\Api\GenerateKeyCommand::class, Command\Api\DisableKeyCommand::NAME => Command\Api\DisableKeyCommand::class, diff --git a/module/CLI/config/dependencies.config.php b/module/CLI/config/dependencies.config.php index 80b26b8d..7d7e2865 100644 --- a/module/CLI/config/dependencies.config.php +++ b/module/CLI/config/dependencies.config.php @@ -44,6 +44,7 @@ return [ Command\ShortUrl\GetVisitsCommand::class => ConfigAbstractFactory::class, Command\ShortUrl\DeleteShortUrlCommand::class => ConfigAbstractFactory::class, + Command\Visit\DownloadGeoLiteDbCommand::class => ConfigAbstractFactory::class, Command\Visit\LocateVisitsCommand::class => ConfigAbstractFactory::class, Command\Api\GenerateKeyCommand::class => ConfigAbstractFactory::class, @@ -80,11 +81,11 @@ return [ Command\ShortUrl\GetVisitsCommand::class => [Visit\VisitsStatsHelper::class], Command\ShortUrl\DeleteShortUrlCommand::class => [Service\ShortUrl\DeleteShortUrlService::class], + Command\Visit\DownloadGeoLiteDbCommand::class => [Util\GeolocationDbUpdater::class], Command\Visit\LocateVisitsCommand::class => [ Visit\VisitLocator::class, IpLocationResolverInterface::class, LockFactory::class, - Util\GeolocationDbUpdater::class, ], Command\Api\GenerateKeyCommand::class => [ApiKeyService::class, ApiKey\RoleResolver::class], diff --git a/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php b/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php new file mode 100644 index 00000000..5f52c3b7 --- /dev/null +++ b/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php @@ -0,0 +1,80 @@ +dbUpdater = $dbUpdater; + } + + protected function configure(): void + { + $this + ->setName(self::NAME) + ->setDescription( + 'Checks if the GeoLite2 db file is too old or it does not exist, and tries to download an up-to-date ' + . 'copy if so.', + ); + } + + protected function execute(InputInterface $input, OutputInterface $output): ?int + { + $io = new SymfonyStyle($input, $output); + + try { + $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists) use ($io): void { + $io->text(sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading')); + $this->progressBar = new ProgressBar($io); + }, function (int $total, int $downloaded): void { + $this->progressBar->setMaxSteps($total); + $this->progressBar->setProgress($downloaded); + }); + + if ($this->progressBar !== null) { + $this->progressBar->finish(); + $io->success('GeoLite2 db file properly downloaded.'); + } else { + $io->info('GeoLite2 db file is up to date.'); + } + + return ExitCodes::EXIT_SUCCESS; + } catch (GeolocationDbUpdateFailedException $e) { + $olderDbExists = $e->olderDbExists(); + + if ($olderDbExists) { + $io->warning( + 'GeoLite2 db file update failed. Visits will continue to be located with the old version.', + ); + } else { + $io->error('GeoLite2 db file download failed. It will not be possible to locate visits.'); + } + + if ($io->isVerbose()) { + $this->getApplication()->renderThrowable($e, $io); + } + + return $olderDbExists ? ExitCodes::EXIT_WARNING : ExitCodes::EXIT_FAILURE; + } + } +} diff --git a/module/CLI/src/Command/Visit/LocateVisitsCommand.php b/module/CLI/src/Command/Visit/LocateVisitsCommand.php index a71ee410..0bcfb1d7 100644 --- a/module/CLI/src/Command/Visit/LocateVisitsCommand.php +++ b/module/CLI/src/Command/Visit/LocateVisitsCommand.php @@ -6,9 +6,7 @@ namespace Shlinkio\Shlink\CLI\Command\Visit; use Shlinkio\Shlink\CLI\Command\Util\AbstractLockedCommand; use Shlinkio\Shlink\CLI\Command\Util\LockedCommandConfig; -use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException; use Shlinkio\Shlink\CLI\Util\ExitCodes; -use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface; use Shlinkio\Shlink\Common\Util\IpAddress; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; @@ -19,7 +17,6 @@ use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; use Symfony\Component\Console\Exception\RuntimeException; -use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; @@ -35,28 +32,26 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat private VisitLocatorInterface $visitLocator; private IpLocationResolverInterface $ipLocationResolver; - private GeolocationDbUpdaterInterface $dbUpdater; private SymfonyStyle $io; - private ?ProgressBar $progressBar = null; public function __construct( VisitLocatorInterface $visitLocator, IpLocationResolverInterface $ipLocationResolver, - LockFactory $locker, - GeolocationDbUpdaterInterface $dbUpdater + LockFactory $locker ) { parent::__construct($locker); $this->visitLocator = $visitLocator; $this->ipLocationResolver = $ipLocationResolver; - $this->dbUpdater = $dbUpdater; } protected function configure(): void { $this ->setName(self::NAME) - ->setDescription('Resolves visits origin locations.') + ->setDescription( + 'Resolves visits origin locations. It implicitly downloads/updates the GeoLite2 db file if needed.', + ) ->addOption( 'retry', 'r', @@ -90,12 +85,12 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat ); } - if ($all && $retry && ! $this->warnAndVerifyContinue()) { + if ($all && $retry && ! $this->warnAndVerifyContinue($input)) { throw new RuntimeException('Execution aborted'); } } - private function warnAndVerifyContinue(): bool + private function warnAndVerifyContinue(InputInterface $input): bool { $this->io->warning([ 'You are about to process the location of all existing visits your short URLs received.', @@ -113,7 +108,7 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat $all = $retry && $input->getOption('all'); try { - $this->checkDbUpdate(); + $this->checkDbUpdate($input); if ($all) { $this->visitLocator->locateAllVisits($this); @@ -128,7 +123,7 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat return ExitCodes::EXIT_SUCCESS; } catch (Throwable $e) { $this->io->error($e->getMessage()); - if ($e instanceof Throwable && $this->io->isVerbose()) { + if ($this->io->isVerbose()) { $this->getApplication()->renderThrowable($e, $this->io); } @@ -176,33 +171,13 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat $this->io->writeln($message); } - private function checkDbUpdate(): void + private function checkDbUpdate(InputInterface $input): void { - try { - $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void { - $this->io->writeln( - sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), - ); - $this->progressBar = new ProgressBar($this->io); - }, function (int $total, int $downloaded): void { - $this->progressBar->setMaxSteps($total); - $this->progressBar->setProgress($downloaded); - }); + $downloadDbCommand = $this->getApplication()->find(DownloadGeoLiteDbCommand::NAME); + $exitCode = $downloadDbCommand->run($input, $this->io); - if ($this->progressBar !== null) { - $this->progressBar->finish(); - $this->io->newLine(); - } - } catch (GeolocationDbUpdateFailedException $e) { - if (! $e->olderDbExists()) { - $this->io->error('GeoLite2 database download failed. It is not possible to locate visits.'); - throw $e; - } - - $this->io->newLine(); - $this->io->writeln( - '[Warning] GeoLite2 database update failed. Proceeding with old version.', - ); + if ($exitCode === ExitCodes::EXIT_FAILURE) { + throw new RuntimeException('It is not possible to locate visits without a GeoLite2 db file.'); } } diff --git a/module/CLI/test/CliTestUtilsTrait.php b/module/CLI/test/CliTestUtilsTrait.php new file mode 100644 index 00000000..b5d81a76 --- /dev/null +++ b/module/CLI/test/CliTestUtilsTrait.php @@ -0,0 +1,32 @@ +prophesize(Command::class); + $command->getName()->willReturn($name); + $command->getDefinition()->willReturn($name); + $command->isEnabled()->willReturn(true); + $command->getAliases()->willReturn([]); + $command->setApplication(Argument::type(Application::class))->willReturn(function (): void { + }); + + return $command; + } +} diff --git a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php index be6846a8..e5632034 100644 --- a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php @@ -6,11 +6,10 @@ namespace ShlinkioTest\Shlink\CLI\Command\Visit; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; +use Shlinkio\Shlink\CLI\Command\Visit\DownloadGeoLiteDbCommand; use Shlinkio\Shlink\CLI\Command\Visit\LocateVisitsCommand; -use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException; -use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface; +use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\Common\Util\IpAddress; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; @@ -21,6 +20,7 @@ use Shlinkio\Shlink\Core\Visit\VisitLocator; use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Application; use Symfony\Component\Console\Exception\RuntimeException; use Symfony\Component\Console\Output\OutputInterface; @@ -33,19 +33,18 @@ use const PHP_EOL; class LocateVisitsCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $visitService; private ObjectProphecy $ipResolver; private ObjectProphecy $lock; - private ObjectProphecy $dbUpdater; + private ObjectProphecy $downloadDbCommand; public function setUp(): void { $this->visitService = $this->prophesize(VisitLocator::class); $this->ipResolver = $this->prophesize(IpLocationResolverInterface::class); - $this->dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class); $locker = $this->prophesize(Lock\LockFactory::class); $this->lock = $this->prophesize(Lock\LockInterface::class); @@ -58,11 +57,14 @@ class LocateVisitsCommandTest extends TestCase $this->visitService->reveal(), $this->ipResolver->reveal(), $locker->reveal(), - $this->dbUpdater->reveal(), ); $app = new Application(); $app->add($command); + $this->downloadDbCommand = $this->createCommandMock(DownloadGeoLiteDbCommand::NAME); + $this->downloadDbCommand->run(Argument::cetera())->willReturn(ExitCodes::EXIT_SUCCESS); + $app->add($this->downloadDbCommand->reveal()); + $this->commandTester = new CommandTester($command); } @@ -202,44 +204,56 @@ class LocateVisitsCommandTest extends TestCase $resolveIpLocation->shouldNotHaveBeenCalled(); } - /** - * @test - * @dataProvider provideParams - */ - public function showsProperMessageWhenGeoLiteUpdateFails(bool $olderDbExists, string $expectedMessage): void + /** @test */ + public function showsProperMessageWhenGeoLiteUpdateFails(): void { - $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will(function (): void { - }); - $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( - function (array $args) use ($olderDbExists): void { - [$mustBeUpdated, $handleProgress] = $args; - - $mustBeUpdated($olderDbExists); - $handleProgress(100, 50); - - throw $olderDbExists - ? GeolocationDbUpdateFailedException::withOlderDb() - : GeolocationDbUpdateFailedException::withoutOlderDb(); - }, - ); + $this->downloadDbCommand->run(Argument::cetera())->willReturn(ExitCodes::EXIT_FAILURE); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); - self::assertStringContainsString( - sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), - $output, - ); - self::assertStringContainsString($expectedMessage, $output); - $locateVisits->shouldHaveBeenCalledTimes((int) $olderDbExists); - $checkDbUpdate->shouldHaveBeenCalledOnce(); + self::assertStringContainsString('It is not possible to locate visits without a GeoLite2 db file.', $output); + $this->visitService->locateUnlocatedVisits(Argument::cetera())->shouldNotHaveBeenCalled(); } - public function provideParams(): iterable - { - yield [true, '[Warning] GeoLite2 database update failed. Proceeding with old version.']; - yield [false, 'GeoLite2 database download failed. It is not possible to locate visits.']; - } +// /** +// * @test +// * @dataProvider provideParams +// */ +// public function showsProperMessageWhenGeoLiteUpdateFails(bool $olderDbExists, string $expectedMessage): void +// { +// $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will(function (): void { +// }); +// $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( +// function (array $args) use ($olderDbExists): void { +// [$mustBeUpdated, $handleProgress] = $args; +// +// $mustBeUpdated($olderDbExists); +// $handleProgress(100, 50); +// +// throw $olderDbExists +// ? GeolocationDbUpdateFailedException::withOlderDb() +// : GeolocationDbUpdateFailedException::withoutOlderDb(); +// }, +// ); +// +// $this->commandTester->execute([]); +// $output = $this->commandTester->getDisplay(); +// +// self::assertStringContainsString( +// sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), +// $output, +// ); +// self::assertStringContainsString($expectedMessage, $output); +// $locateVisits->shouldHaveBeenCalledTimes((int) $olderDbExists); +// $checkDbUpdate->shouldHaveBeenCalledOnce(); +// } +// +// public function provideParams(): iterable +// { +// yield [true, '[Warning] GeoLite2 database update failed. Proceeding with old version.']; +// yield [false, 'GeoLite2 database download failed. It is not possible to locate visits.']; +// } /** @test */ public function providingAllFlagOnItsOwnDisplaysNotice(): void diff --git a/module/CLI/test/Factory/ApplicationFactoryTest.php b/module/CLI/test/Factory/ApplicationFactoryTest.php index ee0793bc..fbb5ace9 100644 --- a/module/CLI/test/Factory/ApplicationFactoryTest.php +++ b/module/CLI/test/Factory/ApplicationFactoryTest.php @@ -6,17 +6,13 @@ namespace ShlinkioTest\Shlink\CLI\Factory; use Laminas\ServiceManager\ServiceManager; use PHPUnit\Framework\TestCase; -use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; -use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Factory\ApplicationFactory; use Shlinkio\Shlink\Core\Options\AppOptions; -use Symfony\Component\Console\Application; -use Symfony\Component\Console\Command\Command; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; class ApplicationFactoryTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private ApplicationFactory $factory; @@ -54,17 +50,4 @@ class ApplicationFactoryTest extends TestCase AppOptions::class => new AppOptions(), ]]); } - - private function createCommandMock(string $name): ObjectProphecy - { - $command = $this->prophesize(Command::class); - $command->getName()->willReturn($name); - $command->getDefinition()->willReturn($name); - $command->isEnabled()->willReturn(true); - $command->getAliases()->willReturn([]); - $command->setApplication(Argument::type(Application::class))->willReturn(function (): void { - }); - - return $command; - } } From 1f8994ca8b5954b82a3c2766feff1bfbc205bdc4 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 13:34:14 +0200 Subject: [PATCH 34/86] Created DownloadGeoLiteDbCommandTest --- .../Visit/DownloadGeoLiteDbCommand.php | 6 +- module/CLI/test/CliTestUtilsTrait.php | 12 ++ .../Visit/DownloadGeoLiteDbCommandTest.php | 107 ++++++++++++++++++ .../Command/Visit/LocateVisitsCommandTest.php | 39 ------- 4 files changed, 122 insertions(+), 42 deletions(-) create mode 100644 module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php diff --git a/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php b/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php index 5f52c3b7..3d76663a 100644 --- a/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php +++ b/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php @@ -51,11 +51,11 @@ class DownloadGeoLiteDbCommand extends Command $this->progressBar->setProgress($downloaded); }); - if ($this->progressBar !== null) { + if ($this->progressBar === null) { + $io->info('GeoLite2 db file is up to date.'); + } else { $this->progressBar->finish(); $io->success('GeoLite2 db file properly downloaded.'); - } else { - $io->info('GeoLite2 db file is up to date.'); } return ExitCodes::EXIT_SUCCESS; diff --git a/module/CLI/test/CliTestUtilsTrait.php b/module/CLI/test/CliTestUtilsTrait.php index b5d81a76..412131dc 100644 --- a/module/CLI/test/CliTestUtilsTrait.php +++ b/module/CLI/test/CliTestUtilsTrait.php @@ -9,6 +9,7 @@ use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Symfony\Component\Console\Application; use Symfony\Component\Console\Command\Command; +use Symfony\Component\Console\Tester\CommandTester; trait CliTestUtilsTrait { @@ -29,4 +30,15 @@ trait CliTestUtilsTrait return $command; } + + private function testerForCommand(Command $mainCommand, Command ...$extraCommands): CommandTester + { + $app = new Application(); + $app->add($mainCommand); + foreach ($extraCommands as $command) { + $app->add($command); + } + + return new CommandTester($mainCommand); + } } diff --git a/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php b/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php new file mode 100644 index 00000000..7ead517d --- /dev/null +++ b/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php @@ -0,0 +1,107 @@ +dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class); + $this->commandTester = $this->testerForCommand(new DownloadGeoLiteDbCommand($this->dbUpdater->reveal())); + } + + /** + * @test + * @dataProvider provideFailureParams + */ + public function showsProperMessageWhenGeoLiteUpdateFails( + bool $olderDbExists, + string $expectedMessage, + int $expectedExitCode + ): void { + $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( + function (array $args) use ($olderDbExists): void { + [$beforeDownload, $handleProgress] = $args; + + $beforeDownload($olderDbExists); + $handleProgress(100, 50); + + throw $olderDbExists + ? GeolocationDbUpdateFailedException::withOlderDb() + : GeolocationDbUpdateFailedException::withoutOlderDb(); + }, + ); + + $this->commandTester->execute([]); + $output = $this->commandTester->getDisplay(); + $exitCode = $this->commandTester->getStatusCode(); + + self::assertStringContainsString( + sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), + $output, + ); + self::assertStringContainsString($expectedMessage, $output); + self::assertSame($expectedExitCode, $exitCode); + $checkDbUpdate->shouldHaveBeenCalledOnce(); + } + + public function provideFailureParams(): iterable + { + yield 'existing db' => [ + true, + '[WARNING] GeoLite2 db file update failed. Visits will continue to be located', + ExitCodes::EXIT_WARNING, + ]; + yield 'not existing db' => [ + false, + '[ERROR] GeoLite2 db file download failed. It will not be possible to locate', + ExitCodes::EXIT_FAILURE, + ]; + } + + /** + * @test + * @dataProvider provideSuccessParams + */ + public function printsExpectedMessageWhenNoErrorOccurs(callable $checkUpdateBehavior, string $expectedMessage): void + { + $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will($checkUpdateBehavior); + + $this->commandTester->execute([]); + $output = $this->commandTester->getDisplay(); + $exitCode = $this->commandTester->getStatusCode(); + + self::assertStringContainsString($expectedMessage, $output); + self::assertSame(ExitCodes::EXIT_SUCCESS, $exitCode); + $checkDbUpdate->shouldHaveBeenCalledOnce(); + } + + public function provideSuccessParams(): iterable + { + yield 'up to date db' => [function (): void { + }, '[INFO] GeoLite2 db file is up to date.']; + yield 'outdated db' => [function (array $args): void { + [$beforeDownload] = $args; + $beforeDownload(true); + }, '[OK] GeoLite2 db file properly downloaded.']; + } +} diff --git a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php index e5632034..e7ccc9d2 100644 --- a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php @@ -216,45 +216,6 @@ class LocateVisitsCommandTest extends TestCase $this->visitService->locateUnlocatedVisits(Argument::cetera())->shouldNotHaveBeenCalled(); } -// /** -// * @test -// * @dataProvider provideParams -// */ -// public function showsProperMessageWhenGeoLiteUpdateFails(bool $olderDbExists, string $expectedMessage): void -// { -// $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will(function (): void { -// }); -// $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( -// function (array $args) use ($olderDbExists): void { -// [$mustBeUpdated, $handleProgress] = $args; -// -// $mustBeUpdated($olderDbExists); -// $handleProgress(100, 50); -// -// throw $olderDbExists -// ? GeolocationDbUpdateFailedException::withOlderDb() -// : GeolocationDbUpdateFailedException::withoutOlderDb(); -// }, -// ); -// -// $this->commandTester->execute([]); -// $output = $this->commandTester->getDisplay(); -// -// self::assertStringContainsString( -// sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), -// $output, -// ); -// self::assertStringContainsString($expectedMessage, $output); -// $locateVisits->shouldHaveBeenCalledTimes((int) $olderDbExists); -// $checkDbUpdate->shouldHaveBeenCalledOnce(); -// } -// -// public function provideParams(): iterable -// { -// yield [true, '[Warning] GeoLite2 database update failed. Proceeding with old version.']; -// yield [false, 'GeoLite2 database download failed. It is not possible to locate visits.']; -// } - /** @test */ public function providingAllFlagOnItsOwnDisplaysNotice(): void { From 86230d9bf3833d7afe3463166944b9c77b73b1e0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 13:42:56 +0200 Subject: [PATCH 35/86] Removed duplicated code during CLI command tests --- .../test/Command/Api/DisableKeyCommandTest.php | 10 +++------- .../test/Command/Api/GenerateKeyCommandTest.php | 16 ++++++---------- .../CLI/test/Command/Api/ListKeysCommandTest.php | 10 +++------- .../Command/Db/CreateDatabaseCommandTest.php | 9 +++------ .../Command/Db/MigrateDatabaseCommandTest.php | 10 +++------- .../Command/Domain/ListDomainsCommandTest.php | 12 +++--------- .../ShortUrl/DeleteShortUrlCommandTest.php | 12 +++--------- .../ShortUrl/GenerateShortUrlCommandTest.php | 9 +++------ .../Command/ShortUrl/GetVisitsCommandTest.php | 10 +++------- .../ShortUrl/ListShortUrlsCommandTest.php | 9 +++------ .../Command/ShortUrl/ResolveUrlCommandTest.php | 11 +++-------- .../test/Command/Tag/CreateTagCommandTest.php | 12 +++--------- .../test/Command/Tag/DeleteTagsCommandTest.php | 12 +++--------- .../CLI/test/Command/Tag/ListTagsCommandTest.php | 12 +++--------- .../test/Command/Tag/RenameTagCommandTest.php | 12 +++--------- .../Command/Visit/LocateVisitsCommandTest.php | 6 +----- 16 files changed, 49 insertions(+), 123 deletions(-) diff --git a/module/CLI/test/Command/Api/DisableKeyCommandTest.php b/module/CLI/test/Command/Api/DisableKeyCommandTest.php index 49835f85..90942dc9 100644 --- a/module/CLI/test/Command/Api/DisableKeyCommandTest.php +++ b/module/CLI/test/Command/Api/DisableKeyCommandTest.php @@ -5,17 +5,16 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\Api; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Api\DisableKeyCommand; use Shlinkio\Shlink\Common\Exception\InvalidArgumentException; use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class DisableKeyCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $apiKeyService; @@ -23,10 +22,7 @@ class DisableKeyCommandTest extends TestCase public function setUp(): void { $this->apiKeyService = $this->prophesize(ApiKeyServiceInterface::class); - $command = new DisableKeyCommand($this->apiKeyService->reveal()); - $app = new Application(); - $app->add($command); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new DisableKeyCommand($this->apiKeyService->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/Api/GenerateKeyCommandTest.php b/module/CLI/test/Command/Api/GenerateKeyCommandTest.php index 5c0c3c8a..e5c543d5 100644 --- a/module/CLI/test/Command/Api/GenerateKeyCommandTest.php +++ b/module/CLI/test/Command/Api/GenerateKeyCommandTest.php @@ -7,34 +7,30 @@ namespace ShlinkioTest\Shlink\CLI\Command\Api; use Cake\Chronos\Chronos; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\ApiKey\RoleResolverInterface; use Shlinkio\Shlink\CLI\Command\Api\GenerateKeyCommand; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Tester\CommandTester; class GenerateKeyCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $apiKeyService; - private ObjectProphecy $roleResolver; public function setUp(): void { $this->apiKeyService = $this->prophesize(ApiKeyServiceInterface::class); - $this->roleResolver = $this->prophesize(RoleResolverInterface::class); - $this->roleResolver->determineRoles(Argument::type(InputInterface::class))->willReturn([]); + $roleResolver = $this->prophesize(RoleResolverInterface::class); + $roleResolver->determineRoles(Argument::type(InputInterface::class))->willReturn([]); - $command = new GenerateKeyCommand($this->apiKeyService->reveal(), $this->roleResolver->reveal()); - $app = new Application(); - $app->add($command); - $this->commandTester = new CommandTester($command); + $command = new GenerateKeyCommand($this->apiKeyService->reveal(), $roleResolver->reveal()); + $this->commandTester = $this->testerForCommand($command); } /** @test */ diff --git a/module/CLI/test/Command/Api/ListKeysCommandTest.php b/module/CLI/test/Command/Api/ListKeysCommandTest.php index fa6816d2..fc845ff7 100644 --- a/module/CLI/test/Command/Api/ListKeysCommandTest.php +++ b/module/CLI/test/Command/Api/ListKeysCommandTest.php @@ -5,7 +5,6 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\Api; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Api\ListKeysCommand; use Shlinkio\Shlink\Core\Entity\Domain; @@ -13,12 +12,12 @@ use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; use Shlinkio\Shlink\Rest\Entity\ApiKey; use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class ListKeysCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $apiKeyService; @@ -26,10 +25,7 @@ class ListKeysCommandTest extends TestCase public function setUp(): void { $this->apiKeyService = $this->prophesize(ApiKeyServiceInterface::class); - $command = new ListKeysCommand($this->apiKeyService->reveal()); - $app = new Application(); - $app->add($command); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new ListKeysCommand($this->apiKeyService->reveal())); } /** diff --git a/module/CLI/test/Command/Db/CreateDatabaseCommandTest.php b/module/CLI/test/Command/Db/CreateDatabaseCommandTest.php index db9dcf66..70d4d5eb 100644 --- a/module/CLI/test/Command/Db/CreateDatabaseCommandTest.php +++ b/module/CLI/test/Command/Db/CreateDatabaseCommandTest.php @@ -9,11 +9,10 @@ use Doctrine\DBAL\Platforms\AbstractPlatform; use Doctrine\DBAL\Schema\AbstractSchemaManager; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Db\CreateDatabaseCommand; use Shlinkio\Shlink\CLI\Util\ProcessRunnerInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; use Symfony\Component\Lock\LockFactory; @@ -22,7 +21,7 @@ use Symfony\Component\Process\PhpExecutableFinder; class CreateDatabaseCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $processHelper; @@ -59,10 +58,8 @@ class CreateDatabaseCommandTest extends TestCase $this->regularConn->reveal(), $noDbNameConn->reveal(), ); - $app = new Application(); - $app->add($command); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand($command); } /** @test */ diff --git a/module/CLI/test/Command/Db/MigrateDatabaseCommandTest.php b/module/CLI/test/Command/Db/MigrateDatabaseCommandTest.php index d25f44f2..d301f55e 100644 --- a/module/CLI/test/Command/Db/MigrateDatabaseCommandTest.php +++ b/module/CLI/test/Command/Db/MigrateDatabaseCommandTest.php @@ -6,11 +6,10 @@ namespace ShlinkioTest\Shlink\CLI\Command\Db; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Db\MigrateDatabaseCommand; use Shlinkio\Shlink\CLI\Util\ProcessRunnerInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; use Symfony\Component\Lock\LockFactory; @@ -19,7 +18,7 @@ use Symfony\Component\Process\PhpExecutableFinder; class MigrateDatabaseCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $processHelper; @@ -43,10 +42,7 @@ class MigrateDatabaseCommandTest extends TestCase $this->processHelper->reveal(), $phpExecutableFinder->reveal(), ); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand($command); } /** @test */ diff --git a/module/CLI/test/Command/Domain/ListDomainsCommandTest.php b/module/CLI/test/Command/Domain/ListDomainsCommandTest.php index a0f79448..04f7eb5d 100644 --- a/module/CLI/test/Command/Domain/ListDomainsCommandTest.php +++ b/module/CLI/test/Command/Domain/ListDomainsCommandTest.php @@ -5,18 +5,17 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\Domain; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Domain\ListDomainsCommand; use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\Core\Domain\DomainServiceInterface; use Shlinkio\Shlink\Core\Domain\Model\DomainItem; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class ListDomainsCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $domainService; @@ -24,12 +23,7 @@ class ListDomainsCommandTest extends TestCase public function setUp(): void { $this->domainService = $this->prophesize(DomainServiceInterface::class); - - $command = new ListDomainsCommand($this->domainService->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new ListDomainsCommand($this->domainService->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/ShortUrl/DeleteShortUrlCommandTest.php b/module/CLI/test/Command/ShortUrl/DeleteShortUrlCommandTest.php index 83fd792d..a6b6fc78 100644 --- a/module/CLI/test/Command/ShortUrl/DeleteShortUrlCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/DeleteShortUrlCommandTest.php @@ -6,13 +6,12 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\ShortUrl\DeleteShortUrlCommand; use Shlinkio\Shlink\Core\Exception; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Service\ShortUrl\DeleteShortUrlServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; use function array_pop; @@ -22,7 +21,7 @@ use const PHP_EOL; class DeleteShortUrlCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $service; @@ -30,12 +29,7 @@ class DeleteShortUrlCommandTest extends TestCase public function setUp(): void { $this->service = $this->prophesize(DeleteShortUrlServiceInterface::class); - - $command = new DeleteShortUrlCommand($this->service->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new DeleteShortUrlCommand($this->service->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/ShortUrl/GenerateShortUrlCommandTest.php b/module/CLI/test/Command/ShortUrl/GenerateShortUrlCommandTest.php index 25953d38..19767dc7 100644 --- a/module/CLI/test/Command/ShortUrl/GenerateShortUrlCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/GenerateShortUrlCommandTest.php @@ -7,7 +7,6 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl; use PHPUnit\Framework\Assert; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\ShortUrl\GenerateShortUrlCommand; use Shlinkio\Shlink\CLI\Util\ExitCodes; @@ -17,12 +16,12 @@ use Shlinkio\Shlink\Core\Exception\NonUniqueSlugException; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Service\UrlShortener; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class GenerateShortUrlCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $urlShortener; @@ -35,9 +34,7 @@ class GenerateShortUrlCommandTest extends TestCase $this->stringifier->stringify(Argument::type(ShortUrl::class))->willReturn(''); $command = new GenerateShortUrlCommand($this->urlShortener->reveal(), $this->stringifier->reveal(), 5); - $app = new Application(); - $app->add($command); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand($command); } /** @test */ diff --git a/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php b/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php index d25d5763..42f008be 100644 --- a/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php @@ -8,7 +8,6 @@ use Cake\Chronos\Chronos; use Pagerfanta\Adapter\ArrayAdapter; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\ShortUrl\GetVisitsCommand; use Shlinkio\Shlink\Common\Paginator\Paginator; @@ -21,14 +20,13 @@ use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface; use Shlinkio\Shlink\IpGeolocation\Model\Location; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; - use function sprintf; class GetVisitsCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $visitsHelper; @@ -37,9 +35,7 @@ class GetVisitsCommandTest extends TestCase { $this->visitsHelper = $this->prophesize(VisitsStatsHelperInterface::class); $command = new GetVisitsCommand($this->visitsHelper->reveal()); - $app = new Application(); - $app->add($command); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand($command); } /** @test */ diff --git a/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php b/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php index 3f2b38b1..08519b62 100644 --- a/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php @@ -8,7 +8,6 @@ use Cake\Chronos\Chronos; use Pagerfanta\Adapter\ArrayAdapter; use PHPUnit\Framework\TestCase; use Prophecy\Argument; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\ShortUrl\ListShortUrlsCommand; use Shlinkio\Shlink\Common\Paginator\Paginator; @@ -17,14 +16,14 @@ use Shlinkio\Shlink\Core\Model\ShortUrlsParams; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifier; use Shlinkio\Shlink\Core\ShortUrl\Transformer\ShortUrlDataTransformer; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; use function explode; class ListShortUrlsCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $shortUrlService; @@ -32,12 +31,10 @@ class ListShortUrlsCommandTest extends TestCase public function setUp(): void { $this->shortUrlService = $this->prophesize(ShortUrlServiceInterface::class); - $app = new Application(); $command = new ListShortUrlsCommand($this->shortUrlService->reveal(), new ShortUrlDataTransformer( new ShortUrlStringifier([]), )); - $app->add($command); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand($command); } /** @test */ diff --git a/module/CLI/test/Command/ShortUrl/ResolveUrlCommandTest.php b/module/CLI/test/Command/ShortUrl/ResolveUrlCommandTest.php index f0025b65..2a816207 100644 --- a/module/CLI/test/Command/ShortUrl/ResolveUrlCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/ResolveUrlCommandTest.php @@ -5,14 +5,13 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\ShortUrl\ResolveUrlCommand; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortUrlResolverInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; use function sprintf; @@ -21,7 +20,7 @@ use const PHP_EOL; class ResolveUrlCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $urlResolver; @@ -29,11 +28,7 @@ class ResolveUrlCommandTest extends TestCase public function setUp(): void { $this->urlResolver = $this->prophesize(ShortUrlResolverInterface::class); - $command = new ResolveUrlCommand($this->urlResolver->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new ResolveUrlCommand($this->urlResolver->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/Tag/CreateTagCommandTest.php b/module/CLI/test/Command/Tag/CreateTagCommandTest.php index 2789c481..7062cb45 100644 --- a/module/CLI/test/Command/Tag/CreateTagCommandTest.php +++ b/module/CLI/test/Command/Tag/CreateTagCommandTest.php @@ -6,16 +6,15 @@ namespace ShlinkioTest\Shlink\CLI\Command\Tag; use Doctrine\Common\Collections\ArrayCollection; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\CreateTagCommand; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class CreateTagCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $tagService; @@ -23,12 +22,7 @@ class CreateTagCommandTest extends TestCase public function setUp(): void { $this->tagService = $this->prophesize(TagServiceInterface::class); - - $command = new CreateTagCommand($this->tagService->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new CreateTagCommand($this->tagService->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php b/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php index 6d3737c1..46f61814 100644 --- a/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/DeleteTagsCommandTest.php @@ -5,16 +5,15 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\Tag; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\DeleteTagsCommand; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class DeleteTagsCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $tagService; @@ -22,12 +21,7 @@ class DeleteTagsCommandTest extends TestCase public function setUp(): void { $this->tagService = $this->prophesize(TagServiceInterface::class); - - $command = new DeleteTagsCommand($this->tagService->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new DeleteTagsCommand($this->tagService->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/Tag/ListTagsCommandTest.php b/module/CLI/test/Command/Tag/ListTagsCommandTest.php index 5b9e14e9..9ec42e54 100644 --- a/module/CLI/test/Command/Tag/ListTagsCommandTest.php +++ b/module/CLI/test/Command/Tag/ListTagsCommandTest.php @@ -5,18 +5,17 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\Tag; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\ListTagsCommand; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class ListTagsCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $tagService; @@ -24,12 +23,7 @@ class ListTagsCommandTest extends TestCase public function setUp(): void { $this->tagService = $this->prophesize(TagServiceInterface::class); - - $command = new ListTagsCommand($this->tagService->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new ListTagsCommand($this->tagService->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/Tag/RenameTagCommandTest.php b/module/CLI/test/Command/Tag/RenameTagCommandTest.php index d457c25d..3a52aba3 100644 --- a/module/CLI/test/Command/Tag/RenameTagCommandTest.php +++ b/module/CLI/test/Command/Tag/RenameTagCommandTest.php @@ -5,19 +5,18 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\CLI\Command\Tag; use PHPUnit\Framework\TestCase; -use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\Tag\RenameTagCommand; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Exception\TagNotFoundException; use Shlinkio\Shlink\Core\Tag\Model\TagRenaming; use Shlinkio\Shlink\Core\Tag\TagServiceInterface; -use Symfony\Component\Console\Application; +use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; class RenameTagCommandTest extends TestCase { - use ProphecyTrait; + use CliTestUtilsTrait; private CommandTester $commandTester; private ObjectProphecy $tagService; @@ -25,12 +24,7 @@ class RenameTagCommandTest extends TestCase public function setUp(): void { $this->tagService = $this->prophesize(TagServiceInterface::class); - - $command = new RenameTagCommand($this->tagService->reveal()); - $app = new Application(); - $app->add($command); - - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand(new RenameTagCommand($this->tagService->reveal())); } /** @test */ diff --git a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php index e7ccc9d2..6e4213f6 100644 --- a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php @@ -21,7 +21,6 @@ use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; -use Symfony\Component\Console\Application; use Symfony\Component\Console\Exception\RuntimeException; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Tester\CommandTester; @@ -58,14 +57,11 @@ class LocateVisitsCommandTest extends TestCase $this->ipResolver->reveal(), $locker->reveal(), ); - $app = new Application(); - $app->add($command); $this->downloadDbCommand = $this->createCommandMock(DownloadGeoLiteDbCommand::NAME); $this->downloadDbCommand->run(Argument::cetera())->willReturn(ExitCodes::EXIT_SUCCESS); - $app->add($this->downloadDbCommand->reveal()); - $this->commandTester = new CommandTester($command); + $this->commandTester = $this->testerForCommand($command, $this->downloadDbCommand->reveal()); } /** From 7b4456e73f6e04d930408646ecacbf0286e90a91 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 14:09:26 +0200 Subject: [PATCH 36/86] Ensured events triggered as a result of a new visit are never skipped --- .gitignore | 3 +-- .../test/Command/ShortUrl/GetVisitsCommandTest.php | 1 + module/Core/src/EventDispatcher/LocateVisit.php | 12 ++++++------ module/Core/src/EventDispatcher/UpdateGeoLiteDb.php | 6 ++++-- module/Core/test/EventDispatcher/LocateVisitTest.php | 2 +- .../test/EventDispatcher/UpdateGeoLiteDbTest.php | 4 ++++ 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 03b2790e..32942a29 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,7 @@ composer.phar vendor/ data/database.sqlite data/shlink-tests.db -data/GeoLite2-City.mmdb -data/GeoLite2-City.mmdb.* +data/GeoLite2-City.* docs/swagger-ui* docs/mercure.html docker-compose.override.yml diff --git a/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php b/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php index 42f008be..044866ed 100644 --- a/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php @@ -22,6 +22,7 @@ use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface; use Shlinkio\Shlink\IpGeolocation\Model\Location; use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; + use function sprintf; class GetVisitsCommandTest extends TestCase diff --git a/module/Core/src/EventDispatcher/LocateVisit.php b/module/Core/src/EventDispatcher/LocateVisit.php index 4d98f454..80dc18eb 100644 --- a/module/Core/src/EventDispatcher/LocateVisit.php +++ b/module/Core/src/EventDispatcher/LocateVisit.php @@ -52,6 +52,12 @@ class LocateVisit return; } + $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit); + $this->eventDispatcher->dispatch(new VisitLocated($visitId)); + } + + private function locateVisit(string $visitId, ?string $originalIpAddress, Visit $visit): void + { if (! $this->dbUpdater->databaseFileExists()) { $this->logger->warning('Tried to locate visit with id "{visitId}", but a GeoLite2 db was not found.', [ 'visitId' => $visitId, @@ -59,12 +65,6 @@ class LocateVisit return; } - $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit); - $this->eventDispatcher->dispatch(new VisitLocated($visitId)); - } - - private function locateVisit(string $visitId, ?string $originalIpAddress, Visit $visit): void - { $isLocatable = $originalIpAddress !== null || $visit->isLocatable(); $addr = $originalIpAddress ?? $visit->getRemoteAddr(); diff --git a/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php b/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php index 7b985026..f17a7ffb 100644 --- a/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php +++ b/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php @@ -26,11 +26,13 @@ class UpdateGeoLiteDb $beforeDownload = fn (bool $olderDbExists) => $this->logger->notice( sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'), ); - $handleProgress = function (int $total, int $downloaded, bool $olderDbExists): void { - if ($total > $downloaded) { + $messageLogged = false; + $handleProgress = function (int $total, int $downloaded, bool $olderDbExists) use (&$messageLogged): void { + if ($messageLogged || $total > $downloaded) { return; } + $messageLogged = true; $this->logger->notice(sprintf('Finished %s GeoLite2 db file', $olderDbExists ? 'updating' : 'downloading')); }; diff --git a/module/Core/test/EventDispatcher/LocateVisitTest.php b/module/Core/test/EventDispatcher/LocateVisitTest.php index 68c7496f..fda45c58 100644 --- a/module/Core/test/EventDispatcher/LocateVisitTest.php +++ b/module/Core/test/EventDispatcher/LocateVisitTest.php @@ -97,7 +97,7 @@ class LocateVisitTest extends TestCase $this->em->flush()->shouldNotHaveBeenCalled(); $this->ipLocationResolver->resolveIpLocation(Argument::cetera())->shouldNotHaveBeenCalled(); $logWarning->shouldHaveBeenCalled(); - $dispatch->shouldNotHaveBeenCalled(); + $dispatch->shouldHaveBeenCalledOnce(); } /** @test */ diff --git a/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php b/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php index a5edcbcb..a492f9dd 100644 --- a/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php +++ b/module/Core/test/EventDispatcher/UpdateGeoLiteDbTest.php @@ -84,6 +84,10 @@ class UpdateGeoLiteDbTest extends TestCase $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will( function (array $args) use ($total, $downloaded, $oldDbExists): void { [, $secondCallback] = $args; + + // Invoke several times to ensure the log is printed only once + $secondCallback($total, $downloaded, $oldDbExists); + $secondCallback($total, $downloaded, $oldDbExists); $secondCallback($total, $downloaded, $oldDbExists); }, ); From 104b7390daa18c121c1fef88b76debb9bc3d35ce Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 14:32:19 +0200 Subject: [PATCH 37/86] Updated docker entry point so that it tries to download the GeoLite2 db file when the license key was provided --- .dockerignore | 2 +- docker/config/shlink_in_docker.local.php | 2 +- docker/docker-entrypoint.sh | 6 ++++++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.dockerignore b/.dockerignore index f9102acb..9fb114c1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,7 +5,7 @@ data/log/* data/locks/* data/proxies/* data/migrations_template.txt -data/GeoLite2-City.* +data/GeoLite2-City* data/database.sqlite data/shlink-tests.db CHANGELOG.md diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index 4ddd52e5..d022f79d 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -170,7 +170,7 @@ return [ ], 'geolite2' => [ - 'license_key' => env('GEOLITE_LICENSE_KEY', 'G4Lm0C60yJsnkdPi'), + 'license_key' => env('GEOLITE_LICENSE_KEY', 'G4Lm0C60yJsnkdPi'), // Deprecated. Remove the default value ], 'mercure' => $helper->getMercureConfig(), diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index df480d2f..1f9337c4 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -15,6 +15,12 @@ php vendor/doctrine/orm/bin/doctrine.php orm:generate-proxies -n -q echo "Clearing entities cache..." php vendor/doctrine/orm/bin/doctrine.php orm:clear-cache:metadata -n -q +# Try to download GeoLite2 db file only if the license key env var was defined +if [ ! -z "${GEOLITE_LICENSE_KEY}" ]; then + echo "Downloading GeoLite2 db file..." + php bin/cli visit:download-db -n -q +fi + # When restarting the container, swoole might think it is already in execution # This forces the app to be started every second until the exit code is 0 until php vendor/bin/laminas mezzio:swoole:start; do sleep 1 ; done From d9b675fc8bd8099981c13209eb15fd5c641db4eb Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 16:56:55 +0200 Subject: [PATCH 38/86] Updated to an installer version with support to download the GeoLite db file --- composer.json | 2 +- config/autoload/installer.global.php | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index bff515f0..41885c62 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", "shlinkio/shlink-importer": "^2.2", - "shlinkio/shlink-installer": "^5.4", + "shlinkio/shlink-installer": "dev-develop#aa50ea9 as 5.5", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", "symfony/filesystem": "^5.1", diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php index d18f31f4..605b16ce 100644 --- a/config/autoload/installer.global.php +++ b/config/autoload/installer.global.php @@ -2,7 +2,10 @@ declare(strict_types=1); +namespace Shlinkio\Shlink\CLI; + use Shlinkio\Shlink\Installer\Config\Option; +use Shlinkio\Shlink\Installer\Util\InstallationCommand; return [ @@ -45,11 +48,14 @@ return [ ], 'installation_commands' => [ - 'db_create_schema' => [ - 'command' => 'bin/cli db:create', + InstallationCommand::DB_CREATE_SCHEMA => [ + 'command' => 'bin/cli ' . Command\Db\CreateDatabaseCommand::NAME, ], - 'db_migrate' => [ - 'command' => 'bin/cli db:migrate', + InstallationCommand::DB_MIGRATE => [ + 'command' => 'bin/cli ' . Command\Db\MigrateDatabaseCommand::NAME, + ], + InstallationCommand::GEOLITE_DOWNLOAD_DB => [ + 'command' => 'bin/cli ' . Command\Visit\DownloadGeoLiteDbCommand::NAME, ], ], ], From 96ff0bffdac81dcfd0ff18192494eef529f08de6 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Thu, 8 Apr 2021 17:00:57 +0200 Subject: [PATCH 39/86] Updated changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97cf925c..1c88f036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this ## [Unreleased] ### Added -* [#1044](https://github.com/shlinkio/shlink/issues/1044) Added ability to set names on API keys, which help identifying them when the list grows. +* [#1044](https://github.com/shlinkio/shlink/issues/1044) Added ability to set names on API keys, which helps to identify them when the list grows. +* [#819](https://github.com/shlinkio/shlink/issues/819) Visits are now always located in real time, even when not using swoole. + + The only side effect is that a GeoLite2 db file is now installed when the docker image starts or during shlink installation or update. + + Also, when using swoole, the file is now updated **after** tracking a visit, which means it will not apply until the next one. ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. From 5d0f306bccffcbeab3d4d9da8015539eaab21620 Mon Sep 17 00:00:00 2001 From: KetchupBomb Date: Sat, 10 Apr 2021 06:50:34 +0000 Subject: [PATCH 40/86] Feature/show API key info in short-url CLI --- .../Command/ShortUrl/ListShortUrlsCommand.php | 105 +++++++++++------- module/Core/src/Entity/ShortUrl.php | 5 + 2 files changed, 72 insertions(+), 38 deletions(-) diff --git a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php index 24689bcb..aa975a40 100644 --- a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php +++ b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php @@ -4,12 +4,15 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\ShortUrl; +use closure; use Shlinkio\Shlink\CLI\Command\Util\AbstractWithDateRangeCommand; use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\CLI\Util\ShlinkTable; use Shlinkio\Shlink\Common\Paginator\Paginator; use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtilsTrait; use Shlinkio\Shlink\Common\Rest\DataTransformerInterface; +use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering; use Shlinkio\Shlink\Core\Model\ShortUrlsParams; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; @@ -19,10 +22,13 @@ use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; use Symfony\Component\Console\Style\SymfonyStyle; +use function array_keys; use function array_pad; +use function count; use function explode; use function Functional\map; -use function implode; +use function is_null; +use function join; use function sprintf; class ListShortUrlsCommand extends AbstractWithDateRangeCommand @@ -30,18 +36,6 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand use PagerfantaUtilsTrait; public const NAME = 'short-url:list'; - private const COLUMNS_TO_SHOW = [ - 'shortCode', - 'title', - 'shortUrl', - 'longUrl', - 'dateCreated', - 'visitsCount', - ]; - private const COLUMNS_TO_SHOW_WITH_TAGS = [ - ...self::COLUMNS_TO_SHOW, - 'tags', - ]; private ShortUrlServiceInterface $shortUrlService; private DataTransformerInterface $transformer; @@ -90,6 +84,18 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand InputOption::VALUE_NONE, 'Whether to display the tags or not.', ) + ->addOption( + 'show-api-key', + 'k', + InputOption::VALUE_NONE, + 'Whether to display the API key from which the URL was generated or not.', + ) + ->addOption( + 'show-api-key-name', + 'm', + InputOption::VALUE_NONE, + 'Whether to display the API key name from which the URL was generated or not.', + ) ->addOption( 'all', 'a', @@ -117,12 +123,38 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand $searchTerm = $this->getOptionWithDeprecatedFallback($input, 'search-term'); $tags = $input->getOption('tags'); $tags = ! empty($tags) ? explode(',', $tags) : []; - $showTags = $this->getOptionWithDeprecatedFallback($input, 'show-tags'); $all = $input->getOption('all'); $startDate = $this->getStartDateOption($input, $output); $endDate = $this->getEndDateOption($input, $output); $orderBy = $this->processOrderBy($input); + + $transformerLookup = fn (string $key): closure + => fn (ShortUrl $shortUrl) + => $this->transformer->transform($shortUrl)[$key]; + + $columnMap = [ + 'Short Code' => $transformerLookup('shortCode'), + 'Title' => $transformerLookup('title'), + 'Short URL' => $transformerLookup('shortUrl'), + 'Long URL' => $transformerLookup('longUrl'), + 'Date created' => $transformerLookup('dateCreated'), + 'Visits count' => $transformerLookup('visitsCount'), + ]; + if ($this->getOptionWithDeprecatedFallback($input, 'show-tags')) { + $columnMap['Tags'] = fn (ShortUrl $shortUrl): string + => join(', ', map($shortUrl->getTags(), fn (Tag $tag): string => (string) $tag)); + } + if ($input->getOption('show-api-key')) { + $columnMap['API Key'] = fn (ShortUrl $shortUrl): string => (string) $shortUrl->authorApiKey(); + } + if ($input->getOption('show-api-key-name')) { + $columnMap['API Key Name'] = fn (ShortUrl $shortUrl): ?string => ! is_null($shortUrl->authorApiKey()) + ? $shortUrl->authorApiKey()->name() + : null; + } + + $data = [ ShortUrlsParamsInputFilter::SEARCH_TERM => $searchTerm, ShortUrlsParamsInputFilter::TAGS => $tags, @@ -137,7 +169,7 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand do { $data[ShortUrlsParamsInputFilter::PAGE] = $page; - $result = $this->renderPage($output, $showTags, ShortUrlsParams::fromRawData($data), $all); + $result = $this->renderPage($output, $columnMap, ShortUrlsParams::fromRawData($data), $all); $page++; $continue = $result->hasNextPage() && $io->confirm( @@ -152,32 +184,29 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand return ExitCodes::EXIT_SUCCESS; } - private function renderPage(OutputInterface $output, bool $showTags, ShortUrlsParams $params, bool $all): Paginator - { - $result = $this->shortUrlService->listShortUrls($params); + private function renderPage( + OutputInterface $output, + array $columnMap, + ShortUrlsParams $params, + bool $all + ): Paginator { + $shortUrls = $this->shortUrlService->listShortUrls($params); - $headers = ['Short code', 'Title', 'Short URL', 'Long URL', 'Date created', 'Visits count']; - if ($showTags) { - $headers[] = 'Tags'; - } + $rows = map($shortUrls, fn (ShortUrl $shortUrl) + => map($columnMap, fn (callable $call) + => $call($shortUrl))); - $rows = []; - foreach ($result as $row) { - $columnsToShow = $showTags ? self::COLUMNS_TO_SHOW_WITH_TAGS : self::COLUMNS_TO_SHOW; - $shortUrl = $this->transformer->transform($row); - if ($showTags) { - $shortUrl['tags'] = implode(', ', $shortUrl['tags']); - } + ShlinkTable::fromOutput($output) + ->render( + array_keys($columnMap), + $rows, + $all ? null : $this->formatCurrentPageMessage( + $shortUrls, + 'Page %s of %s', + ), + ); - $rows[] = map($columnsToShow, fn (string $prop) => $shortUrl[$prop]); - } - - ShlinkTable::fromOutput($output)->render($headers, $rows, $all ? null : $this->formatCurrentPageMessage( - $result, - 'Page %s of %s', - )); - - return $result; + return $shortUrls; } private function processOrderBy(InputInterface $input): ?string diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php index 810281fa..84f215de 100644 --- a/module/Core/src/Entity/ShortUrl.php +++ b/module/Core/src/Entity/ShortUrl.php @@ -132,6 +132,11 @@ class ShortUrl extends AbstractEntity return $this->tags; } + public function authorApiKey(): ?ApiKey + { + return $this->authorApiKey; + } + public function getValidSince(): ?Chronos { return $this->validSince; From 823573cea7565d799c6e9ee879f5c47cb58a3e50 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 10 Apr 2021 10:16:09 +0200 Subject: [PATCH 41/86] Updated PersistenceShortUrlRelationResolver to prevent duplicated tags --- .../PersistenceShortUrlRelationResolver.php | 3 +++ ...ersistenceShortUrlRelationResolverTest.php | 24 +++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php index fd0428bf..1b004a95 100644 --- a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php +++ b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; use function Functional\map; +use function Functional\unique; class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInterface { @@ -42,7 +43,9 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt return new Collections\ArrayCollection(); } + $tags = unique($tags); $repo = $this->em->getRepository(Tag::class); + return new Collections\ArrayCollection(map($tags, function (string $tagName) use ($repo): Tag { $tag = $repo->findOneBy(['name' => $tagName]) ?? new Tag($tagName); $this->em->persist($tag); diff --git a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php index 463ee1ef..187ccbe6 100644 --- a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php +++ b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php @@ -14,6 +14,7 @@ use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Repository\TagRepositoryInterface; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; +use function count; class PersistenceShortUrlRelationResolverTest extends TestCase { @@ -66,10 +67,13 @@ class PersistenceShortUrlRelationResolverTest extends TestCase yield 'found domain' => [new Domain($authority), $authority]; } - /** @test */ - public function findsAndPersistsTagsWrappedIntoCollection(): void + /** + * @test + * @dataProvider provideTags + */ + public function findsAndPersistsTagsWrappedIntoCollection(array $tags, array $expectedTags): void { - $tags = ['foo', 'bar', 'baz']; + $expectedPersistedTags = count($expectedTags); $tagRepo = $this->prophesize(TagRepositoryInterface::class); $findTag = $tagRepo->findOneBy(Argument::type('array'))->will(function (array $args): ?Tag { @@ -81,11 +85,17 @@ class PersistenceShortUrlRelationResolverTest extends TestCase $result = $this->resolver->resolveTags($tags); - self::assertCount(3, $result); - self::assertEquals([new Tag('foo'), new Tag('bar'), new Tag('baz')], $result->toArray()); - $findTag->shouldHaveBeenCalledTimes(3); + self::assertCount($expectedPersistedTags, $result); + self::assertEquals($expectedTags, $result->toArray()); + $findTag->shouldHaveBeenCalledTimes($expectedPersistedTags); $getRepo->shouldHaveBeenCalledOnce(); - $persist->shouldHaveBeenCalledTimes(3); + $persist->shouldHaveBeenCalledTimes($expectedPersistedTags); + } + + public function provideTags(): iterable + { + yield 'no duplicated tags' => [['foo', 'bar', 'baz'], [new Tag('foo'), new Tag('bar'), new Tag('baz')]]; + yield 'duplicated tags' => [['foo', 'bar', 'bar'], [new Tag('foo'), new Tag('bar')]]; } /** @test */ From 28c06de685d274a3809cc1f0450cd1d77a490185 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 10 Apr 2021 11:59:43 +0200 Subject: [PATCH 42/86] Fixed issue when trying to persist several short URLs which include the same new tag/domain at once --- composer.json | 2 +- .../PersistenceShortUrlRelationResolver.php | 30 ++++++++++++- ...ersistenceShortUrlRelationResolverTest.php | 45 +++++++++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index 41885c62..7b72c92a 100644 --- a/composer.json +++ b/composer.json @@ -46,7 +46,7 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.7", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "dev-main#3777189 as 3.7", + "shlinkio/shlink-common": "dev-main#554e370 as 3.7", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", "shlinkio/shlink-importer": "^2.2", diff --git a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php index 1b004a95..eb7fddad 100644 --- a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php +++ b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\ShortUrl\Resolver; use Doctrine\Common\Collections; use Doctrine\Common\Collections\Collection; use Doctrine\ORM\EntityManagerInterface; +use Doctrine\ORM\Events; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; @@ -17,9 +18,15 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt { private EntityManagerInterface $em; + /** @var array */ + private array $memoizedNewDomains = []; + /** @var array */ + private array $memoizedNewTags = []; + public function __construct(EntityManagerInterface $em) { $this->em = $em; + $this->em->getEventManager()->addEventListener(Events::postFlush, $this); } public function resolveDomain(?string $domain): ?Domain @@ -30,7 +37,14 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt /** @var Domain|null $existingDomain */ $existingDomain = $this->em->getRepository(Domain::class)->findOneBy(['authority' => $domain]); - return $existingDomain ?? new Domain($domain); + + // Memoize only new domains, and let doctrine handle objects hydrated from persistence + return $existingDomain ?? $this->memoizeNewDomain($domain); + } + + private function memoizeNewDomain(string $domain): Domain + { + return $this->memoizedNewDomains[$domain] = $this->memoizedNewDomains[$domain] ?? new Domain($domain); } /** @@ -47,10 +61,22 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt $repo = $this->em->getRepository(Tag::class); return new Collections\ArrayCollection(map($tags, function (string $tagName) use ($repo): Tag { - $tag = $repo->findOneBy(['name' => $tagName]) ?? new Tag($tagName); + // Memoize only new tags, and let doctrine handle objects hydrated from persistence + $tag = $repo->findOneBy(['name' => $tagName]) ?? $this->memoizeNewTag($tagName); $this->em->persist($tag); return $tag; })); } + + private function memoizeNewTag(string $tagName): Tag + { + return $this->memoizedNewTags[$tagName] = $this->memoizedNewTags[$tagName] ?? new Tag($tagName); + } + + public function postFlush(): void + { + $this->memoizedNewDomains = []; + $this->memoizedNewTags = []; + } } diff --git a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php index 187ccbe6..8660099c 100644 --- a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php +++ b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\ShortUrl\Resolver; +use Doctrine\Common\EventManager; use Doctrine\ORM\EntityManagerInterface; use Doctrine\Persistence\ObjectRepository; use PHPUnit\Framework\TestCase; @@ -14,6 +15,7 @@ use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Repository\TagRepositoryInterface; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; + use function count; class PersistenceShortUrlRelationResolverTest extends TestCase @@ -26,6 +28,8 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function setUp(): void { $this->em = $this->prophesize(EntityManagerInterface::class); + $this->em->getEventManager()->willReturn(new EventManager()); + $this->resolver = new PersistenceShortUrlRelationResolver($this->em->reveal()); } @@ -113,4 +117,45 @@ class PersistenceShortUrlRelationResolverTest extends TestCase $getRepo->shouldNotHaveBeenCalled(); $persist->shouldNotHaveBeenCalled(); } + + /** @test */ + public function newDomainsAreMemoizedUntilStateIsCleared(): void + { + $repo = $this->prophesize(ObjectRepository::class); + $repo->findOneBy(Argument::type('array'))->willReturn(null); + $this->em->getRepository(Domain::class)->willReturn($repo->reveal()); + + $authority = 'foo.com'; + $domain1 = $this->resolver->resolveDomain($authority); + $domain2 = $this->resolver->resolveDomain($authority); + + self::assertSame($domain1, $domain2); + + $this->resolver->postFlush(); + $domain3 = $this->resolver->resolveDomain($authority); + + self::assertNotSame($domain1, $domain3); + } + + /** @test */ + public function newTagsAreMemoizedUntilStateIsCleared(): void + { + $tagRepo = $this->prophesize(TagRepositoryInterface::class); + $tagRepo->findOneBy(Argument::type('array'))->willReturn(null); + $this->em->getRepository(Tag::class)->willReturn($tagRepo->reveal()); + $this->em->persist(Argument::type(Tag::class))->will(function (): void { + }); + + $tags = ['foo', 'bar']; + [$foo1, $bar1] = $this->resolver->resolveTags($tags); + [$foo2, $bar2] = $this->resolver->resolveTags($tags); + + self::assertSame($foo1, $foo2); + self::assertSame($bar1, $bar2); + + $this->resolver->postFlush(); + [$foo3, $bar3] = $this->resolver->resolveTags($tags); + self::assertNotSame($foo1, $foo3); + self::assertNotSame($bar1, $bar3); + } } From 6387e50276a4cb233e961ea88979fbe8ac8058e0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 10 Apr 2021 12:03:40 +0200 Subject: [PATCH 43/86] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c88f036..bb0b6bb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this ### Fixed * [#1041](https://github.com/shlinkio/shlink/issues/1041) Ensured the default value for the version while building the docker image is `latest`. +* [#1067](https://github.com/shlinkio/shlink/issues/1067) Fixed exception when persisting multiple short URLs in one batch which include the same new tags/domains. This can potentially happen when importing URLs. ## [2.6.2] - 2021-03-12 From a896fbbb9081b1ffa89f1568698f476eed306d3c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 11 Apr 2021 10:50:35 +0200 Subject: [PATCH 44/86] Fixed coding styles --- .../Command/ShortUrl/ListShortUrlsCommand.php | 58 ++++++++----------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php index aa975a40..69fc2556 100644 --- a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php +++ b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php @@ -4,7 +4,6 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\ShortUrl; -use closure; use Shlinkio\Shlink\CLI\Command\Util\AbstractWithDateRangeCommand; use Shlinkio\Shlink\CLI\Util\ExitCodes; use Shlinkio\Shlink\CLI\Util\ShlinkTable; @@ -24,11 +23,9 @@ use Symfony\Component\Console\Style\SymfonyStyle; use function array_keys; use function array_pad; -use function count; use function explode; use function Functional\map; -use function is_null; -use function join; +use function implode; use function sprintf; class ListShortUrlsCommand extends AbstractWithDateRangeCommand @@ -128,32 +125,33 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand $endDate = $this->getEndDateOption($input, $output); $orderBy = $this->processOrderBy($input); - - $transformerLookup = fn (string $key): closure - => fn (ShortUrl $shortUrl) - => $this->transformer->transform($shortUrl)[$key]; + $transformerLookup = fn (string $key): callable => + fn (ShortUrl $shortUrl) => $this->transformer->transform($shortUrl)[$key]; $columnMap = [ - 'Short Code' => $transformerLookup('shortCode'), - 'Title' => $transformerLookup('title'), - 'Short URL' => $transformerLookup('shortUrl'), - 'Long URL' => $transformerLookup('longUrl'), - 'Date created' => $transformerLookup('dateCreated'), - 'Visits count' => $transformerLookup('visitsCount'), + 'Short Code' => $transformerLookup('shortCode'), + 'Title' => $transformerLookup('title'), + 'Short URL' => $transformerLookup('shortUrl'), + 'Long URL' => $transformerLookup('longUrl'), + 'Date created' => $transformerLookup('dateCreated'), + 'Visits count' => $transformerLookup('visitsCount'), ]; if ($this->getOptionWithDeprecatedFallback($input, 'show-tags')) { - $columnMap['Tags'] = fn (ShortUrl $shortUrl): string - => join(', ', map($shortUrl->getTags(), fn (Tag $tag): string => (string) $tag)); + $columnMap['Tags'] = static fn (ShortUrl $shortUrl): string => implode( + ', ', + map($shortUrl->getTags(), fn (Tag $tag): string => (string) $tag), + ); } if ($input->getOption('show-api-key')) { - $columnMap['API Key'] = fn (ShortUrl $shortUrl): string => (string) $shortUrl->authorApiKey(); + $columnMap['API Key'] = static fn (ShortUrl $shortUrl): string => (string) $shortUrl->authorApiKey(); } if ($input->getOption('show-api-key-name')) { - $columnMap['API Key Name'] = fn (ShortUrl $shortUrl): ?string => ! is_null($shortUrl->authorApiKey()) - ? $shortUrl->authorApiKey()->name() - : null; - } + $columnMap['API Key Name'] = static function (ShortUrl $shortUrl): ?string { + $apiKey = $shortUrl->authorApiKey(); + return $apiKey !== null ? $apiKey->name() : null; + }; + } $data = [ ShortUrlsParamsInputFilter::SEARCH_TERM => $searchTerm, @@ -192,19 +190,13 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand ): Paginator { $shortUrls = $this->shortUrlService->listShortUrls($params); - $rows = map($shortUrls, fn (ShortUrl $shortUrl) - => map($columnMap, fn (callable $call) - => $call($shortUrl))); + $rows = map($shortUrls, fn (ShortUrl $shortUrl) => map($columnMap, fn (callable $call) => $call($shortUrl))); - ShlinkTable::fromOutput($output) - ->render( - array_keys($columnMap), - $rows, - $all ? null : $this->formatCurrentPageMessage( - $shortUrls, - 'Page %s of %s', - ), - ); + ShlinkTable::fromOutput($output)->render( + array_keys($columnMap), + $rows, + $all ? null : $this->formatCurrentPageMessage($shortUrls, 'Page %s of %s'), + ); return $shortUrls; } From 5ddac7866b4ce1e673e1b142a59833fd55aa10e7 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 11 Apr 2021 11:06:29 +0200 Subject: [PATCH 45/86] Ensured short URL transformation happens only once per short URL when listing from CLI --- .../Command/ShortUrl/ListShortUrlsCommand.php | 32 +++++++++---------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php index 69fc2556..8aa6bb1d 100644 --- a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php +++ b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php @@ -11,7 +11,6 @@ use Shlinkio\Shlink\Common\Paginator\Paginator; use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtilsTrait; use Shlinkio\Shlink\Common\Rest\DataTransformerInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; -use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering; use Shlinkio\Shlink\Core\Model\ShortUrlsParams; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; @@ -125,28 +124,24 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand $endDate = $this->getEndDateOption($input, $output); $orderBy = $this->processOrderBy($input); - $transformerLookup = fn (string $key): callable => - fn (ShortUrl $shortUrl) => $this->transformer->transform($shortUrl)[$key]; - + $pickProp = static fn (string $prop): callable => static fn (array $shortUrl) => $shortUrl[$prop]; $columnMap = [ - 'Short Code' => $transformerLookup('shortCode'), - 'Title' => $transformerLookup('title'), - 'Short URL' => $transformerLookup('shortUrl'), - 'Long URL' => $transformerLookup('longUrl'), - 'Date created' => $transformerLookup('dateCreated'), - 'Visits count' => $transformerLookup('visitsCount'), + 'Short Code' => $pickProp('shortCode'), + 'Title' => $pickProp('title'), + 'Short URL' => $pickProp('shortUrl'), + 'Long URL' => $pickProp('longUrl'), + 'Date created' => $pickProp('dateCreated'), + 'Visits count' => $pickProp('visitsCount'), ]; if ($this->getOptionWithDeprecatedFallback($input, 'show-tags')) { - $columnMap['Tags'] = static fn (ShortUrl $shortUrl): string => implode( - ', ', - map($shortUrl->getTags(), fn (Tag $tag): string => (string) $tag), - ); + $columnMap['Tags'] = static fn (array $shortUrl): string => implode(', ', $shortUrl['tags']); } if ($input->getOption('show-api-key')) { - $columnMap['API Key'] = static fn (ShortUrl $shortUrl): string => (string) $shortUrl->authorApiKey(); + $columnMap['API Key'] = static fn (array $_, ShortUrl $shortUrl): string => + (string) $shortUrl->authorApiKey(); } if ($input->getOption('show-api-key-name')) { - $columnMap['API Key Name'] = static function (ShortUrl $shortUrl): ?string { + $columnMap['API Key Name'] = static function (array $_, ShortUrl $shortUrl): ?string { $apiKey = $shortUrl->authorApiKey(); return $apiKey !== null ? $apiKey->name() : null; @@ -190,7 +185,10 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand ): Paginator { $shortUrls = $this->shortUrlService->listShortUrls($params); - $rows = map($shortUrls, fn (ShortUrl $shortUrl) => map($columnMap, fn (callable $call) => $call($shortUrl))); + $rows = map($shortUrls, function (ShortUrl $shortUrl) use ($columnMap) { + $rawShortUrl = $this->transformer->transform($shortUrl); + return map($columnMap, fn (callable $call) => $call($rawShortUrl, $shortUrl)); + }); ShlinkTable::fromOutput($output)->render( array_keys($columnMap), From 334d95c843de9c70e84512e12234e5312acd93ef Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 11 Apr 2021 11:29:42 +0200 Subject: [PATCH 46/86] Improved test covering ListSHortUrlsCommand with optional tags --- .../Command/ShortUrl/ListShortUrlsCommand.php | 64 +++++++++------- .../ShortUrl/ListShortUrlsCommandTest.php | 76 +++++++++++++++++-- 2 files changed, 105 insertions(+), 35 deletions(-) diff --git a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php index 8aa6bb1d..0d637f5f 100644 --- a/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php +++ b/module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php @@ -123,30 +123,7 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand $startDate = $this->getStartDateOption($input, $output); $endDate = $this->getEndDateOption($input, $output); $orderBy = $this->processOrderBy($input); - - $pickProp = static fn (string $prop): callable => static fn (array $shortUrl) => $shortUrl[$prop]; - $columnMap = [ - 'Short Code' => $pickProp('shortCode'), - 'Title' => $pickProp('title'), - 'Short URL' => $pickProp('shortUrl'), - 'Long URL' => $pickProp('longUrl'), - 'Date created' => $pickProp('dateCreated'), - 'Visits count' => $pickProp('visitsCount'), - ]; - if ($this->getOptionWithDeprecatedFallback($input, 'show-tags')) { - $columnMap['Tags'] = static fn (array $shortUrl): string => implode(', ', $shortUrl['tags']); - } - if ($input->getOption('show-api-key')) { - $columnMap['API Key'] = static fn (array $_, ShortUrl $shortUrl): string => - (string) $shortUrl->authorApiKey(); - } - if ($input->getOption('show-api-key-name')) { - $columnMap['API Key Name'] = static function (array $_, ShortUrl $shortUrl): ?string { - $apiKey = $shortUrl->authorApiKey(); - - return $apiKey !== null ? $apiKey->name() : null; - }; - } + $columnsMap = $this->resolveColumnsMap($input); $data = [ ShortUrlsParamsInputFilter::SEARCH_TERM => $searchTerm, @@ -162,7 +139,7 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand do { $data[ShortUrlsParamsInputFilter::PAGE] = $page; - $result = $this->renderPage($output, $columnMap, ShortUrlsParams::fromRawData($data), $all); + $result = $this->renderPage($output, $columnsMap, ShortUrlsParams::fromRawData($data), $all); $page++; $continue = $result->hasNextPage() && $io->confirm( @@ -179,19 +156,19 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand private function renderPage( OutputInterface $output, - array $columnMap, + array $columnsMap, ShortUrlsParams $params, bool $all ): Paginator { $shortUrls = $this->shortUrlService->listShortUrls($params); - $rows = map($shortUrls, function (ShortUrl $shortUrl) use ($columnMap) { + $rows = map($shortUrls, function (ShortUrl $shortUrl) use ($columnsMap) { $rawShortUrl = $this->transformer->transform($shortUrl); - return map($columnMap, fn (callable $call) => $call($rawShortUrl, $shortUrl)); + return map($columnsMap, fn (callable $call) => $call($rawShortUrl, $shortUrl)); }); ShlinkTable::fromOutput($output)->render( - array_keys($columnMap), + array_keys($columnsMap), $rows, $all ? null : $this->formatCurrentPageMessage($shortUrls, 'Page %s of %s'), ); @@ -209,4 +186,33 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand [$field, $dir] = array_pad(explode(',', $orderBy), 2, null); return $dir === null ? $field : sprintf('%s-%s', $field, $dir); } + + private function resolveColumnsMap(InputInterface $input): array + { + $pickProp = static fn (string $prop): callable => static fn (array $shortUrl) => $shortUrl[$prop]; + $columnsMap = [ + 'Short Code' => $pickProp('shortCode'), + 'Title' => $pickProp('title'), + 'Short URL' => $pickProp('shortUrl'), + 'Long URL' => $pickProp('longUrl'), + 'Date created' => $pickProp('dateCreated'), + 'Visits count' => $pickProp('visitsCount'), + ]; + if ($this->getOptionWithDeprecatedFallback($input, 'show-tags')) { + $columnsMap['Tags'] = static fn (array $shortUrl): string => implode(', ', $shortUrl['tags']); + } + if ($input->getOption('show-api-key')) { + $columnsMap['API Key'] = static fn (array $_, ShortUrl $shortUrl): string => + (string) $shortUrl->authorApiKey(); + } + if ($input->getOption('show-api-key-name')) { + $columnsMap['API Key Name'] = static function (array $_, ShortUrl $shortUrl): ?string { + $apiKey = $shortUrl->authorApiKey(); + + return $apiKey !== null ? $apiKey->name() : null; + }; + } + + return $columnsMap; + } } diff --git a/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php b/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php index 08519b62..6f7b11a6 100644 --- a/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php @@ -12,13 +12,17 @@ use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\CLI\Command\ShortUrl\ListShortUrlsCommand; use Shlinkio\Shlink\Common\Paginator\Paginator; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\ShortUrlsParams; use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifier; use Shlinkio\Shlink\Core\ShortUrl\Transformer\ShortUrlDataTransformer; +use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; +use Shlinkio\Shlink\Rest\Entity\ApiKey; use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait; use Symfony\Component\Console\Tester\CommandTester; +use function count; use function explode; class ListShortUrlsCommandTest extends TestCase @@ -98,17 +102,77 @@ class ListShortUrlsCommandTest extends TestCase $this->commandTester->execute(['--page' => $page]); } - /** @test */ - public function ifTagsFlagIsProvidedTagsColumnIsIncluded(): void - { + /** + * @test + * @dataProvider provideOptionalFlags + */ + public function provideOptionalFlagsMakesNewColumnsToBeIncluded( + array $input, + array $expectedContents, + array $notExpectedContents, + ApiKey $apiKey + ): void { $this->shortUrlService->listShortUrls(ShortUrlsParams::emptyInstance()) - ->willReturn(new Paginator(new ArrayAdapter([]))) + ->willReturn(new Paginator(new ArrayAdapter([ + ShortUrl::fromMeta(ShortUrlMeta::fromRawData([ + 'longUrl' => 'foo.com', + 'tags' => ['foo', 'bar', 'baz'], + 'apiKey' => $apiKey, + ])), + ]))) ->shouldBeCalledOnce(); $this->commandTester->setInputs(['y']); - $this->commandTester->execute(['--show-tags' => true]); + $this->commandTester->execute($input); $output = $this->commandTester->getDisplay(); - self::assertStringContainsString('Tags', $output); + + if (count($expectedContents) === 0 && count($notExpectedContents) === 0) { + self::fail('No expectations were run'); + } + + foreach ($expectedContents as $column) { + self::assertStringContainsString($column, $output); + } + foreach ($notExpectedContents as $column) { + self::assertStringNotContainsString($column, $output); + } + } + + public function provideOptionalFlags(): iterable + { + $apiKey = ApiKey::fromMeta(ApiKeyMeta::withName('my api key')); + $key = $apiKey->toString(); + + yield 'tags only' => [ + ['--show-tags' => true], + ['| Tags ', '| foo, bar, baz'], + ['| API Key ', '| API Key Name |', $key, '| my api key'], + $apiKey, + ]; + yield 'api key only' => [ + ['--show-api-key' => true], + ['| API Key ', $key], + ['| Tags ', '| foo, bar, baz', '| API Key Name |', '| my api key'], + $apiKey, + ]; + yield 'api key name only' => [ + ['--show-api-key-name' => true], + ['| API Key Name |', '| my api key'], + ['| Tags ', '| foo, bar, baz', '| API Key ', $key], + $apiKey, + ]; + yield 'tags and api key' => [ + ['--show-tags' => true, '--show-api-key' => true], + ['| API Key ', '| Tags ', '| foo, bar, baz', $key], + ['| API Key Name |', '| my api key'], + $apiKey, + ]; + yield 'all' => [ + ['--show-tags' => true, '--show-api-key' => true, '--show-api-key-name' => true], + ['| API Key ', '| Tags ', '| API Key Name |', '| foo, bar, baz', $key, '| my api key'], + [], + $apiKey, + ]; } /** From d751df70fdca8230c71ccac35e3be0a8f2e09c33 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 11 Apr 2021 11:30:43 +0200 Subject: [PATCH 47/86] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c88f036..5c0b0075 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this Also, when using swoole, the file is now updated **after** tracking a visit, which means it will not apply until the next one. +* [#1059](https://github.com/shlinkio/shlink/issues/1059) Added ability to optionally display author API key and its name when listing short URLs from the command line. + ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. * [#1039](https://github.com/shlinkio/shlink/issues/1039) Updated to `endroid/qr-code` 4.0. From 743bb7a6ee2eed56c9cbbfd84d67583859230a05 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 10 Apr 2021 10:04:53 +0200 Subject: [PATCH 48/86] Updated ShortUrl importing to take metadata into account --- composer.json | 2 +- module/Core/src/Entity/ShortUrl.php | 14 +++++++++++++- .../Core/src/Importer/ImportedLinksProcessor.php | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index 7b72c92a..aadb06ba 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "shlinkio/shlink-common": "dev-main#554e370 as 3.7", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", - "shlinkio/shlink-importer": "^2.2", + "shlinkio/shlink-importer": "dev-main#d7e2762 as 2.3", "shlinkio/shlink-installer": "dev-develop#aa50ea9 as 5.5", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php index 84f215de..b25b90e5 100644 --- a/module/Core/src/Entity/ShortUrl.php +++ b/module/Core/src/Entity/ShortUrl.php @@ -86,17 +86,29 @@ class ShortUrl extends AbstractEntity ?ShortUrlRelationResolverInterface $relationResolver = null ): self { $meta = [ + ShortUrlInputFilter::VALIDATE_URL => false, ShortUrlInputFilter::LONG_URL => $url->longUrl(), ShortUrlInputFilter::DOMAIN => $url->domain(), ShortUrlInputFilter::TAGS => $url->tags(), ShortUrlInputFilter::TITLE => $url->title(), - ShortUrlInputFilter::VALIDATE_URL => false, + ShortUrlInputFilter::MAX_VISITS => $url->meta()->maxVisits(), ]; if ($importShortCode) { $meta[ShortUrlInputFilter::CUSTOM_SLUG] = $url->shortCode(); } $instance = self::fromMeta(ShortUrlMeta::fromRawData($meta), $relationResolver); + + $validSince = $url->meta()->validSince(); + if ($validSince !== null) { + $instance->validSince = Chronos::instance($validSince); + } + + $validUntil = $url->meta()->validUntil(); + if ($validUntil !== null) { + $instance->validUntil = Chronos::instance($validUntil); + } + $instance->importSource = $url->source(); $instance->importOriginalShortCode = $url->shortCode(); $instance->dateCreated = Chronos::instance($url->createdAt()); diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index 2b5cde17..e88020c7 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -51,17 +51,24 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface // Skip already imported URLs if ($shortUrlRepo->importedUrlExists($url)) { + // TODO If the URL exists, allow to merge visits instead of just skipping completely $io->text(sprintf('%s: Skipped', $longUrl)); continue; } $shortUrl = ShortUrl::fromImport($url, $importShortCodes, $this->relationResolver); if (! $this->handleShortCodeUniqueness($url, $shortUrl, $io, $importShortCodes)) { + $io->text(sprintf('%s: Skipped', $longUrl)); continue; } $this->em->persist($shortUrl); $io->text(sprintf('%s: Imported', $longUrl)); + + // Process only missing visits when possible + if ($url->visitsCount() !== null) { + + } } } @@ -84,7 +91,6 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface ), ['Generate new short-code', 'Skip'], 1); if ($action === 'Skip') { - $io->text(sprintf('%s: Skipped', $longUrl)); return false; } From e23cd6a85630df5bdec49c0ccfa247dfcad4f398 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 10 Apr 2021 13:47:51 +0200 Subject: [PATCH 49/86] Removed old MySQL connection options --- config/autoload/entity-manager.local.php.dist | 9 --------- config/test/test_config.global.php | 4 ---- docker/config/shlink_in_docker.local.php | 7 ------- 3 files changed, 20 deletions(-) diff --git a/config/autoload/entity-manager.local.php.dist b/config/autoload/entity-manager.local.php.dist index 1faed328..f3cca338 100644 --- a/config/autoload/entity-manager.local.php.dist +++ b/config/autoload/entity-manager.local.php.dist @@ -2,14 +2,6 @@ declare(strict_types=1); -use function Shlinkio\Shlink\Common\env; - -// When running tests, any mysql-specific option can interfere with other drivers -$driverOptions = env('APP_ENV') === 'test' ? [] : [ - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', - PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, -]; - return [ 'entity_manager' => [ @@ -18,7 +10,6 @@ return [ 'password' => 'root', 'driver' => 'pdo_mysql', 'host' => 'shlink_db', - 'driverOptions' => $driverOptions, ], ], diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index c6d90e39..c8e9510b 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -55,10 +55,6 @@ $buildDbConnection = function (): array { 'password' => 'root', 'dbname' => 'shlink_test', 'charset' => 'utf8', - 'driverOptions' => [ - PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8', - PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true, - ], ], 'postgres' => [ 'driver' => 'pdo_pgsql', diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index d022f79d..acecda14 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -42,12 +42,6 @@ $helper = new class { ]; } - $driverOptions = ! $isMysql ? [] : [ - // 1002 -> PDO::MYSQL_ATTR_INIT_COMMAND - 1002 => 'SET NAMES utf8', - // 1000 -> PDO::MYSQL_ATTR_USE_BUFFERED_QUERY - 1000 => true, - ]; return [ 'driver' => self::DB_DRIVERS_MAP[$driver], 'dbname' => env('DB_NAME', 'shlink'), @@ -55,7 +49,6 @@ $helper = new class { 'password' => env('DB_PASSWORD'), 'host' => env('DB_HOST', $driver === 'postgres' ? env('DB_UNIX_SOCKET') : null), 'port' => env('DB_PORT', self::DB_PORTS_MAP[$driver]), - 'driverOptions' => $driverOptions, 'unix_socket' => $isMysql ? env('DB_UNIX_SOCKET') : null, ]; } From 1efa9735076f4cba2fe5f32f9e00c68b71bd4e41 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 10 Apr 2021 23:24:01 +0200 Subject: [PATCH 50/86] Updated ImportedLinksProcessor to support importing visits if provided --- composer.json | 2 +- config/test/test_config.global.php | 1 - .../Command/ShortUrl/GetVisitsCommandTest.php | 2 +- .../Command/Visit/LocateVisitsCommandTest.php | 6 +- module/Core/src/Entity/Visit.php | 78 +++++++++++++------ module/Core/src/Entity/VisitLocation.php | 69 ++++++++++------ .../Core/src/EventDispatcher/LocateVisit.php | 2 +- .../src/Importer/ImportedLinksProcessor.php | 29 ++++--- module/Core/src/Visit/VisitLocator.php | 3 +- .../Repository/VisitRepositoryTest.php | 2 +- module/Core/test/Entity/VisitLocationTest.php | 2 +- .../test/EventDispatcher/LocateVisitTest.php | 4 +- .../OrphanVisitDataTransformerTest.php | 2 +- 13 files changed, 132 insertions(+), 70 deletions(-) diff --git a/composer.json b/composer.json index aadb06ba..9782b137 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "shlinkio/shlink-common": "dev-main#554e370 as 3.7", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", - "shlinkio/shlink-importer": "dev-main#d7e2762 as 2.3", + "shlinkio/shlink-importer": "dev-main#174e352 as 2.3", "shlinkio/shlink-installer": "dev-develop#aa50ea9 as 5.5", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", diff --git a/config/test/test_config.global.php b/config/test/test_config.global.php index c8e9510b..c2375cba 100644 --- a/config/test/test_config.global.php +++ b/config/test/test_config.global.php @@ -11,7 +11,6 @@ use Laminas\ServiceManager\Factory\InvokableFactory; use Laminas\Stdlib\Glob; use Monolog\Handler\StreamHandler; use Monolog\Logger; -use PDO; use PHPUnit\Runner\Version; use SebastianBergmann\CodeCoverage\CodeCoverage; use SebastianBergmann\CodeCoverage\Driver\Selector; diff --git a/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php b/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php index 044866ed..b9262217 100644 --- a/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php @@ -103,7 +103,7 @@ class GetVisitsCommandTest extends TestCase $this->visitsHelper->visitsForShortUrl(new ShortUrlIdentifier($shortCode), Argument::any())->willReturn( new Paginator(new ArrayAdapter([ Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('bar', 'foo', '', ''))->locate( - new VisitLocation(new Location('', 'Spain', '', '', 0, 0, '')), + VisitLocation::fromGeolocation(new Location('', 'Spain', '', '', 0, 0, '')), ), ])), )->shouldBeCalledOnce(); diff --git a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php index 6e4213f6..74148f9c 100644 --- a/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/LocateVisitsCommandTest.php @@ -76,7 +76,7 @@ class LocateVisitsCommandTest extends TestCase array $args ): void { $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', '1.2.3.4', '')); - $location = new VisitLocation(Location::emptyInstance()); + $location = VisitLocation::fromGeolocation(Location::emptyInstance()); $mockMethodBehavior = $this->invokeHelperMethods($visit, $location); $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will($mockMethodBehavior); @@ -120,7 +120,7 @@ class LocateVisitsCommandTest extends TestCase public function localhostAndEmptyAddressesAreIgnored(?string $address, string $message): void { $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $address, '')); - $location = new VisitLocation(Location::emptyInstance()); + $location = VisitLocation::fromGeolocation(Location::emptyInstance()); $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will( $this->invokeHelperMethods($visit, $location), @@ -153,7 +153,7 @@ class LocateVisitsCommandTest extends TestCase public function errorWhileLocatingIpIsDisplayed(): void { $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', '1.2.3.4', '')); - $location = new VisitLocation(Location::emptyInstance()); + $location = VisitLocation::fromGeolocation(Location::emptyInstance()); $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will( $this->invokeHelperMethods($visit, $location), diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index 61739dec..e2d5f09e 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Common\Exception\InvalidArgumentException; use Shlinkio\Shlink\Common\Util\IpAddress; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Visit\Model\VisitLocationInterface; +use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit; class Visit extends AbstractEntity implements JsonSerializable { @@ -21,22 +22,71 @@ class Visit extends AbstractEntity implements JsonSerializable private string $referer; private Chronos $date; - private ?string $remoteAddr; - private ?string $visitedUrl; + private ?string $remoteAddr = null; + private ?string $visitedUrl = null; private string $userAgent; private string $type; private ?ShortUrl $shortUrl; private ?VisitLocation $visitLocation = null; - private function __construct(?ShortUrl $shortUrl, Visitor $visitor, string $type, bool $anonymize = true) + private function __construct(?ShortUrl $shortUrl, string $type) { $this->shortUrl = $shortUrl; $this->date = Chronos::now(); + $this->type = $type; + } + + public static function forValidShortUrl(ShortUrl $shortUrl, Visitor $visitor, bool $anonymize = true): self + { + $instance = new self($shortUrl, self::TYPE_VALID_SHORT_URL); + $instance->hydrateFromVisitor($visitor, $anonymize); + + return $instance; + } + + public static function fromImport(ImportedShlinkVisit $importedVisit, ShortUrl $shortUrl): self + { + $instance = new self($shortUrl, self::TYPE_VALID_SHORT_URL); + $instance->userAgent = $importedVisit->userAgent(); + $instance->referer = $importedVisit->referer(); + $instance->date = Chronos::instance($importedVisit->date()); + + $importedLocation = $importedVisit->location(); + $instance->visitLocation = $importedLocation !== null ? VisitLocation::fromImport($importedLocation) : null; + + return $instance; + } + + public static function forBasePath(Visitor $visitor, bool $anonymize = true): self + { + $instance = new self(null, self::TYPE_BASE_URL); + $instance->hydrateFromVisitor($visitor, $anonymize); + + return $instance; + } + + public static function forInvalidShortUrl(Visitor $visitor, bool $anonymize = true): self + { + $instance = new self(null, self::TYPE_INVALID_SHORT_URL); + $instance->hydrateFromVisitor($visitor, $anonymize); + + return $instance; + } + + public static function forRegularNotFound(Visitor $visitor, bool $anonymize = true): self + { + $instance = new self(null, self::TYPE_REGULAR_404); + $instance->hydrateFromVisitor($visitor, $anonymize); + + return $instance; + } + + private function hydrateFromVisitor(Visitor $visitor, bool $anonymize = true): void + { $this->userAgent = $visitor->getUserAgent(); $this->referer = $visitor->getReferer(); $this->remoteAddr = $this->processAddress($anonymize, $visitor->getRemoteAddress()); $this->visitedUrl = $visitor->getVisitedUrl(); - $this->type = $type; } private function processAddress(bool $anonymize, ?string $address): ?string @@ -53,26 +103,6 @@ class Visit extends AbstractEntity implements JsonSerializable } } - public static function forValidShortUrl(ShortUrl $shortUrl, Visitor $visitor, bool $anonymize = true): self - { - return new self($shortUrl, $visitor, self::TYPE_VALID_SHORT_URL, $anonymize); - } - - public static function forBasePath(Visitor $visitor, bool $anonymize = true): self - { - return new self(null, $visitor, self::TYPE_BASE_URL, $anonymize); - } - - public static function forInvalidShortUrl(Visitor $visitor, bool $anonymize = true): self - { - return new self(null, $visitor, self::TYPE_INVALID_SHORT_URL, $anonymize); - } - - public static function forRegularNotFound(Visitor $visitor, bool $anonymize = true): self - { - return new self(null, $visitor, self::TYPE_REGULAR_404, $anonymize); - } - public function getRemoteAddr(): ?string { return $this->remoteAddr; diff --git a/module/Core/src/Entity/VisitLocation.php b/module/Core/src/Entity/VisitLocation.php index ef545bba..594126a7 100644 --- a/module/Core/src/Entity/VisitLocation.php +++ b/module/Core/src/Entity/VisitLocation.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Entity; use Shlinkio\Shlink\Common\Entity\AbstractEntity; use Shlinkio\Shlink\Core\Visit\Model\VisitLocationInterface; +use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisitLocation; use Shlinkio\Shlink\IpGeolocation\Model\Location; class VisitLocation extends AbstractEntity implements VisitLocationInterface @@ -19,9 +20,53 @@ class VisitLocation extends AbstractEntity implements VisitLocationInterface private string $timezone; private bool $isEmpty; - public function __construct(Location $location) + private function __construct() { - $this->exchangeLocationInfo($location); + } + + public static function fromGeolocation(Location $location): self + { + $instance = new self(); + + $instance->countryCode = $location->countryCode(); + $instance->countryName = $location->countryName(); + $instance->regionName = $location->regionName(); + $instance->cityName = $location->city(); + $instance->latitude = $location->latitude(); + $instance->longitude = $location->longitude(); + $instance->timezone = $location->timeZone(); + $instance->computeIsEmpty(); + + return $instance; + } + + public static function fromImport(ImportedShlinkVisitLocation $location): self + { + $instance = new self(); + + $instance->countryCode = $location->countryCode(); + $instance->countryName = $location->countryName(); + $instance->regionName = $location->regionName(); + $instance->cityName = $location->cityName(); + $instance->latitude = $location->latitude(); + $instance->longitude = $location->longitude(); + $instance->timezone = $location->timeZone(); + $instance->computeIsEmpty(); + + return $instance; + } + + private function computeIsEmpty(): void + { + $this->isEmpty = ( + $this->countryCode === '' && + $this->countryName === '' && + $this->regionName === '' && + $this->cityName === '' && + $this->latitude === 0.0 && + $this->longitude === 0.0 && + $this->timezone === '' + ); } public function getCountryName(): string @@ -49,26 +94,6 @@ class VisitLocation extends AbstractEntity implements VisitLocationInterface return $this->isEmpty; } - private function exchangeLocationInfo(Location $info): void - { - $this->countryCode = $info->countryCode(); - $this->countryName = $info->countryName(); - $this->regionName = $info->regionName(); - $this->cityName = $info->city(); - $this->latitude = $info->latitude(); - $this->longitude = $info->longitude(); - $this->timezone = $info->timeZone(); - $this->isEmpty = ( - $this->countryCode === '' && - $this->countryName === '' && - $this->regionName === '' && - $this->cityName === '' && - $this->latitude === 0.0 && - $this->longitude === 0.0 && - $this->timezone === '' - ); - } - public function jsonSerialize(): array { return [ diff --git a/module/Core/src/EventDispatcher/LocateVisit.php b/module/Core/src/EventDispatcher/LocateVisit.php index 80dc18eb..0150c529 100644 --- a/module/Core/src/EventDispatcher/LocateVisit.php +++ b/module/Core/src/EventDispatcher/LocateVisit.php @@ -71,7 +71,7 @@ class LocateVisit try { $location = $isLocatable ? $this->ipLocationResolver->resolveIpLocation($addr) : Location::emptyInstance(); - $visit->locate(new VisitLocation($location)); + $visit->locate(VisitLocation::fromGeolocation($location)); $this->em->flush(); } catch (WrongIpException $e) { $this->logger->warning( diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index e88020c7..4a5562bc 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Importer; use Doctrine\ORM\EntityManagerInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortCodeHelperInterface; use Shlinkio\Shlink\Core\ShortUrl\Resolver\ShortUrlRelationResolverInterface; @@ -45,30 +46,38 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface $importShortCodes = $params['import_short_codes']; $iterable = $this->batchHelper->wrapIterable($shlinkUrls, 100); - /** @var ImportedShlinkUrl $url */ - foreach ($iterable as $url) { - $longUrl = $url->longUrl(); + /** @var ImportedShlinkUrl $importedUrl */ + foreach ($iterable as $importedUrl) { + $longUrl = $importedUrl->longUrl(); // Skip already imported URLs - if ($shortUrlRepo->importedUrlExists($url)) { + if ($shortUrlRepo->importedUrlExists($importedUrl)) { // TODO If the URL exists, allow to merge visits instead of just skipping completely $io->text(sprintf('%s: Skipped', $longUrl)); continue; } - $shortUrl = ShortUrl::fromImport($url, $importShortCodes, $this->relationResolver); - if (! $this->handleShortCodeUniqueness($url, $shortUrl, $io, $importShortCodes)) { + $shortUrl = ShortUrl::fromImport($importedUrl, $importShortCodes, $this->relationResolver); + if (! $this->handleShortCodeUniqueness($importedUrl, $shortUrl, $io, $importShortCodes)) { $io->text(sprintf('%s: Skipped', $longUrl)); continue; } $this->em->persist($shortUrl); - $io->text(sprintf('%s: Imported', $longUrl)); - - // Process only missing visits when possible - if ($url->visitsCount() !== null) { + // TODO Process only missing visits when possible: $importedUrl->visitsCount(); + // TODO Make importing visits optional based on params + $importedVisits = 0; + foreach ($importedUrl->visits() as $importedVisit) { + $this->em->persist(Visit::fromImport($importedVisit, $shortUrl)); + $importedVisits++; } + + $io->text( + $importedVisits === 0 + ? sprintf('%s: Imported', $longUrl) + : sprintf('%s: Imported with %s visits', $longUrl, $importedVisits), + ); } } diff --git a/module/Core/src/Visit/VisitLocator.php b/module/Core/src/Visit/VisitLocator.php index 46a30559..d7f0e426 100644 --- a/module/Core/src/Visit/VisitLocator.php +++ b/module/Core/src/Visit/VisitLocator.php @@ -63,8 +63,7 @@ class VisitLocator implements VisitLocatorInterface $location = Location::emptyInstance(); } - $location = new VisitLocation($location); - $this->locateVisit($visit, $location, $helper); + $this->locateVisit($visit, VisitLocation::fromGeolocation($location), $helper); // Flush and clear after X iterations if ($count % $persistBlock === 0) { diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 4e634493..27ab3252 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -57,7 +57,7 @@ class VisitRepositoryTest extends DatabaseTestCase $visit = Visit::forValidShortUrl($shortUrl, Visitor::emptyInstance()); if ($i >= 2) { - $location = new VisitLocation(Location::emptyInstance()); + $location = VisitLocation::fromGeolocation(Location::emptyInstance()); $this->getEntityManager()->persist($location); $visit->locate($location); } diff --git a/module/Core/test/Entity/VisitLocationTest.php b/module/Core/test/Entity/VisitLocationTest.php index 057a1920..6021b124 100644 --- a/module/Core/test/Entity/VisitLocationTest.php +++ b/module/Core/test/Entity/VisitLocationTest.php @@ -17,7 +17,7 @@ class VisitLocationTest extends TestCase public function isEmptyReturnsTrueWhenAllValuesAreEmpty(array $args, bool $isEmpty): void { $payload = new Location(...$args); - $location = new VisitLocation($payload); + $location = VisitLocation::fromGeolocation($payload); self::assertEquals($isEmpty, $location->isEmpty()); } diff --git a/module/Core/test/EventDispatcher/LocateVisitTest.php b/module/Core/test/EventDispatcher/LocateVisitTest.php index fda45c58..406e8146 100644 --- a/module/Core/test/EventDispatcher/LocateVisitTest.php +++ b/module/Core/test/EventDispatcher/LocateVisitTest.php @@ -168,7 +168,7 @@ class LocateVisitTest extends TestCase ($this->locateVisit)($event); - self::assertEquals($visit->getVisitLocation(), new VisitLocation(Location::emptyInstance())); + self::assertEquals($visit->getVisitLocation(), VisitLocation::fromGeolocation(Location::emptyInstance())); $findVisit->shouldHaveBeenCalledOnce(); $flush->shouldHaveBeenCalledOnce(); $resolveIp->shouldNotHaveBeenCalled(); @@ -204,7 +204,7 @@ class LocateVisitTest extends TestCase ($this->locateVisit)($event); - self::assertEquals($visit->getVisitLocation(), new VisitLocation($location)); + self::assertEquals($visit->getVisitLocation(), VisitLocation::fromGeolocation($location)); $findVisit->shouldHaveBeenCalledOnce(); $flush->shouldHaveBeenCalledOnce(); $resolveIp->shouldHaveBeenCalledOnce(); diff --git a/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php b/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php index cf36c052..61193c86 100644 --- a/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php +++ b/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php @@ -68,7 +68,7 @@ class OrphanVisitDataTransformerTest extends TestCase ->withHeader('Referer', 'referer') ->withUri(new Uri('https://doma.in/foo/bar')), ), - )->locate($location = new VisitLocation(Location::emptyInstance())), + )->locate($location = VisitLocation::fromGeolocation(Location::emptyInstance())), [ 'referer' => 'referer', 'date' => $visit->getDate()->toAtomString(), From 09414a8834d07066bd1ae1d0cc19905bd23e5162 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 11 Apr 2021 13:30:12 +0200 Subject: [PATCH 51/86] Allowed to optionally import visits from other shlink instance --- composer.json | 2 +- .../src/Importer/ImportedLinksProcessor.php | 21 ++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/composer.json b/composer.json index 9782b137..0c088a77 100644 --- a/composer.json +++ b/composer.json @@ -49,7 +49,7 @@ "shlinkio/shlink-common": "dev-main#554e370 as 3.7", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", - "shlinkio/shlink-importer": "dev-main#174e352 as 2.3", + "shlinkio/shlink-importer": "dev-main#39928b6 as 2.3", "shlinkio/shlink-installer": "dev-develop#aa50ea9 as 5.5", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index 4a5562bc..7cecd5fe 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -64,14 +64,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface } $this->em->persist($shortUrl); - - // TODO Process only missing visits when possible: $importedUrl->visitsCount(); - // TODO Make importing visits optional based on params - $importedVisits = 0; - foreach ($importedUrl->visits() as $importedVisit) { - $this->em->persist(Visit::fromImport($importedVisit, $shortUrl)); - $importedVisits++; - } + $importedVisits = $this->importVisits($importedUrl, $shortUrl); $io->text( $importedVisits === 0 @@ -105,4 +98,16 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface return $this->shortCodeHelper->ensureShortCodeUniqueness($shortUrl, false); } + + private function importVisits(ImportedShlinkUrl $importedUrl, ShortUrl $shortUrl): int + { + // TODO Process only missing visits when possible: $importedUrl->visitsCount(); + $importedVisits = 0; + foreach ($importedUrl->visits() as $importedVisit) { + $this->em->persist(Visit::fromImport($importedVisit, $shortUrl)); + $importedVisits++; + } + + return $importedVisits; + } } From 9a78d1585d340662700cfa398aec4e27973d6669 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 11 Apr 2021 19:56:00 +0200 Subject: [PATCH 52/86] Ensured only pending visits are imported when processing a short URL which already has imported visits --- module/Core/src/Entity/ShortUrl.php | 10 ++++++++++ module/Core/src/Entity/Visit.php | 5 +++-- .../src/Importer/ImportedLinksProcessor.php | 18 +++++++++++++++--- .../Core/src/Repository/ShortUrlRepository.php | 11 ++++------- .../Repository/ShortUrlRepositoryInterface.php | 2 +- .../Repository/ShortUrlRepositoryTest.php | 14 +++++++------- .../Importer/ImportedLinksProcessorTest.php | 16 +++++++++------- 7 files changed, 49 insertions(+), 27 deletions(-) diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php index b25b90e5..9c24bdc3 100644 --- a/module/Core/src/Entity/ShortUrl.php +++ b/module/Core/src/Entity/ShortUrl.php @@ -7,6 +7,8 @@ namespace Shlinkio\Shlink\Core\Entity; use Cake\Chronos\Chronos; use Doctrine\Common\Collections\ArrayCollection; use Doctrine\Common\Collections\Collection; +use Doctrine\Common\Collections\Criteria; +use Doctrine\Common\Collections\Selectable; use Shlinkio\Shlink\Common\Entity\AbstractEntity; use Shlinkio\Shlink\Core\Exception\ShortCodeCannotBeRegeneratedException; use Shlinkio\Shlink\Core\Model\ShortUrlEdit; @@ -164,6 +166,14 @@ class ShortUrl extends AbstractEntity return count($this->visits); } + public function importedVisitsCount(): int + { + /** @var Selectable $visits */ + $visits = $this->visits; + $criteria = Criteria::create()->where(Criteria::expr()->eq('type', Visit::TYPE_IMPORTED)); + return count($visits->matching($criteria)); + } + /** * @param Collection|Visit[] $visits * @internal diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index e2d5f09e..fd1b830b 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -16,6 +16,7 @@ use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit; class Visit extends AbstractEntity implements JsonSerializable { public const TYPE_VALID_SHORT_URL = 'valid_short_url'; + public const TYPE_IMPORTED = 'imported'; public const TYPE_INVALID_SHORT_URL = 'invalid_short_url'; public const TYPE_BASE_URL = 'base_url'; public const TYPE_REGULAR_404 = 'regular_404'; @@ -44,9 +45,9 @@ class Visit extends AbstractEntity implements JsonSerializable return $instance; } - public static function fromImport(ImportedShlinkVisit $importedVisit, ShortUrl $shortUrl): self + public static function fromImport(ShortUrl $shortUrl, ImportedShlinkVisit $importedVisit): self { - $instance = new self($shortUrl, self::TYPE_VALID_SHORT_URL); + $instance = new self($shortUrl, self::TYPE_IMPORTED); $instance->userAgent = $importedVisit->userAgent(); $instance->referer = $importedVisit->referer(); $instance->date = Chronos::instance($importedVisit->date()); diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index 7cecd5fe..802e33b4 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -51,7 +51,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface $longUrl = $importedUrl->longUrl(); // Skip already imported URLs - if ($shortUrlRepo->importedUrlExists($importedUrl)) { + if ($shortUrlRepo->findOneByImportedUrl($importedUrl) !== null) { // TODO If the URL exists, allow to merge visits instead of just skipping completely $io->text(sprintf('%s: Skipped', $longUrl)); continue; @@ -74,6 +74,11 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface } } +// private function getOrCreateShortUrl(ImportedShlinkUrl $url, bool $importShortCodes): ?ShortUrl +// { +// +// } + private function handleShortCodeUniqueness( ImportedShlinkUrl $url, ShortUrl $shortUrl, @@ -101,10 +106,17 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface private function importVisits(ImportedShlinkUrl $importedUrl, ShortUrl $shortUrl): int { - // TODO Process only missing visits when possible: $importedUrl->visitsCount(); + // If we know the amount of visits that can be imported, import only those left. Import all otherwise. + $importVisitsCount = $importedUrl->visitsCount(); + $visitsLeft = $importVisitsCount !== null ? $importVisitsCount - $shortUrl->importedVisitsCount() : null; + $importedVisits = 0; foreach ($importedUrl->visits() as $importedVisit) { - $this->em->persist(Visit::fromImport($importedVisit, $shortUrl)); + if ($visitsLeft !== null && $importedVisits >= $visitsLeft) { + break; + } + + $this->em->persist(Visit::fromImport($shortUrl, $importedVisit)); $importedVisits++; } diff --git a/module/Core/src/Repository/ShortUrlRepository.php b/module/Core/src/Repository/ShortUrlRepository.php index 24b20a38..fe3b170c 100644 --- a/module/Core/src/Repository/ShortUrlRepository.php +++ b/module/Core/src/Repository/ShortUrlRepository.php @@ -264,12 +264,10 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU return $qb->getQuery()->getOneOrNullResult(); } - public function importedUrlExists(ImportedShlinkUrl $url): bool + public function findOneByImportedUrl(ImportedShlinkUrl $url): ?ShortUrl { - $qb = $this->getEntityManager()->createQueryBuilder(); - $qb->select('COUNT(DISTINCT s.id)') - ->from(ShortUrl::class, 's') - ->andWhere($qb->expr()->eq('s.importOriginalShortCode', ':shortCode')) + $qb = $this->createQueryBuilder('s'); + $qb->andWhere($qb->expr()->eq('s.importOriginalShortCode', ':shortCode')) ->setParameter('shortCode', $url->shortCode()) ->andWhere($qb->expr()->eq('s.importSource', ':importSource')) ->setParameter('importSource', $url->source()) @@ -277,8 +275,7 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU $this->whereDomainIs($qb, $url->domain()); - $result = (int) $qb->getQuery()->getSingleScalarResult(); - return $result > 0; + return $qb->getQuery()->getOneOrNullResult(); } private function whereDomainIs(QueryBuilder $qb, ?string $domain): void diff --git a/module/Core/src/Repository/ShortUrlRepositoryInterface.php b/module/Core/src/Repository/ShortUrlRepositoryInterface.php index ca04ffda..29485eeb 100644 --- a/module/Core/src/Repository/ShortUrlRepositoryInterface.php +++ b/module/Core/src/Repository/ShortUrlRepositoryInterface.php @@ -40,5 +40,5 @@ interface ShortUrlRepositoryInterface extends ObjectRepository, EntitySpecificat public function findOneMatching(ShortUrlMeta $meta): ?ShortUrl; - public function importedUrlExists(ImportedShlinkUrl $url): bool; + public function findOneByImportedUrl(ImportedShlinkUrl $url): ?ShortUrl; } diff --git a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php index 35a6b709..cf082d85 100644 --- a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php +++ b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php @@ -416,7 +416,7 @@ class ShortUrlRepositoryTest extends DatabaseTestCase } /** @test */ - public function importedShortUrlsAreSearchedAsExpected(): void + public function importedShortUrlsAreFoundWhenExpected(): void { $buildImported = static fn (string $shortCode, ?String $domain = null) => new ImportedShlinkUrl('', 'foo', [], Chronos::now(), $domain, $shortCode, null); @@ -429,11 +429,11 @@ class ShortUrlRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - self::assertTrue($this->repo->importedUrlExists($buildImported('my-cool-slug'))); - self::assertTrue($this->repo->importedUrlExists($buildImported('another-slug', 'doma.in'))); - self::assertFalse($this->repo->importedUrlExists($buildImported('non-existing-slug'))); - self::assertFalse($this->repo->importedUrlExists($buildImported('non-existing-slug', 'doma.in'))); - self::assertFalse($this->repo->importedUrlExists($buildImported('my-cool-slug', 'doma.in'))); - self::assertFalse($this->repo->importedUrlExists($buildImported('another-slug'))); + self::assertNotNull($this->repo->findOneByImportedUrl($buildImported('my-cool-slug'))); + self::assertNotNull($this->repo->findOneByImportedUrl($buildImported('another-slug', 'doma.in'))); + self::assertNull($this->repo->findOneByImportedUrl($buildImported('non-existing-slug'))); + self::assertNull($this->repo->findOneByImportedUrl($buildImported('non-existing-slug', 'doma.in'))); + self::assertNull($this->repo->findOneByImportedUrl($buildImported('my-cool-slug', 'doma.in'))); + self::assertNull($this->repo->findOneByImportedUrl($buildImported('another-slug'))); } } diff --git a/module/Core/test/Importer/ImportedLinksProcessorTest.php b/module/Core/test/Importer/ImportedLinksProcessorTest.php index c294ffe5..df650e18 100644 --- a/module/Core/test/Importer/ImportedLinksProcessorTest.php +++ b/module/Core/test/Importer/ImportedLinksProcessorTest.php @@ -64,7 +64,7 @@ class ImportedLinksProcessorTest extends TestCase ]; $expectedCalls = count($urls); - $importedUrlExists = $this->repo->importedUrlExists(Argument::cetera())->willReturn(false); + $importedUrlExists = $this->repo->findOneByImportedUrl(Argument::cetera())->willReturn(null); $ensureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness(Argument::cetera())->willReturn(true); $persist = $this->em->persist(Argument::type(ShortUrl::class)); @@ -88,12 +88,14 @@ class ImportedLinksProcessorTest extends TestCase ]; $contains = fn (string $needle) => fn (string $text) => str_contains($text, $needle); - $importedUrlExists = $this->repo->importedUrlExists(Argument::cetera())->will(function (array $args): bool { - /** @var ImportedShlinkUrl $url */ - [$url] = $args; + $importedUrlExists = $this->repo->findOneByImportedUrl(Argument::cetera())->will( + function (array $args): ?ShortUrl { + /** @var ImportedShlinkUrl $url */ + [$url] = $args; - return contains(['foo', 'baz2', 'baz3'], $url->longUrl()); - }); + return contains(['foo', 'baz2', 'baz3'], $url->longUrl()) ? ShortUrl::fromImport($url, true) : null; + }, + ); $ensureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness(Argument::cetera())->willReturn(true); $persist = $this->em->persist(Argument::type(ShortUrl::class)); @@ -118,7 +120,7 @@ class ImportedLinksProcessorTest extends TestCase ]; $contains = fn (string $needle) => fn (string $text) => str_contains($text, $needle); - $importedUrlExists = $this->repo->importedUrlExists(Argument::cetera())->willReturn(false); + $importedUrlExists = $this->repo->findOneByImportedUrl(Argument::cetera())->willReturn(null); $failingEnsureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness( Argument::any(), true, From c8b8947b1fca7cff77b50988318a81466bd25d86 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 18 Apr 2021 11:58:59 +0200 Subject: [PATCH 53/86] Allowed to import visits to existing already imported short URLs --- module/Core/src/Entity/ShortUrl.php | 14 ++- .../src/Importer/ImportedLinksProcessor.php | 101 +++++++++++------- .../Core/src/Repository/VisitRepository.php | 3 + .../Transformer/ShortUrlDataTransformer.php | 2 +- 4 files changed, 76 insertions(+), 44 deletions(-) diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php index 9c24bdc3..3fe2932b 100644 --- a/module/Core/src/Entity/ShortUrl.php +++ b/module/Core/src/Entity/ShortUrl.php @@ -166,12 +166,18 @@ class ShortUrl extends AbstractEntity return count($this->visits); } - public function importedVisitsCount(): int + public function mostRecentImportedVisitDate(): ?Chronos { /** @var Selectable $visits */ $visits = $this->visits; - $criteria = Criteria::create()->where(Criteria::expr()->eq('type', Visit::TYPE_IMPORTED)); - return count($visits->matching($criteria)); + $criteria = Criteria::create()->where(Criteria::expr()->eq('type', Visit::TYPE_IMPORTED)) + ->orderBy(['id' => 'DESC']) + ->setMaxResults(1); + + /** @var Visit|false $visit */ + $visit = $visits->matching($criteria)->last(); + + return $visit === false ? null : $visit->getDate(); } /** @@ -189,7 +195,7 @@ class ShortUrl extends AbstractEntity return $this->maxVisits; } - public function getTitle(): ?string + public function title(): ?string { return $this->title; } diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index 802e33b4..4091906d 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Importer; +use Cake\Chronos\Chronos; use Doctrine\ORM\EntityManagerInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; @@ -23,6 +24,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface private ShortUrlRelationResolverInterface $relationResolver; private ShortCodeHelperInterface $shortCodeHelper; private DoctrineBatchHelperInterface $batchHelper; + private ShortUrlRepositoryInterface $shortUrlRepo; public function __construct( EntityManagerInterface $em, @@ -34,6 +36,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface $this->relationResolver = $relationResolver; $this->shortCodeHelper = $shortCodeHelper; $this->batchHelper = $batchHelper; + $this->shortUrlRepo = $this->em->getRepository(ShortUrl::class); // @phpstan-ignore-line } /** @@ -41,8 +44,6 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface */ public function process(StyleInterface $io, iterable $shlinkUrls, array $params): void { - /** @var ShortUrlRepositoryInterface $shortUrlRepo */ - $shortUrlRepo = $this->em->getRepository(ShortUrl::class); $importShortCodes = $params['import_short_codes']; $iterable = $this->batchHelper->wrapIterable($shlinkUrls, 100); @@ -50,54 +51,74 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface foreach ($iterable as $importedUrl) { $longUrl = $importedUrl->longUrl(); - // Skip already imported URLs - if ($shortUrlRepo->findOneByImportedUrl($importedUrl) !== null) { - // TODO If the URL exists, allow to merge visits instead of just skipping completely - $io->text(sprintf('%s: Skipped', $longUrl)); + $generateNewIfDuplicated = static function () use ($io, $importedUrl): bool { + $action = $io->choice(sprintf( + 'Failed to import URL "%s" because its short-code "%s" is already in use. Do you want to generate ' + . 'a new one or skip it?', + $importedUrl->longUrl(), + $importedUrl->shortCode(), + ), ['Generate new short-code', 'Skip'], 1); + + return $action !== 'Skip'; + }; + [$shortUrl, $isNew] = $this->getOrCreateShortUrl($importedUrl, $importShortCodes, $generateNewIfDuplicated); + + if ($shortUrl === null) { + $io->text(sprintf('%s: Error', $longUrl)); continue; } - $shortUrl = ShortUrl::fromImport($importedUrl, $importShortCodes, $this->relationResolver); - if (! $this->handleShortCodeUniqueness($importedUrl, $shortUrl, $io, $importShortCodes)) { - $io->text(sprintf('%s: Skipped', $longUrl)); - continue; - } - - $this->em->persist($shortUrl); $importedVisits = $this->importVisits($importedUrl, $shortUrl); - $io->text( - $importedVisits === 0 - ? sprintf('%s: Imported', $longUrl) - : sprintf('%s: Imported with %s visits', $longUrl, $importedVisits), - ); + if ($importedVisits === 0) { + $io->text( + $isNew + ? sprintf('%s: Imported', $longUrl) + : sprintf('%s: Skipped', $longUrl), + ); + } else { + $io->text( + $isNew + ? sprintf('%s: Imported with %s visits', $longUrl, $importedVisits) + : sprintf( + '%s: Skipped. Imported %s visits', + $longUrl, + $importedVisits, + ), + ); + } } } -// private function getOrCreateShortUrl(ImportedShlinkUrl $url, bool $importShortCodes): ?ShortUrl -// { -// -// } + private function getOrCreateShortUrl( + ImportedShlinkUrl $importedUrl, + bool $importShortCodes, + callable $generateNewIfDuplicated + ): array { + $existingShortUrl = $this->shortUrlRepo->findOneByImportedUrl($importedUrl); + if ($existingShortUrl !== null) { + return [$existingShortUrl, false]; + } + + $shortUrl = ShortUrl::fromImport($importedUrl, $importShortCodes, $this->relationResolver); + if (! $this->handleShortCodeUniqueness($shortUrl, $importShortCodes, $generateNewIfDuplicated)) { + return [null, false]; + } + + $this->em->persist($shortUrl); + return [$shortUrl, true]; + } private function handleShortCodeUniqueness( - ImportedShlinkUrl $url, ShortUrl $shortUrl, - StyleInterface $io, - bool $importShortCodes + bool $importShortCodes, + callable $generateNewIfDuplicated ): bool { if ($this->shortCodeHelper->ensureShortCodeUniqueness($shortUrl, $importShortCodes)) { return true; } - $longUrl = $url->longUrl(); - $action = $io->choice(sprintf( - 'Failed to import URL "%s" because its short-code "%s" is already in use. Do you want to generate a new ' - . 'one or skip it?', - $longUrl, - $url->shortCode(), - ), ['Generate new short-code', 'Skip'], 1); - - if ($action === 'Skip') { + if (! $generateNewIfDuplicated()) { return false; } @@ -106,14 +127,16 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface private function importVisits(ImportedShlinkUrl $importedUrl, ShortUrl $shortUrl): int { - // If we know the amount of visits that can be imported, import only those left. Import all otherwise. - $importVisitsCount = $importedUrl->visitsCount(); - $visitsLeft = $importVisitsCount !== null ? $importVisitsCount - $shortUrl->importedVisitsCount() : null; + $mostRecentImportedDate = $shortUrl->mostRecentImportedVisitDate(); $importedVisits = 0; foreach ($importedUrl->visits() as $importedVisit) { - if ($visitsLeft !== null && $importedVisits >= $visitsLeft) { - break; + // Skip visits which are older than the most recent already imported visit's date + if ( + $mostRecentImportedDate !== null + && $mostRecentImportedDate->gte(Chronos::instance($importedVisit->date())) + ) { + continue; } $this->em->persist(Visit::fromImport($shortUrl, $importedVisit)); diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index cd51f60d..d9c18977 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -203,6 +203,9 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo private function resolveVisitsWithNativeQuery(QueryBuilder $qb, ?int $limit, ?int $offset): array { + // TODO Order by date and ID, not just by ID (order by date DESC, id DESC). + // That ensures imported visits are properly ordered even if inserted in wrong chronological order. + $qb->select('v.id') ->orderBy('v.id', 'DESC') // Falling back to values that will behave as no limit/offset, but will workaround MS SQL not allowing diff --git a/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php b/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php index ce459714..49918867 100644 --- a/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php +++ b/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php @@ -34,7 +34,7 @@ class ShortUrlDataTransformer implements DataTransformerInterface 'tags' => invoke($shortUrl->getTags(), '__toString'), 'meta' => $this->buildMeta($shortUrl), 'domain' => $shortUrl->getDomain(), - 'title' => $shortUrl->getTitle(), + 'title' => $shortUrl->title(), ]; } From b277f431c20f2ffa1050a9c1e1d850c2539141c1 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 18 Apr 2021 12:44:02 +0200 Subject: [PATCH 54/86] Added test covering imported short URLs with visits --- composer.json | 2 +- module/Core/src/Entity/Visit.php | 9 +++ .../src/Importer/ImportedLinksProcessor.php | 23 +++--- .../Importer/ImportedLinksProcessorTest.php | 76 +++++++++++++++++-- 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/composer.json b/composer.json index 0c088a77..87275e92 100644 --- a/composer.json +++ b/composer.json @@ -124,6 +124,7 @@ ], "test:unit": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox", "test:unit:ci": "@test:unit --coverage-xml=build/coverage-unit/coverage-xml --log-junit=build/coverage-unit/junit.xml", + "test:unit:pretty": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-html build/coverage-unit-html", "test:db": "@parallel test:db:sqlite:ci test:db:mysql test:db:maria test:db:postgres test:db:ms", "test:db:sqlite": "APP_ENV=test php vendor/bin/phpunit --order-by=random --colors=always --testdox -c phpunit-db.xml", "test:db:sqlite:ci": "@test:db:sqlite --coverage-php build/coverage-db.cov --coverage-xml=build/coverage-db/coverage-xml --log-junit=build/coverage-db/junit.xml", @@ -132,7 +133,6 @@ "test:db:postgres": "DB_DRIVER=postgres composer test:db:sqlite", "test:db:ms": "DB_DRIVER=mssql composer test:db:sqlite", "test:api": "bin/test/run-api-tests.sh", - "test:unit:pretty": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-html build/coverage-unit-html", "infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --skip-initial-tests", "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=80", "infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json", diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index fd1b830b..98d1a4c5 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -150,6 +150,15 @@ class Visit extends AbstractEntity implements JsonSerializable return $this->type; } + /** + * Needed only for ArrayCollections to be able to apply criteria filtering + * @internal + */ + public function getType(): string + { + return $this->type(); + } + public function jsonSerialize(): array { return [ diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index 4091906d..25393055 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -49,9 +49,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface /** @var ImportedShlinkUrl $importedUrl */ foreach ($iterable as $importedUrl) { - $longUrl = $importedUrl->longUrl(); - - $generateNewIfDuplicated = static function () use ($io, $importedUrl): bool { + $skipOnShortCodeConflict = static function () use ($io, $importedUrl): bool { $action = $io->choice(sprintf( 'Failed to import URL "%s" because its short-code "%s" is already in use. Do you want to generate ' . 'a new one or skip it?', @@ -59,10 +57,11 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface $importedUrl->shortCode(), ), ['Generate new short-code', 'Skip'], 1); - return $action !== 'Skip'; + return $action === 'Skip'; }; - [$shortUrl, $isNew] = $this->getOrCreateShortUrl($importedUrl, $importShortCodes, $generateNewIfDuplicated); + [$shortUrl, $isNew] = $this->getOrCreateShortUrl($importedUrl, $importShortCodes, $skipOnShortCodeConflict); + $longUrl = $importedUrl->longUrl(); if ($shortUrl === null) { $io->text(sprintf('%s: Error', $longUrl)); continue; @@ -93,15 +92,15 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface private function getOrCreateShortUrl( ImportedShlinkUrl $importedUrl, bool $importShortCodes, - callable $generateNewIfDuplicated + callable $skipOnShortCodeConflict ): array { - $existingShortUrl = $this->shortUrlRepo->findOneByImportedUrl($importedUrl); - if ($existingShortUrl !== null) { - return [$existingShortUrl, false]; + $alreadyImportedShortUrl = $this->shortUrlRepo->findOneByImportedUrl($importedUrl); + if ($alreadyImportedShortUrl !== null) { + return [$alreadyImportedShortUrl, false]; } $shortUrl = ShortUrl::fromImport($importedUrl, $importShortCodes, $this->relationResolver); - if (! $this->handleShortCodeUniqueness($shortUrl, $importShortCodes, $generateNewIfDuplicated)) { + if (! $this->handleShortCodeUniqueness($shortUrl, $importShortCodes, $skipOnShortCodeConflict)) { return [null, false]; } @@ -112,13 +111,13 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface private function handleShortCodeUniqueness( ShortUrl $shortUrl, bool $importShortCodes, - callable $generateNewIfDuplicated + callable $skipOnShortCodeConflict ): bool { if ($this->shortCodeHelper->ensureShortCodeUniqueness($shortUrl, $importShortCodes)) { return true; } - if (! $generateNewIfDuplicated()) { + if ($skipOnShortCodeConflict()) { return false; } diff --git a/module/Core/test/Importer/ImportedLinksProcessorTest.php b/module/Core/test/Importer/ImportedLinksProcessorTest.php index df650e18..170e21b7 100644 --- a/module/Core/test/Importer/ImportedLinksProcessorTest.php +++ b/module/Core/test/Importer/ImportedLinksProcessorTest.php @@ -5,18 +5,21 @@ declare(strict_types=1); namespace ShlinkioTest\Shlink\Core\Importer; use Cake\Chronos\Chronos; +use Doctrine\Common\Collections\ArrayCollection; use Doctrine\ORM\EntityManagerInterface; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Importer\ImportedLinksProcessor; use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortCodeHelperInterface; use Shlinkio\Shlink\Core\ShortUrl\Resolver\SimpleShortUrlRelationResolver; use Shlinkio\Shlink\Core\Util\DoctrineBatchHelperInterface; use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; +use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit; use Symfony\Component\Console\Style\StyleInterface; use function count; @@ -86,7 +89,6 @@ class ImportedLinksProcessorTest extends TestCase new ImportedShlinkUrl('', 'baz2', [], Chronos::now(), null, 'baz2', null), new ImportedShlinkUrl('', 'baz3', [], Chronos::now(), null, 'baz3', null), ]; - $contains = fn (string $needle) => fn (string $text) => str_contains($text, $needle); $importedUrlExists = $this->repo->findOneByImportedUrl(Argument::cetera())->will( function (array $args): ?ShortUrl { @@ -104,8 +106,8 @@ class ImportedLinksProcessorTest extends TestCase $importedUrlExists->shouldHaveBeenCalledTimes(count($urls)); $ensureUniqueness->shouldHaveBeenCalledTimes(2); $persist->shouldHaveBeenCalledTimes(2); - $this->io->text(Argument::that($contains('Skipped')))->shouldHaveBeenCalledTimes(3); - $this->io->text(Argument::that($contains('Imported')))->shouldHaveBeenCalledTimes(2); + $this->io->text(Argument::containingString('Skipped'))->shouldHaveBeenCalledTimes(3); + $this->io->text(Argument::containingString('Imported'))->shouldHaveBeenCalledTimes(2); } /** @test */ @@ -118,7 +120,6 @@ class ImportedLinksProcessorTest extends TestCase new ImportedShlinkUrl('', 'baz2', [], Chronos::now(), null, 'baz2', null), new ImportedShlinkUrl('', 'baz3', [], Chronos::now(), null, 'baz3', 'bar'), ]; - $contains = fn (string $needle) => fn (string $text) => str_contains($text, $needle); $importedUrlExists = $this->repo->findOneByImportedUrl(Argument::cetera())->willReturn(null); $failingEnsureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness( @@ -144,7 +145,70 @@ class ImportedLinksProcessorTest extends TestCase $successEnsureUniqueness->shouldHaveBeenCalledTimes(2); $choice->shouldHaveBeenCalledTimes(5); $persist->shouldHaveBeenCalledTimes(2); - $this->io->text(Argument::that($contains('Skipped')))->shouldHaveBeenCalledTimes(3); - $this->io->text(Argument::that($contains('Imported')))->shouldHaveBeenCalledTimes(2); + $this->io->text(Argument::containingString('Error'))->shouldHaveBeenCalledTimes(3); + $this->io->text(Argument::containingString('Imported'))->shouldHaveBeenCalledTimes(2); + } + + /** + * @test + * @dataProvider provideUrlsWithVisits + */ + public function properAmountOfVisitsIsImported( + ImportedShlinkUrl $importedUrl, + string $expectedOutput, + int $amountOfPersistedVisits, + ?ShortUrl $foundShortUrl + ): void { + $findExisting = $this->repo->findOneByImportedUrl(Argument::cetera())->willReturn($foundShortUrl); + $ensureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness(Argument::cetera())->willReturn(true); + $persistUrl = $this->em->persist(Argument::type(ShortUrl::class)); + $persistVisits = $this->em->persist(Argument::type(Visit::class)); + + $this->processor->process($this->io->reveal(), [$importedUrl], ['import_short_codes' => true]); + + $findExisting->shouldHaveBeenCalledOnce(); + $ensureUniqueness->shouldHaveBeenCalledTimes($foundShortUrl === null ? 1 : 0); + $persistUrl->shouldHaveBeenCalledTimes($foundShortUrl === null ? 1 : 0); + $persistVisits->shouldHaveBeenCalledTimes($amountOfPersistedVisits); + $this->io->text(Argument::containingString($expectedOutput))->shouldHaveBeenCalledOnce(); + } + + public function provideUrlsWithVisits(): iterable + { + $now = Chronos::now(); + $createImportedUrl = fn (array $visits) => new ImportedShlinkUrl('', 's', [], $now, null, 's', null, $visits); + + yield 'new short URL' => [$createImportedUrl([ + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + ]), 'Imported with 5 visits', 5, null]; + yield 'existing short URL without previous imported visits' => [ + $createImportedUrl([ + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now->addDays(3), null), + new ImportedShlinkVisit('', '', $now->addDays(3), null), + ]), + 'Skipped. Imported 4 visits', + 4, + ShortUrl::createEmpty(), + ]; + yield 'existing short URL with previous imported visits' => [ + $createImportedUrl([ + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now, null), + new ImportedShlinkVisit('', '', $now->addDays(3), null), + new ImportedShlinkVisit('', '', $now->addDays(3), null), + ]), + 'Skipped. Imported 2 visits', + 2, + ShortUrl::createEmpty()->setVisits(new ArrayCollection([ + Visit::fromImport(ShortUrl::createEmpty(), new ImportedShlinkVisit('', '', $now, null)), + ])), + ]; } } From e9a5284dde4d87896b76b603d4ad81344576c1f8 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 18 Apr 2021 17:07:56 +0200 Subject: [PATCH 55/86] Encapsulated logic to get rid of nested ifs --- .../src/Exception/NonUniqueSlugException.php | 6 ++ .../src/Importer/ImportedLinksProcessor.php | 66 +++++-------------- .../Core/src/Importer/ShortUrlImporting.php | 65 ++++++++++++++++++ .../src/Service/ShortUrl/ShortCodeHelper.php | 2 +- .../ShortUrl/ShortCodeHelperInterface.php | 2 +- .../Importer/ImportedLinksProcessorTest.php | 11 ++-- 6 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 module/Core/src/Importer/ShortUrlImporting.php diff --git a/module/Core/src/Exception/NonUniqueSlugException.php b/module/Core/src/Exception/NonUniqueSlugException.php index 8887f961..3b8b0b15 100644 --- a/module/Core/src/Exception/NonUniqueSlugException.php +++ b/module/Core/src/Exception/NonUniqueSlugException.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Exception; use Fig\Http\Message\StatusCodeInterface; use Mezzio\ProblemDetails\Exception\CommonProblemDetailsExceptionTrait; use Mezzio\ProblemDetails\Exception\ProblemDetailsExceptionInterface; +use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; use function sprintf; @@ -34,4 +35,9 @@ class NonUniqueSlugException extends InvalidArgumentException implements Problem return $e; } + + public static function fromImport(ImportedShlinkUrl $importedUrl): self + { + return self::fromSlug($importedUrl->shortCode(), $importedUrl->domain()); + } } diff --git a/module/Core/src/Importer/ImportedLinksProcessor.php b/module/Core/src/Importer/ImportedLinksProcessor.php index 25393055..e700e8a8 100644 --- a/module/Core/src/Importer/ImportedLinksProcessor.php +++ b/module/Core/src/Importer/ImportedLinksProcessor.php @@ -4,16 +4,16 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Importer; -use Cake\Chronos\Chronos; use Doctrine\ORM\EntityManagerInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; -use Shlinkio\Shlink\Core\Entity\Visit; +use Shlinkio\Shlink\Core\Exception\NonUniqueSlugException; use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortCodeHelperInterface; use Shlinkio\Shlink\Core\ShortUrl\Resolver\ShortUrlRelationResolverInterface; use Shlinkio\Shlink\Core\Util\DoctrineBatchHelperInterface; use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface; use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; +use Shlinkio\Shlink\Importer\Sources\ImportSources; use Symfony\Component\Console\Style\StyleInterface; use function sprintf; @@ -45,7 +45,8 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface public function process(StyleInterface $io, iterable $shlinkUrls, array $params): void { $importShortCodes = $params['import_short_codes']; - $iterable = $this->batchHelper->wrapIterable($shlinkUrls, 100); + $source = $params['source']; + $iterable = $this->batchHelper->wrapIterable($shlinkUrls, $source === ImportSources::SHLINK ? 10 : 100); /** @var ImportedShlinkUrl $importedUrl */ foreach ($iterable as $importedUrl) { @@ -59,53 +60,37 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface return $action === 'Skip'; }; - [$shortUrl, $isNew] = $this->getOrCreateShortUrl($importedUrl, $importShortCodes, $skipOnShortCodeConflict); - $longUrl = $importedUrl->longUrl(); - if ($shortUrl === null) { + + try { + $shortUrlImporting = $this->resolveShortUrl($importedUrl, $importShortCodes, $skipOnShortCodeConflict); + } catch (NonUniqueSlugException $e) { $io->text(sprintf('%s: Error', $longUrl)); continue; } - $importedVisits = $this->importVisits($importedUrl, $shortUrl); - - if ($importedVisits === 0) { - $io->text( - $isNew - ? sprintf('%s: Imported', $longUrl) - : sprintf('%s: Skipped', $longUrl), - ); - } else { - $io->text( - $isNew - ? sprintf('%s: Imported with %s visits', $longUrl, $importedVisits) - : sprintf( - '%s: Skipped. Imported %s visits', - $longUrl, - $importedVisits, - ), - ); - } + $resultMessage = $shortUrlImporting->importVisits($importedUrl->visits(), $this->em); + $io->text(sprintf('%s: %s', $longUrl, $resultMessage)); } } - private function getOrCreateShortUrl( + private function resolveShortUrl( ImportedShlinkUrl $importedUrl, bool $importShortCodes, callable $skipOnShortCodeConflict - ): array { + ): ShortUrlImporting { $alreadyImportedShortUrl = $this->shortUrlRepo->findOneByImportedUrl($importedUrl); if ($alreadyImportedShortUrl !== null) { - return [$alreadyImportedShortUrl, false]; + return ShortUrlImporting::fromExistingShortUrl($alreadyImportedShortUrl); } $shortUrl = ShortUrl::fromImport($importedUrl, $importShortCodes, $this->relationResolver); if (! $this->handleShortCodeUniqueness($shortUrl, $importShortCodes, $skipOnShortCodeConflict)) { - return [null, false]; + throw NonUniqueSlugException::fromImport($importedUrl); } $this->em->persist($shortUrl); - return [$shortUrl, true]; + return ShortUrlImporting::fromNewShortUrl($shortUrl); } private function handleShortCodeUniqueness( @@ -123,25 +108,4 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface return $this->shortCodeHelper->ensureShortCodeUniqueness($shortUrl, false); } - - private function importVisits(ImportedShlinkUrl $importedUrl, ShortUrl $shortUrl): int - { - $mostRecentImportedDate = $shortUrl->mostRecentImportedVisitDate(); - - $importedVisits = 0; - foreach ($importedUrl->visits() as $importedVisit) { - // Skip visits which are older than the most recent already imported visit's date - if ( - $mostRecentImportedDate !== null - && $mostRecentImportedDate->gte(Chronos::instance($importedVisit->date())) - ) { - continue; - } - - $this->em->persist(Visit::fromImport($shortUrl, $importedVisit)); - $importedVisits++; - } - - return $importedVisits; - } } diff --git a/module/Core/src/Importer/ShortUrlImporting.php b/module/Core/src/Importer/ShortUrlImporting.php new file mode 100644 index 00000000..b5ae4651 --- /dev/null +++ b/module/Core/src/Importer/ShortUrlImporting.php @@ -0,0 +1,65 @@ +shortUrl = $shortUrl; + $this->isNew = $isNew; + } + + public static function fromExistingShortUrl(ShortUrl $shortUrl): self + { + return new self($shortUrl, false); + } + + public static function fromNewShortUrl(ShortUrl $shortUrl): self + { + return new self($shortUrl, true); + } + + /** + * @param iterable|ImportedShlinkVisit[] $visits + */ + public function importVisits(iterable $visits, EntityManagerInterface $em): string + { + $mostRecentImportedDate = $this->shortUrl->mostRecentImportedVisitDate(); + + $importedVisits = 0; + foreach ($visits as $importedVisit) { + // Skip visits which are older than the most recent already imported visit's date + if ( + $mostRecentImportedDate !== null + && $mostRecentImportedDate->gte(Chronos::instance($importedVisit->date())) + ) { + continue; + } + + $em->persist(Visit::fromImport($this->shortUrl, $importedVisit)); + $importedVisits++; + } + + if ($importedVisits === 0) { + return $this->isNew ? 'Imported' : 'Skipped'; + } + + return $this->isNew + ? sprintf('Imported with %s visits', $importedVisits) + : sprintf('Skipped. Imported %s visits', $importedVisits); + } +} diff --git a/module/Core/src/Service/ShortUrl/ShortCodeHelper.php b/module/Core/src/Service/ShortUrl/ShortCodeHelper.php index 6e4e57ac..3df4c016 100644 --- a/module/Core/src/Service/ShortUrl/ShortCodeHelper.php +++ b/module/Core/src/Service/ShortUrl/ShortCodeHelper.php @@ -8,7 +8,7 @@ use Doctrine\ORM\EntityManagerInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Repository\ShortUrlRepository; -class ShortCodeHelper implements ShortCodeHelperInterface +class ShortCodeHelper implements ShortCodeHelperInterface // TODO Rename to ShortCodeUniquenessHelper { private EntityManagerInterface $em; diff --git a/module/Core/src/Service/ShortUrl/ShortCodeHelperInterface.php b/module/Core/src/Service/ShortUrl/ShortCodeHelperInterface.php index af3f2aa5..a020a30c 100644 --- a/module/Core/src/Service/ShortUrl/ShortCodeHelperInterface.php +++ b/module/Core/src/Service/ShortUrl/ShortCodeHelperInterface.php @@ -6,7 +6,7 @@ namespace Shlinkio\Shlink\Core\Service\ShortUrl; use Shlinkio\Shlink\Core\Entity\ShortUrl; -interface ShortCodeHelperInterface +interface ShortCodeHelperInterface // TODO Rename to ShortCodeUniquenessHelperInterface { public function ensureShortCodeUniqueness(ShortUrl $shortUrlToBeCreated, bool $hasCustomSlug): bool; } diff --git a/module/Core/test/Importer/ImportedLinksProcessorTest.php b/module/Core/test/Importer/ImportedLinksProcessorTest.php index 170e21b7..d17c5720 100644 --- a/module/Core/test/Importer/ImportedLinksProcessorTest.php +++ b/module/Core/test/Importer/ImportedLinksProcessorTest.php @@ -20,6 +20,7 @@ use Shlinkio\Shlink\Core\ShortUrl\Resolver\SimpleShortUrlRelationResolver; use Shlinkio\Shlink\Core\Util\DoctrineBatchHelperInterface; use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit; +use Shlinkio\Shlink\Importer\Sources\ImportSources; use Symfony\Component\Console\Style\StyleInterface; use function count; @@ -31,6 +32,8 @@ class ImportedLinksProcessorTest extends TestCase { use ProphecyTrait; + private const PARAMS = ['import_short_codes' => true, 'source' => ImportSources::BITLY]; + private ImportedLinksProcessor $processor; private ObjectProphecy $em; private ObjectProphecy $shortCodeHelper; @@ -71,7 +74,7 @@ class ImportedLinksProcessorTest extends TestCase $ensureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness(Argument::cetera())->willReturn(true); $persist = $this->em->persist(Argument::type(ShortUrl::class)); - $this->processor->process($this->io->reveal(), $urls, ['import_short_codes' => true]); + $this->processor->process($this->io->reveal(), $urls, self::PARAMS); $importedUrlExists->shouldHaveBeenCalledTimes($expectedCalls); $ensureUniqueness->shouldHaveBeenCalledTimes($expectedCalls); @@ -101,7 +104,7 @@ class ImportedLinksProcessorTest extends TestCase $ensureUniqueness = $this->shortCodeHelper->ensureShortCodeUniqueness(Argument::cetera())->willReturn(true); $persist = $this->em->persist(Argument::type(ShortUrl::class)); - $this->processor->process($this->io->reveal(), $urls, ['import_short_codes' => true]); + $this->processor->process($this->io->reveal(), $urls, self::PARAMS); $importedUrlExists->shouldHaveBeenCalledTimes(count($urls)); $ensureUniqueness->shouldHaveBeenCalledTimes(2); @@ -138,7 +141,7 @@ class ImportedLinksProcessorTest extends TestCase }); $persist = $this->em->persist(Argument::type(ShortUrl::class)); - $this->processor->process($this->io->reveal(), $urls, ['import_short_codes' => true]); + $this->processor->process($this->io->reveal(), $urls, self::PARAMS); $importedUrlExists->shouldHaveBeenCalledTimes(count($urls)); $failingEnsureUniqueness->shouldHaveBeenCalledTimes(5); @@ -164,7 +167,7 @@ class ImportedLinksProcessorTest extends TestCase $persistUrl = $this->em->persist(Argument::type(ShortUrl::class)); $persistVisits = $this->em->persist(Argument::type(Visit::class)); - $this->processor->process($this->io->reveal(), [$importedUrl], ['import_short_codes' => true]); + $this->processor->process($this->io->reveal(), [$importedUrl], self::PARAMS); $findExisting->shouldHaveBeenCalledOnce(); $ensureUniqueness->shouldHaveBeenCalledTimes($foundShortUrl === null ? 1 : 0); From 111fc3c37dc1c17fac7593eff56da1f59fbd087a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 18 Apr 2021 17:09:06 +0200 Subject: [PATCH 56/86] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 971895bc..78ba489c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this Also, when using swoole, the file is now updated **after** tracking a visit, which means it will not apply until the next one. * [#1059](https://github.com/shlinkio/shlink/issues/1059) Added ability to optionally display author API key and its name when listing short URLs from the command line. +* [#1066](https://github.com/shlinkio/shlink/issues/1066) Added support to import short URLs and their visits from another Shlink instance using its API. ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. From d423d18249d2f1edbd3e83cd3e0fd48523f4877b Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 16 May 2021 09:30:04 +0200 Subject: [PATCH 57/86] Defined new structure for tracking config, together with new options --- config/autoload/app_options.global.php | 1 - config/autoload/tracking.global.php | 34 +++++++ config/autoload/url-shortener.global.php | 2 - docker/config/shlink_in_docker.local.php | 15 ++- module/Core/config/dependencies.config.php | 8 +- .../src/Action/AbstractTrackingAction.php | 10 +- module/Core/src/Action/RedirectAction.php | 4 +- .../src/Config/DeprecatedConfigParser.php | 1 + .../src/Config/SimplifiedConfigParser.php | 4 +- module/Core/src/Options/AppOptions.php | 11 +-- module/Core/src/Options/TrackingOptions.php | 99 +++++++++++++++++++ .../Core/src/Options/UrlShortenerOptions.php | 18 +--- module/Core/src/Visit/VisitsTracker.php | 6 +- module/Core/test/Action/PixelActionTest.php | 4 +- .../Core/test/Action/RedirectActionTest.php | 2 +- .../Config/SimplifiedConfigParserTest.php | 6 +- module/Core/test/Visit/VisitsTrackerTest.php | 6 +- 17 files changed, 176 insertions(+), 55 deletions(-) create mode 100644 config/autoload/tracking.global.php create mode 100644 module/Core/src/Options/TrackingOptions.php diff --git a/config/autoload/app_options.global.php b/config/autoload/app_options.global.php index f64f9cff..0b7ec937 100644 --- a/config/autoload/app_options.global.php +++ b/config/autoload/app_options.global.php @@ -7,7 +7,6 @@ return [ 'app_options' => [ 'name' => 'Shlink', 'version' => '%SHLINK_VERSION%', - 'disable_track_param' => null, ], ]; diff --git a/config/autoload/tracking.global.php b/config/autoload/tracking.global.php new file mode 100644 index 00000000..1ddec8b8 --- /dev/null +++ b/config/autoload/tracking.global.php @@ -0,0 +1,34 @@ + [ + // Tells if IP addresses should be anonymized before persisting, to fulfil data protection regulations + // This applies only if IP address tracking is enabled + 'anonymize_remote_addr' => true, + + // Tells if visits to not-found URLs should be tracked. The disable_tracking option takes precedence + 'track_orphan_visits' => true, + + // A query param that, if provided, will disable tracking of one particular visit. Always takes precedence + 'disable_track_param' => null, + + // If true, visits will not be tracked at all + 'disable_tracking' => false, + + // If true, visits will be tracked, but neither the IP address or the location will be resolved + 'disable_ip_tracking' => false, + + // If true, visits will be tracked including the IP address, but the location won't be resolved + 'disable_location_tracking' => false, + + // If true, the referrer will not be tracked + 'disable_referrer_tracking' => false, + + // If true, the user agent will not be tracked + 'disable_ua_tracking' => false, + ], + +]; diff --git a/config/autoload/url-shortener.global.php b/config/autoload/url-shortener.global.php index 3751b1e9..d7cd8b02 100644 --- a/config/autoload/url-shortener.global.php +++ b/config/autoload/url-shortener.global.php @@ -14,13 +14,11 @@ return [ 'hostname' => '', ], 'validate_url' => false, // Deprecated - 'anonymize_remote_addr' => true, 'visits_webhooks' => [], 'default_short_codes_length' => DEFAULT_SHORT_CODES_LENGTH, 'redirect_status_code' => DEFAULT_REDIRECT_STATUS_CODE, 'redirect_cache_lifetime' => DEFAULT_REDIRECT_CACHE_LIFETIME, 'auto_resolve_titles' => false, - 'track_orphan_visits' => true, ], ]; diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index acecda14..b467237e 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -94,10 +94,6 @@ $helper = new class { return [ - 'app_options' => [ - 'disable_track_param' => env('DISABLE_TRACK_PARAM'), - ], - 'delete_short_urls' => [ 'check_visits_threshold' => true, 'visits_threshold' => (int) env('DELETE_SHORT_URL_THRESHOLD', DEFAULT_DELETE_SHORT_URL_THRESHOLD), @@ -113,13 +109,22 @@ return [ 'hostname' => env('SHORT_DOMAIN_HOST', ''), ], 'validate_url' => (bool) env('VALIDATE_URLS', false), - 'anonymize_remote_addr' => (bool) env('ANONYMIZE_REMOTE_ADDR', true), 'visits_webhooks' => $helper->getVisitsWebhooks(), 'default_short_codes_length' => $helper->getDefaultShortCodesLength(), 'redirect_status_code' => (int) env('REDIRECT_STATUS_CODE', DEFAULT_REDIRECT_STATUS_CODE), 'redirect_cache_lifetime' => (int) env('REDIRECT_CACHE_LIFETIME', DEFAULT_REDIRECT_CACHE_LIFETIME), 'auto_resolve_titles' => (bool) env('AUTO_RESOLVE_TITLES', false), + ], + + 'tracking' => [ + 'anonymize_remote_addr' => (bool) env('ANONYMIZE_REMOTE_ADDR', true), 'track_orphan_visits' => (bool) env('TRACK_ORPHAN_VISITS', true), + 'disable_track_param' => env('DISABLE_TRACK_PARAM'), + 'disable_tracking' => (bool) env('DISABLE_TRACKING', false), + 'disable_ip_tracking' => (bool) env('DISABLE_IP_TRACKING', false), + 'disable_location_tracking' => (bool) env('DISABLE_LOCATION_TRACKING', false), + 'disable_referrer_tracking' => (bool) env('DISABLE_REFERRER_TRACKING', false), + 'disable_ua_tracking' => (bool) env('DISABLE_UA_TRACKING', false), ], 'not_found_redirects' => $helper->getNotFoundRedirectsConfig(), diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 479b497a..b84c74a4 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -24,6 +24,7 @@ return [ Options\DeleteShortUrlsOptions::class => ConfigAbstractFactory::class, Options\NotFoundRedirectOptions::class => ConfigAbstractFactory::class, Options\UrlShortenerOptions::class => ConfigAbstractFactory::class, + Options\TrackingOptions::class => ConfigAbstractFactory::class, Service\UrlShortener::class => ConfigAbstractFactory::class, Service\ShortUrlService::class => ConfigAbstractFactory::class, @@ -75,6 +76,7 @@ return [ Options\DeleteShortUrlsOptions::class => ['config.delete_short_urls'], Options\NotFoundRedirectOptions::class => ['config.not_found_redirects'], Options\UrlShortenerOptions::class => ['config.url_shortener'], + Options\TrackingOptions::class => ['config.tracking'], Service\UrlShortener::class => [ ShortUrl\Helper\ShortUrlTitleResolutionHelper::class, @@ -85,7 +87,7 @@ return [ Visit\VisitsTracker::class => [ 'em', EventDispatcherInterface::class, - Options\UrlShortenerOptions::class, + Options\TrackingOptions::class, ], Service\ShortUrlService::class => [ 'em', @@ -112,14 +114,14 @@ return [ Action\RedirectAction::class => [ Service\ShortUrl\ShortUrlResolver::class, Visit\VisitsTracker::class, - Options\AppOptions::class, + Options\TrackingOptions::class, Util\RedirectResponseHelper::class, 'Logger_Shlink', ], Action\PixelAction::class => [ Service\ShortUrl\ShortUrlResolver::class, Visit\VisitsTracker::class, - Options\AppOptions::class, + Options\TrackingOptions::class, 'Logger_Shlink', ], Action\QrCodeAction::class => [ diff --git a/module/Core/src/Action/AbstractTrackingAction.php b/module/Core/src/Action/AbstractTrackingAction.php index b6a119b2..567e930c 100644 --- a/module/Core/src/Action/AbstractTrackingAction.php +++ b/module/Core/src/Action/AbstractTrackingAction.php @@ -18,7 +18,7 @@ use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\Visitor; -use Shlinkio\Shlink\Core\Options\AppOptions; +use Shlinkio\Shlink\Core\Options\TrackingOptions; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortUrlResolverInterface; use Shlinkio\Shlink\Core\Visit\VisitsTrackerInterface; @@ -29,18 +29,18 @@ abstract class AbstractTrackingAction implements MiddlewareInterface, RequestMet { private ShortUrlResolverInterface $urlResolver; private VisitsTrackerInterface $visitTracker; - private AppOptions $appOptions; + private TrackingOptions $trackingOptions; private LoggerInterface $logger; public function __construct( ShortUrlResolverInterface $urlResolver, VisitsTrackerInterface $visitTracker, - AppOptions $appOptions, + TrackingOptions $trackingOptions, ?LoggerInterface $logger = null ) { $this->urlResolver = $urlResolver; $this->visitTracker = $visitTracker; - $this->appOptions = $appOptions; + $this->trackingOptions = $trackingOptions; $this->logger = $logger ?? new NullLogger(); } @@ -48,7 +48,7 @@ abstract class AbstractTrackingAction implements MiddlewareInterface, RequestMet { $identifier = ShortUrlIdentifier::fromRedirectRequest($request); $query = $request->getQueryParams(); - $disableTrackParam = $this->appOptions->getDisableTrackParam(); + $disableTrackParam = $this->trackingOptions->getDisableTrackParam(); try { $shortUrl = $this->urlResolver->resolveEnabledShortUrl($identifier); diff --git a/module/Core/src/Action/RedirectAction.php b/module/Core/src/Action/RedirectAction.php index d346456b..7da67b59 100644 --- a/module/Core/src/Action/RedirectAction.php +++ b/module/Core/src/Action/RedirectAction.php @@ -21,11 +21,11 @@ class RedirectAction extends AbstractTrackingAction implements StatusCodeInterfa public function __construct( ShortUrlResolverInterface $urlResolver, VisitsTrackerInterface $visitTracker, - Options\AppOptions $appOptions, + Options\TrackingOptions $trackingOptions, RedirectResponseHelperInterface $redirectResponseHelper, ?LoggerInterface $logger = null ) { - parent::__construct($urlResolver, $visitTracker, $appOptions, $logger); + parent::__construct($urlResolver, $visitTracker, $trackingOptions, $logger); $this->redirectResponseHelper = $redirectResponseHelper; } diff --git a/module/Core/src/Config/DeprecatedConfigParser.php b/module/Core/src/Config/DeprecatedConfigParser.php index 92074bfc..b3421146 100644 --- a/module/Core/src/Config/DeprecatedConfigParser.php +++ b/module/Core/src/Config/DeprecatedConfigParser.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Config; use function Functional\compose; +/** @deprecated */ class DeprecatedConfigParser { public function __invoke(array $config): array diff --git a/module/Core/src/Config/SimplifiedConfigParser.php b/module/Core/src/Config/SimplifiedConfigParser.php index b578799b..2b0b1d71 100644 --- a/module/Core/src/Config/SimplifiedConfigParser.php +++ b/module/Core/src/Config/SimplifiedConfigParser.php @@ -19,7 +19,7 @@ use function uksort; class SimplifiedConfigParser { private const SIMPLIFIED_CONFIG_MAPPING = [ - 'disable_track_param' => ['app_options', 'disable_track_param'], + 'disable_track_param' => ['tracking', 'disable_track_param'], 'short_domain_schema' => ['url_shortener', 'domain', 'schema'], 'short_domain_host' => ['url_shortener', 'domain', 'hostname'], 'validate_url' => ['url_shortener', 'validate_url'], @@ -38,7 +38,7 @@ class SimplifiedConfigParser 'mercure_public_hub_url' => ['mercure', 'public_hub_url'], 'mercure_internal_hub_url' => ['mercure', 'internal_hub_url'], 'mercure_jwt_secret' => ['mercure', 'jwt_secret'], - 'anonymize_remote_addr' => ['url_shortener', 'anonymize_remote_addr'], + 'anonymize_remote_addr' => ['tracking', 'anonymize_remote_addr'], 'redirect_status_code' => ['url_shortener', 'redirect_status_code'], 'redirect_cache_lifetime' => ['url_shortener', 'redirect_cache_lifetime'], 'port' => ['mezzio-swoole', 'swoole-http-server', 'port'], diff --git a/module/Core/src/Options/AppOptions.php b/module/Core/src/Options/AppOptions.php index 66d76126..8fde2663 100644 --- a/module/Core/src/Options/AppOptions.php +++ b/module/Core/src/Options/AppOptions.php @@ -12,7 +12,6 @@ class AppOptions extends AbstractOptions { private string $name = ''; private string $version = '1.0'; - private ?string $disableTrackParam = null; public function getName(): string { @@ -36,16 +35,10 @@ class AppOptions extends AbstractOptions return $this; } - /** - */ - public function getDisableTrackParam(): ?string - { - return $this->disableTrackParam; - } - + /** @deprecated */ protected function setDisableTrackParam(?string $disableTrackParam): self { - $this->disableTrackParam = $disableTrackParam; + // Keep just for backwards compatibility during hydration return $this; } diff --git a/module/Core/src/Options/TrackingOptions.php b/module/Core/src/Options/TrackingOptions.php new file mode 100644 index 00000000..0e1762b0 --- /dev/null +++ b/module/Core/src/Options/TrackingOptions.php @@ -0,0 +1,99 @@ +anonymizeRemoteAddr; + } + + protected function setAnonymizeRemoteAddr(bool $anonymizeRemoteAddr): void + { + $this->anonymizeRemoteAddr = $anonymizeRemoteAddr; + } + + public function trackOrphanVisits(): bool + { + return $this->trackOrphanVisits; + } + + protected function setTrackOrphanVisits(bool $trackOrphanVisits): void + { + $this->trackOrphanVisits = $trackOrphanVisits; + } + + public function getDisableTrackParam(): ?string + { + return $this->disableTrackParam; + } + + protected function setDisableTrackParam(?string $disableTrackParam): void + { + $this->disableTrackParam = $disableTrackParam; + } + + public function disableTracking(): bool + { + return $this->disableTracking; + } + + protected function setDisableTracking(bool $disableTracking): void + { + $this->disableTracking = $disableTracking; + } + + public function disableIpTracking(): bool + { + return $this->disableIpTracking; + } + + protected function setDisableIpTracking(bool $disableIpTracking): void + { + $this->disableIpTracking = $disableIpTracking; + } + + public function disableLocationTracking(): bool + { + return $this->disableLocationTracking; + } + + protected function setDisableLocationTracking(bool $disableLocationTracking): void + { + $this->disableLocationTracking = $disableLocationTracking; + } + + public function disableReferrerTracking(): bool + { + return $this->disableReferrerTracking; + } + + protected function setDisableReferrerTracking(bool $disableReferrerTracking): void + { + $this->disableReferrerTracking = $disableReferrerTracking; + } + + public function disableUaTracking(): bool + { + return $this->disableUaTracking; + } + + protected function setDisableUaTracking(bool $disableUaTracking): void + { + $this->disableUaTracking = $disableUaTracking; + } +} diff --git a/module/Core/src/Options/UrlShortenerOptions.php b/module/Core/src/Options/UrlShortenerOptions.php index e1956203..a0005da2 100644 --- a/module/Core/src/Options/UrlShortenerOptions.php +++ b/module/Core/src/Options/UrlShortenerOptions.php @@ -19,8 +19,6 @@ class UrlShortenerOptions extends AbstractOptions private int $redirectStatusCode = DEFAULT_REDIRECT_STATUS_CODE; private int $redirectCacheLifetime = DEFAULT_REDIRECT_CACHE_LIFETIME; private bool $autoResolveTitles = false; - private bool $anonymizeRemoteAddr = true; - private bool $trackOrphanVisits = true; public function isUrlValidationEnabled(): bool { @@ -69,23 +67,15 @@ class UrlShortenerOptions extends AbstractOptions $this->autoResolveTitles = $autoResolveTitles; } - public function anonymizeRemoteAddr(): bool - { - return $this->anonymizeRemoteAddr; - } - + /** @deprecated */ protected function setAnonymizeRemoteAddr(bool $anonymizeRemoteAddr): void { - $this->anonymizeRemoteAddr = $anonymizeRemoteAddr; - } - - public function trackOrphanVisits(): bool - { - return $this->trackOrphanVisits; + // Keep just for backwards compatibility during hydration } + /** @deprecated */ protected function setTrackOrphanVisits(bool $trackOrphanVisits): void { - $this->trackOrphanVisits = $trackOrphanVisits; + // Keep just for backwards compatibility during hydration } } diff --git a/module/Core/src/Visit/VisitsTracker.php b/module/Core/src/Visit/VisitsTracker.php index f8c82b49..fb820b24 100644 --- a/module/Core/src/Visit/VisitsTracker.php +++ b/module/Core/src/Visit/VisitsTracker.php @@ -10,18 +10,18 @@ use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\EventDispatcher\Event\UrlVisited; use Shlinkio\Shlink\Core\Model\Visitor; -use Shlinkio\Shlink\Core\Options\UrlShortenerOptions; +use Shlinkio\Shlink\Core\Options\TrackingOptions; class VisitsTracker implements VisitsTrackerInterface { private ORM\EntityManagerInterface $em; private EventDispatcherInterface $eventDispatcher; - private UrlShortenerOptions $options; + private TrackingOptions $options; public function __construct( ORM\EntityManagerInterface $em, EventDispatcherInterface $eventDispatcher, - UrlShortenerOptions $options + TrackingOptions $options ) { $this->em = $em; $this->eventDispatcher = $eventDispatcher; diff --git a/module/Core/test/Action/PixelActionTest.php b/module/Core/test/Action/PixelActionTest.php index 065cc2c4..6df2498a 100644 --- a/module/Core/test/Action/PixelActionTest.php +++ b/module/Core/test/Action/PixelActionTest.php @@ -14,7 +14,7 @@ use Shlinkio\Shlink\Common\Response\PixelResponse; use Shlinkio\Shlink\Core\Action\PixelAction; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; -use Shlinkio\Shlink\Core\Options\AppOptions; +use Shlinkio\Shlink\Core\Options\TrackingOptions; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortUrlResolverInterface; use Shlinkio\Shlink\Core\Visit\VisitsTracker; @@ -34,7 +34,7 @@ class PixelActionTest extends TestCase $this->action = new PixelAction( $this->urlResolver->reveal(), $this->visitTracker->reveal(), - new AppOptions(), + new TrackingOptions(), ); } diff --git a/module/Core/test/Action/RedirectActionTest.php b/module/Core/test/Action/RedirectActionTest.php index f869e2c4..dde9144c 100644 --- a/module/Core/test/Action/RedirectActionTest.php +++ b/module/Core/test/Action/RedirectActionTest.php @@ -42,7 +42,7 @@ class RedirectActionTest extends TestCase $this->action = new RedirectAction( $this->urlResolver->reveal(), $this->visitTracker->reveal(), - new Options\AppOptions(['disableTrackParam' => 'foobar']), + new Options\TrackingOptions(['disableTrackParam' => 'foobar']), $this->redirectRespHelper->reveal(), ); } diff --git a/module/Core/test/Config/SimplifiedConfigParserTest.php b/module/Core/test/Config/SimplifiedConfigParserTest.php index 6f040bb6..f4e5c8f0 100644 --- a/module/Core/test/Config/SimplifiedConfigParserTest.php +++ b/module/Core/test/Config/SimplifiedConfigParserTest.php @@ -22,7 +22,7 @@ class SimplifiedConfigParserTest extends TestCase public function properlyMapsSimplifiedConfig(): void { $config = [ - 'app_options' => [ + 'tracking' => [ 'disable_track_param' => 'foo', ], @@ -70,8 +70,9 @@ class SimplifiedConfigParserTest extends TestCase 'port' => 8888, ]; $expected = [ - 'app_options' => [ + 'tracking' => [ 'disable_track_param' => 'bar', + 'anonymize_remote_addr' => false, ], 'entity_manager' => [ @@ -96,7 +97,6 @@ class SimplifiedConfigParserTest extends TestCase 'https://third-party.io/foo', ], 'default_short_codes_length' => 8, - 'anonymize_remote_addr' => false, 'redirect_status_code' => 301, 'redirect_cache_lifetime' => 90, ], diff --git a/module/Core/test/Visit/VisitsTrackerTest.php b/module/Core/test/Visit/VisitsTrackerTest.php index bba4e919..e39a4522 100644 --- a/module/Core/test/Visit/VisitsTrackerTest.php +++ b/module/Core/test/Visit/VisitsTrackerTest.php @@ -14,7 +14,7 @@ use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\EventDispatcher\Event\UrlVisited; use Shlinkio\Shlink\Core\Model\Visitor; -use Shlinkio\Shlink\Core\Options\UrlShortenerOptions; +use Shlinkio\Shlink\Core\Options\TrackingOptions; use Shlinkio\Shlink\Core\Visit\VisitsTracker; class VisitsTrackerTest extends TestCase @@ -24,7 +24,7 @@ class VisitsTrackerTest extends TestCase private VisitsTracker $visitsTracker; private ObjectProphecy $em; private ObjectProphecy $eventDispatcher; - private UrlShortenerOptions $options; + private TrackingOptions $options; public function setUp(): void { @@ -35,7 +35,7 @@ class VisitsTrackerTest extends TestCase }); $this->eventDispatcher = $this->prophesize(EventDispatcherInterface::class); - $this->options = new UrlShortenerOptions(); + $this->options = new TrackingOptions(); $this->visitsTracker = new VisitsTracker($this->em->reveal(), $this->eventDispatcher->reveal(), $this->options); } From f0dc32b6e5049456570d7861cb72dc9bc3c03aaf Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 16 May 2021 09:51:52 +0200 Subject: [PATCH 58/86] Added logic for new tracking options --- config/autoload/tracking.global.php | 3 -- docker/config/shlink_in_docker.local.php | 1 - module/Core/src/Model/Visitor.php | 13 ++++++++ module/Core/src/Options/TrackingOptions.php | 11 ------- module/Core/src/Visit/VisitsTracker.php | 34 +++++++++++++------- module/Core/test/Model/VisitorTest.php | 25 ++++++++++++++ module/Core/test/Visit/VisitsTrackerTest.php | 16 +++++++++ 7 files changed, 76 insertions(+), 27 deletions(-) diff --git a/config/autoload/tracking.global.php b/config/autoload/tracking.global.php index 1ddec8b8..4fdb5b19 100644 --- a/config/autoload/tracking.global.php +++ b/config/autoload/tracking.global.php @@ -21,9 +21,6 @@ return [ // If true, visits will be tracked, but neither the IP address or the location will be resolved 'disable_ip_tracking' => false, - // If true, visits will be tracked including the IP address, but the location won't be resolved - 'disable_location_tracking' => false, - // If true, the referrer will not be tracked 'disable_referrer_tracking' => false, diff --git a/docker/config/shlink_in_docker.local.php b/docker/config/shlink_in_docker.local.php index b467237e..2a8369d7 100644 --- a/docker/config/shlink_in_docker.local.php +++ b/docker/config/shlink_in_docker.local.php @@ -122,7 +122,6 @@ return [ 'disable_track_param' => env('DISABLE_TRACK_PARAM'), 'disable_tracking' => (bool) env('DISABLE_TRACKING', false), 'disable_ip_tracking' => (bool) env('DISABLE_IP_TRACKING', false), - 'disable_location_tracking' => (bool) env('DISABLE_LOCATION_TRACKING', false), 'disable_referrer_tracking' => (bool) env('DISABLE_REFERRER_TRACKING', false), 'disable_ua_tracking' => (bool) env('DISABLE_UA_TRACKING', false), ], diff --git a/module/Core/src/Model/Visitor.php b/module/Core/src/Model/Visitor.php index 7438bdce..9564a41c 100644 --- a/module/Core/src/Model/Visitor.php +++ b/module/Core/src/Model/Visitor.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Model; use Psr\Http\Message\ServerRequestInterface; use Shlinkio\Shlink\Common\Middleware\IpAddressMiddlewareFactory; +use Shlinkio\Shlink\Core\Options\TrackingOptions; use function substr; @@ -68,4 +69,16 @@ final class Visitor { return $this->visitedUrl; } + + public function normalizeForTrackingOptions(TrackingOptions $options): self + { + $instance = self::emptyInstance(); + + $instance->userAgent = $options->disableUaTracking() ? '' : $this->userAgent; + $instance->referer = $options->disableReferrerTracking() ? '' : $this->referer; + $instance->remoteAddress = $options->disableIpTracking() ? null : $this->remoteAddress; + $instance->visitedUrl = $this->visitedUrl; + + return $instance; + } } diff --git a/module/Core/src/Options/TrackingOptions.php b/module/Core/src/Options/TrackingOptions.php index 0e1762b0..98e09085 100644 --- a/module/Core/src/Options/TrackingOptions.php +++ b/module/Core/src/Options/TrackingOptions.php @@ -13,7 +13,6 @@ class TrackingOptions extends AbstractOptions private ?string $disableTrackParam = null; private bool $disableTracking = false; private bool $disableIpTracking = false; - private bool $disableLocationTracking = false; private bool $disableReferrerTracking = false; private bool $disableUaTracking = false; @@ -67,16 +66,6 @@ class TrackingOptions extends AbstractOptions $this->disableIpTracking = $disableIpTracking; } - public function disableLocationTracking(): bool - { - return $this->disableLocationTracking; - } - - protected function setDisableLocationTracking(bool $disableLocationTracking): void - { - $this->disableLocationTracking = $disableLocationTracking; - } - public function disableReferrerTracking(): bool { return $this->disableReferrerTracking; diff --git a/module/Core/src/Visit/VisitsTracker.php b/module/Core/src/Visit/VisitsTracker.php index fb820b24..68c1891a 100644 --- a/module/Core/src/Visit/VisitsTracker.php +++ b/module/Core/src/Visit/VisitsTracker.php @@ -32,39 +32,49 @@ class VisitsTracker implements VisitsTrackerInterface { $this->trackVisit( Visit::forValidShortUrl($shortUrl, $visitor, $this->options->anonymizeRemoteAddr()), - $visitor, + $visitor->normalizeForTrackingOptions($this->options), ); } public function trackInvalidShortUrlVisit(Visitor $visitor): void { - if (! $this->options->trackOrphanVisits()) { - return; - } - - $this->trackVisit(Visit::forInvalidShortUrl($visitor, $this->options->anonymizeRemoteAddr()), $visitor); + $this->trackOrphanVisit( + Visit::forInvalidShortUrl($visitor, $this->options->anonymizeRemoteAddr()), + $visitor->normalizeForTrackingOptions($this->options), + ); } public function trackBaseUrlVisit(Visitor $visitor): void { - if (! $this->options->trackOrphanVisits()) { - return; - } - - $this->trackVisit(Visit::forBasePath($visitor, $this->options->anonymizeRemoteAddr()), $visitor); + $this->trackOrphanVisit( + Visit::forBasePath($visitor, $this->options->anonymizeRemoteAddr()), + $visitor->normalizeForTrackingOptions($this->options), + ); } public function trackRegularNotFoundVisit(Visitor $visitor): void + { + $this->trackOrphanVisit( + Visit::forRegularNotFound($visitor, $this->options->anonymizeRemoteAddr()), + $visitor->normalizeForTrackingOptions($this->options), + ); + } + + private function trackOrphanVisit(Visit $visit, Visitor $visitor): void { if (! $this->options->trackOrphanVisits()) { return; } - $this->trackVisit(Visit::forRegularNotFound($visitor, $this->options->anonymizeRemoteAddr()), $visitor); + $this->trackVisit($visit, $visitor); } private function trackVisit(Visit $visit, Visitor $visitor): void { + if ($this->options->disableTracking()) { + return; + } + $this->em->transactional(function () use ($visit, $visitor): void { $this->em->persist($visit); $this->em->flush(); diff --git a/module/Core/test/Model/VisitorTest.php b/module/Core/test/Model/VisitorTest.php index e1003056..50c277c4 100644 --- a/module/Core/test/Model/VisitorTest.php +++ b/module/Core/test/Model/VisitorTest.php @@ -6,6 +6,7 @@ namespace ShlinkioTest\Shlink\Core\Model; use PHPUnit\Framework\TestCase; use Shlinkio\Shlink\Core\Model\Visitor; +use Shlinkio\Shlink\Core\Options\TrackingOptions; use function random_int; use function str_repeat; @@ -71,4 +72,28 @@ class VisitorTest extends TestCase } return $randomString; } + + /** @test */ + public function newNormalizedInstanceIsCreatedFromTrackingOptions(): void + { + $visitor = new Visitor( + $this->generateRandomString(2000), + $this->generateRandomString(2000), + $this->generateRandomString(2000), + $this->generateRandomString(2000), + ); + $normalizedVisitor = $visitor->normalizeForTrackingOptions(new TrackingOptions([ + 'disableIpTracking' => true, + 'disableReferrerTracking' => true, + 'disableUaTracking' => true, + ])); + + self::assertNotSame($visitor, $normalizedVisitor); + self::assertEmpty($normalizedVisitor->getUserAgent()); + self::assertNotEmpty($visitor->getUserAgent()); + self::assertEmpty($normalizedVisitor->getReferer()); + self::assertNotEmpty($visitor->getReferer()); + self::assertNull($normalizedVisitor->getRemoteAddress()); + self::assertNotNull($visitor->getRemoteAddress()); + } } diff --git a/module/Core/test/Visit/VisitsTrackerTest.php b/module/Core/test/Visit/VisitsTrackerTest.php index e39a4522..45188f6c 100644 --- a/module/Core/test/Visit/VisitsTrackerTest.php +++ b/module/Core/test/Visit/VisitsTrackerTest.php @@ -57,6 +57,22 @@ class VisitsTrackerTest extends TestCase $this->eventDispatcher->dispatch(Argument::type(UrlVisited::class))->shouldHaveBeenCalled(); } + /** + * @test + * @dataProvider provideTrackingMethodNames + */ + public function trackingIsSkippedCompletelyWhenDisabledFromOptions(string $method, array $args): void + { + $this->options->disableTracking = true; + + $this->visitsTracker->{$method}(...$args); + + $this->eventDispatcher->dispatch(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->em->transactional(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->em->persist(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->em->flush()->shouldNotHaveBeenCalled(); + } + public function provideTrackingMethodNames(): iterable { yield 'track' => ['track', [ShortUrl::createEmpty(), Visitor::emptyInstance()]]; From 15c028e151ca652277f68d621f61f25b6b03f19d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 16 May 2021 10:08:05 +0200 Subject: [PATCH 59/86] Ensured visitor is normalized before creating the visit --- module/Core/src/Visit/VisitsTracker.php | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/module/Core/src/Visit/VisitsTracker.php b/module/Core/src/Visit/VisitsTracker.php index 68c1891a..f77cd624 100644 --- a/module/Core/src/Visit/VisitsTracker.php +++ b/module/Core/src/Visit/VisitsTracker.php @@ -31,50 +31,51 @@ class VisitsTracker implements VisitsTrackerInterface public function track(ShortUrl $shortUrl, Visitor $visitor): void { $this->trackVisit( - Visit::forValidShortUrl($shortUrl, $visitor, $this->options->anonymizeRemoteAddr()), - $visitor->normalizeForTrackingOptions($this->options), + fn (Visitor $v) => Visit::forValidShortUrl($shortUrl, $v, $this->options->anonymizeRemoteAddr()), + $visitor, ); } public function trackInvalidShortUrlVisit(Visitor $visitor): void { $this->trackOrphanVisit( - Visit::forInvalidShortUrl($visitor, $this->options->anonymizeRemoteAddr()), - $visitor->normalizeForTrackingOptions($this->options), + fn (Visitor $v) => Visit::forInvalidShortUrl($v, $this->options->anonymizeRemoteAddr()), + $visitor, ); } public function trackBaseUrlVisit(Visitor $visitor): void { $this->trackOrphanVisit( - Visit::forBasePath($visitor, $this->options->anonymizeRemoteAddr()), - $visitor->normalizeForTrackingOptions($this->options), + fn (Visitor $v) => Visit::forBasePath($v, $this->options->anonymizeRemoteAddr()), + $visitor, ); } public function trackRegularNotFoundVisit(Visitor $visitor): void { $this->trackOrphanVisit( - Visit::forRegularNotFound($visitor, $this->options->anonymizeRemoteAddr()), - $visitor->normalizeForTrackingOptions($this->options), + fn (Visitor $v) => Visit::forRegularNotFound($v, $this->options->anonymizeRemoteAddr()), + $visitor, ); } - private function trackOrphanVisit(Visit $visit, Visitor $visitor): void + private function trackOrphanVisit(callable $createVisit, Visitor $visitor): void { if (! $this->options->trackOrphanVisits()) { return; } - $this->trackVisit($visit, $visitor); + $this->trackVisit($createVisit, $visitor); } - private function trackVisit(Visit $visit, Visitor $visitor): void + private function trackVisit(callable $createVisit, Visitor $visitor): void { if ($this->options->disableTracking()) { return; } + $visit = $createVisit($visitor->normalizeForTrackingOptions($this->options)); $this->em->transactional(function () use ($visit, $visitor): void { $this->em->persist($visit); $this->em->flush(); From 9cff570c45fb15d0162ce22ef2e2c6af19b20d85 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 16 May 2021 10:12:35 +0200 Subject: [PATCH 60/86] Updated changelog --- CHANGELOG.md | 8 ++++++++ config/autoload/tracking.global.php | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78ba489c..5bcfe116 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * [#1059](https://github.com/shlinkio/shlink/issues/1059) Added ability to optionally display author API key and its name when listing short URLs from the command line. * [#1066](https://github.com/shlinkio/shlink/issues/1066) Added support to import short URLs and their visits from another Shlink instance using its API. +* [#898](https://github.com/shlinkio/shlink/issues/898) Improved tracking granularity, allowing to disable visits tracking completely, or just parts of it. + + In order to achieve it, Shlink now supports 4 new tracking-related options, that can be customized via env vars for docker, or via installer: + + * `disable_tracking`: If true, visits will not be tracked at all. + * `disable_ip_tracking`: If true, visits will be tracked, but neither the IP address, nor the location will be resolved. + * `disable_referrer_tracking`: If true, the referrer will not be tracked. + * `disable_ua_tracking`: If true, the user agent will not be tracked. ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. diff --git a/config/autoload/tracking.global.php b/config/autoload/tracking.global.php index 4fdb5b19..4fdf0ba6 100644 --- a/config/autoload/tracking.global.php +++ b/config/autoload/tracking.global.php @@ -18,7 +18,7 @@ return [ // If true, visits will not be tracked at all 'disable_tracking' => false, - // If true, visits will be tracked, but neither the IP address or the location will be resolved + // If true, visits will be tracked, but neither the IP address, nor the location will be resolved 'disable_ip_tracking' => false, // If true, the referrer will not be tracked From 3535688c3b3f099b5a071c9a53accf774fae2c8b Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 16 May 2021 13:21:12 +0200 Subject: [PATCH 61/86] Updated to latest installer with support for all tracking options --- composer.json | 2 +- config/autoload/installer.global.php | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/composer.json b/composer.json index 87275e92..2474cebf 100644 --- a/composer.json +++ b/composer.json @@ -50,7 +50,7 @@ "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", "shlinkio/shlink-importer": "dev-main#39928b6 as 2.3", - "shlinkio/shlink-installer": "dev-develop#aa50ea9 as 5.5", + "shlinkio/shlink-installer": "dev-develop#15cf3b3 as 6.0", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", "symfony/filesystem": "^5.1", diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php index 605b16ce..0a72c6fa 100644 --- a/config/autoload/installer.global.php +++ b/config/autoload/installer.global.php @@ -27,7 +27,6 @@ return [ Option\Redirect\BaseUrlRedirectConfigOption::class, Option\Redirect\InvalidShortUrlRedirectConfigOption::class, Option\Redirect\Regular404RedirectConfigOption::class, - Option\DisableTrackParamConfigOption::class, Option\Visit\CheckVisitsThresholdConfigOption::class, Option\Visit\VisitsThresholdConfigOption::class, Option\BasePathConfigOption::class, @@ -40,11 +39,16 @@ return [ Option\Mercure\MercureInternalUrlConfigOption::class, Option\Mercure\MercureJwtSecretConfigOption::class, Option\UrlShortener\GeoLiteLicenseKeyConfigOption::class, - Option\UrlShortener\IpAnonymizationConfigOption::class, Option\UrlShortener\RedirectStatusCodeConfigOption::class, Option\UrlShortener\RedirectCacheLifeTimeConfigOption::class, Option\UrlShortener\AutoResolveTitlesConfigOption::class, - Option\UrlShortener\OrphanVisitsTrackingConfigOption::class, + Option\Tracking\IpAnonymizationConfigOption::class, + Option\Tracking\OrphanVisitsTrackingConfigOption::class, + Option\Tracking\DisableTrackParamConfigOption::class, + Option\Tracking\DisableTrackingConfigOption::class, + Option\Tracking\DisableIpTrackingConfigOption::class, + Option\Tracking\DisableReferrerTrackingConfigOption::class, + Option\Tracking\DisableUaTrackingConfigOption::class, ], 'installation_commands' => [ From 7280b48cdcc19a2558200d20929ab0126d851247 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 07:15:34 +0200 Subject: [PATCH 62/86] Created action to dynamically build the robots.txt --- module/Core/config/dependencies.config.php | 4 + module/Core/config/routes.config.php | 8 ++ module/Core/src/Action/RobotsAction.php | 49 ++++++++++++ module/Core/src/Crawling/CrawlingHelper.php | 13 ++++ .../src/Crawling/CrawlingHelperInterface.php | 13 ++++ module/Core/test/Action/RobotsActionTest.php | 75 +++++++++++++++++++ public/robots.txt | 5 -- 7 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 module/Core/src/Action/RobotsAction.php create mode 100644 module/Core/src/Crawling/CrawlingHelper.php create mode 100644 module/Core/src/Crawling/CrawlingHelperInterface.php create mode 100644 module/Core/test/Action/RobotsActionTest.php delete mode 100644 public/robots.txt diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index b84c74a4..4eb3d60d 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -48,6 +48,7 @@ return [ Action\RedirectAction::class => ConfigAbstractFactory::class, Action\PixelAction::class => ConfigAbstractFactory::class, Action\QrCodeAction::class => ConfigAbstractFactory::class, + Action\RobotsAction::class => ConfigAbstractFactory::class, ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ConfigAbstractFactory::class, ShortUrl\Helper\ShortUrlStringifier::class => ConfigAbstractFactory::class, @@ -57,6 +58,8 @@ return [ Mercure\MercureUpdatesGenerator::class => ConfigAbstractFactory::class, Importer\ImportedLinksProcessor::class => ConfigAbstractFactory::class, + + Crawling\CrawlingHelper::class => InvokableFactory::class, ], 'aliases' => [ @@ -129,6 +132,7 @@ return [ ShortUrl\Helper\ShortUrlStringifier::class, 'Logger_Shlink', ], + Action\RobotsAction::class => [Crawling\CrawlingHelper::class], ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ['em'], ShortUrl\Helper\ShortUrlStringifier::class => ['config.url_shortener.domain', 'config.router.base_path'], diff --git a/module/Core/config/routes.config.php b/module/Core/config/routes.config.php index a95e8e96..c3f4b66a 100644 --- a/module/Core/config/routes.config.php +++ b/module/Core/config/routes.config.php @@ -9,6 +9,14 @@ use Shlinkio\Shlink\Core\Action; return [ 'routes' => [ + [ + 'name' => Action\RobotsAction::class, + 'path' => '/robots.txt', + 'middleware' => [ + Action\RobotsAction::class, + ], + 'allowed_methods' => [RequestMethod::METHOD_GET], + ], [ 'name' => Action\RedirectAction::class, 'path' => '/{shortCode}', diff --git a/module/Core/src/Action/RobotsAction.php b/module/Core/src/Action/RobotsAction.php new file mode 100644 index 00000000..31539b92 --- /dev/null +++ b/module/Core/src/Action/RobotsAction.php @@ -0,0 +1,49 @@ +crawlingHelper = $crawlingHelper; + } + + public function handle(ServerRequestInterface $request): ResponseInterface + { + return new Response(self::STATUS_OK, ['Content-type' => 'text/plain'], $this->buildRobots()); + } + + private function buildRobots(): iterable + { + yield <<crawlingHelper->listCrawlableShortCodes(); + foreach ($shortCodes as $shortCode) { + yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL); + } + + yield 'Disallow: /'; + } +} diff --git a/module/Core/src/Crawling/CrawlingHelper.php b/module/Core/src/Crawling/CrawlingHelper.php new file mode 100644 index 00000000..ef54761b --- /dev/null +++ b/module/Core/src/Crawling/CrawlingHelper.php @@ -0,0 +1,13 @@ +helper = $this->prophesize(CrawlingHelperInterface::class); + $this->action = new RobotsAction($this->helper->reveal()); + } + + /** + * @test + * @dataProvider provideShortCodes + */ + public function buildsRobotsLinesFromCrawlableShortCodes(array $shortCodes, string $expected): void + { + $getShortCodes = $this->helper->listCrawlableShortCodes()->willReturn($shortCodes); + + $response = $this->action->handle(ServerRequestFactory::fromGlobals()); + + self::assertEquals(200, $response->getStatusCode()); + self::assertEquals($expected, $response->getBody()->__toString()); + self::assertEquals('text/plain', $response->getHeaderLine('Content-Type')); + $getShortCodes->shouldHaveBeenCalledOnce(); + } + + public function provideShortCodes(): iterable + { + yield 'three short codes' => [['foo', 'bar', 'baz'], << [['foo', 'bar', 'some', 'thing', 'baz'], << [[], << Date: Sat, 22 May 2021 07:32:47 +0200 Subject: [PATCH 63/86] Documented crawlable prop in API specs --- docs/async-api/async-api.json | 13 ++++++++++++- docs/swagger/definitions/ShortUrl.json | 4 ++++ docs/swagger/paths/v1_short-urls.json | 13 +++++++++---- docs/swagger/paths/v1_short-urls_shorten.json | 3 ++- docs/swagger/paths/v1_short-urls_{shortCode}.json | 6 ++++-- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/async-api/async-api.json b/docs/async-api/async-api.json index df9bc6d6..3360d897 100644 --- a/docs/async-api/async-api.json +++ b/docs/async-api/async-api.json @@ -116,6 +116,15 @@ "domain": { "type": "string", "description": "The domain in which the short URL was created. Null if it belongs to default domain." + }, + "title": { + "type": "string", + "nullable": true, + "description": "A descriptive title of the short URL." + }, + "crawlable": { + "type": "boolean", + "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt." } }, "example": { @@ -133,7 +142,9 @@ "validUntil": null, "maxVisits": 100 }, - "domain": "example.com" + "domain": "example.com", + "title": "The title", + "crawlable": false } }, "ShortUrlMeta": { diff --git a/docs/swagger/definitions/ShortUrl.json b/docs/swagger/definitions/ShortUrl.json index 3e4c6ead..b2ffd3f6 100644 --- a/docs/swagger/definitions/ShortUrl.json +++ b/docs/swagger/definitions/ShortUrl.json @@ -41,6 +41,10 @@ "type": "string", "nullable": true, "description": "A descriptive title of the short URL." + }, + "crawlable": { + "type": "boolean", + "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt." } } } diff --git a/docs/swagger/paths/v1_short-urls.json b/docs/swagger/paths/v1_short-urls.json index b034dcf3..4d30e4de 100644 --- a/docs/swagger/paths/v1_short-urls.json +++ b/docs/swagger/paths/v1_short-urls.json @@ -140,7 +140,8 @@ "maxVisits": 100 }, "domain": null, - "title": "Welcome to Steam" + "title": "Welcome to Steam", + "crawlable": false }, { "shortCode": "12Kb3", @@ -157,7 +158,8 @@ "maxVisits": null }, "domain": null, - "title": null + "title": null, + "crawlable": false }, { "shortCode": "123bA", @@ -172,7 +174,8 @@ "maxVisits": null }, "domain": "example.com", - "title": null + "title": null, + "crawlable": false } ], "pagination": { @@ -305,7 +308,9 @@ "validUntil": null, "maxVisits": 500 }, - "domain": null + "domain": null, + "title": null, + "crawlable": false } } }, diff --git a/docs/swagger/paths/v1_short-urls_shorten.json b/docs/swagger/paths/v1_short-urls_shorten.json index b6184d8d..90c3eda5 100644 --- a/docs/swagger/paths/v1_short-urls_shorten.json +++ b/docs/swagger/paths/v1_short-urls_shorten.json @@ -74,7 +74,8 @@ "maxVisits": 100 }, "domain": null, - "title": null + "title": null, + "crawlable": false }, "text/plain": "https://doma.in/abc123" } diff --git a/docs/swagger/paths/v1_short-urls_{shortCode}.json b/docs/swagger/paths/v1_short-urls_{shortCode}.json index 2281d9b8..11ff276f 100644 --- a/docs/swagger/paths/v1_short-urls_{shortCode}.json +++ b/docs/swagger/paths/v1_short-urls_{shortCode}.json @@ -54,7 +54,8 @@ "maxVisits": 100 }, "domain": null, - "title": null + "title": null, + "crawlable": false } } }, @@ -184,7 +185,8 @@ "maxVisits": 100 }, "domain": null, - "title": "Shlink - The URL shortener" + "title": "Shlink - The URL shortener", + "crawlable": false } } }, From 348e34d52e20b663f231642a3657dd984b47d992 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 07:35:47 +0200 Subject: [PATCH 64/86] Added new crawlable flag to Short URLs --- data/migrations/Version20210522051601.php | 26 +++++++++++++++++++ .../Shlinkio.Shlink.Core.Entity.ShortUrl.php | 5 ++++ module/Core/src/Entity/ShortUrl.php | 10 +++++++ module/Core/src/Model/ShortUrlEdit.php | 14 ++++++++++ module/Core/src/Model/ShortUrlMeta.php | 7 +++++ .../Transformer/ShortUrlDataTransformer.php | 1 + .../src/Validation/ShortUrlInputFilter.php | 3 +++ .../Mercure/MercureUpdatesGeneratorTest.php | 1 + .../test-api/Action/ListShortUrlsTest.php | 6 +++++ .../test-api/Fixtures/ShortUrlsFixture.php | 1 + 10 files changed, 74 insertions(+) create mode 100644 data/migrations/Version20210522051601.php diff --git a/data/migrations/Version20210522051601.php b/data/migrations/Version20210522051601.php new file mode 100644 index 00000000..9e2bd19e --- /dev/null +++ b/data/migrations/Version20210522051601.php @@ -0,0 +1,26 @@ +getTable('short_urls'); + $this->skipIf($shortUrls->hasColumn('crawlable')); + $shortUrls->addColumn('crawlable', Types::BOOLEAN, ['default' => false]); + } + + public function down(Schema $schema): void + { + $shortUrls = $schema->getTable('short_urls'); + $this->skipIf(! $shortUrls->hasColumn('crawlable')); + $shortUrls->dropColumn('crawlable'); + } +} diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php index 751e513c..a9269d36 100644 --- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php @@ -95,4 +95,9 @@ return static function (ClassMetadata $metadata, array $emConfig): void { ->columnName('title_was_auto_resolved') ->option('default', false) ->build(); + + $builder->createField('crawlable', Types::BOOLEAN) + ->columnName('crawlable') + ->option('default', false) + ->build(); }; diff --git a/module/Core/src/Entity/ShortUrl.php b/module/Core/src/Entity/ShortUrl.php index 3fe2932b..6f502da3 100644 --- a/module/Core/src/Entity/ShortUrl.php +++ b/module/Core/src/Entity/ShortUrl.php @@ -42,6 +42,7 @@ class ShortUrl extends AbstractEntity private ?ApiKey $authorApiKey = null; private ?string $title = null; private bool $titleWasAutoResolved = false; + private bool $crawlable = false; private function __construct() { @@ -78,6 +79,7 @@ class ShortUrl extends AbstractEntity $instance->authorApiKey = $meta->getApiKey(); $instance->title = $meta->getTitle(); $instance->titleWasAutoResolved = $meta->titleWasAutoResolved(); + $instance->crawlable = $meta->isCrawlable(); return $instance; } @@ -200,6 +202,11 @@ class ShortUrl extends AbstractEntity return $this->title; } + public function crawlable(): bool + { + return $this->crawlable; + } + public function update( ShortUrlEdit $shortUrlEdit, ?ShortUrlRelationResolverInterface $relationResolver = null @@ -220,6 +227,9 @@ class ShortUrl extends AbstractEntity $relationResolver = $relationResolver ?? new SimpleShortUrlRelationResolver(); $this->tags = $relationResolver->resolveTags($shortUrlEdit->tags()); } + if ($shortUrlEdit->crawlableWasProvided()) { + $this->crawlable = $shortUrlEdit->crawlable(); + } if ( $this->title === null || $shortUrlEdit->titleWasProvided() diff --git a/module/Core/src/Model/ShortUrlEdit.php b/module/Core/src/Model/ShortUrlEdit.php index 3327aad4..32c1ca1e 100644 --- a/module/Core/src/Model/ShortUrlEdit.php +++ b/module/Core/src/Model/ShortUrlEdit.php @@ -30,6 +30,8 @@ final class ShortUrlEdit implements TitleResolutionModelInterface private ?string $title = null; private bool $titleWasAutoResolved = false; private ?bool $validateUrl = null; + private bool $crawlablePropWasProvided = false; + private bool $crawlable = false; private function __construct() { @@ -61,6 +63,7 @@ final class ShortUrlEdit implements TitleResolutionModelInterface $this->maxVisitsPropWasProvided = array_key_exists(ShortUrlInputFilter::MAX_VISITS, $data); $this->tagsPropWasProvided = array_key_exists(ShortUrlInputFilter::TAGS, $data); $this->titlePropWasProvided = array_key_exists(ShortUrlInputFilter::TITLE, $data); + $this->crawlablePropWasProvided = array_key_exists(ShortUrlInputFilter::CRAWLABLE, $data); $this->longUrl = $inputFilter->getValue(ShortUrlInputFilter::LONG_URL); $this->validSince = parseDateField($inputFilter->getValue(ShortUrlInputFilter::VALID_SINCE)); @@ -69,6 +72,7 @@ final class ShortUrlEdit implements TitleResolutionModelInterface $this->validateUrl = getOptionalBoolFromInputFilter($inputFilter, ShortUrlInputFilter::VALIDATE_URL); $this->tags = $inputFilter->getValue(ShortUrlInputFilter::TAGS); $this->title = $inputFilter->getValue(ShortUrlInputFilter::TITLE); + $this->crawlable = $inputFilter->getValue(ShortUrlInputFilter::CRAWLABLE); } public function longUrl(): ?string @@ -162,4 +166,14 @@ final class ShortUrlEdit implements TitleResolutionModelInterface { return $this->validateUrl; } + + public function crawlable(): bool + { + return $this->crawlable; + } + + public function crawlableWasProvided(): bool + { + return $this->crawlablePropWasProvided; + } } diff --git a/module/Core/src/Model/ShortUrlMeta.php b/module/Core/src/Model/ShortUrlMeta.php index df25735c..06e0eee7 100644 --- a/module/Core/src/Model/ShortUrlMeta.php +++ b/module/Core/src/Model/ShortUrlMeta.php @@ -31,6 +31,7 @@ final class ShortUrlMeta implements TitleResolutionModelInterface private array $tags = []; private ?string $title = null; private bool $titleWasAutoResolved = false; + private bool $crawlable = false; private function __construct() { @@ -80,6 +81,7 @@ final class ShortUrlMeta implements TitleResolutionModelInterface $this->apiKey = $inputFilter->getValue(ShortUrlInputFilter::API_KEY); $this->tags = $inputFilter->getValue(ShortUrlInputFilter::TAGS); $this->title = $inputFilter->getValue(ShortUrlInputFilter::TITLE); + $this->crawlable = $inputFilter->getValue(ShortUrlInputFilter::CRAWLABLE); } public function getLongUrl(): string @@ -188,4 +190,9 @@ final class ShortUrlMeta implements TitleResolutionModelInterface return $copy; } + + public function isCrawlable(): bool + { + return $this->crawlable; + } } diff --git a/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php b/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php index 49918867..52b98c36 100644 --- a/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php +++ b/module/Core/src/ShortUrl/Transformer/ShortUrlDataTransformer.php @@ -35,6 +35,7 @@ class ShortUrlDataTransformer implements DataTransformerInterface 'meta' => $this->buildMeta($shortUrl), 'domain' => $shortUrl->getDomain(), 'title' => $shortUrl->title(), + 'crawlable' => $shortUrl->crawlable(), ]; } diff --git a/module/Core/src/Validation/ShortUrlInputFilter.php b/module/Core/src/Validation/ShortUrlInputFilter.php index b5d4fa07..c7cdaa43 100644 --- a/module/Core/src/Validation/ShortUrlInputFilter.php +++ b/module/Core/src/Validation/ShortUrlInputFilter.php @@ -32,6 +32,7 @@ class ShortUrlInputFilter extends InputFilter public const API_KEY = 'apiKey'; public const TAGS = 'tags'; public const TITLE = 'title'; + public const CRAWLABLE = 'crawlable'; private function __construct(array $data, bool $requireLongUrl) { @@ -105,5 +106,7 @@ class ShortUrlInputFilter extends InputFilter $this->add($this->createTagsInput(self::TAGS, false)); $this->add($this->createInput(self::TITLE, false)); + + $this->add($this->createBooleanInput(self::CRAWLABLE, false)); } } diff --git a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php index b4361ca5..1d460623 100644 --- a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php +++ b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php @@ -59,6 +59,7 @@ class MercureUpdatesGeneratorTest extends TestCase ], 'domain' => null, 'title' => $title, + 'crawlable' => false, ], 'visit' => [ 'referer' => '', diff --git a/module/Rest/test-api/Action/ListShortUrlsTest.php b/module/Rest/test-api/Action/ListShortUrlsTest.php index f81524ae..95d77dc6 100644 --- a/module/Rest/test-api/Action/ListShortUrlsTest.php +++ b/module/Rest/test-api/Action/ListShortUrlsTest.php @@ -26,6 +26,7 @@ class ListShortUrlsTest extends ApiTestCase ], 'domain' => null, 'title' => 'My cool title', + 'crawlable' => true, ]; private const SHORT_URL_DOCS = [ 'shortCode' => 'ghi789', @@ -41,6 +42,7 @@ class ListShortUrlsTest extends ApiTestCase ], 'domain' => null, 'title' => null, + 'crawlable' => false, ]; private const SHORT_URL_CUSTOM_SLUG_AND_DOMAIN = [ 'shortCode' => 'custom-with-domain', @@ -56,6 +58,7 @@ class ListShortUrlsTest extends ApiTestCase ], 'domain' => 'some-domain.com', 'title' => null, + 'crawlable' => false, ]; private const SHORT_URL_META = [ 'shortCode' => 'def456', @@ -73,6 +76,7 @@ class ListShortUrlsTest extends ApiTestCase ], 'domain' => null, 'title' => null, + 'crawlable' => false, ]; private const SHORT_URL_CUSTOM_SLUG = [ 'shortCode' => 'custom', @@ -88,6 +92,7 @@ class ListShortUrlsTest extends ApiTestCase ], 'domain' => null, 'title' => null, + 'crawlable' => false, ]; private const SHORT_URL_CUSTOM_DOMAIN = [ 'shortCode' => 'ghi789', @@ -105,6 +110,7 @@ class ListShortUrlsTest extends ApiTestCase ], 'domain' => 'example.com', 'title' => null, + 'crawlable' => false, ]; /** diff --git a/module/Rest/test-api/Fixtures/ShortUrlsFixture.php b/module/Rest/test-api/Fixtures/ShortUrlsFixture.php index bfc65aa0..ccc83525 100644 --- a/module/Rest/test-api/Fixtures/ShortUrlsFixture.php +++ b/module/Rest/test-api/Fixtures/ShortUrlsFixture.php @@ -35,6 +35,7 @@ class ShortUrlsFixture extends AbstractFixture implements DependentFixtureInterf 'longUrl' => 'https://shlink.io', 'tags' => ['foo'], 'title' => 'My cool title', + 'crawlable' => true, ]), $relationResolver), '2018-05-01', ); From e6ce84aa14ca52979b7ce237a4b536d4113530de Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 07:40:21 +0200 Subject: [PATCH 65/86] Added more missing API spec docs --- docs/swagger/paths/v1_short-urls.json | 4 ++++ docs/swagger/paths/v1_short-urls_{shortCode}.json | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/docs/swagger/paths/v1_short-urls.json b/docs/swagger/paths/v1_short-urls.json index 4d30e4de..8cf22045 100644 --- a/docs/swagger/paths/v1_short-urls.json +++ b/docs/swagger/paths/v1_short-urls.json @@ -276,6 +276,10 @@ "title": { "type": "string", "description": "A descriptive title of the short URL." + }, + "crawlable": { + "type": "boolean", + "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt." } } } diff --git a/docs/swagger/paths/v1_short-urls_{shortCode}.json b/docs/swagger/paths/v1_short-urls_{shortCode}.json index 11ff276f..8691c0b5 100644 --- a/docs/swagger/paths/v1_short-urls_{shortCode}.json +++ b/docs/swagger/paths/v1_short-urls_{shortCode}.json @@ -148,6 +148,10 @@ "type": "string", "description": "A descriptive title of the short URL.", "nullable": true + }, + "crawlable": { + "type": "boolean", + "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt." } } } From 3ef02d46c08b846516c3d4385c33180d63ddb9e9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 09:34:42 +0200 Subject: [PATCH 66/86] Added logic to resolve crawlable short codes --- module/Core/config/dependencies.config.php | 4 +- module/Core/src/Crawling/CrawlingHelper.php | 15 ++++++- .../src/Repository/ShortUrlRepository.php | 24 +++++++++++ .../ShortUrlRepositoryInterface.php | 2 + .../Core/src/Repository/VisitRepository.php | 4 +- .../Core/test/Crawling/CrawlingHelperTest.php | 43 +++++++++++++++++++ 6 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 module/Core/test/Crawling/CrawlingHelperTest.php diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index 4eb3d60d..7dfd5df2 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -59,7 +59,7 @@ return [ Importer\ImportedLinksProcessor::class => ConfigAbstractFactory::class, - Crawling\CrawlingHelper::class => InvokableFactory::class, + Crawling\CrawlingHelper::class => ConfigAbstractFactory::class, ], 'aliases' => [ @@ -150,6 +150,8 @@ return [ Service\ShortUrl\ShortCodeHelper::class, Util\DoctrineBatchHelper::class, ], + + Crawling\CrawlingHelper::class => ['em'], ], ]; diff --git a/module/Core/src/Crawling/CrawlingHelper.php b/module/Core/src/Crawling/CrawlingHelper.php index ef54761b..5f688645 100644 --- a/module/Core/src/Crawling/CrawlingHelper.php +++ b/module/Core/src/Crawling/CrawlingHelper.php @@ -4,10 +4,23 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Crawling; +use Doctrine\ORM\EntityManagerInterface; +use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; + class CrawlingHelper implements CrawlingHelperInterface { + private EntityManagerInterface $em; + + public function __construct(EntityManagerInterface $em) + { + $this->em = $em; + } + public function listCrawlableShortCodes(): iterable { - return []; + /** @var ShortUrlRepositoryInterface $repo */ + $repo = $this->em->getRepository(ShortUrl::class); + yield from $repo->findCrawlableShortCodes(); } } diff --git a/module/Core/src/Repository/ShortUrlRepository.php b/module/Core/src/Repository/ShortUrlRepository.php index fe3b170c..eacf293b 100644 --- a/module/Core/src/Repository/ShortUrlRepository.php +++ b/module/Core/src/Repository/ShortUrlRepository.php @@ -288,4 +288,28 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU $qb->andWhere($qb->expr()->isNull('s.domain')); } } + + public function findCrawlableShortCodes(): iterable + { + $blockSize = 1000; + $qb = $this->getEntityManager()->createQueryBuilder(); + $qb->select('DISTINCT s.shortCode') + ->from(ShortUrl::class, 's') + ->where($qb->expr()->eq('s.crawlable', ':crawlable')) + ->setParameter('crawlable', true) + ->setMaxResults($blockSize); + + $page = 0; + do { + $qbClone = (clone $qb)->setFirstResult($blockSize * $page); + $iterator = $qbClone->getQuery()->toIterable(); + $resultsFound = false; + $page++; + + foreach ($iterator as ['shortCode' => $shortCode]) { + $resultsFound = true; + yield $shortCode; + } + } while ($resultsFound); + } } diff --git a/module/Core/src/Repository/ShortUrlRepositoryInterface.php b/module/Core/src/Repository/ShortUrlRepositoryInterface.php index 29485eeb..5d8fa924 100644 --- a/module/Core/src/Repository/ShortUrlRepositoryInterface.php +++ b/module/Core/src/Repository/ShortUrlRepositoryInterface.php @@ -41,4 +41,6 @@ interface ShortUrlRepositoryInterface extends ObjectRepository, EntitySpecificat public function findOneMatching(ShortUrlMeta $meta): ?ShortUrl; public function findOneByImportedUrl(ImportedShlinkUrl $url): ?ShortUrl; + + public function findCrawlableShortCodes(): iterable; } diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index d9c18977..35d6a535 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -66,11 +66,11 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo do { $qb = (clone $originalQueryBuilder)->andWhere($qb->expr()->gt('v.id', $lastId)); - $iterator = $qb->getQuery()->iterate(); + $iterator = $qb->getQuery()->toIterable(); $resultsFound = false; /** @var Visit $visit */ - foreach ($iterator as $key => [$visit]) { + foreach ($iterator as $key => $visit) { $resultsFound = true; yield $key => $visit; } diff --git a/module/Core/test/Crawling/CrawlingHelperTest.php b/module/Core/test/Crawling/CrawlingHelperTest.php new file mode 100644 index 00000000..2c65ebac --- /dev/null +++ b/module/Core/test/Crawling/CrawlingHelperTest.php @@ -0,0 +1,43 @@ +em = $this->prophesize(EntityManagerInterface::class); + $this->helper = new CrawlingHelper($this->em->reveal()); + } + + /** @test */ + public function listCrawlableShortCodesDelegatesIntoRepository(): void + { + $repo = $this->prophesize(ShortUrlRepositoryInterface::class); + $findCrawlableShortCodes = $repo->findCrawlableShortCodes()->willReturn([]); + $getRepo = $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal()); + + $result = $this->helper->listCrawlableShortCodes(); + foreach ($result as $shortCode) { + // Result is a generator and therefore, it needs to be iterated + } + + $findCrawlableShortCodes->shouldHaveBeenCalledOnce(); + $getRepo->shouldHaveBeenCalledOnce(); + } +} From 36e4a0dd325ceffcb01a18ed11c7b2314eecbe0c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 09:41:12 +0200 Subject: [PATCH 67/86] Added tests for findCrawlableShortCodes --- .../Repository/ShortUrlRepositoryTest.php | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php index cf082d85..bd5b22d4 100644 --- a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php +++ b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php @@ -436,4 +436,37 @@ class ShortUrlRepositoryTest extends DatabaseTestCase self::assertNull($this->repo->findOneByImportedUrl($buildImported('my-cool-slug', 'doma.in'))); self::assertNull($this->repo->findOneByImportedUrl($buildImported('another-slug'))); } + + /** @test */ + public function findCrawlableShortCodesReturnsExpectedResult(): void + { + $createShortUrl = fn (bool $crawlable) => ShortUrl::fromMeta( + ShortUrlMeta::fromRawData(['crawlable' => $crawlable, 'longUrl' => 'foo.com']), + ); + + $shortUrl1 = $createShortUrl(true); + $this->getEntityManager()->persist($shortUrl1); + $shortUrl2 = $createShortUrl(false); + $this->getEntityManager()->persist($shortUrl2); + $shortUrl3 = $createShortUrl(true); + $this->getEntityManager()->persist($shortUrl3); + $shortUrl4 = $createShortUrl(true); + $this->getEntityManager()->persist($shortUrl4); + $shortUrl5 = $createShortUrl(false); + $this->getEntityManager()->persist($shortUrl5); + $this->getEntityManager()->flush(); + + $iterable = $this->repo->findCrawlableShortCodes(); + $results = []; + foreach ($iterable as $shortCode) { + $results[] = $shortCode; + } + + self::assertCount(3, $results); + self::assertContains($shortUrl1->getShortCode(), $results); + self::assertContains($shortUrl3->getShortCode(), $results); + self::assertContains($shortUrl4->getShortCode(), $results); + self::assertNotContains($shortUrl2->getShortCode(), $results); + self::assertNotContains($shortUrl5->getShortCode(), $results); + } } From 70384237c1811cf8f1206d2e9de9dc30f8e7f326 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 09:42:24 +0200 Subject: [PATCH 68/86] Updated changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bcfe116..c5d6f749 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * `disable_referrer_tracking`: If true, the referrer will not be tracked. * `disable_ua_tracking`: If true, the user agent will not be tracked. +* [#955](https://github.com/shlinkio/shlink/issues/955) Added new option to set short URLs as crawlable, making them be listed in the robots.txt as Allowed. + ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. * [#1039](https://github.com/shlinkio/shlink/issues/1039) Updated to `endroid/qr-code` 4.0. From 05d73552cfd6371e9294c5d1b2e88babeddd6fce Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 09:49:24 +0200 Subject: [PATCH 69/86] Used PHP 8.0 in ci workflow when running against just one PHP version --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 23b76317..188a7f02 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - php-version: ['7.4'] + php-version: ['8.0'] steps: - name: Checkout code uses: actions/checkout@v2 @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - php-version: ['7.4'] + php-version: ['8.0'] steps: - name: Checkout code uses: actions/checkout@v2 @@ -242,7 +242,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - php-version: ['7.4'] + php-version: ['8.0'] steps: - name: Checkout code uses: actions/checkout@v2 From 70c73bc5d6cfdabc4d94f995c9ca2c1a4cea67c5 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 10:08:33 +0200 Subject: [PATCH 70/86] Removed no-longer valid false positive for static analysis --- phpstan.neon | 1 - 1 file changed, 1 deletion(-) diff --git a/phpstan.neon b/phpstan.neon index fbddc81c..80f1b083 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -3,4 +3,3 @@ parameters: checkGenericClassInNonGenericObjectType: false ignoreErrors: - '#If condition is always false#' - - '#setOrderBy\(\) expects array\, array\ given#' From 9fa32b5b6b5f8682e615d5b56457a06e13e6dae4 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 15:09:14 +0200 Subject: [PATCH 71/86] Added detection of visits from potential bots --- composer.json | 1 + data/migrations/Version20210522124633.php | 28 +++++++++++++++++++ docs/async-api/async-api.json | 8 +++++- docs/swagger/definitions/Visit.json | 4 +++ .../v1_short-urls_{shortCode}_visits.json | 9 ++++-- docs/swagger/paths/v2_tags_{tag}_visits.json | 9 ++++-- docs/swagger/paths/v2_visits_orphan.json | 3 ++ .../Shlinkio.Shlink.Core.Entity.Visit.php | 5 ++++ module/Core/functions/functions.php | 11 ++++++++ module/Core/src/Entity/Visit.php | 6 ++++ module/Core/src/Model/Visitor.php | 21 ++++++++++---- module/Core/test/Entity/VisitTest.php | 24 +++++++++++++--- .../Mercure/MercureUpdatesGeneratorTest.php | 2 ++ .../OrphanVisitDataTransformerTest.php | 3 ++ .../Rest/test-api/Action/OrphanVisitsTest.php | 6 ++-- .../Rest/test-api/Fixtures/VisitsFixture.php | 2 +- 16 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 data/migrations/Version20210522124633.php diff --git a/composer.json b/composer.json index 2474cebf..0bb1186a 100644 --- a/composer.json +++ b/composer.json @@ -25,6 +25,7 @@ "geoip2/geoip2": "^2.9", "guzzlehttp/guzzle": "^7.0", "happyr/doctrine-specification": "^2.0", + "jaybizzle/crawler-detect": "^1.2", "laminas/laminas-config": "^3.3", "laminas/laminas-config-aggregator": "^1.1", "laminas/laminas-diactoros": "^2.1.3", diff --git a/data/migrations/Version20210522124633.php b/data/migrations/Version20210522124633.php new file mode 100644 index 00000000..ea486e93 --- /dev/null +++ b/data/migrations/Version20210522124633.php @@ -0,0 +1,28 @@ +getTable('visits'); + $this->skipIf($visits->hasColumn(self::POTENTIAL_BOT_COLUMN)); + $visits->addColumn(self::POTENTIAL_BOT_COLUMN, Types::BOOLEAN, ['default' => false]); + } + + public function down(Schema $schema): void + { + $visits = $schema->getTable('visits'); + $this->skipIf(! $visits->hasColumn(self::POTENTIAL_BOT_COLUMN)); + $visits->dropColumn(self::POTENTIAL_BOT_COLUMN); + } +} diff --git a/docs/async-api/async-api.json b/docs/async-api/async-api.json index 3360d897..b1313d1e 100644 --- a/docs/async-api/async-api.json +++ b/docs/async-api/async-api.json @@ -190,6 +190,10 @@ }, "visitLocation": { "$ref": "#/components/schemas/VisitLocation" + }, + "potentialBot": { + "type": "boolean", + "description": "Tells if Shlink thinks this visits comes potentially from a bot or crawler" } }, "example": { @@ -204,7 +208,8 @@ "longitude": -122.0946, "regionName": "California", "timezone": "America/Los_Angeles" - } + }, + "potentialBot": false } }, "OrphanVisit": { @@ -243,6 +248,7 @@ "regionName": "California", "timezone": "America/Los_Angeles" }, + "potentialBot": false, "visitedUrl": "https://doma.in", "type": "base_url" } diff --git a/docs/swagger/definitions/Visit.json b/docs/swagger/definitions/Visit.json index e004e4fe..ad5fd97b 100644 --- a/docs/swagger/definitions/Visit.json +++ b/docs/swagger/definitions/Visit.json @@ -17,6 +17,10 @@ }, "visitLocation": { "$ref": "./VisitLocation.json" + }, + "potentialBot": { + "type": "boolean", + "description": "Tells if Shlink thinks this visits comes potentially from a bot or crawler" } } } diff --git a/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json b/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json index 03d66a99..1bef6110 100644 --- a/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json +++ b/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json @@ -98,7 +98,8 @@ "referer": "https://twitter.com", "date": "2015-08-20T05:05:03+04:00", "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0", - "visitLocation": null + "visitLocation": null, + "potentialBot": false }, { "referer": "https://t.co", @@ -112,13 +113,15 @@ "longitude": -122.0946, "regionName": "California", "timezone": "America/Los_Angeles" - } + }, + "potentialBot": false }, { "referer": null, "date": "2015-08-20T05:05:03+04:00", "userAgent": "some_web_crawler/1.4", - "visitLocation": null + "visitLocation": null, + "potentialBot": true } ], "pagination": { diff --git a/docs/swagger/paths/v2_tags_{tag}_visits.json b/docs/swagger/paths/v2_tags_{tag}_visits.json index d9d9dda7..ab442793 100644 --- a/docs/swagger/paths/v2_tags_{tag}_visits.json +++ b/docs/swagger/paths/v2_tags_{tag}_visits.json @@ -95,7 +95,8 @@ "referer": "https://twitter.com", "date": "2015-08-20T05:05:03+04:00", "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0", - "visitLocation": null + "visitLocation": null, + "potentialBot": false }, { "referer": "https://t.co", @@ -109,13 +110,15 @@ "longitude": -122.0946, "regionName": "California", "timezone": "America/Los_Angeles" - } + }, + "potentialBot": false }, { "referer": null, "date": "2015-08-20T05:05:03+04:00", "userAgent": "some_web_crawler/1.4", - "visitLocation": null + "visitLocation": null, + "potentialBot": true } ], "pagination": { diff --git a/docs/swagger/paths/v2_visits_orphan.json b/docs/swagger/paths/v2_visits_orphan.json index 683f40ec..7876e703 100644 --- a/docs/swagger/paths/v2_visits_orphan.json +++ b/docs/swagger/paths/v2_visits_orphan.json @@ -87,6 +87,7 @@ "date": "2015-08-20T05:05:03+04:00", "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0", "visitLocation": null, + "potentialBot": false, "visitedUrl": "https://doma.in", "type": "base_url" }, @@ -103,6 +104,7 @@ "regionName": "California", "timezone": "America/Los_Angeles" }, + "potentialBot": false, "visitedUrl": "https://doma.in/foo", "type": "invalid_short_url" }, @@ -111,6 +113,7 @@ "date": "2015-08-20T05:05:03+04:00", "userAgent": "some_web_crawler/1.4", "visitLocation": null, + "potentialBot": true, "visitedUrl": "https://doma.in/foo/bar/baz", "type": "regular_404" } diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php index efcccb65..8886e141 100644 --- a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php @@ -65,4 +65,9 @@ return static function (ClassMetadata $metadata, array $emConfig): void { ->columnName('type') ->length(255) ->build(); + + $builder->createField('potentialBot', Types::BOOLEAN) + ->columnName('potential_bot') + ->option('default', false) + ->build(); }; diff --git a/module/Core/functions/functions.php b/module/Core/functions/functions.php index 62df2070..867f7c7d 100644 --- a/module/Core/functions/functions.php +++ b/module/Core/functions/functions.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core; use Cake\Chronos\Chronos; use DateTimeInterface; use Fig\Http\Message\StatusCodeInterface; +use Jaybizzle\CrawlerDetect\CrawlerDetect; use Laminas\InputFilter\InputFilter; use PUGX\Shortid\Factory as ShortIdFactory; use Shlinkio\Shlink\Common\Util\DateRange; @@ -128,3 +129,13 @@ function kebabCaseToCamelCase(string $name): string { return lcfirst(str_replace(' ', '', ucwords(str_replace('-', ' ', $name)))); } + +function isCrawler(string $userAgent): bool +{ + static $detector; + if ($detector === null) { + $detector = new CrawlerDetect(); + } + + return $detector->isCrawler($userAgent); +} diff --git a/module/Core/src/Entity/Visit.php b/module/Core/src/Entity/Visit.php index 98d1a4c5..358bedde 100644 --- a/module/Core/src/Entity/Visit.php +++ b/module/Core/src/Entity/Visit.php @@ -13,6 +13,8 @@ use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Visit\Model\VisitLocationInterface; use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit; +use function Shlinkio\Shlink\Core\isCrawler; + class Visit extends AbstractEntity implements JsonSerializable { public const TYPE_VALID_SHORT_URL = 'valid_short_url'; @@ -29,6 +31,7 @@ class Visit extends AbstractEntity implements JsonSerializable private string $type; private ?ShortUrl $shortUrl; private ?VisitLocation $visitLocation = null; + private bool $potentialBot; private function __construct(?ShortUrl $shortUrl, string $type) { @@ -49,6 +52,7 @@ class Visit extends AbstractEntity implements JsonSerializable { $instance = new self($shortUrl, self::TYPE_IMPORTED); $instance->userAgent = $importedVisit->userAgent(); + $instance->potentialBot = isCrawler($instance->userAgent); $instance->referer = $importedVisit->referer(); $instance->date = Chronos::instance($importedVisit->date()); @@ -88,6 +92,7 @@ class Visit extends AbstractEntity implements JsonSerializable $this->referer = $visitor->getReferer(); $this->remoteAddr = $this->processAddress($anonymize, $visitor->getRemoteAddress()); $this->visitedUrl = $visitor->getVisitedUrl(); + $this->potentialBot = $visitor->isPotentialBot(); } private function processAddress(bool $anonymize, ?string $address): ?string @@ -166,6 +171,7 @@ class Visit extends AbstractEntity implements JsonSerializable 'date' => $this->date->toAtomString(), 'userAgent' => $this->userAgent, 'visitLocation' => $this->visitLocation, + 'potentialBot' => $this->potentialBot, ]; } diff --git a/module/Core/src/Model/Visitor.php b/module/Core/src/Model/Visitor.php index 9564a41c..e9bdc36e 100644 --- a/module/Core/src/Model/Visitor.php +++ b/module/Core/src/Model/Visitor.php @@ -8,6 +8,7 @@ use Psr\Http\Message\ServerRequestInterface; use Shlinkio\Shlink\Common\Middleware\IpAddressMiddlewareFactory; use Shlinkio\Shlink\Core\Options\TrackingOptions; +use function Shlinkio\Shlink\Core\isCrawler; use function substr; final class Visitor @@ -21,6 +22,7 @@ final class Visitor private string $referer; private string $visitedUrl; private ?string $remoteAddress; + private bool $potentialBot; public function __construct(string $userAgent, string $referer, ?string $remoteAddress, string $visitedUrl) { @@ -28,6 +30,7 @@ final class Visitor $this->referer = $this->cropToLength($referer, self::REFERER_MAX_LENGTH); $this->visitedUrl = $this->cropToLength($visitedUrl, self::VISITED_URL_MAX_LENGTH); $this->remoteAddress = $this->cropToLength($remoteAddress, self::REMOTE_ADDRESS_MAX_LENGTH); + $this->potentialBot = isCrawler($userAgent); } private function cropToLength(?string $value, int $length): ?string @@ -70,14 +73,22 @@ final class Visitor return $this->visitedUrl; } + public function isPotentialBot(): bool + { + return $this->potentialBot; + } + public function normalizeForTrackingOptions(TrackingOptions $options): self { - $instance = self::emptyInstance(); + $instance = new self( + $options->disableUaTracking() ? '' : $this->userAgent, + $options->disableReferrerTracking() ? '' : $this->referer, + $options->disableIpTracking() ? null : $this->remoteAddress, + $this->visitedUrl, + ); - $instance->userAgent = $options->disableUaTracking() ? '' : $this->userAgent; - $instance->referer = $options->disableReferrerTracking() ? '' : $this->referer; - $instance->remoteAddress = $options->disableIpTracking() ? null : $this->remoteAddress; - $instance->visitedUrl = $this->visitedUrl; + // Keep the fact that the visit was a potential bot, even if we no longer save the user agent + $instance->potentialBot = $this->potentialBot; return $instance; } diff --git a/module/Core/test/Entity/VisitTest.php b/module/Core/test/Entity/VisitTest.php index 7be3c3fc..2d2cb4f8 100644 --- a/module/Core/test/Entity/VisitTest.php +++ b/module/Core/test/Entity/VisitTest.php @@ -12,19 +12,35 @@ use Shlinkio\Shlink\Core\Model\Visitor; class VisitTest extends TestCase { - /** @test */ - public function isProperlyJsonSerialized(): void + /** + * @test + * @dataProvider provideUserAgents + */ + public function isProperlyJsonSerialized(string $userAgent, bool $expectedToBePotentialBot): void { - $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('Chrome', 'some site', '1.2.3.4', '')); + $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor($userAgent, 'some site', '1.2.3.4', '')); self::assertEquals([ 'referer' => 'some site', 'date' => $visit->getDate()->toAtomString(), - 'userAgent' => 'Chrome', + 'userAgent' => $userAgent, 'visitLocation' => null, + 'potentialBot' => $expectedToBePotentialBot, ], $visit->jsonSerialize()); } + public function provideUserAgents(): iterable + { + yield 'Chrome' => [ + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36', + false, + ]; + yield 'Firefox' => ['Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0', false]; + yield 'Facebook' => ['cf-facebook', true]; + yield 'Twitter' => ['IDG Twitter Links Resolver', true]; + yield 'Guzzle' => ['guzzlehttp', true]; + } + /** * @test * @dataProvider provideAddresses diff --git a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php index 1d460623..86d1b3d5 100644 --- a/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php +++ b/module/Core/test/Mercure/MercureUpdatesGeneratorTest.php @@ -66,6 +66,7 @@ class MercureUpdatesGeneratorTest extends TestCase 'userAgent' => '', 'visitLocation' => null, 'date' => $visit->getDate()->toAtomString(), + 'potentialBot' => false, ], ], json_decode($update->getData())); } @@ -91,6 +92,7 @@ class MercureUpdatesGeneratorTest extends TestCase 'userAgent' => '', 'visitLocation' => null, 'date' => $orphanVisit->getDate()->toAtomString(), + 'potentialBot' => false, 'visitedUrl' => $orphanVisit->visitedUrl(), 'type' => $orphanVisit->type(), ], diff --git a/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php b/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php index 61193c86..c836cd7c 100644 --- a/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php +++ b/module/Core/test/Visit/Transformer/OrphanVisitDataTransformerTest.php @@ -42,6 +42,7 @@ class OrphanVisitDataTransformerTest extends TestCase 'date' => $visit->getDate()->toAtomString(), 'userAgent' => '', 'visitLocation' => null, + 'potentialBot' => false, 'visitedUrl' => '', 'type' => Visit::TYPE_BASE_URL, ], @@ -57,6 +58,7 @@ class OrphanVisitDataTransformerTest extends TestCase 'date' => $visit->getDate()->toAtomString(), 'userAgent' => 'foo', 'visitLocation' => null, + 'potentialBot' => false, 'visitedUrl' => 'https://example.com/foo', 'type' => Visit::TYPE_INVALID_SHORT_URL, ], @@ -74,6 +76,7 @@ class OrphanVisitDataTransformerTest extends TestCase 'date' => $visit->getDate()->toAtomString(), 'userAgent' => 'user-agent', 'visitLocation' => $location, + 'potentialBot' => false, 'visitedUrl' => 'https://doma.in/foo/bar', 'type' => Visit::TYPE_REGULAR_404, ], diff --git a/module/Rest/test-api/Action/OrphanVisitsTest.php b/module/Rest/test-api/Action/OrphanVisitsTest.php index ea890f9f..06857653 100644 --- a/module/Rest/test-api/Action/OrphanVisitsTest.php +++ b/module/Rest/test-api/Action/OrphanVisitsTest.php @@ -12,17 +12,18 @@ class OrphanVisitsTest extends ApiTestCase private const INVALID_SHORT_URL = [ 'referer' => 'https://doma.in/foo', 'date' => '2020-03-01T00:00:00+00:00', - 'userAgent' => 'shlink-tests-agent', + 'userAgent' => 'cf-facebook', 'visitLocation' => null, + 'potentialBot' => true, 'visitedUrl' => 'foo.com', 'type' => 'invalid_short_url', - ]; private const REGULAR_NOT_FOUND = [ 'referer' => 'https://doma.in/foo/bar', 'date' => '2020-02-01T00:00:00+00:00', 'userAgent' => 'shlink-tests-agent', 'visitLocation' => null, + 'potentialBot' => false, 'visitedUrl' => '', 'type' => 'regular_404', ]; @@ -31,6 +32,7 @@ class OrphanVisitsTest extends ApiTestCase 'date' => '2020-01-01T00:00:00+00:00', 'userAgent' => 'shlink-tests-agent', 'visitLocation' => null, + 'potentialBot' => false, 'visitedUrl' => '', 'type' => 'base_url', ]; diff --git a/module/Rest/test-api/Fixtures/VisitsFixture.php b/module/Rest/test-api/Fixtures/VisitsFixture.php index 412c79d5..62e1527d 100644 --- a/module/Rest/test-api/Fixtures/VisitsFixture.php +++ b/module/Rest/test-api/Fixtures/VisitsFixture.php @@ -58,7 +58,7 @@ class VisitsFixture extends AbstractFixture implements DependentFixtureInterface '2020-02-01', )); $manager->persist($this->setVisitDate( - Visit::forInvalidShortUrl(new Visitor('shlink-tests-agent', 'https://doma.in/foo', '1.2.3.4', 'foo.com')), + Visit::forInvalidShortUrl(new Visitor('cf-facebook', 'https://doma.in/foo', '1.2.3.4', 'foo.com')), '2020-03-01', )); From 6327ed814afa090489e8a6aeb7bdba1e676b787c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 20:16:32 +0200 Subject: [PATCH 72/86] Added new models to pass to repositories when counting visits of any kind --- module/Core/src/Model/VisitsParams.php | 16 +++++- .../Adapter/OrphanVisitsPaginatorAdapter.php | 3 +- .../Adapter/VisitsForTagPaginatorAdapter.php | 8 ++- .../Adapter/VisitsPaginatorAdapter.php | 8 ++- .../Core/src/Repository/VisitRepository.php | 19 +++---- .../Repository/VisitRepositoryInterface.php | 12 ++--- .../Persistence/VisitsCountFiltering.php | 37 ++++++++++++++ .../Visit/Persistence/VisitsListFiltering.php | 36 +++++++++++++ module/Core/src/Visit/VisitsStatsHelper.php | 6 ++- .../Repository/VisitRepositoryTest.php | 51 ++++++++++--------- .../OrphanVisitsPaginatorAdapterTest.php | 5 +- .../VisitsForTagPaginatorAdapterTest.php | 6 ++- .../Adapter/VisitsPaginatorAdapterTest.php | 7 ++- .../Core/test/Visit/VisitsStatsHelperTest.php | 11 ++-- 14 files changed, 168 insertions(+), 57 deletions(-) create mode 100644 module/Core/src/Visit/Persistence/VisitsCountFiltering.php create mode 100644 module/Core/src/Visit/Persistence/VisitsListFiltering.php diff --git a/module/Core/src/Model/VisitsParams.php b/module/Core/src/Model/VisitsParams.php index b579239b..f8498c7a 100644 --- a/module/Core/src/Model/VisitsParams.php +++ b/module/Core/src/Model/VisitsParams.php @@ -16,12 +16,18 @@ final class VisitsParams private ?DateRange $dateRange; private int $page; private int $itemsPerPage; + private bool $excludeBots; - public function __construct(?DateRange $dateRange = null, int $page = self::FIRST_PAGE, ?int $itemsPerPage = null) - { + public function __construct( + ?DateRange $dateRange = null, + int $page = self::FIRST_PAGE, + ?int $itemsPerPage = null, + bool $excludeBots = false + ) { $this->dateRange = $dateRange ?? new DateRange(); $this->page = $page; $this->itemsPerPage = $this->determineItemsPerPage($itemsPerPage); + $this->excludeBots = $excludeBots; } private function determineItemsPerPage(?int $itemsPerPage): int @@ -39,6 +45,7 @@ final class VisitsParams parseDateRangeFromQuery($query, 'startDate', 'endDate'), (int) ($query['page'] ?? 1), isset($query['itemsPerPage']) ? (int) $query['itemsPerPage'] : null, + isset($query['excludeBots']), ); } @@ -56,4 +63,9 @@ final class VisitsParams { return $this->itemsPerPage; } + + public function excludeBots(): bool + { + return $this->excludeBots; + } } diff --git a/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php index 7167b9e7..9d96d5ab 100644 --- a/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Paginator\Adapter; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; class OrphanVisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter { @@ -20,7 +21,7 @@ class OrphanVisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapte protected function doCount(): int { - return $this->repo->countOrphanVisits($this->params->getDateRange()); + return $this->repo->countOrphanVisits(new VisitsCountFiltering($this->params->getDateRange())); } public function getSlice($offset, $length): iterable // phpcs:ignore diff --git a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php index 4c4e718b..df1eba11 100644 --- a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Paginator\Adapter; use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsForTagPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter @@ -43,8 +44,11 @@ class VisitsForTagPaginatorAdapter extends AbstractCacheableCountPaginatorAdapte { return $this->visitRepository->countVisitsByTag( $this->tag, - $this->params->getDateRange(), - $this->resolveSpec(), + new VisitsCountFiltering( + $this->params->getDateRange(), + $this->params->excludeBots(), + $this->resolveSpec(), + ), ); } diff --git a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php index 02ba37b3..a0f002ff 100644 --- a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php @@ -8,6 +8,7 @@ use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter { @@ -45,8 +46,11 @@ class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter return $this->visitRepository->countVisitsByShortCode( $this->identifier->shortCode(), $this->identifier->domain(), - $this->params->getDateRange(), - $this->spec, + new VisitsCountFiltering( + $this->params->getDateRange(), + $this->params->excludeBots(), + $this->spec, + ), ); } } diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 35d6a535..bab885f6 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -12,6 +12,7 @@ use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Core\Visit\Spec\CountOfOrphanVisits; use Shlinkio\Shlink\Core\Visit\Spec\CountOfShortUrlVisits; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -95,13 +96,9 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); } - public function countVisitsByShortCode( - string $shortCode, - ?string $domain = null, - ?DateRange $dateRange = null, - ?Specification $spec = null - ): int { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange, $spec); + public function countVisitsByShortCode(string $shortCode, ?string $domain, VisitsCountFiltering $filtering): int + { + $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $filtering->dateRange(), $filtering->spec()); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); @@ -141,9 +138,9 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); } - public function countVisitsByTag(string $tag, ?DateRange $dateRange = null, ?Specification $spec = null): int + public function countVisitsByTag(string $tag, VisitsCountFiltering $filtering): int { - $qb = $this->createVisitsByTagQueryBuilder($tag, $dateRange, $spec); + $qb = $this->createVisitsByTagQueryBuilder($tag, $filtering->dateRange(), $filtering->spec()); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); @@ -181,9 +178,9 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); } - public function countOrphanVisits(?DateRange $dateRange = null): int + public function countOrphanVisits(VisitsCountFiltering $filtering): int { - return (int) $this->matchSingleScalarResult(new CountOfOrphanVisits($dateRange)); + return (int) $this->matchSingleScalarResult(new CountOfOrphanVisits($filtering->dateRange())); } public function countVisits(?ApiKey $apiKey = null): int diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index 96fb21ee..a3a6ca1a 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -9,6 +9,7 @@ use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterfa use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Visit; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; interface VisitRepositoryInterface extends ObjectRepository, EntitySpecificationRepositoryInterface @@ -42,12 +43,7 @@ interface VisitRepositoryInterface extends ObjectRepository, EntitySpecification ?Specification $spec = null ): array; - public function countVisitsByShortCode( - string $shortCode, - ?string $domain = null, - ?DateRange $dateRange = null, - ?Specification $spec = null - ): int; + public function countVisitsByShortCode(string $shortCode, ?string $domain, VisitsCountFiltering $filtering): int; /** * @return Visit[] @@ -60,14 +56,14 @@ interface VisitRepositoryInterface extends ObjectRepository, EntitySpecification ?Specification $spec = null ): array; - public function countVisitsByTag(string $tag, ?DateRange $dateRange = null, ?Specification $spec = null): int; + public function countVisitsByTag(string $tag, VisitsCountFiltering $filtering): int; /** * @return Visit[] */ public function findOrphanVisits(?DateRange $dateRange = null, ?int $limit = null, ?int $offset = null): array; - public function countOrphanVisits(?DateRange $dateRange = null): int; + public function countOrphanVisits(VisitsCountFiltering $filtering): int; public function countVisits(?ApiKey $apiKey = null): int; } diff --git a/module/Core/src/Visit/Persistence/VisitsCountFiltering.php b/module/Core/src/Visit/Persistence/VisitsCountFiltering.php new file mode 100644 index 00000000..bc9ac5de --- /dev/null +++ b/module/Core/src/Visit/Persistence/VisitsCountFiltering.php @@ -0,0 +1,37 @@ +dateRange = $dateRange; + $this->excludeBots = $excludeBots; + $this->spec = $spec; + } + + public function dateRange(): ?DateRange + { + return $this->dateRange; + } + + public function excludeBots(): bool + { + return $this->excludeBots; + } + + public function spec(): ?Specification + { + return $this->spec; + } +} diff --git a/module/Core/src/Visit/Persistence/VisitsListFiltering.php b/module/Core/src/Visit/Persistence/VisitsListFiltering.php new file mode 100644 index 00000000..4f67967d --- /dev/null +++ b/module/Core/src/Visit/Persistence/VisitsListFiltering.php @@ -0,0 +1,36 @@ +limit = $limit; + $this->offset = $offset; + } + + public function limit(): ?int + { + return $this->limit; + } + + public function offset(): ?int + { + return $this->offset; + } +} diff --git a/module/Core/src/Visit/VisitsStatsHelper.php b/module/Core/src/Visit/VisitsStatsHelper.php index 61d879fd..1aa03c46 100644 --- a/module/Core/src/Visit/VisitsStatsHelper.php +++ b/module/Core/src/Visit/VisitsStatsHelper.php @@ -22,6 +22,7 @@ use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Model\VisitsStats; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsStatsHelper implements VisitsStatsHelperInterface @@ -38,7 +39,10 @@ class VisitsStatsHelper implements VisitsStatsHelperInterface /** @var VisitRepository $visitsRepo */ $visitsRepo = $this->em->getRepository(Visit::class); - return new VisitsStats($visitsRepo->countVisits($apiKey), $visitsRepo->countOrphanVisits()); + return new VisitsStats( + $visitsRepo->countVisits($apiKey), + $visitsRepo->countOrphanVisits(new VisitsCountFiltering()), + ); } /** diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 27ab3252..9bf32182 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -15,6 +15,7 @@ use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; @@ -110,18 +111,17 @@ class VisitRepositoryTest extends DatabaseTestCase { [$shortCode, $domain] = $this->createShortUrlsAndVisits(); - self::assertEquals(0, $this->repo->countVisitsByShortCode('invalid')); - self::assertEquals(6, $this->repo->countVisitsByShortCode($shortCode)); - self::assertEquals(3, $this->repo->countVisitsByShortCode($shortCode, $domain)); - self::assertEquals(2, $this->repo->countVisitsByShortCode($shortCode, null, new DateRange( - Chronos::parse('2016-01-02'), - Chronos::parse('2016-01-03'), + self::assertEquals(0, $this->repo->countVisitsByShortCode('invalid', null, new VisitsCountFiltering())); + self::assertEquals(6, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering())); + self::assertEquals(3, $this->repo->countVisitsByShortCode($shortCode, $domain, new VisitsCountFiltering())); + self::assertEquals(2, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), ))); - self::assertEquals(4, $this->repo->countVisitsByShortCode($shortCode, null, new DateRange( - Chronos::parse('2016-01-03'), + self::assertEquals(4, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( + DateRange::withStartDate(Chronos::parse('2016-01-03')), ))); - self::assertEquals(1, $this->repo->countVisitsByShortCode($shortCode, $domain, new DateRange( - Chronos::parse('2016-01-03'), + self::assertEquals(1, $this->repo->countVisitsByShortCode($shortCode, $domain, new VisitsCountFiltering( + DateRange::withStartDate(Chronos::parse('2016-01-03')), ))); } @@ -160,13 +160,14 @@ class VisitRepositoryTest extends DatabaseTestCase $this->createShortUrlsAndVisits(false, [$foo]); $this->getEntityManager()->flush(); - self::assertEquals(0, $this->repo->countVisitsByTag('invalid')); - self::assertEquals(12, $this->repo->countVisitsByTag($foo)); - self::assertEquals(4, $this->repo->countVisitsByTag($foo, new DateRange( - Chronos::parse('2016-01-02'), - Chronos::parse('2016-01-03'), + self::assertEquals(0, $this->repo->countVisitsByTag('invalid', new VisitsCountFiltering())); + self::assertEquals(12, $this->repo->countVisitsByTag($foo, new VisitsCountFiltering())); + self::assertEquals(4, $this->repo->countVisitsByTag($foo, new VisitsCountFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), + ))); + self::assertEquals(8, $this->repo->countVisitsByTag($foo, new VisitsCountFiltering( + DateRange::withStartDate(Chronos::parse('2016-01-03')), ))); - self::assertEquals(8, $this->repo->countVisitsByTag($foo, new DateRange(Chronos::parse('2016-01-03')))); } /** @test */ @@ -213,7 +214,7 @@ class VisitRepositoryTest extends DatabaseTestCase self::assertEquals(4, $this->repo->countVisits($apiKey1)); self::assertEquals(5 + 7, $this->repo->countVisits($apiKey2)); self::assertEquals(4 + 7, $this->repo->countVisits($domainApiKey)); - self::assertEquals(3, $this->repo->countOrphanVisits()); + self::assertEquals(3, $this->repo->countOrphanVisits(new VisitsCountFiltering())); } /** @test */ @@ -276,13 +277,17 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - self::assertEquals(18, $this->repo->countOrphanVisits()); - self::assertEquals(18, $this->repo->countOrphanVisits(DateRange::emptyInstance())); - self::assertEquals(9, $this->repo->countOrphanVisits(DateRange::withStartDate(Chronos::parse('2020-01-04')))); - self::assertEquals(6, $this->repo->countOrphanVisits( - DateRange::withStartAndEndDate(Chronos::parse('2020-01-02'), Chronos::parse('2020-01-03')), + self::assertEquals(18, $this->repo->countOrphanVisits(new VisitsCountFiltering())); + self::assertEquals(18, $this->repo->countOrphanVisits(new VisitsCountFiltering(DateRange::emptyInstance()))); + self::assertEquals(9, $this->repo->countOrphanVisits( + new VisitsCountFiltering(DateRange::withStartDate(Chronos::parse('2020-01-04'))), + )); + self::assertEquals(6, $this->repo->countOrphanVisits(new VisitsCountFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2020-01-02'), Chronos::parse('2020-01-03')), + ))); + self::assertEquals(3, $this->repo->countOrphanVisits( + new VisitsCountFiltering(DateRange::withEndDate(Chronos::parse('2020-01-01'))), )); - self::assertEquals(3, $this->repo->countOrphanVisits(DateRange::withEndDate(Chronos::parse('2020-01-01')))); } private function createShortUrlsAndVisits(bool $withDomain = true, array $tags = []): array diff --git a/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php index 6b28aa68..7ec20d9c 100644 --- a/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php @@ -12,6 +12,7 @@ use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Paginator\Adapter\OrphanVisitsPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; class OrphanVisitsPaginatorAdapterTest extends TestCase { @@ -32,7 +33,9 @@ class OrphanVisitsPaginatorAdapterTest extends TestCase public function countDelegatesToRepository(): void { $expectedCount = 5; - $repoCount = $this->repo->countOrphanVisits($this->params->getDateRange())->willReturn($expectedCount); + $repoCount = $this->repo->countOrphanVisits( + new VisitsCountFiltering($this->params->getDateRange()), + )->willReturn($expectedCount); $result = $this->adapter->getNbResults(); diff --git a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php index 6c95a60f..d9f4a492 100644 --- a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Paginator\Adapter\VisitsForTagPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsForTagPaginatorAdapterTest extends TestCase @@ -46,7 +47,10 @@ class VisitsForTagPaginatorAdapterTest extends TestCase $count = 3; $apiKey = ApiKey::create(); $adapter = $this->createAdapter($apiKey); - $countVisits = $this->repo->countVisitsByTag('foo', new DateRange(), $apiKey->spec())->willReturn(3); + $countVisits = $this->repo->countVisitsByTag( + 'foo', + new VisitsCountFiltering(DateRange::emptyInstance(), false, $apiKey->spec()), + )->willReturn(3); for ($i = 0; $i < $count; $i++) { $adapter->getNbResults(); diff --git a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php index 955e5ac5..91ddeaae 100644 --- a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php @@ -12,6 +12,7 @@ use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Paginator\Adapter\VisitsPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsPaginatorAdapterTest extends TestCase @@ -49,7 +50,11 @@ class VisitsPaginatorAdapterTest extends TestCase $count = 3; $apiKey = ApiKey::create(); $adapter = $this->createAdapter($apiKey); - $countVisits = $this->repo->countVisitsByShortCode('', null, new DateRange(), $apiKey->spec())->willReturn(3); + $countVisits = $this->repo->countVisitsByShortCode( + '', + null, + new VisitsCountFiltering(new DateRange(), false, $apiKey->spec()), + )->willReturn(3); for ($i = 0; $i < $count; $i++) { $adapter->getNbResults(); diff --git a/module/Core/test/Visit/VisitsStatsHelperTest.php b/module/Core/test/Visit/VisitsStatsHelperTest.php index 8e90a447..7b525d1a 100644 --- a/module/Core/test/Visit/VisitsStatsHelperTest.php +++ b/module/Core/test/Visit/VisitsStatsHelperTest.php @@ -23,6 +23,7 @@ use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface; use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\Visit\Model\VisitsStats; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Core\Visit\VisitsStatsHelper; use Shlinkio\Shlink\Rest\Entity\ApiKey; use ShlinkioTest\Shlink\Core\Util\ApiKeyHelpersTrait; @@ -53,7 +54,9 @@ class VisitsStatsHelperTest extends TestCase { $repo = $this->prophesize(VisitRepository::class); $count = $repo->countVisits(null)->willReturn($expectedCount * 3); - $countOrphan = $repo->countOrphanVisits()->willReturn($expectedCount); + $countOrphan = $repo->countOrphanVisits(Argument::type(VisitsCountFiltering::class))->willReturn( + $expectedCount, + ); $getRepo = $this->em->getRepository(Visit::class)->willReturn($repo->reveal()); $stats = $this->helper->getVisitsStats(); @@ -86,7 +89,7 @@ class VisitsStatsHelperTest extends TestCase $repo2->findVisitsByShortCode($shortCode, null, Argument::type(DateRange::class), 1, 0, $spec)->willReturn( $list, ); - $repo2->countVisitsByShortCode($shortCode, null, Argument::type(DateRange::class), $spec)->willReturn(1); + $repo2->countVisitsByShortCode($shortCode, null, Argument::type(VisitsCountFiltering::class))->willReturn(1); $this->em->getRepository(Visit::class)->willReturn($repo2->reveal())->shouldBeCalledOnce(); $paginator = $this->helper->visitsForShortUrl(new ShortUrlIdentifier($shortCode), new VisitsParams(), $apiKey); @@ -140,7 +143,7 @@ class VisitsStatsHelperTest extends TestCase $list = map(range(0, 1), fn () => Visit::forValidShortUrl(ShortUrl::createEmpty(), Visitor::emptyInstance())); $repo2 = $this->prophesize(VisitRepository::class); $repo2->findVisitsByTag($tag, Argument::type(DateRange::class), 1, 0, $spec)->willReturn($list); - $repo2->countVisitsByTag($tag, Argument::type(DateRange::class), $spec)->willReturn(1); + $repo2->countVisitsByTag($tag, Argument::type(VisitsCountFiltering::class))->willReturn(1); $this->em->getRepository(Visit::class)->willReturn($repo2->reveal())->shouldBeCalledOnce(); $paginator = $this->helper->visitsForTag($tag, new VisitsParams(), $apiKey); @@ -155,7 +158,7 @@ class VisitsStatsHelperTest extends TestCase { $list = map(range(0, 3), fn () => Visit::forBasePath(Visitor::emptyInstance())); $repo = $this->prophesize(VisitRepository::class); - $countVisits = $repo->countOrphanVisits(Argument::type(DateRange::class))->willReturn(count($list)); + $countVisits = $repo->countOrphanVisits(Argument::type(VisitsCountFiltering::class))->willReturn(count($list)); $listVisits = $repo->findOrphanVisits(Argument::type(DateRange::class), Argument::cetera())->willReturn($list); $getRepo = $this->em->getRepository(Visit::class)->willReturn($repo->reveal()); From db3c5a303181012325825b5da002ecbd2690b7e9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 20:32:30 +0200 Subject: [PATCH 73/86] Added new models to pass to repositories when listing visits of any kind --- .../Adapter/OrphanVisitsPaginatorAdapter.php | 9 ++- .../Adapter/VisitsForTagPaginatorAdapter.php | 12 ++- .../Adapter/VisitsPaginatorAdapter.php | 12 ++- .../Core/src/Repository/VisitRepository.php | 57 ++++++-------- .../Repository/VisitRepositoryInterface.php | 22 +----- .../Repository/VisitRepositoryTest.php | 74 ++++++++++++------- .../OrphanVisitsPaginatorAdapterTest.php | 5 +- .../VisitsForTagPaginatorAdapterTest.php | 6 +- .../Adapter/VisitsPaginatorAdapterTest.php | 9 ++- .../Core/test/Visit/VisitsStatsHelperTest.php | 11 +-- 10 files changed, 116 insertions(+), 101 deletions(-) diff --git a/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php index 9d96d5ab..0cc9ae7d 100644 --- a/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Paginator\Adapter; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; class OrphanVisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter { @@ -26,6 +27,12 @@ class OrphanVisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapte public function getSlice($offset, $length): iterable // phpcs:ignore { - return $this->repo->findOrphanVisits($this->params->getDateRange(), $length, $offset); + return $this->repo->findOrphanVisits(new VisitsListFiltering( + $this->params->getDateRange(), + $this->params->excludeBots(), + null, + $length, + $offset, + )); } } diff --git a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php index df1eba11..d7c0580f 100644 --- a/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsForTagPaginatorAdapter.php @@ -8,6 +8,7 @@ use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsForTagPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter @@ -33,10 +34,13 @@ class VisitsForTagPaginatorAdapter extends AbstractCacheableCountPaginatorAdapte { return $this->visitRepository->findVisitsByTag( $this->tag, - $this->params->getDateRange(), - $length, - $offset, - $this->resolveSpec(), + new VisitsListFiltering( + $this->params->getDateRange(), + $this->params->excludeBots(), + $this->resolveSpec(), + $length, + $offset, + ), ); } diff --git a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php index a0f002ff..5369bd05 100644 --- a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php @@ -9,6 +9,7 @@ use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter { @@ -34,10 +35,13 @@ class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter return $this->visitRepository->findVisitsByShortCode( $this->identifier->shortCode(), $this->identifier->domain(), - $this->params->getDateRange(), - $length, - $offset, - $this->spec, + new VisitsListFiltering( + $this->params->getDateRange(), + $this->params->excludeBots(), + $this->spec, + $length, + $offset, + ), ); } diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index bab885f6..5d3bacbc 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -7,12 +7,12 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\Query\ResultSetMappingBuilder; use Doctrine\ORM\QueryBuilder; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; -use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Core\Visit\Spec\CountOfOrphanVisits; use Shlinkio\Shlink\Core\Visit\Spec\CountOfShortUrlVisits; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -84,21 +84,15 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo /** * @return Visit[] */ - public function findVisitsByShortCode( - string $shortCode, - ?string $domain = null, - ?DateRange $dateRange = null, - ?int $limit = null, - ?int $offset = null, - ?Specification $spec = null - ): array { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $dateRange, $spec); - return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); + public function findVisitsByShortCode(string $shortCode, ?string $domain, VisitsListFiltering $filtering): array + { + $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $filtering); + return $this->resolveVisitsWithNativeQuery($qb, $filtering->limit(), $filtering->offset()); } public function countVisitsByShortCode(string $shortCode, ?string $domain, VisitsCountFiltering $filtering): int { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $filtering->dateRange(), $filtering->spec()); + $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $filtering); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); @@ -107,12 +101,11 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo private function createVisitsByShortCodeQueryBuilder( string $shortCode, ?string $domain, - ?DateRange $dateRange, - ?Specification $spec = null + VisitsCountFiltering $filtering ): QueryBuilder { /** @var ShortUrlRepositoryInterface $shortUrlRepo */ $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); - $shortUrl = $shortUrlRepo->findOne($shortCode, $domain, $spec); + $shortUrl = $shortUrlRepo->findOne($shortCode, $domain, $filtering->spec()); $shortUrlId = $shortUrl !== null ? $shortUrl->getId() : -1; // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later @@ -122,35 +115,27 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo ->where($qb->expr()->eq('v.shortUrl', $shortUrlId)); // Apply date range filtering - $this->applyDatesInline($qb, $dateRange); + $this->applyDatesInline($qb, $filtering->dateRange()); return $qb; } - public function findVisitsByTag( - string $tag, - ?DateRange $dateRange = null, - ?int $limit = null, - ?int $offset = null, - ?Specification $spec = null - ): array { - $qb = $this->createVisitsByTagQueryBuilder($tag, $dateRange, $spec); - return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); + public function findVisitsByTag(string $tag, VisitsListFiltering $filtering): array + { + $qb = $this->createVisitsByTagQueryBuilder($tag, $filtering); + return $this->resolveVisitsWithNativeQuery($qb, $filtering->limit(), $filtering->offset()); } public function countVisitsByTag(string $tag, VisitsCountFiltering $filtering): int { - $qb = $this->createVisitsByTagQueryBuilder($tag, $filtering->dateRange(), $filtering->spec()); + $qb = $this->createVisitsByTagQueryBuilder($tag, $filtering); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); } - private function createVisitsByTagQueryBuilder( - string $tag, - ?DateRange $dateRange, - ?Specification $spec - ): QueryBuilder { + private function createVisitsByTagQueryBuilder(string $tag, VisitsCountFiltering $filtering): QueryBuilder + { // Parameters in this query need to be inlined, not bound, as we need to use it as sub-query later // Since they are not strictly provided by the caller, it's reasonably safe $qb = $this->getEntityManager()->createQueryBuilder(); @@ -159,13 +144,13 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo ->join('s.tags', 't') ->where($qb->expr()->eq('t.name', '\'' . $tag . '\'')); // This needs to be concatenated, not bound - $this->applyDatesInline($qb, $dateRange); - $this->applySpecification($qb, $spec, 'v'); + $this->applyDatesInline($qb, $filtering->dateRange()); + $this->applySpecification($qb, $filtering->spec(), 'v'); return $qb; } - public function findOrphanVisits(?DateRange $dateRange = null, ?int $limit = null, ?int $offset = null): array + public function findOrphanVisits(VisitsListFiltering $filtering): array { // Parameters in this query need to be inlined, not bound, as we need to use it as sub-query later // Since they are not strictly provided by the caller, it's reasonably safe @@ -173,9 +158,9 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo $qb->from(Visit::class, 'v') ->where($qb->expr()->isNull('v.shortUrl')); - $this->applyDatesInline($qb, $dateRange); + $this->applyDatesInline($qb, $filtering->dateRange()); - return $this->resolveVisitsWithNativeQuery($qb, $limit, $offset); + return $this->resolveVisitsWithNativeQuery($qb, $filtering->limit(), $filtering->offset()); } public function countOrphanVisits(VisitsCountFiltering $filtering): int diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index a3a6ca1a..05a59720 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -6,10 +6,9 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; -use Happyr\DoctrineSpecification\Specification\Specification; -use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; interface VisitRepositoryInterface extends ObjectRepository, EntitySpecificationRepositoryInterface @@ -34,34 +33,21 @@ interface VisitRepositoryInterface extends ObjectRepository, EntitySpecification /** * @return Visit[] */ - public function findVisitsByShortCode( - string $shortCode, - ?string $domain = null, - ?DateRange $dateRange = null, - ?int $limit = null, - ?int $offset = null, - ?Specification $spec = null - ): array; + public function findVisitsByShortCode(string $shortCode, ?string $domain, VisitsListFiltering $filtering): array; public function countVisitsByShortCode(string $shortCode, ?string $domain, VisitsCountFiltering $filtering): int; /** * @return Visit[] */ - public function findVisitsByTag( - string $tag, - ?DateRange $dateRange = null, - ?int $limit = null, - ?int $offset = null, - ?Specification $spec = null - ): array; + public function findVisitsByTag(string $tag, VisitsListFiltering $filtering): array; public function countVisitsByTag(string $tag, VisitsCountFiltering $filtering): int; /** * @return Visit[] */ - public function findOrphanVisits(?DateRange $dateRange = null, ?int $limit = null, ?int $offset = null): array; + public function findOrphanVisits(VisitsListFiltering $filtering): array; public function countOrphanVisits(VisitsCountFiltering $filtering): int; diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 9bf32182..02be1260 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -16,6 +16,7 @@ use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\ShortUrl\Resolver\PersistenceShortUrlRelationResolver; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\IpGeolocation\Model\Location; use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta; use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition; @@ -88,22 +89,33 @@ class VisitRepositoryTest extends DatabaseTestCase { [$shortCode, $domain] = $this->createShortUrlsAndVisits(); - self::assertCount(0, $this->repo->findVisitsByShortCode('invalid')); - self::assertCount(6, $this->repo->findVisitsByShortCode($shortCode)); - self::assertCount(3, $this->repo->findVisitsByShortCode($shortCode, $domain)); - self::assertCount(2, $this->repo->findVisitsByShortCode($shortCode, null, new DateRange( - Chronos::parse('2016-01-02'), - Chronos::parse('2016-01-03'), + self::assertCount(0, $this->repo->findVisitsByShortCode('invalid', null, new VisitsListFiltering())); + self::assertCount(6, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering())); + self::assertCount(3, $this->repo->findVisitsByShortCode($shortCode, $domain, new VisitsListFiltering())); + self::assertCount(2, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), ))); - self::assertCount(4, $this->repo->findVisitsByShortCode($shortCode, null, new DateRange( - Chronos::parse('2016-01-03'), + self::assertCount(4, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering( + DateRange::withStartDate(Chronos::parse('2016-01-03')), ))); - self::assertCount(1, $this->repo->findVisitsByShortCode($shortCode, $domain, new DateRange( - Chronos::parse('2016-01-03'), + self::assertCount(1, $this->repo->findVisitsByShortCode($shortCode, $domain, new VisitsListFiltering( + DateRange::withStartDate(Chronos::parse('2016-01-03')), ))); - self::assertCount(3, $this->repo->findVisitsByShortCode($shortCode, null, null, 3, 2)); - self::assertCount(2, $this->repo->findVisitsByShortCode($shortCode, null, null, 5, 4)); - self::assertCount(1, $this->repo->findVisitsByShortCode($shortCode, $domain, null, 3, 2)); + self::assertCount(3, $this->repo->findVisitsByShortCode( + $shortCode, + null, + new VisitsListFiltering(null, false, null, 3, 2), + )); + self::assertCount(2, $this->repo->findVisitsByShortCode( + $shortCode, + null, + new VisitsListFiltering(null, false, null, 5, 4), + )); + self::assertCount(1, $this->repo->findVisitsByShortCode( + $shortCode, + $domain, + new VisitsListFiltering(null, false, null, 3, 2), + )); } /** @test */ @@ -140,13 +152,14 @@ class VisitRepositoryTest extends DatabaseTestCase $this->createShortUrlsAndVisits(false, [$foo]); $this->getEntityManager()->flush(); - self::assertCount(0, $this->repo->findVisitsByTag('invalid')); - self::assertCount(18, $this->repo->findVisitsByTag($foo)); - self::assertCount(6, $this->repo->findVisitsByTag($foo, new DateRange( - Chronos::parse('2016-01-02'), - Chronos::parse('2016-01-03'), + self::assertCount(0, $this->repo->findVisitsByTag('invalid', new VisitsListFiltering())); + self::assertCount(18, $this->repo->findVisitsByTag($foo, new VisitsListFiltering())); + self::assertCount(6, $this->repo->findVisitsByTag($foo, new VisitsListFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), + ))); + self::assertCount(12, $this->repo->findVisitsByTag($foo, new VisitsListFiltering( + DateRange::withStartDate(Chronos::parse('2016-01-03')), ))); - self::assertCount(12, $this->repo->findVisitsByTag($foo, new DateRange(Chronos::parse('2016-01-03')))); } /** @test */ @@ -241,16 +254,25 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - self::assertCount(18, $this->repo->findOrphanVisits()); - self::assertCount(5, $this->repo->findOrphanVisits(null, 5)); - self::assertCount(10, $this->repo->findOrphanVisits(null, 15, 8)); - self::assertCount(9, $this->repo->findOrphanVisits(DateRange::withStartDate(Chronos::parse('2020-01-04')), 15)); - self::assertCount(2, $this->repo->findOrphanVisits( + self::assertCount(18, $this->repo->findOrphanVisits(new VisitsListFiltering())); + self::assertCount(5, $this->repo->findOrphanVisits(new VisitsListFiltering(null, false, null, 5))); + self::assertCount(10, $this->repo->findOrphanVisits(new VisitsListFiltering(null, false, null, 15, 8))); + self::assertCount(9, $this->repo->findOrphanVisits(new VisitsListFiltering( + DateRange::withStartDate(Chronos::parse('2020-01-04')), + false, + null, + 15, + ))); + self::assertCount(2, $this->repo->findOrphanVisits(new VisitsListFiltering( DateRange::withStartAndEndDate(Chronos::parse('2020-01-02'), Chronos::parse('2020-01-03')), + false, + null, 6, 4, - )); - self::assertCount(3, $this->repo->findOrphanVisits(DateRange::withEndDate(Chronos::parse('2020-01-01')))); + ))); + self::assertCount(3, $this->repo->findOrphanVisits(new VisitsListFiltering( + DateRange::withEndDate(Chronos::parse('2020-01-01')), + ))); } /** @test */ diff --git a/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php index 7ec20d9c..1cc21eef 100644 --- a/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/OrphanVisitsPaginatorAdapterTest.php @@ -13,6 +13,7 @@ use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Paginator\Adapter\OrphanVisitsPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; class OrphanVisitsPaginatorAdapterTest extends TestCase { @@ -51,7 +52,9 @@ class OrphanVisitsPaginatorAdapterTest extends TestCase { $visitor = Visitor::emptyInstance(); $list = [Visit::forRegularNotFound($visitor), Visit::forInvalidShortUrl($visitor)]; - $repoFind = $this->repo->findOrphanVisits($this->params->getDateRange(), $limit, $offset)->willReturn($list); + $repoFind = $this->repo->findOrphanVisits( + new VisitsListFiltering($this->params->getDateRange(), $this->params->excludeBots(), null, $limit, $offset), + )->willReturn($list); $result = $this->adapter->getSlice($offset, $limit); diff --git a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php index d9f4a492..aa684b70 100644 --- a/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsForTagPaginatorAdapterTest.php @@ -12,6 +12,7 @@ use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Paginator\Adapter\VisitsForTagPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsForTagPaginatorAdapterTest extends TestCase @@ -32,7 +33,10 @@ class VisitsForTagPaginatorAdapterTest extends TestCase $limit = 1; $offset = 5; $adapter = $this->createAdapter(null); - $findVisits = $this->repo->findVisitsByTag('foo', new DateRange(), $limit, $offset, null)->willReturn([]); + $findVisits = $this->repo->findVisitsByTag( + 'foo', + new VisitsListFiltering(new DateRange(), false, null, $limit, $offset), + )->willReturn([]); for ($i = 0; $i < $count; $i++) { $adapter->getSlice($offset, $limit); diff --git a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php index 91ddeaae..d7b454b0 100644 --- a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php @@ -13,6 +13,7 @@ use Shlinkio\Shlink\Core\Model\VisitsParams; use Shlinkio\Shlink\Core\Paginator\Adapter\VisitsPaginatorAdapter; use Shlinkio\Shlink\Core\Repository\VisitRepositoryInterface; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; class VisitsPaginatorAdapterTest extends TestCase @@ -33,9 +34,11 @@ class VisitsPaginatorAdapterTest extends TestCase $limit = 1; $offset = 5; $adapter = $this->createAdapter(null); - $findVisits = $this->repo->findVisitsByShortCode('', null, new DateRange(), $limit, $offset, null)->willReturn( - [], - ); + $findVisits = $this->repo->findVisitsByShortCode( + '', + null, + new VisitsListFiltering(new DateRange(), false, null, $limit, $offset), + )->willReturn([]); for ($i = 0; $i < $count; $i++) { $adapter->getSlice($offset, $limit); diff --git a/module/Core/test/Visit/VisitsStatsHelperTest.php b/module/Core/test/Visit/VisitsStatsHelperTest.php index 7b525d1a..e6f067da 100644 --- a/module/Core/test/Visit/VisitsStatsHelperTest.php +++ b/module/Core/test/Visit/VisitsStatsHelperTest.php @@ -10,7 +10,6 @@ use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; -use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Entity\Visit; @@ -24,6 +23,7 @@ use Shlinkio\Shlink\Core\Repository\TagRepository; use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\Visit\Model\VisitsStats; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Core\Visit\VisitsStatsHelper; use Shlinkio\Shlink\Rest\Entity\ApiKey; use ShlinkioTest\Shlink\Core\Util\ApiKeyHelpersTrait; @@ -86,9 +86,7 @@ class VisitsStatsHelperTest extends TestCase $list = map(range(0, 1), fn () => Visit::forValidShortUrl(ShortUrl::createEmpty(), Visitor::emptyInstance())); $repo2 = $this->prophesize(VisitRepository::class); - $repo2->findVisitsByShortCode($shortCode, null, Argument::type(DateRange::class), 1, 0, $spec)->willReturn( - $list, - ); + $repo2->findVisitsByShortCode($shortCode, null, Argument::type(VisitsListFiltering::class))->willReturn($list); $repo2->countVisitsByShortCode($shortCode, null, Argument::type(VisitsCountFiltering::class))->willReturn(1); $this->em->getRepository(Visit::class)->willReturn($repo2->reveal())->shouldBeCalledOnce(); @@ -139,10 +137,9 @@ class VisitsStatsHelperTest extends TestCase $tagExists = $repo->tagExists($tag, $apiKey)->willReturn(true); $getRepo = $this->em->getRepository(Tag::class)->willReturn($repo->reveal()); - $spec = $apiKey === null ? null : $apiKey->spec(); $list = map(range(0, 1), fn () => Visit::forValidShortUrl(ShortUrl::createEmpty(), Visitor::emptyInstance())); $repo2 = $this->prophesize(VisitRepository::class); - $repo2->findVisitsByTag($tag, Argument::type(DateRange::class), 1, 0, $spec)->willReturn($list); + $repo2->findVisitsByTag($tag, Argument::type(VisitsListFiltering::class))->willReturn($list); $repo2->countVisitsByTag($tag, Argument::type(VisitsCountFiltering::class))->willReturn(1); $this->em->getRepository(Visit::class)->willReturn($repo2->reveal())->shouldBeCalledOnce(); @@ -159,7 +156,7 @@ class VisitsStatsHelperTest extends TestCase $list = map(range(0, 3), fn () => Visit::forBasePath(Visitor::emptyInstance())); $repo = $this->prophesize(VisitRepository::class); $countVisits = $repo->countOrphanVisits(Argument::type(VisitsCountFiltering::class))->willReturn(count($list)); - $listVisits = $repo->findOrphanVisits(Argument::type(DateRange::class), Argument::cetera())->willReturn($list); + $listVisits = $repo->findOrphanVisits(Argument::type(VisitsListFiltering::class))->willReturn($list); $getRepo = $this->em->getRepository(Visit::class)->willReturn($repo->reveal()); $paginator = $this->helper->orphanVisits(new VisitsParams()); From 69d72e754fc8d1ba744b48de27bfc3a208652996 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 20:49:24 +0200 Subject: [PATCH 74/86] Added logic to exclude bots from visits when requested --- module/Core/src/Model/Visitor.php | 5 ++++ .../Core/src/Repository/VisitRepository.php | 14 ++++++++++- .../src/Visit/Spec/CountOfOrphanVisits.php | 20 +++++++++------ .../Repository/VisitRepositoryTest.php | 25 ++++++++++++++++--- 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/module/Core/src/Model/Visitor.php b/module/Core/src/Model/Visitor.php index e9bdc36e..b73ed68a 100644 --- a/module/Core/src/Model/Visitor.php +++ b/module/Core/src/Model/Visitor.php @@ -53,6 +53,11 @@ final class Visitor return new self('', '', null, ''); } + public static function botInstance(): self + { + return new self('cf-facebook', '', null, ''); + } + public function getUserAgent(): string { return $this->userAgent; diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 5d3bacbc..bc3f958b 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -114,6 +114,10 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo $qb->from(Visit::class, 'v') ->where($qb->expr()->eq('v.shortUrl', $shortUrlId)); + if ($filtering->excludeBots()) { + $qb->andWhere($qb->expr()->eq('v.potentialBot', 'false')); + } + // Apply date range filtering $this->applyDatesInline($qb, $filtering->dateRange()); @@ -144,6 +148,10 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo ->join('s.tags', 't') ->where($qb->expr()->eq('t.name', '\'' . $tag . '\'')); // This needs to be concatenated, not bound + if ($filtering->excludeBots()) { + $qb->andWhere($qb->expr()->eq('v.potentialBot', 'false')); + } + $this->applyDatesInline($qb, $filtering->dateRange()); $this->applySpecification($qb, $filtering->spec(), 'v'); @@ -158,6 +166,10 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo $qb->from(Visit::class, 'v') ->where($qb->expr()->isNull('v.shortUrl')); + if ($filtering->excludeBots()) { + $qb->andWhere($qb->expr()->eq('v.potentialBot', 'false')); + } + $this->applyDatesInline($qb, $filtering->dateRange()); return $this->resolveVisitsWithNativeQuery($qb, $filtering->limit(), $filtering->offset()); @@ -165,7 +177,7 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo public function countOrphanVisits(VisitsCountFiltering $filtering): int { - return (int) $this->matchSingleScalarResult(new CountOfOrphanVisits($filtering->dateRange())); + return (int) $this->matchSingleScalarResult(new CountOfOrphanVisits($filtering)); } public function countVisits(?ApiKey $apiKey = null): int diff --git a/module/Core/src/Visit/Spec/CountOfOrphanVisits.php b/module/Core/src/Visit/Spec/CountOfOrphanVisits.php index 97712944..b2cc9efd 100644 --- a/module/Core/src/Visit/Spec/CountOfOrphanVisits.php +++ b/module/Core/src/Visit/Spec/CountOfOrphanVisits.php @@ -7,24 +7,30 @@ namespace Shlinkio\Shlink\Core\Visit\Spec; use Happyr\DoctrineSpecification\Spec; use Happyr\DoctrineSpecification\Specification\BaseSpecification; use Happyr\DoctrineSpecification\Specification\Specification; -use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Spec\InDateRange; +use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; class CountOfOrphanVisits extends BaseSpecification { - private ?DateRange $dateRange; + private VisitsCountFiltering $filtering; - public function __construct(?DateRange $dateRange) + public function __construct(VisitsCountFiltering $filtering) { parent::__construct(); - $this->dateRange = $dateRange; + $this->filtering = $filtering; } protected function getSpec(): Specification { - return Spec::countOf(Spec::andX( + $conditions = [ Spec::isNull('shortUrl'), - new InDateRange($this->dateRange), - )); + new InDateRange($this->filtering->dateRange()), + ]; + + if ($this->filtering->excludeBots()) { + $conditions[] = Spec::eq('potentialBot', false); + } + + return Spec::countOf(Spec::andX(...$conditions)); } } diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 02be1260..f77d7fd0 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -91,6 +91,7 @@ class VisitRepositoryTest extends DatabaseTestCase self::assertCount(0, $this->repo->findVisitsByShortCode('invalid', null, new VisitsListFiltering())); self::assertCount(6, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering())); + self::assertCount(4, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering(null, true))); self::assertCount(3, $this->repo->findVisitsByShortCode($shortCode, $domain, new VisitsListFiltering())); self::assertCount(2, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering( DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), @@ -125,6 +126,10 @@ class VisitRepositoryTest extends DatabaseTestCase self::assertEquals(0, $this->repo->countVisitsByShortCode('invalid', null, new VisitsCountFiltering())); self::assertEquals(6, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering())); + self::assertEquals(4, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( + null, + true, + ))); self::assertEquals(3, $this->repo->countVisitsByShortCode($shortCode, $domain, new VisitsCountFiltering())); self::assertEquals(2, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), @@ -154,6 +159,7 @@ class VisitRepositoryTest extends DatabaseTestCase self::assertCount(0, $this->repo->findVisitsByTag('invalid', new VisitsListFiltering())); self::assertCount(18, $this->repo->findVisitsByTag($foo, new VisitsListFiltering())); + self::assertCount(12, $this->repo->findVisitsByTag($foo, new VisitsListFiltering(null, true))); self::assertCount(6, $this->repo->findVisitsByTag($foo, new VisitsListFiltering( DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), ))); @@ -175,6 +181,7 @@ class VisitRepositoryTest extends DatabaseTestCase self::assertEquals(0, $this->repo->countVisitsByTag('invalid', new VisitsCountFiltering())); self::assertEquals(12, $this->repo->countVisitsByTag($foo, new VisitsCountFiltering())); + self::assertEquals(8, $this->repo->countVisitsByTag($foo, new VisitsCountFiltering(null, true))); self::assertEquals(4, $this->repo->countVisitsByTag($foo, new VisitsCountFiltering( DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), ))); @@ -220,6 +227,7 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->persist(Visit::forBasePath(Visitor::emptyInstance())); $this->getEntityManager()->persist(Visit::forInvalidShortUrl(Visitor::emptyInstance())); $this->getEntityManager()->persist(Visit::forRegularNotFound(Visitor::emptyInstance())); + $this->getEntityManager()->persist(Visit::forRegularNotFound(Visitor::botInstance())); $this->getEntityManager()->flush(); @@ -227,7 +235,8 @@ class VisitRepositoryTest extends DatabaseTestCase self::assertEquals(4, $this->repo->countVisits($apiKey1)); self::assertEquals(5 + 7, $this->repo->countVisits($apiKey2)); self::assertEquals(4 + 7, $this->repo->countVisits($domainApiKey)); - self::assertEquals(3, $this->repo->countOrphanVisits(new VisitsCountFiltering())); + self::assertEquals(4, $this->repo->countOrphanVisits(new VisitsCountFiltering())); + self::assertEquals(3, $this->repo->countOrphanVisits(new VisitsCountFiltering(null, true))); } /** @test */ @@ -237,9 +246,10 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->persist($shortUrl); $this->createVisitsForShortUrl($shortUrl, 7); + $botsCount = 3; for ($i = 0; $i < 6; $i++) { $this->getEntityManager()->persist($this->setDateOnVisit( - Visit::forBasePath(Visitor::emptyInstance()), + Visit::forBasePath($botsCount < 1 ? Visitor::emptyInstance() : Visitor::botInstance()), Chronos::parse(sprintf('2020-01-0%s', $i + 1)), )); $this->getEntityManager()->persist($this->setDateOnVisit( @@ -250,11 +260,14 @@ class VisitRepositoryTest extends DatabaseTestCase Visit::forRegularNotFound(Visitor::emptyInstance()), Chronos::parse(sprintf('2020-01-0%s', $i + 1)), )); + + $botsCount--; } $this->getEntityManager()->flush(); self::assertCount(18, $this->repo->findOrphanVisits(new VisitsListFiltering())); + self::assertCount(15, $this->repo->findOrphanVisits(new VisitsListFiltering(null, true))); self::assertCount(5, $this->repo->findOrphanVisits(new VisitsListFiltering(null, false, null, 5))); self::assertCount(10, $this->repo->findOrphanVisits(new VisitsListFiltering(null, false, null, 15, 8))); self::assertCount(9, $this->repo->findOrphanVisits(new VisitsListFiltering( @@ -338,13 +351,17 @@ class VisitRepositoryTest extends DatabaseTestCase return [$shortCode, $domain, $shortUrl]; } - private function createVisitsForShortUrl(ShortUrl $shortUrl, int $amount = 6): void + private function createVisitsForShortUrl(ShortUrl $shortUrl, int $amount = 6, int $botsAmount = 2): void { for ($i = 0; $i < $amount; $i++) { $visit = $this->setDateOnVisit( - Visit::forValidShortUrl($shortUrl, Visitor::emptyInstance()), + Visit::forValidShortUrl( + $shortUrl, + $botsAmount < 1 ? Visitor::emptyInstance() : Visitor::botInstance(), + ), Chronos::parse(sprintf('2016-01-0%s', $i + 1)), ); + $botsAmount--; $this->getEntityManager()->persist($visit); } From a12c9f54c4c91af2bcbf7b7c2435e6fd24ad2c7d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 21:05:54 +0200 Subject: [PATCH 75/86] Added API tests covering the excludion of bot visits --- .../Adapter/OrphanVisitsPaginatorAdapter.php | 5 +++- .../Rest/test-api/Action/OrphanVisitsTest.php | 23 +++++++++++---- .../test-api/Action/ShortUrlVisitsTest.php | 26 +++++++++++++++++ module/Rest/test-api/Action/TagVisitsTest.php | 29 +++++++++++++------ .../Rest/test-api/Fixtures/VisitsFixture.php | 2 +- 5 files changed, 68 insertions(+), 17 deletions(-) diff --git a/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php index 0cc9ae7d..d7361fb3 100644 --- a/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/OrphanVisitsPaginatorAdapter.php @@ -22,7 +22,10 @@ class OrphanVisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapte protected function doCount(): int { - return $this->repo->countOrphanVisits(new VisitsCountFiltering($this->params->getDateRange())); + return $this->repo->countOrphanVisits(new VisitsCountFiltering( + $this->params->getDateRange(), + $this->params->excludeBots(), + )); } public function getSlice($offset, $length): iterable // phpcs:ignore diff --git a/module/Rest/test-api/Action/OrphanVisitsTest.php b/module/Rest/test-api/Action/OrphanVisitsTest.php index 06857653..067cf9a4 100644 --- a/module/Rest/test-api/Action/OrphanVisitsTest.php +++ b/module/Rest/test-api/Action/OrphanVisitsTest.php @@ -41,21 +41,32 @@ class OrphanVisitsTest extends ApiTestCase * @test * @dataProvider provideQueries */ - public function properVisitsAreReturnedBasedInQuery(array $query, int $expectedAmount, array $expectedVisits): void - { + public function properVisitsAreReturnedBasedInQuery( + array $query, + int $totalItems, + int $expectedAmount, + array $expectedVisits + ): void { $resp = $this->callApiWithKey(self::METHOD_GET, '/visits/orphan', [RequestOptions::QUERY => $query]); $payload = $this->getJsonResponsePayload($resp); $visits = $payload['visits']['data'] ?? []; - self::assertEquals(3, $payload['visits']['pagination']['totalItems'] ?? -1); + self::assertEquals($totalItems, $payload['visits']['pagination']['totalItems'] ?? -1); self::assertCount($expectedAmount, $visits); self::assertEquals($expectedVisits, $visits); } public function provideQueries(): iterable { - yield 'all data' => [[], 3, [self::INVALID_SHORT_URL, self::REGULAR_NOT_FOUND, self::BASE_URL]]; - yield 'limit items' => [['itemsPerPage' => 2], 2, [self::INVALID_SHORT_URL, self::REGULAR_NOT_FOUND]]; - yield 'limit items and page' => [['itemsPerPage' => 2, 'page' => 2], 1, [self::BASE_URL]]; + yield 'all data' => [[], 3, 3, [self::INVALID_SHORT_URL, self::REGULAR_NOT_FOUND, self::BASE_URL]]; + yield 'limit items' => [['itemsPerPage' => 2], 3, 2, [self::INVALID_SHORT_URL, self::REGULAR_NOT_FOUND]]; + yield 'limit items and page' => [['itemsPerPage' => 2, 'page' => 2], 3, 1, [self::BASE_URL]]; + yield 'exclude bots' => [['excludeBots' => true], 2, 2, [self::REGULAR_NOT_FOUND, self::BASE_URL]]; + yield 'exclude bots and limit items' => [ + ['excludeBots' => true, 'itemsPerPage' => 1], + 2, + 1, + [self::REGULAR_NOT_FOUND], + ]; } } diff --git a/module/Rest/test-api/Action/ShortUrlVisitsTest.php b/module/Rest/test-api/Action/ShortUrlVisitsTest.php index c578d48d..1d572004 100644 --- a/module/Rest/test-api/Action/ShortUrlVisitsTest.php +++ b/module/Rest/test-api/Action/ShortUrlVisitsTest.php @@ -67,4 +67,30 @@ class ShortUrlVisitsTest extends ApiTestCase yield 'domain' => ['example.com', 0]; yield 'no domain' => [null, 2]; } + + /** + * @test + * @dataProvider provideVisitsForBots + */ + public function properVisitsAreReturnedWhenExcludingBots(bool $excludeBots, int $expectedAmountOfVisits): void + { + $shortCode = 'def456'; + $url = new Uri(sprintf('/short-urls/%s/visits', $shortCode)); + + if ($excludeBots) { + $url = $url->withQuery(Query::build(['excludeBots' => true])); + } + + $resp = $this->callApiWithKey(self::METHOD_GET, (string) $url); + $payload = $this->getJsonResponsePayload($resp); + + self::assertEquals($expectedAmountOfVisits, $payload['visits']['pagination']['totalItems'] ?? -1); + self::assertCount($expectedAmountOfVisits, $payload['visits']['data'] ?? []); + } + + public function provideVisitsForBots(): iterable + { + yield 'bots excluded' => [true, 1]; + yield 'bots not excluded' => [false, 2]; + } } diff --git a/module/Rest/test-api/Action/TagVisitsTest.php b/module/Rest/test-api/Action/TagVisitsTest.php index b30b787f..07b0576d 100644 --- a/module/Rest/test-api/Action/TagVisitsTest.php +++ b/module/Rest/test-api/Action/TagVisitsTest.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace ShlinkioApiTest\Shlink\Rest\Action; +use GuzzleHttp\RequestOptions; use Shlinkio\Shlink\TestUtils\ApiTest\ApiTestCase; use function sprintf; @@ -14,9 +15,15 @@ class TagVisitsTest extends ApiTestCase * @test * @dataProvider provideTags */ - public function expectedVisitsAreReturned(string $apiKey, string $tag, int $expectedVisitsAmount): void - { - $resp = $this->callApiWithKey(self::METHOD_GET, sprintf('/tags/%s/visits', $tag), [], $apiKey); + public function expectedVisitsAreReturned( + string $apiKey, + string $tag, + bool $excludeBots, + int $expectedVisitsAmount + ): void { + $resp = $this->callApiWithKey(self::METHOD_GET, sprintf('/tags/%s/visits', $tag), [ + RequestOptions::QUERY => $excludeBots ? ['excludeBots' => true] : [], + ], $apiKey); $payload = $this->getJsonResponsePayload($resp); self::assertEquals(self::STATUS_OK, $resp->getStatusCode()); @@ -27,12 +34,16 @@ class TagVisitsTest extends ApiTestCase public function provideTags(): iterable { - yield 'foo with admin API key' => ['valid_api_key', 'foo', 5]; - yield 'bar with admin API key' => ['valid_api_key', 'bar', 2]; - yield 'baz with admin API key' => ['valid_api_key', 'baz', 0]; - yield 'foo with author API key' => ['author_api_key', 'foo', 5]; - yield 'bar with author API key' => ['author_api_key', 'bar', 2]; - yield 'foo with domain API key' => ['domain_api_key', 'foo', 0]; + yield 'foo with admin API key' => ['valid_api_key', 'foo', false, 5]; + yield 'foo with admin API key and no bots' => ['valid_api_key', 'foo', true, 4]; + yield 'bar with admin API key' => ['valid_api_key', 'bar', false, 2]; + yield 'bar with admin API key and no bots' => ['valid_api_key', 'bar', true, 1]; + yield 'baz with admin API key' => ['valid_api_key', 'baz', false, 0]; + yield 'foo with author API key' => ['author_api_key', 'foo', false, 5]; + yield 'foo with author API key and no bots' => ['author_api_key', 'foo', true, 4]; + yield 'bar with author API key' => ['author_api_key', 'bar', false, 2]; + yield 'bar with author API key and no bots' => ['author_api_key', 'bar', true, 1]; + yield 'foo with domain API key' => ['domain_api_key', 'foo', false, 0]; } /** diff --git a/module/Rest/test-api/Fixtures/VisitsFixture.php b/module/Rest/test-api/Fixtures/VisitsFixture.php index 62e1527d..4432df92 100644 --- a/module/Rest/test-api/Fixtures/VisitsFixture.php +++ b/module/Rest/test-api/Fixtures/VisitsFixture.php @@ -36,7 +36,7 @@ class VisitsFixture extends AbstractFixture implements DependentFixtureInterface /** @var ShortUrl $defShortUrl */ $defShortUrl = $this->getReference('def456_short_url'); $manager->persist( - Visit::forValidShortUrl($defShortUrl, new Visitor('shlink-tests-agent', '', '127.0.0.1', '')), + Visit::forValidShortUrl($defShortUrl, new Visitor('cf-facebook', '', '127.0.0.1', '')), ); $manager->persist( Visit::forValidShortUrl($defShortUrl, new Visitor('shlink-tests-agent', 'https://app.shlink.io', '', '')), From 1c861fecfc62d948234ac2292bb221ae24fa16ad Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 21:14:15 +0200 Subject: [PATCH 76/86] Documented the excludeBots query param for visits endpoints --- .../paths/v1_short-urls_{shortCode}_visits.json | 10 ++++++++++ docs/swagger/paths/v2_tags_{tag}_visits.json | 10 ++++++++++ docs/swagger/paths/v2_visits_orphan.json | 10 ++++++++++ 3 files changed, 30 insertions(+) diff --git a/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json b/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json index 1bef6110..e5bbbe86 100644 --- a/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json +++ b/docs/swagger/paths/v1_short-urls_{shortCode}_visits.json @@ -57,6 +57,16 @@ "schema": { "type": "number" } + }, + { + "name": "excludeBots", + "in": "query", + "description": "Tells if visits from potential bots should be excluded from the result set", + "required": false, + "schema": { + "type": "string", + "enum": ["true"] + } } ], "security": [ diff --git a/docs/swagger/paths/v2_tags_{tag}_visits.json b/docs/swagger/paths/v2_tags_{tag}_visits.json index ab442793..df1242f6 100644 --- a/docs/swagger/paths/v2_tags_{tag}_visits.json +++ b/docs/swagger/paths/v2_tags_{tag}_visits.json @@ -54,6 +54,16 @@ "schema": { "type": "number" } + }, + { + "name": "excludeBots", + "in": "query", + "description": "Tells if visits from potential bots should be excluded from the result set", + "required": false, + "schema": { + "type": "string", + "enum": ["true"] + } } ], "security": [ diff --git a/docs/swagger/paths/v2_visits_orphan.json b/docs/swagger/paths/v2_visits_orphan.json index 7876e703..ce52b197 100644 --- a/docs/swagger/paths/v2_visits_orphan.json +++ b/docs/swagger/paths/v2_visits_orphan.json @@ -45,6 +45,16 @@ "schema": { "type": "number" } + }, + { + "name": "excludeBots", + "in": "query", + "description": "Tells if visits from potential bots should be excluded from the result set", + "required": false, + "schema": { + "type": "string", + "enum": ["true"] + } } ], "security": [ From 4b89687e45d22383f5783a2862dc0b43ad3a7cda Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 21:17:00 +0200 Subject: [PATCH 77/86] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5d6f749..22176f7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * `disable_ua_tracking`: If true, the user agent will not be tracked. * [#955](https://github.com/shlinkio/shlink/issues/955) Added new option to set short URLs as crawlable, making them be listed in the robots.txt as Allowed. +* [#900](https://github.com/shlinkio/shlink/issues/900) Shlink now tries to detect if the visit is coming from a potential bot or crawler, and allows to exclude those visits from visits lists if desired. ### Changed * [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0. From a1cab4ca7d040a553b25122aacaf33f5c0068358 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 21:22:15 +0200 Subject: [PATCH 78/86] Fixed typos --- docs/async-api/async-api.json | 2 +- docs/swagger/definitions/Visit.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/async-api/async-api.json b/docs/async-api/async-api.json index b1313d1e..0b546377 100644 --- a/docs/async-api/async-api.json +++ b/docs/async-api/async-api.json @@ -193,7 +193,7 @@ }, "potentialBot": { "type": "boolean", - "description": "Tells if Shlink thinks this visits comes potentially from a bot or crawler" + "description": "Tells if Shlink thinks this visit comes potentially from a bot or crawler" } }, "example": { diff --git a/docs/swagger/definitions/Visit.json b/docs/swagger/definitions/Visit.json index ad5fd97b..ecb6b9f9 100644 --- a/docs/swagger/definitions/Visit.json +++ b/docs/swagger/definitions/Visit.json @@ -20,7 +20,7 @@ }, "potentialBot": { "type": "boolean", - "description": "Tells if Shlink thinks this visits comes potentially from a bot or crawler" + "description": "Tells if Shlink thinks this visit comes potentially from a bot or crawler" } } } From 73aead01b4ca95df70920e39d2bf5d70e951ae19 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 21:33:50 +0200 Subject: [PATCH 79/86] Split execution of db and unit mutation tests during ci workflow --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 188a7f02..7e278bda 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -217,6 +217,7 @@ jobs: strategy: matrix: php-version: ['7.4', '8.0'] + test-group: ['unit', 'db'] steps: - name: Checkout code uses: actions/checkout@v2 @@ -232,7 +233,7 @@ jobs: - uses: actions/download-artifact@v2 with: path: build - - run: composer infect:ci + - run: composer infect:ci:${{ matrix.test-group }} upload-coverage: needs: From bf0c679a48f9eba5600e7fb7c17579dec620c01c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 22 May 2021 22:06:08 +0200 Subject: [PATCH 80/86] Added real versions from some shlink dependencies --- composer.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/composer.json b/composer.json index 0bb1186a..7a84c886 100644 --- a/composer.json +++ b/composer.json @@ -47,11 +47,11 @@ "predis/predis": "^1.1", "pugx/shortid-php": "^0.7", "ramsey/uuid": "^3.9", - "shlinkio/shlink-common": "dev-main#554e370 as 3.7", + "shlinkio/shlink-common": "^3.7", "shlinkio/shlink-config": "^1.0", "shlinkio/shlink-event-dispatcher": "^2.1", - "shlinkio/shlink-importer": "dev-main#39928b6 as 2.3", - "shlinkio/shlink-installer": "dev-develop#15cf3b3 as 6.0", + "shlinkio/shlink-importer": "^2.3", + "shlinkio/shlink-installer": "^6.0", "shlinkio/shlink-ip-geolocation": "^1.5", "symfony/console": "^5.1", "symfony/filesystem": "^5.1", From 3ff4ac84c499c820342e0eb118532cdf87282b86 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 2 May 2021 10:33:27 +0200 Subject: [PATCH 81/86] Added locking to short URL creation when checking if URL exists --- module/Core/src/Model/ShortUrlIdentifier.php | 14 ++++++++++++ .../src/Repository/ShortUrlRepository.php | 22 +++++++++++++++---- .../ShortUrlRepositoryInterface.php | 5 ++++- .../src/Service/ShortUrl/ShortCodeHelper.php | 7 ++---- module/Core/src/Visit/VisitsStatsHelper.php | 2 +- .../Repository/ShortUrlRepositoryTest.php | 19 +++++++++++----- .../Service/ShortUrl/ShortCodeHelperTest.php | 17 ++++++++------ module/Core/test/Service/UrlShortenerTest.php | 1 - .../Core/test/Visit/VisitsStatsHelperTest.php | 8 +++++-- 9 files changed, 68 insertions(+), 27 deletions(-) diff --git a/module/Core/src/Model/ShortUrlIdentifier.php b/module/Core/src/Model/ShortUrlIdentifier.php index 4a74ba07..a277782c 100644 --- a/module/Core/src/Model/ShortUrlIdentifier.php +++ b/module/Core/src/Model/ShortUrlIdentifier.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Model; use Psr\Http\Message\ServerRequestInterface; +use Shlinkio\Shlink\Core\Entity\ShortUrl; use Symfony\Component\Console\Input\InputInterface; final class ShortUrlIdentifier @@ -42,6 +43,19 @@ final class ShortUrlIdentifier return new self($shortCode, $domain); } + public static function fromShortUrl(ShortUrl $shortUrl): self + { + $domain = $shortUrl->getDomain(); + $domainAuthority = $domain !== null ? $domain->getAuthority() : null; + + return new self($shortUrl->getShortCode(), $domainAuthority); + } + + public static function fromShortCodeAndDomain(string $shortCode, ?string $domain = null): self + { + return new self($shortCode, $domain); + } + public function shortCode(): string { return $this->shortCode; diff --git a/module/Core/src/Repository/ShortUrlRepository.php b/module/Core/src/Repository/ShortUrlRepository.php index eacf293b..a1b93f4d 100644 --- a/module/Core/src/Repository/ShortUrlRepository.php +++ b/module/Core/src/Repository/ShortUrlRepository.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; +use Doctrine\DBAL\LockMode; use Doctrine\ORM\Query\Expr\Join; use Doctrine\ORM\QueryBuilder; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; @@ -11,6 +12,7 @@ use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Doctrine\Type\ChronosDateTimeType; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering; use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; @@ -180,12 +182,24 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU return $qb->getQuery()->getOneOrNullResult(); } - public function shortCodeIsInUse(string $slug, ?string $domain = null, ?Specification $spec = null): bool + public function shortCodeIsInUse(ShortUrlIdentifier $identifier, ?Specification $spec = null): bool { - $qb = $this->createFindOneQueryBuilder($slug, $domain, $spec); - $qb->select('COUNT(DISTINCT s.id)'); + return $this->doShortCodeIsInUse($identifier, $spec, null); + } - return ((int) $qb->getQuery()->getSingleScalarResult()) > 0; + public function shortCodeIsInUseWithLock(ShortUrlIdentifier $identifier, ?Specification $spec = null): bool + { + return $this->doShortCodeIsInUse($identifier, $spec, LockMode::PESSIMISTIC_WRITE); + } + + private function doShortCodeIsInUse(ShortUrlIdentifier $identifier, ?Specification $spec, ?int $lockMode): bool + { + $qb = $this->createFindOneQueryBuilder($identifier->shortCode(), $identifier->domain(), $spec); + $qb->select('s.id'); + + $query = $qb->getQuery()->setLockMode($lockMode); + + return $query->getOneOrNullResult() !== null; } private function createFindOneQueryBuilder(string $slug, ?string $domain, ?Specification $spec): QueryBuilder diff --git a/module/Core/src/Repository/ShortUrlRepositoryInterface.php b/module/Core/src/Repository/ShortUrlRepositoryInterface.php index 5d8fa924..9b6dffc8 100644 --- a/module/Core/src/Repository/ShortUrlRepositoryInterface.php +++ b/module/Core/src/Repository/ShortUrlRepositoryInterface.php @@ -9,6 +9,7 @@ use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterfa use Happyr\DoctrineSpecification\Specification\Specification; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering; use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl; @@ -36,7 +37,9 @@ interface ShortUrlRepositoryInterface extends ObjectRepository, EntitySpecificat public function findOne(string $shortCode, ?string $domain = null, ?Specification $spec = null): ?ShortUrl; - public function shortCodeIsInUse(string $slug, ?string $domain, ?Specification $spec = null): bool; + public function shortCodeIsInUse(ShortUrlIdentifier $identifier, ?Specification $spec = null): bool; + + public function shortCodeIsInUseWithLock(ShortUrlIdentifier $identifier, ?Specification $spec = null): bool; public function findOneMatching(ShortUrlMeta $meta): ?ShortUrl; diff --git a/module/Core/src/Service/ShortUrl/ShortCodeHelper.php b/module/Core/src/Service/ShortUrl/ShortCodeHelper.php index 3df4c016..83c3397e 100644 --- a/module/Core/src/Service/ShortUrl/ShortCodeHelper.php +++ b/module/Core/src/Service/ShortUrl/ShortCodeHelper.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Service\ShortUrl; use Doctrine\ORM\EntityManagerInterface; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Repository\ShortUrlRepository; class ShortCodeHelper implements ShortCodeHelperInterface // TODO Rename to ShortCodeUniquenessHelper @@ -19,13 +20,9 @@ class ShortCodeHelper implements ShortCodeHelperInterface // TODO Rename to Shor public function ensureShortCodeUniqueness(ShortUrl $shortUrlToBeCreated, bool $hasCustomSlug): bool { - $shortCode = $shortUrlToBeCreated->getShortCode(); - $domain = $shortUrlToBeCreated->getDomain(); - $domainAuthority = $domain !== null ? $domain->getAuthority() : null; - /** @var ShortUrlRepository $repo */ $repo = $this->em->getRepository(ShortUrl::class); - $otherShortUrlsExist = $repo->shortCodeIsInUse($shortCode, $domainAuthority); + $otherShortUrlsExist = $repo->shortCodeIsInUseWithLock(ShortUrlIdentifier::fromShortUrl($shortUrlToBeCreated)); if (! $otherShortUrlsExist) { return true; diff --git a/module/Core/src/Visit/VisitsStatsHelper.php b/module/Core/src/Visit/VisitsStatsHelper.php index 1aa03c46..dfa00a4c 100644 --- a/module/Core/src/Visit/VisitsStatsHelper.php +++ b/module/Core/src/Visit/VisitsStatsHelper.php @@ -58,7 +58,7 @@ class VisitsStatsHelper implements VisitsStatsHelperInterface /** @var ShortUrlRepositoryInterface $repo */ $repo = $this->em->getRepository(ShortUrl::class); - if (! $repo->shortCodeIsInUse($identifier->shortCode(), $identifier->domain(), $spec)) { + if (! $repo->shortCodeIsInUse($identifier, $spec)) { throw ShortUrlNotFoundException::fromNotFound($identifier); } diff --git a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php index bd5b22d4..345e0911 100644 --- a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php +++ b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering; use Shlinkio\Shlink\Core\Model\Visitor; @@ -180,12 +181,18 @@ class ShortUrlRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - self::assertTrue($this->repo->shortCodeIsInUse('my-cool-slug')); - self::assertFalse($this->repo->shortCodeIsInUse('my-cool-slug', 'doma.in')); - self::assertFalse($this->repo->shortCodeIsInUse('slug-not-in-use')); - self::assertFalse($this->repo->shortCodeIsInUse('another-slug')); - self::assertFalse($this->repo->shortCodeIsInUse('another-slug', 'example.com')); - self::assertTrue($this->repo->shortCodeIsInUse('another-slug', 'doma.in')); + self::assertTrue($this->repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain('my-cool-slug'))); + self::assertFalse($this->repo->shortCodeIsInUse( + ShortUrlIdentifier::fromShortCodeAndDomain('my-cool-slug', 'doma.in'), + )); + self::assertFalse($this->repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain('slug-not-in-use'))); + self::assertFalse($this->repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain('another-slug'))); + self::assertFalse($this->repo->shortCodeIsInUse( + ShortUrlIdentifier::fromShortCodeAndDomain('another-slug', 'example.com'), + )); + self::assertTrue($this->repo->shortCodeIsInUse( + ShortUrlIdentifier::fromShortCodeAndDomain('another-slug', 'doma.in'), + )); } /** @test */ diff --git a/module/Core/test/Service/ShortUrl/ShortCodeHelperTest.php b/module/Core/test/Service/ShortUrl/ShortCodeHelperTest.php index 047dbc96..ca3b463f 100644 --- a/module/Core/test/Service/ShortUrl/ShortCodeHelperTest.php +++ b/module/Core/test/Service/ShortUrl/ShortCodeHelperTest.php @@ -10,6 +10,7 @@ use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\ShortUrl; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Repository\ShortUrlRepository; use Shlinkio\Shlink\Core\Service\ShortUrl\ShortCodeHelper; @@ -39,12 +40,12 @@ class ShortCodeHelperTest extends TestCase $callIndex = 0; $expectedCalls = 3; $repo = $this->prophesize(ShortUrlRepository::class); - $shortCodeIsInUse = $repo->shortCodeIsInUse('abc123', $expectedAuthority)->will( - function () use (&$callIndex, $expectedCalls) { - $callIndex++; - return $callIndex < $expectedCalls; - }, - ); + $shortCodeIsInUse = $repo->shortCodeIsInUseWithLock( + ShortUrlIdentifier::fromShortCodeAndDomain('abc123', $expectedAuthority), + )->will(function () use (&$callIndex, $expectedCalls) { + $callIndex++; + return $callIndex < $expectedCalls; + }); $getRepo = $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal()); $this->shortUrl->getDomain()->willReturn($domain); @@ -66,7 +67,9 @@ class ShortCodeHelperTest extends TestCase public function inUseSlugReturnsError(): void { $repo = $this->prophesize(ShortUrlRepository::class); - $shortCodeIsInUse = $repo->shortCodeIsInUse('abc123', null)->willReturn(true); + $shortCodeIsInUse = $repo->shortCodeIsInUseWithLock( + ShortUrlIdentifier::fromShortCodeAndDomain('abc123'), + )->willReturn(true); $getRepo = $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal()); $this->shortUrl->getDomain()->willReturn(null); diff --git a/module/Core/test/Service/UrlShortenerTest.php b/module/Core/test/Service/UrlShortenerTest.php index 7e319314..6bac432e 100644 --- a/module/Core/test/Service/UrlShortenerTest.php +++ b/module/Core/test/Service/UrlShortenerTest.php @@ -46,7 +46,6 @@ class UrlShortenerTest extends TestCase return $callback(); }); $repo = $this->prophesize(ShortUrlRepository::class); - $repo->shortCodeIsInUse(Argument::cetera())->willReturn(false); $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal()); $this->shortCodeHelper = $this->prophesize(ShortCodeHelperInterface::class); diff --git a/module/Core/test/Visit/VisitsStatsHelperTest.php b/module/Core/test/Visit/VisitsStatsHelperTest.php index e6f067da..372bda9a 100644 --- a/module/Core/test/Visit/VisitsStatsHelperTest.php +++ b/module/Core/test/Visit/VisitsStatsHelperTest.php @@ -81,7 +81,9 @@ class VisitsStatsHelperTest extends TestCase $shortCode = '123ABC'; $spec = $apiKey === null ? null : $apiKey->spec(); $repo = $this->prophesize(ShortUrlRepositoryInterface::class); - $count = $repo->shortCodeIsInUse($shortCode, null, $spec)->willReturn(true); + $count = $repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), $spec)->willReturn( + true, + ); $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal())->shouldBeCalledOnce(); $list = map(range(0, 1), fn () => Visit::forValidShortUrl(ShortUrl::createEmpty(), Visitor::emptyInstance())); @@ -101,7 +103,9 @@ class VisitsStatsHelperTest extends TestCase { $shortCode = '123ABC'; $repo = $this->prophesize(ShortUrlRepositoryInterface::class); - $count = $repo->shortCodeIsInUse($shortCode, null, null)->willReturn(false); + $count = $repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), null)->willReturn( + false, + ); $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal())->shouldBeCalledOnce(); $this->expectException(ShortUrlNotFoundException::class); From f82e103bc559d1b50c0006982fd1980ccb49af41 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 15 May 2021 12:11:45 +0200 Subject: [PATCH 82/86] Added locks to tag and domain creation during short URL creation --- .../src/Domain/Repository/DomainRepository.php | 13 +++++++++++++ .../Repository/DomainRepositoryInterface.php | 2 ++ module/Core/src/Repository/TagRepository.php | 13 +++++++++++++ .../src/Repository/TagRepositoryInterface.php | 3 +++ .../PersistenceShortUrlRelationResolver.php | 10 +++++++--- .../PersistenceShortUrlRelationResolverTest.php | 16 ++++++++-------- 6 files changed, 46 insertions(+), 11 deletions(-) diff --git a/module/Core/src/Domain/Repository/DomainRepository.php b/module/Core/src/Domain/Repository/DomainRepository.php index 2e4f3bb2..99a7a927 100644 --- a/module/Core/src/Domain/Repository/DomainRepository.php +++ b/module/Core/src/Domain/Repository/DomainRepository.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Domain\Repository; +use Doctrine\DBAL\LockMode; use Doctrine\ORM\Query\Expr\Join; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Shlinkio\Shlink\Core\Entity\Domain; @@ -32,4 +33,16 @@ class DomainRepository extends EntitySpecificationRepository implements DomainRe return $qb->getQuery()->getResult(); } + + public function findOneByAuthorityWithLock(string $authority): ?Domain + { + $qb = $this->createQueryBuilder('d'); + $qb->where($qb->expr()->eq('d.authority', ':authority')) + ->setParameter('authority', $authority) + ->setMaxResults(1); + + $query = $qb->getQuery()->setLockMode(LockMode::PESSIMISTIC_WRITE); + + return $query->getOneOrNullResult(); + } } diff --git a/module/Core/src/Domain/Repository/DomainRepositoryInterface.php b/module/Core/src/Domain/Repository/DomainRepositoryInterface.php index 1d201520..5dc2f873 100644 --- a/module/Core/src/Domain/Repository/DomainRepositoryInterface.php +++ b/module/Core/src/Domain/Repository/DomainRepositoryInterface.php @@ -15,4 +15,6 @@ interface DomainRepositoryInterface extends ObjectRepository, EntitySpecificatio * @return Domain[] */ public function findDomainsWithout(?string $excludedAuthority, ?ApiKey $apiKey = null): array; + + public function findOneByAuthorityWithLock(string $authority): ?Domain; } diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index d21122d0..38aae1dc 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -4,6 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; +use Doctrine\DBAL\LockMode; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Happyr\DoctrineSpecification\Spec; use Shlinkio\Shlink\Core\Entity\Tag; @@ -62,4 +63,16 @@ class TagRepository extends EntitySpecificationRepository implements TagReposito return $result > 0; } + + public function findOneByNameWithLock(string $name): ?Tag + { + $qb = $this->createQueryBuilder('t'); + $qb->where($qb->expr()->eq('t.name', ':name')) + ->setParameter('name', $name) + ->setMaxResults(1); + + $query = $qb->getQuery()->setLockMode(LockMode::PESSIMISTIC_WRITE); + + return $query->getOneOrNullResult(); + } } diff --git a/module/Core/src/Repository/TagRepositoryInterface.php b/module/Core/src/Repository/TagRepositoryInterface.php index 924706ff..b340c1fc 100644 --- a/module/Core/src/Repository/TagRepositoryInterface.php +++ b/module/Core/src/Repository/TagRepositoryInterface.php @@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; +use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -19,4 +20,6 @@ interface TagRepositoryInterface extends ObjectRepository, EntitySpecificationRe public function findTagsWithInfo(?ApiKey $apiKey = null): array; public function tagExists(string $tag, ?ApiKey $apiKey = null): bool; + + public function findOneByNameWithLock(string $name): ?Tag; } diff --git a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php index eb7fddad..b7e27a82 100644 --- a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php +++ b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php @@ -8,8 +8,10 @@ use Doctrine\Common\Collections; use Doctrine\Common\Collections\Collection; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Events; +use Shlinkio\Shlink\Core\Domain\Repository\DomainRepositoryInterface; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; +use Shlinkio\Shlink\Core\Repository\TagRepositoryInterface; use function Functional\map; use function Functional\unique; @@ -35,8 +37,9 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt return null; } - /** @var Domain|null $existingDomain */ - $existingDomain = $this->em->getRepository(Domain::class)->findOneBy(['authority' => $domain]); + /** @var DomainRepositoryInterface $repo */ + $repo = $this->em->getRepository(Domain::class); + $existingDomain = $repo->findOneByAuthorityWithLock($domain); // Memoize only new domains, and let doctrine handle objects hydrated from persistence return $existingDomain ?? $this->memoizeNewDomain($domain); @@ -58,11 +61,12 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt } $tags = unique($tags); + /** @var TagRepositoryInterface $repo */ $repo = $this->em->getRepository(Tag::class); return new Collections\ArrayCollection(map($tags, function (string $tagName) use ($repo): Tag { // Memoize only new tags, and let doctrine handle objects hydrated from persistence - $tag = $repo->findOneBy(['name' => $tagName]) ?? $this->memoizeNewTag($tagName); + $tag = $repo->findOneByNameWithLock($tagName) ?? $this->memoizeNewTag($tagName); $this->em->persist($tag); return $tag; diff --git a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php index 8660099c..8698bec0 100644 --- a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php +++ b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php @@ -6,11 +6,11 @@ namespace ShlinkioTest\Shlink\Core\ShortUrl\Resolver; use Doctrine\Common\EventManager; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\Persistence\ObjectRepository; use PHPUnit\Framework\TestCase; use Prophecy\Argument; use Prophecy\PhpUnit\ProphecyTrait; use Prophecy\Prophecy\ObjectProphecy; +use Shlinkio\Shlink\Core\Domain\Repository\DomainRepositoryInterface; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Repository\TagRepositoryInterface; @@ -48,8 +48,8 @@ class PersistenceShortUrlRelationResolverTest extends TestCase */ public function findsOrCreatesDomainWhenValueIsProvided(?Domain $foundDomain, string $authority): void { - $repo = $this->prophesize(ObjectRepository::class); - $findDomain = $repo->findOneBy(['authority' => $authority])->willReturn($foundDomain); + $repo = $this->prophesize(DomainRepositoryInterface::class); + $findDomain = $repo->findOneByAuthorityWithLock($authority)->willReturn($foundDomain); $getRepository = $this->em->getRepository(Domain::class)->willReturn($repo->reveal()); $result = $this->resolver->resolveDomain($authority); @@ -80,7 +80,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase $expectedPersistedTags = count($expectedTags); $tagRepo = $this->prophesize(TagRepositoryInterface::class); - $findTag = $tagRepo->findOneBy(Argument::type('array'))->will(function (array $args): ?Tag { + $findTag = $tagRepo->findOneByNameWithLock(Argument::type('string'))->will(function (array $args): ?Tag { ['name' => $name] = $args[0]; return $name === 'foo' ? new Tag($name) : null; }); @@ -106,7 +106,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function returnsEmptyCollectionWhenProvidingEmptyListOfTags(): void { $tagRepo = $this->prophesize(TagRepositoryInterface::class); - $findTag = $tagRepo->findOneBy(Argument::type('array'))->willReturn(null); + $findTag = $tagRepo->findOneByNameWithLock(Argument::type('string'))->willReturn(null); $getRepo = $this->em->getRepository(Tag::class)->willReturn($tagRepo->reveal()); $persist = $this->em->persist(Argument::type(Tag::class)); @@ -121,8 +121,8 @@ class PersistenceShortUrlRelationResolverTest extends TestCase /** @test */ public function newDomainsAreMemoizedUntilStateIsCleared(): void { - $repo = $this->prophesize(ObjectRepository::class); - $repo->findOneBy(Argument::type('array'))->willReturn(null); + $repo = $this->prophesize(DomainRepositoryInterface::class); + $repo->findOneByAuthorityWithLock(Argument::type('string'))->willReturn(null); $this->em->getRepository(Domain::class)->willReturn($repo->reveal()); $authority = 'foo.com'; @@ -141,7 +141,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function newTagsAreMemoizedUntilStateIsCleared(): void { $tagRepo = $this->prophesize(TagRepositoryInterface::class); - $tagRepo->findOneBy(Argument::type('array'))->willReturn(null); + $tagRepo->findOneByNameWithLock(Argument::type('string'))->willReturn(null); $this->em->getRepository(Tag::class)->willReturn($tagRepo->reveal()); $this->em->persist(Argument::type(Tag::class))->will(function (): void { }); From cd198764198d8c40ea8187d3058b241e3685cd03 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 23 May 2021 08:21:40 +0200 Subject: [PATCH 83/86] Removed methods to create tags and domains with lock, as they do not really lock as expected --- .../Core/src/Domain/Repository/DomainRepository.php | 13 ------------- .../Domain/Repository/DomainRepositoryInterface.php | 2 -- module/Core/src/Repository/TagRepository.php | 13 ------------- .../Core/src/Repository/TagRepositoryInterface.php | 3 --- .../PersistenceShortUrlRelationResolver.php | 10 +++------- .../PersistenceShortUrlRelationResolverTest.php | 10 +++++----- 6 files changed, 8 insertions(+), 43 deletions(-) diff --git a/module/Core/src/Domain/Repository/DomainRepository.php b/module/Core/src/Domain/Repository/DomainRepository.php index 99a7a927..2e4f3bb2 100644 --- a/module/Core/src/Domain/Repository/DomainRepository.php +++ b/module/Core/src/Domain/Repository/DomainRepository.php @@ -4,7 +4,6 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Domain\Repository; -use Doctrine\DBAL\LockMode; use Doctrine\ORM\Query\Expr\Join; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Shlinkio\Shlink\Core\Entity\Domain; @@ -33,16 +32,4 @@ class DomainRepository extends EntitySpecificationRepository implements DomainRe return $qb->getQuery()->getResult(); } - - public function findOneByAuthorityWithLock(string $authority): ?Domain - { - $qb = $this->createQueryBuilder('d'); - $qb->where($qb->expr()->eq('d.authority', ':authority')) - ->setParameter('authority', $authority) - ->setMaxResults(1); - - $query = $qb->getQuery()->setLockMode(LockMode::PESSIMISTIC_WRITE); - - return $query->getOneOrNullResult(); - } } diff --git a/module/Core/src/Domain/Repository/DomainRepositoryInterface.php b/module/Core/src/Domain/Repository/DomainRepositoryInterface.php index 5dc2f873..1d201520 100644 --- a/module/Core/src/Domain/Repository/DomainRepositoryInterface.php +++ b/module/Core/src/Domain/Repository/DomainRepositoryInterface.php @@ -15,6 +15,4 @@ interface DomainRepositoryInterface extends ObjectRepository, EntitySpecificatio * @return Domain[] */ public function findDomainsWithout(?string $excludedAuthority, ?ApiKey $apiKey = null): array; - - public function findOneByAuthorityWithLock(string $authority): ?Domain; } diff --git a/module/Core/src/Repository/TagRepository.php b/module/Core/src/Repository/TagRepository.php index 38aae1dc..d21122d0 100644 --- a/module/Core/src/Repository/TagRepository.php +++ b/module/Core/src/Repository/TagRepository.php @@ -4,7 +4,6 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core\Repository; -use Doctrine\DBAL\LockMode; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository; use Happyr\DoctrineSpecification\Spec; use Shlinkio\Shlink\Core\Entity\Tag; @@ -63,16 +62,4 @@ class TagRepository extends EntitySpecificationRepository implements TagReposito return $result > 0; } - - public function findOneByNameWithLock(string $name): ?Tag - { - $qb = $this->createQueryBuilder('t'); - $qb->where($qb->expr()->eq('t.name', ':name')) - ->setParameter('name', $name) - ->setMaxResults(1); - - $query = $qb->getQuery()->setLockMode(LockMode::PESSIMISTIC_WRITE); - - return $query->getOneOrNullResult(); - } } diff --git a/module/Core/src/Repository/TagRepositoryInterface.php b/module/Core/src/Repository/TagRepositoryInterface.php index b340c1fc..924706ff 100644 --- a/module/Core/src/Repository/TagRepositoryInterface.php +++ b/module/Core/src/Repository/TagRepositoryInterface.php @@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; -use Shlinkio\Shlink\Core\Entity\Tag; use Shlinkio\Shlink\Core\Tag\Model\TagInfo; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -20,6 +19,4 @@ interface TagRepositoryInterface extends ObjectRepository, EntitySpecificationRe public function findTagsWithInfo(?ApiKey $apiKey = null): array; public function tagExists(string $tag, ?ApiKey $apiKey = null): bool; - - public function findOneByNameWithLock(string $name): ?Tag; } diff --git a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php index b7e27a82..8601a045 100644 --- a/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php +++ b/module/Core/src/ShortUrl/Resolver/PersistenceShortUrlRelationResolver.php @@ -8,10 +8,8 @@ use Doctrine\Common\Collections; use Doctrine\Common\Collections\Collection; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Events; -use Shlinkio\Shlink\Core\Domain\Repository\DomainRepositoryInterface; use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\Tag; -use Shlinkio\Shlink\Core\Repository\TagRepositoryInterface; use function Functional\map; use function Functional\unique; @@ -37,9 +35,8 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt return null; } - /** @var DomainRepositoryInterface $repo */ - $repo = $this->em->getRepository(Domain::class); - $existingDomain = $repo->findOneByAuthorityWithLock($domain); + /** @var Domain|null $existingDomain */ + $existingDomain = $this->em->getRepository(Domain::class)->findOneBy(['authority' => $domain]); // Memoize only new domains, and let doctrine handle objects hydrated from persistence return $existingDomain ?? $this->memoizeNewDomain($domain); @@ -61,12 +58,11 @@ class PersistenceShortUrlRelationResolver implements ShortUrlRelationResolverInt } $tags = unique($tags); - /** @var TagRepositoryInterface $repo */ $repo = $this->em->getRepository(Tag::class); return new Collections\ArrayCollection(map($tags, function (string $tagName) use ($repo): Tag { // Memoize only new tags, and let doctrine handle objects hydrated from persistence - $tag = $repo->findOneByNameWithLock($tagName) ?? $this->memoizeNewTag($tagName); + $tag = $repo->findOneBy(['name' => $tagName]) ?? $this->memoizeNewTag($tagName); $this->em->persist($tag); return $tag; diff --git a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php index 8698bec0..aeef3f47 100644 --- a/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php +++ b/module/Core/test/ShortUrl/Resolver/PersistenceShortUrlRelationResolverTest.php @@ -49,7 +49,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function findsOrCreatesDomainWhenValueIsProvided(?Domain $foundDomain, string $authority): void { $repo = $this->prophesize(DomainRepositoryInterface::class); - $findDomain = $repo->findOneByAuthorityWithLock($authority)->willReturn($foundDomain); + $findDomain = $repo->findOneBy(['authority' => $authority])->willReturn($foundDomain); $getRepository = $this->em->getRepository(Domain::class)->willReturn($repo->reveal()); $result = $this->resolver->resolveDomain($authority); @@ -80,7 +80,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase $expectedPersistedTags = count($expectedTags); $tagRepo = $this->prophesize(TagRepositoryInterface::class); - $findTag = $tagRepo->findOneByNameWithLock(Argument::type('string'))->will(function (array $args): ?Tag { + $findTag = $tagRepo->findOneBy(Argument::type('array'))->will(function (array $args): ?Tag { ['name' => $name] = $args[0]; return $name === 'foo' ? new Tag($name) : null; }); @@ -106,7 +106,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function returnsEmptyCollectionWhenProvidingEmptyListOfTags(): void { $tagRepo = $this->prophesize(TagRepositoryInterface::class); - $findTag = $tagRepo->findOneByNameWithLock(Argument::type('string'))->willReturn(null); + $findTag = $tagRepo->findOneBy(Argument::type('array'))->willReturn(null); $getRepo = $this->em->getRepository(Tag::class)->willReturn($tagRepo->reveal()); $persist = $this->em->persist(Argument::type(Tag::class)); @@ -122,7 +122,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function newDomainsAreMemoizedUntilStateIsCleared(): void { $repo = $this->prophesize(DomainRepositoryInterface::class); - $repo->findOneByAuthorityWithLock(Argument::type('string'))->willReturn(null); + $repo->findOneBy(Argument::type('array'))->willReturn(null); $this->em->getRepository(Domain::class)->willReturn($repo->reveal()); $authority = 'foo.com'; @@ -141,7 +141,7 @@ class PersistenceShortUrlRelationResolverTest extends TestCase public function newTagsAreMemoizedUntilStateIsCleared(): void { $tagRepo = $this->prophesize(TagRepositoryInterface::class); - $tagRepo->findOneByNameWithLock(Argument::type('string'))->willReturn(null); + $tagRepo->findOneBy(Argument::type('array'))->willReturn(null); $this->em->getRepository(Tag::class)->willReturn($tagRepo->reveal()); $this->em->persist(Argument::type(Tag::class))->will(function (): void { }); From 5e6d2881bcaaf006ed9d1403d1ea29abe10086b9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 23 May 2021 08:41:42 +0200 Subject: [PATCH 84/86] Used ShorturlIdentifier model whenever possible --- .../Adapter/VisitsPaginatorAdapter.php | 6 +- .../src/Repository/ShortUrlRepository.php | 12 +-- .../ShortUrlRepositoryInterface.php | 2 +- .../Core/src/Repository/VisitRepository.php | 14 +-- .../Repository/VisitRepositoryInterface.php | 5 +- .../src/Service/ShortUrl/ShortUrlResolver.php | 6 +- .../Repository/ShortUrlRepositoryTest.php | 16 +-- .../Repository/VisitRepositoryTest.php | 99 ++++++++++++------- .../Adapter/VisitsPaginatorAdapterTest.php | 6 +- .../Service/ShortUrl/ShortUrlResolverTest.php | 10 +- .../Core/test/Visit/VisitsStatsHelperTest.php | 16 +-- 11 files changed, 112 insertions(+), 80 deletions(-) diff --git a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php index 5369bd05..d651b1b5 100644 --- a/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php +++ b/module/Core/src/Paginator/Adapter/VisitsPaginatorAdapter.php @@ -33,8 +33,7 @@ class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter public function getSlice($offset, $length): array // phpcs:ignore { return $this->visitRepository->findVisitsByShortCode( - $this->identifier->shortCode(), - $this->identifier->domain(), + $this->identifier, new VisitsListFiltering( $this->params->getDateRange(), $this->params->excludeBots(), @@ -48,8 +47,7 @@ class VisitsPaginatorAdapter extends AbstractCacheableCountPaginatorAdapter protected function doCount(): int { return $this->visitRepository->countVisitsByShortCode( - $this->identifier->shortCode(), - $this->identifier->domain(), + $this->identifier, new VisitsCountFiltering( $this->params->getDateRange(), $this->params->excludeBots(), diff --git a/module/Core/src/Repository/ShortUrlRepository.php b/module/Core/src/Repository/ShortUrlRepository.php index a1b93f4d..c658d478 100644 --- a/module/Core/src/Repository/ShortUrlRepository.php +++ b/module/Core/src/Repository/ShortUrlRepository.php @@ -174,9 +174,9 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU return $query->getOneOrNullResult(); } - public function findOne(string $shortCode, ?string $domain = null, ?Specification $spec = null): ?ShortUrl + public function findOne(ShortUrlIdentifier $identifier, ?Specification $spec = null): ?ShortUrl { - $qb = $this->createFindOneQueryBuilder($shortCode, $domain, $spec); + $qb = $this->createFindOneQueryBuilder($identifier, $spec); $qb->select('s'); return $qb->getQuery()->getOneOrNullResult(); @@ -194,7 +194,7 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU private function doShortCodeIsInUse(ShortUrlIdentifier $identifier, ?Specification $spec, ?int $lockMode): bool { - $qb = $this->createFindOneQueryBuilder($identifier->shortCode(), $identifier->domain(), $spec); + $qb = $this->createFindOneQueryBuilder($identifier, $spec); $qb->select('s.id'); $query = $qb->getQuery()->setLockMode($lockMode); @@ -202,16 +202,16 @@ class ShortUrlRepository extends EntitySpecificationRepository implements ShortU return $query->getOneOrNullResult() !== null; } - private function createFindOneQueryBuilder(string $slug, ?string $domain, ?Specification $spec): QueryBuilder + private function createFindOneQueryBuilder(ShortUrlIdentifier $identifier, ?Specification $spec): QueryBuilder { $qb = $this->getEntityManager()->createQueryBuilder(); $qb->from(ShortUrl::class, 's') ->where($qb->expr()->isNotNull('s.shortCode')) ->andWhere($qb->expr()->eq('s.shortCode', ':slug')) - ->setParameter('slug', $slug) + ->setParameter('slug', $identifier->shortCode()) ->setMaxResults(1); - $this->whereDomainIs($qb, $domain); + $this->whereDomainIs($qb, $identifier->domain()); $this->applySpecification($qb, $spec, 's'); diff --git a/module/Core/src/Repository/ShortUrlRepositoryInterface.php b/module/Core/src/Repository/ShortUrlRepositoryInterface.php index 9b6dffc8..7489f2a0 100644 --- a/module/Core/src/Repository/ShortUrlRepositoryInterface.php +++ b/module/Core/src/Repository/ShortUrlRepositoryInterface.php @@ -35,7 +35,7 @@ interface ShortUrlRepositoryInterface extends ObjectRepository, EntitySpecificat public function findOneWithDomainFallback(string $shortCode, ?string $domain = null): ?ShortUrl; - public function findOne(string $shortCode, ?string $domain = null, ?Specification $spec = null): ?ShortUrl; + public function findOne(ShortUrlIdentifier $identifier, ?Specification $spec = null): ?ShortUrl; public function shortCodeIsInUse(ShortUrlIdentifier $identifier, ?Specification $spec = null): bool; diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index bc3f958b..61cd108e 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Core\Visit\Spec\CountOfOrphanVisits; @@ -84,28 +85,27 @@ class VisitRepository extends EntitySpecificationRepository implements VisitRepo /** * @return Visit[] */ - public function findVisitsByShortCode(string $shortCode, ?string $domain, VisitsListFiltering $filtering): array + public function findVisitsByShortCode(ShortUrlIdentifier $identifier, VisitsListFiltering $filtering): array { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $filtering); + $qb = $this->createVisitsByShortCodeQueryBuilder($identifier, $filtering); return $this->resolveVisitsWithNativeQuery($qb, $filtering->limit(), $filtering->offset()); } - public function countVisitsByShortCode(string $shortCode, ?string $domain, VisitsCountFiltering $filtering): int + public function countVisitsByShortCode(ShortUrlIdentifier $identifier, VisitsCountFiltering $filtering): int { - $qb = $this->createVisitsByShortCodeQueryBuilder($shortCode, $domain, $filtering); + $qb = $this->createVisitsByShortCodeQueryBuilder($identifier, $filtering); $qb->select('COUNT(v.id)'); return (int) $qb->getQuery()->getSingleScalarResult(); } private function createVisitsByShortCodeQueryBuilder( - string $shortCode, - ?string $domain, + ShortUrlIdentifier $identifier, VisitsCountFiltering $filtering ): QueryBuilder { /** @var ShortUrlRepositoryInterface $shortUrlRepo */ $shortUrlRepo = $this->getEntityManager()->getRepository(ShortUrl::class); - $shortUrl = $shortUrlRepo->findOne($shortCode, $domain, $filtering->spec()); + $shortUrl = $shortUrlRepo->findOne($identifier, $filtering->spec()); $shortUrlId = $shortUrl !== null ? $shortUrl->getId() : -1; // Parameters in this query need to be part of the query itself, as we need to use it a sub-query later diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index 05a59720..28f1e9a8 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\Persistence\ObjectRepository; use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface; use Shlinkio\Shlink\Core\Entity\Visit; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsCountFiltering; use Shlinkio\Shlink\Core\Visit\Persistence\VisitsListFiltering; use Shlinkio\Shlink\Rest\Entity\ApiKey; @@ -33,9 +34,9 @@ interface VisitRepositoryInterface extends ObjectRepository, EntitySpecification /** * @return Visit[] */ - public function findVisitsByShortCode(string $shortCode, ?string $domain, VisitsListFiltering $filtering): array; + public function findVisitsByShortCode(ShortUrlIdentifier $identifier, VisitsListFiltering $filtering): array; - public function countVisitsByShortCode(string $shortCode, ?string $domain, VisitsCountFiltering $filtering): int; + public function countVisitsByShortCode(ShortUrlIdentifier $identifier, VisitsCountFiltering $filtering): int; /** * @return Visit[] diff --git a/module/Core/src/Service/ShortUrl/ShortUrlResolver.php b/module/Core/src/Service/ShortUrl/ShortUrlResolver.php index 6e03114c..1394e1ab 100644 --- a/module/Core/src/Service/ShortUrl/ShortUrlResolver.php +++ b/module/Core/src/Service/ShortUrl/ShortUrlResolver.php @@ -27,11 +27,7 @@ class ShortUrlResolver implements ShortUrlResolverInterface { /** @var ShortUrlRepository $shortUrlRepo */ $shortUrlRepo = $this->em->getRepository(ShortUrl::class); - $shortUrl = $shortUrlRepo->findOne( - $identifier->shortCode(), - $identifier->domain(), - $apiKey !== null ? $apiKey->spec() : null, - ); + $shortUrl = $shortUrlRepo->findOne($identifier, $apiKey !== null ? $apiKey->spec() : null); if ($shortUrl === null) { throw ShortUrlNotFoundException::fromNotFound($identifier); } diff --git a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php index 345e0911..867ff3f2 100644 --- a/module/Core/test-db/Repository/ShortUrlRepositoryTest.php +++ b/module/Core/test-db/Repository/ShortUrlRepositoryTest.php @@ -210,12 +210,16 @@ class ShortUrlRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); - self::assertNotNull($this->repo->findOne('my-cool-slug')); - self::assertNull($this->repo->findOne('my-cool-slug', 'doma.in')); - self::assertNull($this->repo->findOne('slug-not-in-use')); - self::assertNull($this->repo->findOne('another-slug')); - self::assertNull($this->repo->findOne('another-slug', 'example.com')); - self::assertNotNull($this->repo->findOne('another-slug', 'doma.in')); + self::assertNotNull($this->repo->findOne(ShortUrlIdentifier::fromShortCodeAndDomain('my-cool-slug'))); + self::assertNull($this->repo->findOne(ShortUrlIdentifier::fromShortCodeAndDomain('my-cool-slug', 'doma.in'))); + self::assertNull($this->repo->findOne(ShortUrlIdentifier::fromShortCodeAndDomain('slug-not-in-use'))); + self::assertNull($this->repo->findOne(ShortUrlIdentifier::fromShortCodeAndDomain('another-slug'))); + self::assertNull($this->repo->findOne( + ShortUrlIdentifier::fromShortCodeAndDomain('another-slug', 'example.com'), + )); + self::assertNotNull($this->repo->findOne( + ShortUrlIdentifier::fromShortCodeAndDomain('another-slug', 'doma.in'), + )); } /** @test */ diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index f77d7fd0..15fe34f4 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -11,6 +11,7 @@ use Shlinkio\Shlink\Core\Entity\Domain; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\Entity\VisitLocation; +use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\VisitRepository; @@ -89,32 +90,46 @@ class VisitRepositoryTest extends DatabaseTestCase { [$shortCode, $domain] = $this->createShortUrlsAndVisits(); - self::assertCount(0, $this->repo->findVisitsByShortCode('invalid', null, new VisitsListFiltering())); - self::assertCount(6, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering())); - self::assertCount(4, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering(null, true))); - self::assertCount(3, $this->repo->findVisitsByShortCode($shortCode, $domain, new VisitsListFiltering())); - self::assertCount(2, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering( - DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), - ))); - self::assertCount(4, $this->repo->findVisitsByShortCode($shortCode, null, new VisitsListFiltering( - DateRange::withStartDate(Chronos::parse('2016-01-03')), - ))); - self::assertCount(1, $this->repo->findVisitsByShortCode($shortCode, $domain, new VisitsListFiltering( - DateRange::withStartDate(Chronos::parse('2016-01-03')), - ))); + self::assertCount(0, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain('invalid'), + new VisitsListFiltering(), + )); + self::assertCount(6, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsListFiltering(), + )); + self::assertCount(4, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsListFiltering(null, true), + )); self::assertCount(3, $this->repo->findVisitsByShortCode( - $shortCode, - null, + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain), + new VisitsListFiltering(), + )); + self::assertCount(2, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsListFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), + ), + )); + self::assertCount(4, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsListFiltering(DateRange::withStartDate(Chronos::parse('2016-01-03'))), + )); + self::assertCount(1, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain), + new VisitsListFiltering(DateRange::withStartDate(Chronos::parse('2016-01-03'))), + )); + self::assertCount(3, $this->repo->findVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), new VisitsListFiltering(null, false, null, 3, 2), )); self::assertCount(2, $this->repo->findVisitsByShortCode( - $shortCode, - null, + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), new VisitsListFiltering(null, false, null, 5, 4), )); self::assertCount(1, $this->repo->findVisitsByShortCode( - $shortCode, - $domain, + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain), new VisitsListFiltering(null, false, null, 3, 2), )); } @@ -124,22 +139,36 @@ class VisitRepositoryTest extends DatabaseTestCase { [$shortCode, $domain] = $this->createShortUrlsAndVisits(); - self::assertEquals(0, $this->repo->countVisitsByShortCode('invalid', null, new VisitsCountFiltering())); - self::assertEquals(6, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering())); - self::assertEquals(4, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( - null, - true, - ))); - self::assertEquals(3, $this->repo->countVisitsByShortCode($shortCode, $domain, new VisitsCountFiltering())); - self::assertEquals(2, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( - DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), - ))); - self::assertEquals(4, $this->repo->countVisitsByShortCode($shortCode, null, new VisitsCountFiltering( - DateRange::withStartDate(Chronos::parse('2016-01-03')), - ))); - self::assertEquals(1, $this->repo->countVisitsByShortCode($shortCode, $domain, new VisitsCountFiltering( - DateRange::withStartDate(Chronos::parse('2016-01-03')), - ))); + self::assertEquals(0, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain('invalid'), + new VisitsCountFiltering(), + )); + self::assertEquals(6, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsCountFiltering(), + )); + self::assertEquals(4, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsCountFiltering(null, true), + )); + self::assertEquals(3, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain), + new VisitsCountFiltering(), + )); + self::assertEquals(2, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsCountFiltering( + DateRange::withStartAndEndDate(Chronos::parse('2016-01-02'), Chronos::parse('2016-01-03')), + ), + )); + self::assertEquals(4, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), + new VisitsCountFiltering(DateRange::withStartDate(Chronos::parse('2016-01-03'))), + )); + self::assertEquals(1, $this->repo->countVisitsByShortCode( + ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain), + new VisitsCountFiltering(DateRange::withStartDate(Chronos::parse('2016-01-03'))), + )); } /** @test */ diff --git a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php index d7b454b0..2a9e5fc4 100644 --- a/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php +++ b/module/Core/test/Paginator/Adapter/VisitsPaginatorAdapterTest.php @@ -35,8 +35,7 @@ class VisitsPaginatorAdapterTest extends TestCase $offset = 5; $adapter = $this->createAdapter(null); $findVisits = $this->repo->findVisitsByShortCode( - '', - null, + ShortUrlIdentifier::fromShortCodeAndDomain(''), new VisitsListFiltering(new DateRange(), false, null, $limit, $offset), )->willReturn([]); @@ -54,8 +53,7 @@ class VisitsPaginatorAdapterTest extends TestCase $apiKey = ApiKey::create(); $adapter = $this->createAdapter($apiKey); $countVisits = $this->repo->countVisitsByShortCode( - '', - null, + ShortUrlIdentifier::fromShortCodeAndDomain(''), new VisitsCountFiltering(new DateRange(), false, $apiKey->spec()), )->willReturn(3); diff --git a/module/Core/test/Service/ShortUrl/ShortUrlResolverTest.php b/module/Core/test/Service/ShortUrl/ShortUrlResolverTest.php index cf2330b3..73823729 100644 --- a/module/Core/test/Service/ShortUrl/ShortUrlResolverTest.php +++ b/module/Core/test/Service/ShortUrl/ShortUrlResolverTest.php @@ -46,12 +46,13 @@ class ShortUrlResolverTest extends TestCase { $shortUrl = ShortUrl::withLongUrl('expected_url'); $shortCode = $shortUrl->getShortCode(); + $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode); $repo = $this->prophesize(ShortUrlRepositoryInterface::class); - $findOne = $repo->findOne($shortCode, null, $apiKey !== null ? $apiKey->spec() : null)->willReturn($shortUrl); + $findOne = $repo->findOne($identifier, $apiKey !== null ? $apiKey->spec() : null)->willReturn($shortUrl); $getRepo = $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal()); - $result = $this->urlResolver->resolveShortUrl(new ShortUrlIdentifier($shortCode), $apiKey); + $result = $this->urlResolver->resolveShortUrl($identifier, $apiKey); self::assertSame($shortUrl, $result); $findOne->shouldHaveBeenCalledOnce(); @@ -65,16 +66,17 @@ class ShortUrlResolverTest extends TestCase public function exceptionIsThrownIfShortcodeIsNotFound(?ApiKey $apiKey): void { $shortCode = 'abc123'; + $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode); $repo = $this->prophesize(ShortUrlRepositoryInterface::class); - $findOne = $repo->findOne($shortCode, null, $apiKey !== null ? $apiKey->spec() : null)->willReturn(null); + $findOne = $repo->findOne($identifier, $apiKey !== null ? $apiKey->spec() : null)->willReturn(null); $getRepo = $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal(), $apiKey); $this->expectException(ShortUrlNotFoundException::class); $findOne->shouldBeCalledOnce(); $getRepo->shouldBeCalledOnce(); - $this->urlResolver->resolveShortUrl(new ShortUrlIdentifier($shortCode), $apiKey); + $this->urlResolver->resolveShortUrl($identifier, $apiKey); } /** @test */ diff --git a/module/Core/test/Visit/VisitsStatsHelperTest.php b/module/Core/test/Visit/VisitsStatsHelperTest.php index 372bda9a..cae3fbb1 100644 --- a/module/Core/test/Visit/VisitsStatsHelperTest.php +++ b/module/Core/test/Visit/VisitsStatsHelperTest.php @@ -79,20 +79,22 @@ class VisitsStatsHelperTest extends TestCase public function infoReturnsVisitsForCertainShortCode(?ApiKey $apiKey): void { $shortCode = '123ABC'; + $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode); $spec = $apiKey === null ? null : $apiKey->spec(); + $repo = $this->prophesize(ShortUrlRepositoryInterface::class); - $count = $repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), $spec)->willReturn( + $count = $repo->shortCodeIsInUse($identifier, $spec)->willReturn( true, ); $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal())->shouldBeCalledOnce(); $list = map(range(0, 1), fn () => Visit::forValidShortUrl(ShortUrl::createEmpty(), Visitor::emptyInstance())); $repo2 = $this->prophesize(VisitRepository::class); - $repo2->findVisitsByShortCode($shortCode, null, Argument::type(VisitsListFiltering::class))->willReturn($list); - $repo2->countVisitsByShortCode($shortCode, null, Argument::type(VisitsCountFiltering::class))->willReturn(1); + $repo2->findVisitsByShortCode($identifier, Argument::type(VisitsListFiltering::class))->willReturn($list); + $repo2->countVisitsByShortCode($identifier, Argument::type(VisitsCountFiltering::class))->willReturn(1); $this->em->getRepository(Visit::class)->willReturn($repo2->reveal())->shouldBeCalledOnce(); - $paginator = $this->helper->visitsForShortUrl(new ShortUrlIdentifier($shortCode), new VisitsParams(), $apiKey); + $paginator = $this->helper->visitsForShortUrl($identifier, new VisitsParams(), $apiKey); self::assertEquals($list, ArrayUtils::iteratorToArray($paginator->getCurrentPageResults())); $count->shouldHaveBeenCalledOnce(); @@ -102,8 +104,10 @@ class VisitsStatsHelperTest extends TestCase public function throwsExceptionWhenRequestingVisitsForInvalidShortCode(): void { $shortCode = '123ABC'; + $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode); + $repo = $this->prophesize(ShortUrlRepositoryInterface::class); - $count = $repo->shortCodeIsInUse(ShortUrlIdentifier::fromShortCodeAndDomain($shortCode), null)->willReturn( + $count = $repo->shortCodeIsInUse($identifier, null)->willReturn( false, ); $this->em->getRepository(ShortUrl::class)->willReturn($repo->reveal())->shouldBeCalledOnce(); @@ -111,7 +115,7 @@ class VisitsStatsHelperTest extends TestCase $this->expectException(ShortUrlNotFoundException::class); $count->shouldBeCalledOnce(); - $this->helper->visitsForShortUrl(new ShortUrlIdentifier($shortCode), new VisitsParams()); + $this->helper->visitsForShortUrl($identifier, new VisitsParams()); } /** @test */ From 46bea241e632fbd5c6d51c9ab49c0503d39fbf9a Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 23 May 2021 08:42:26 +0200 Subject: [PATCH 85/86] Updated changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22176f7e..b7df04c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). -## [Unreleased] +## [2.7.0] - 2021-05-23 ### Added * [#1044](https://github.com/shlinkio/shlink/issues/1044) Added ability to set names on API keys, which helps to identify them when the list grows. * [#819](https://github.com/shlinkio/shlink/issues/819) Visits are now always located in real time, even when not using swoole. From b6aca82da6afa17a54b6aae24cd35a2d54a18ebc Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 23 May 2021 09:05:04 +0200 Subject: [PATCH 86/86] Updated docker dependencies --- .github/workflows/ci.yml | 20 ++++++++++---------- .github/workflows/publish-release.yml | 2 +- Dockerfile | 11 ++++++----- data/infra/php.Dockerfile | 9 +++++---- data/infra/swoole.Dockerfile | 11 ++++++----- 5 files changed, 28 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7e278bda..f1aefb80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: none - run: composer install --no-interaction --prefer-dist - run: composer cs @@ -39,7 +39,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: none - run: composer install --no-interaction --prefer-dist - run: composer stan @@ -57,7 +57,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: pcov ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist @@ -83,7 +83,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: pcov ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist @@ -111,7 +111,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: none - run: composer install --no-interaction --prefer-dist - run: composer test:db:mysql @@ -131,7 +131,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: none - run: composer install --no-interaction --prefer-dist - run: composer test:db:maria @@ -151,7 +151,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: none - run: composer install --no-interaction --prefer-dist - run: composer test:db:postgres @@ -173,7 +173,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3, pdo_sqlsrv-5.9.0 + extensions: swoole-4.6.7, pdo_sqlsrv-5.9.0 coverage: none - run: composer install --no-interaction --prefer-dist - name: Create test database @@ -195,7 +195,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: pcov ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist @@ -226,7 +226,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 coverage: pcov ini-values: pcov.directory=module - run: composer install --no-interaction --prefer-dist diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 18c174c8..26ee4ac0 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -20,7 +20,7 @@ jobs: with: php-version: ${{ matrix.php-version }} tools: composer - extensions: swoole-4.6.3 + extensions: swoole-4.6.7 - if: ${{ matrix.swoole == 'yes' }} run: ./build.sh ${GITHUB_REF#refs/tags/v} - if: ${{ matrix.swoole == 'no' }} diff --git a/Dockerfile b/Dockerfile index 49bb28a2..c07adc28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,10 @@ -FROM php:8.0.2-alpine3.13 as base +FROM php:8.0.6-alpine3.13 as base ARG SHLINK_VERSION=latest ENV SHLINK_VERSION ${SHLINK_VERSION} -ENV SWOOLE_VERSION 4.6.3 +ENV SWOOLE_VERSION 4.6.7 ENV PDO_SQLSRV_VERSION 5.9.0 +ENV MS_ODBC_SQL_VERSION 17.5.2.1 ENV LC_ALL "C" WORKDIR /etc/shlink @@ -30,13 +31,13 @@ RUN \ # Install sqlsrv driver RUN if [ $(uname -m) == "x86_64" ]; then \ - wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ + wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \ + apk add --allow-untrusted msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \ apk add --no-cache --virtual .phpize-deps ${PHPIZE_DEPS} unixodbc-dev && \ pecl install pdo_sqlsrv-${PDO_SQLSRV_VERSION} && \ docker-php-ext-enable pdo_sqlsrv && \ apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk ; \ + rm msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk ; \ fi # Install swoole diff --git a/data/infra/php.Dockerfile b/data/infra/php.Dockerfile index dc4930ec..8972e1ac 100644 --- a/data/infra/php.Dockerfile +++ b/data/infra/php.Dockerfile @@ -1,8 +1,9 @@ -FROM php:8.0.2-fpm-alpine3.13 +FROM php:8.0.6-fpm-alpine3.13 MAINTAINER Alejandro Celaya ENV APCU_VERSION 5.1.19 ENV PDO_SQLSRV_VERSION 5.9.0 +ENV MS_ODBC_SQL_VERSION 17.5.2.1 RUN apk update @@ -44,13 +45,13 @@ RUN mkdir -p /usr/src/php/ext/apcu \ && echo extension=apcu.so > /usr/local/etc/php/conf.d/20-php-ext-apcu.ini # Install pcov and sqlsrv driver -RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ +RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \ + apk add --allow-untrusted msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \ apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ pecl install pdo_sqlsrv-${PDO_SQLSRV_VERSION} pcov && \ docker-php-ext-enable pdo_sqlsrv pcov && \ apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk + rm msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk # Install composer COPY --from=composer:2 /usr/bin/composer /usr/local/bin/composer diff --git a/data/infra/swoole.Dockerfile b/data/infra/swoole.Dockerfile index 7cbfacb0..f0f2ca74 100644 --- a/data/infra/swoole.Dockerfile +++ b/data/infra/swoole.Dockerfile @@ -1,10 +1,11 @@ -FROM php:8.0.2-alpine3.13 +FROM php:8.0.6-alpine3.13 MAINTAINER Alejandro Celaya ENV APCU_VERSION 5.1.19 ENV PDO_SQLSRV_VERSION 5.9.0 ENV INOTIFY_VERSION 3.0.0 -ENV SWOOLE_VERSION 4.6.3 +ENV SWOOLE_VERSION 4.6.7 +ENV MS_ODBC_SQL_VERSION 17.5.2.1 RUN apk update @@ -54,13 +55,13 @@ RUN mkdir -p /usr/src/php/ext/inotify \ && rm /tmp/inotify.tar.gz # Install swoole, pcov and mssql driver -RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \ - apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \ +RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \ + apk add --allow-untrusted msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \ apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \ pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv-${PDO_SQLSRV_VERSION} pcov && \ docker-php-ext-enable swoole pdo_sqlsrv pcov && \ apk del .phpize-deps && \ - rm msodbcsql17_17.5.1.1-1_amd64.apk + rm msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk # Install composer COPY --from=composer:2 /usr/bin/composer /usr/local/bin/composer