From 69f4daa9d2a009cc95a769ebf82d79eff9adb740 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 16:19:38 +0100 Subject: [PATCH 1/7] Added dev container with RabbitMQ --- composer.json | 1 + docker-compose.yml | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/composer.json b/composer.json index b40d24ba..f7da28f9 100644 --- a/composer.json +++ b/composer.json @@ -42,6 +42,7 @@ "nikolaposa/monolog-factory": "^3.1", "ocramius/proxy-manager": "^2.11", "pagerfanta/core": "^3.5", + "php-amqplib/php-amqplib": "^2.0", "php-middleware/request-id": "^4.1", "predis/predis": "^1.1", "pugx/shortid-php": "^1.0", diff --git a/docker-compose.yml b/docker-compose.yml index d9dd776c..3d552f9a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,7 @@ services: - shlink_redis - shlink_mercure - shlink_mercure_proxy + - shlink_rabbitmq environment: LC_ALL: C extra_hosts: @@ -64,6 +65,7 @@ services: - shlink_redis - shlink_mercure - shlink_mercure_proxy + - shlink_rabbitmq environment: LC_ALL: C extra_hosts: @@ -143,3 +145,13 @@ services: MERCURE_PUBLISHER_JWT_KEY: mercure_jwt_key MERCURE_SUBSCRIBER_JWT_KEY: mercure_jwt_key MERCURE_EXTRA_DIRECTIVES: "cors_origins https://app.shlink.io http://localhost:3000 http://127.0.0.1:3000" + + shlink_rabbitmq: + container_name: shlink_rabbitmq + image: rabbitmq:3.9-management-alpine + ports: + - "15672:15672" + - "5672:5672" + environment: + RABBITMQ_DEFAULT_USER: "rabbit" + RABBITMQ_DEFAULT_PASS: "rabbit" From bd3bb67949d366fde8609f17fc7a2f9b7aa7f18d Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 17:07:40 +0100 Subject: [PATCH 2/7] Added dependencies and config to integrate with Rabbit MQ --- composer.json | 2 +- config/autoload/rabbit.global.php | 49 +++++++++++++++++++++++++++ config/autoload/rabbit.local.php.dist | 13 +++++++ data/infra/php.Dockerfile | 3 ++ data/infra/swoole.Dockerfile | 3 ++ 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 config/autoload/rabbit.global.php create mode 100644 config/autoload/rabbit.local.php.dist diff --git a/composer.json b/composer.json index f7da28f9..2cfe4e14 100644 --- a/composer.json +++ b/composer.json @@ -42,7 +42,7 @@ "nikolaposa/monolog-factory": "^3.1", "ocramius/proxy-manager": "^2.11", "pagerfanta/core": "^3.5", - "php-amqplib/php-amqplib": "^2.0", + "php-amqplib/php-amqplib": "^3.1", "php-middleware/request-id": "^4.1", "predis/predis": "^1.1", "pugx/shortid-php": "^1.0", diff --git a/config/autoload/rabbit.global.php b/config/autoload/rabbit.global.php new file mode 100644 index 00000000..113a0048 --- /dev/null +++ b/config/autoload/rabbit.global.php @@ -0,0 +1,49 @@ + [ + 'host' => env('RABBITMQ_HOST'), + 'port' => env('RABBITMQ_PORT', '5672'), + 'user' => env('RABBITMQ_USER'), + 'password' => env('RABBITMQ_PASSWORD'), + 'vhost' => env('RABBITMQ_VHOST', '/'), + 'exchange' => env('RABBITMQ_EXCHANGE', 'shlink-exchange'), + 'queue' => env('RABBITMQ_QUEUE', 'shlink-queue'), + ], + + 'dependencies' => [ + 'factories' => [ + AMQPStreamConnection::class => ConfigAbstractFactory::class, + ], + 'delegators' => [ + AMQPStreamConnection::class => [ + LazyServiceFactory::class, + ], + ], + 'lazy_services' => [ + 'class_map' => [ + AMQPStreamConnection::class => AMQPStreamConnection::class, + ], + ], + ], + + ConfigAbstractFactory::class => [ + AMQPStreamConnection::class => [ + 'config.rabbit.host', + 'config.rabbit.port', + 'config.rabbit.user', + 'config.rabbit.password', + 'config.rabbit.vhost', + ], + ], + +]; diff --git a/config/autoload/rabbit.local.php.dist b/config/autoload/rabbit.local.php.dist new file mode 100644 index 00000000..2425a2c5 --- /dev/null +++ b/config/autoload/rabbit.local.php.dist @@ -0,0 +1,13 @@ + [ + 'host' => 'shlink_rabbitmq', + 'user' => 'rabbit', + 'password' => 'rabbit', + ], + +]; diff --git a/data/infra/php.Dockerfile b/data/infra/php.Dockerfile index 86f95361..96556869 100644 --- a/data/infra/php.Dockerfile +++ b/data/infra/php.Dockerfile @@ -34,6 +34,9 @@ RUN docker-php-ext-install pdo_pgsql RUN apk add --no-cache gmp-dev RUN docker-php-ext-install gmp +RUN docker-php-ext-install sockets +RUN docker-php-ext-install bcmath + # Install APCu extension ADD https://pecl.php.net/get/apcu-$APCU_VERSION.tgz /tmp/apcu.tar.gz RUN mkdir -p /usr/src/php/ext/apcu \ diff --git a/data/infra/swoole.Dockerfile b/data/infra/swoole.Dockerfile index 74b83d07..570ca2a9 100644 --- a/data/infra/swoole.Dockerfile +++ b/data/infra/swoole.Dockerfile @@ -36,6 +36,9 @@ RUN docker-php-ext-install pdo_pgsql RUN apk add --no-cache gmp-dev RUN docker-php-ext-install gmp +RUN docker-php-ext-install sockets +RUN docker-php-ext-install bcmath + # Install APCu extension ADD https://pecl.php.net/get/apcu-$APCU_VERSION.tgz /tmp/apcu.tar.gz RUN mkdir -p /usr/src/php/ext/apcu \ From 966620f840e199acca81e8f12f19a9bac94cbdf0 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 21:04:16 +0100 Subject: [PATCH 3/7] Created event listener to send visits to a RabbitMQ instance --- composer.json | 2 +- config/autoload/rabbit.global.php | 3 +- config/autoload/rabbit.local.php.dist | 1 + docs/async-api/async-api.json | 6 +- .../Core/config/event_dispatcher.config.php | 13 +++ .../EventDispatcher/NotifyVisitToRabbit.php | 103 ++++++++++++++++++ .../src/Mercure/MercureUpdatesGenerator.php | 15 +-- 7 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 module/Core/src/EventDispatcher/NotifyVisitToRabbit.php diff --git a/composer.json b/composer.json index 2cfe4e14..7a0a8542 100644 --- a/composer.json +++ b/composer.json @@ -48,7 +48,7 @@ "pugx/shortid-php": "^1.0", "ramsey/uuid": "^4.2", "rlanvin/php-ip": "dev-master#6b3a785 as 3.0", - "shlinkio/shlink-common": "dev-main#c2e3442 as 4.2", + "shlinkio/shlink-common": "dev-main#e7fdff3 as 4.2", "shlinkio/shlink-config": "^1.4", "shlinkio/shlink-event-dispatcher": "dev-main#3925299 as 2.3", "shlinkio/shlink-importer": "dev-main#d099072 as 2.5", diff --git a/config/autoload/rabbit.global.php b/config/autoload/rabbit.global.php index 113a0048..a17e9887 100644 --- a/config/autoload/rabbit.global.php +++ b/config/autoload/rabbit.global.php @@ -11,13 +11,12 @@ use function Shlinkio\Shlink\Common\env; return [ 'rabbit' => [ + 'enabled' => (bool) env('RABBITMQ_ENABLED', false), 'host' => env('RABBITMQ_HOST'), 'port' => env('RABBITMQ_PORT', '5672'), 'user' => env('RABBITMQ_USER'), 'password' => env('RABBITMQ_PASSWORD'), 'vhost' => env('RABBITMQ_VHOST', '/'), - 'exchange' => env('RABBITMQ_EXCHANGE', 'shlink-exchange'), - 'queue' => env('RABBITMQ_QUEUE', 'shlink-queue'), ], 'dependencies' => [ diff --git a/config/autoload/rabbit.local.php.dist b/config/autoload/rabbit.local.php.dist index 2425a2c5..141b4b8b 100644 --- a/config/autoload/rabbit.local.php.dist +++ b/config/autoload/rabbit.local.php.dist @@ -5,6 +5,7 @@ declare(strict_types=1); return [ 'rabbit' => [ + 'enabled' => true, 'host' => 'shlink_rabbitmq', 'user' => 'rabbit', 'password' => 'rabbit', diff --git a/docs/async-api/async-api.json b/docs/async-api/async-api.json index 0b546377..82da91c5 100644 --- a/docs/async-api/async-api.json +++ b/docs/async-api/async-api.json @@ -11,7 +11,7 @@ }, "defaultContentType": "application/json", "channels": { - "http://shlink.io/new-visit": { + "https://shlink.io/new-visit": { "subscribe": { "summary": "Receive information about any new visit occurring on any short URL.", "operationId": "newVisit", @@ -31,7 +31,7 @@ } } }, - "http://shlink.io/new-visit/{shortCode}": { + "https://shlink.io/new-visit/{shortCode}": { "parameters": { "shortCode": { "description": "The short code of the short URL", @@ -59,7 +59,7 @@ } } }, - "http://shlink.io/new-orphan-visit": { + "https://shlink.io/new-orphan-visit": { "subscribe": { "summary": "Receive information about any new orphan visit.", "operationId": "newOrphanVisit", diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index 5256bc92..a0f09beb 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -5,6 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\Core; use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory; +use PhpAmqpLib\Connection\AMQPStreamConnection; use Psr\EventDispatcher\EventDispatcherInterface; use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater; use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater; @@ -22,6 +23,7 @@ return [ 'async' => [ EventDispatcher\Event\VisitLocated::class => [ EventDispatcher\NotifyVisitToMercure::class, + EventDispatcher\NotifyVisitToRabbit::class, EventDispatcher\NotifyVisitToWebHooks::class, EventDispatcher\UpdateGeoLiteDb::class, ], @@ -33,6 +35,7 @@ return [ EventDispatcher\LocateVisit::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToMercure::class => ConfigAbstractFactory::class, + EventDispatcher\NotifyVisitToRabbit::class => ConfigAbstractFactory::class, EventDispatcher\UpdateGeoLiteDb::class => ConfigAbstractFactory::class, ], @@ -40,6 +43,9 @@ return [ EventDispatcher\NotifyVisitToMercure::class => [ EventDispatcher\CloseDbConnectionEventListenerDelegator::class, ], + EventDispatcher\NotifyVisitToRabbit::class => [ + EventDispatcher\CloseDbConnectionEventListenerDelegator::class, + ], EventDispatcher\NotifyVisitToWebHooks::class => [ EventDispatcher\CloseDbConnectionEventListenerDelegator::class, ], @@ -68,6 +74,13 @@ return [ 'em', 'Logger_Shlink', ], + EventDispatcher\NotifyVisitToRabbit::class => [ + AMQPStreamConnection::class, + 'em', + 'Logger_Shlink', + Visit\Transformer\OrphanVisitDataTransformer::class, + 'config.rabbit.enabled', + ], EventDispatcher\UpdateGeoLiteDb::class => [GeolocationDbUpdater::class, 'Logger_Shlink'], ], diff --git a/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php b/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php new file mode 100644 index 00000000..426b02bb --- /dev/null +++ b/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php @@ -0,0 +1,103 @@ +isEnabled) { + return; + } + + $visitId = $shortUrlLocated->visitId(); + + /** @var Visit|null $visit */ + $visit = $this->em->find(Visit::class, $visitId); + if ($visit === null) { + $this->logger->warning('Tried to notify RabbitMQ for visit with id "{visitId}", but it does not exist.', [ + 'visitId' => $visitId, + ]); + return; + } + + if (! $this->connection->isConnected()) { + $this->connection->reconnect(); + } + + $queues = $this->determineQueuesToPublishTo($visit); + $message = $this->visitToMessage($visit); + + try { + $channel = $this->connection->channel(); + + foreach ($queues as $queue) { + // Declare an exchange and a queue that will persist server restarts + $exchange = $queue; // We use the same name for the exchange and the queue + $channel->exchange_declare($exchange, AMQPExchangeType::DIRECT, false, true, false); + $channel->queue_declare($queue, false, true, false, false); + + // Bind the exchange and the queue together, and publish the message + $channel->queue_bind($queue, $exchange); + $channel->basic_publish($message, $exchange); + } + + $channel->close(); + } catch (Throwable $e) { + $this->logger->debug('Error while trying to notify RabbitMQ with new visit. {e}', ['e' => $e]); + } finally { + $this->connection->close(); + } + } + + /** + * @return string[] + */ + private function determineQueuesToPublishTo(Visit $visit): array + { + if ($visit->isOrphan()) { + return [self::NEW_ORPHAN_VISIT_QUEUE]; + } + + return [ + self::NEW_VISIT_QUEUE, + sprintf('%s/%s', self::NEW_VISIT_QUEUE, $visit->getShortUrl()?->getShortCode()), + ]; + } + + private function visitToMessage(Visit $visit): AMQPMessage + { + $messageBody = json_encode(! $visit->isOrphan() ? $visit : $this->orphanVisitTransformer->transform($visit)); + return new AMQPMessage($messageBody, [ + 'content_type' => 'application/json', + 'delivery_mode' => AMQPMessage::DELIVERY_MODE_PERSISTENT, + ]); + } +} diff --git a/module/Core/src/Mercure/MercureUpdatesGenerator.php b/module/Core/src/Mercure/MercureUpdatesGenerator.php index cc0f785a..74b85388 100644 --- a/module/Core/src/Mercure/MercureUpdatesGenerator.php +++ b/module/Core/src/Mercure/MercureUpdatesGenerator.php @@ -8,11 +8,9 @@ use Shlinkio\Shlink\Common\Rest\DataTransformerInterface; use Shlinkio\Shlink\Core\Entity\Visit; use Symfony\Component\Mercure\Update; -use function json_encode; +use function Shlinkio\Shlink\Common\json_encode; use function sprintf; -use const JSON_THROW_ON_ERROR; - final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface { private const NEW_VISIT_TOPIC = 'https://shlink.io/new-visit'; @@ -26,7 +24,7 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface public function newVisitUpdate(Visit $visit): Update { - return new Update(self::NEW_VISIT_TOPIC, $this->serialize([ + return new Update(self::NEW_VISIT_TOPIC, json_encode([ 'shortUrl' => $this->shortUrlTransformer->transform($visit->getShortUrl()), 'visit' => $visit, ])); @@ -34,7 +32,7 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface public function newOrphanVisitUpdate(Visit $visit): Update { - return new Update(self::NEW_ORPHAN_VISIT_TOPIC, $this->serialize([ + return new Update(self::NEW_ORPHAN_VISIT_TOPIC, json_encode([ 'visit' => $this->orphanVisitTransformer->transform($visit), ])); } @@ -44,14 +42,9 @@ final class MercureUpdatesGenerator implements MercureUpdatesGeneratorInterface $shortUrl = $visit->getShortUrl(); $topic = sprintf('%s/%s', self::NEW_VISIT_TOPIC, $shortUrl?->getShortCode()); - return new Update($topic, $this->serialize([ + return new Update($topic, json_encode([ 'shortUrl' => $this->shortUrlTransformer->transform($shortUrl), 'visit' => $visit, ])); } - - private function serialize(array $data): string - { - return json_encode($data, JSON_THROW_ON_ERROR); - } } From 0bcefda60d51571b35a7250a3942ac201ebd9967 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 21:44:56 +0100 Subject: [PATCH 4/7] Added sockets and bcmath extensions to docker image --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d26c7848..30ca29e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,8 @@ WORKDIR /etc/shlink # Install required PHP extensions RUN \ - # Install mysql and calendar - docker-php-ext-install -j"$(nproc)" pdo_mysql calendar && \ + # Install extensions with no extra dependencies + docker-php-ext-install -j"$(nproc)" pdo_mysql calendar sockets bcmath && \ # Install sqlite apk add --no-cache sqlite-libs sqlite-dev && \ docker-php-ext-install -j"$(nproc)" pdo_sqlite && \ From cb1705b6e801bf3b193970f1ebce8d9c87fcadf7 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 11 Dec 2021 22:18:46 +0100 Subject: [PATCH 5/7] Created NotifyVisitToRabbitTest --- CHANGELOG.md | 1 - composer.json | 2 +- .../EventDispatcher/NotifyVisitToRabbit.php | 3 +- .../NotifyVisitToRabbitTest.php | 178 ++++++++++++++++++ 4 files changed, 180 insertions(+), 4 deletions(-) create mode 100644 module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php diff --git a/CHANGELOG.md b/CHANGELOG.md index d0cbf02b..2f6d10c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,7 +29,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * [#1218](https://github.com/shlinkio/shlink/issues/1218) Updated to symfony/mercure 0.6. * [#1223](https://github.com/shlinkio/shlink/issues/1223) Updated to phpstan 1.0. * Added `domain` field to `DeleteShortUrlException` exception. -* [#1001](https://github.com/shlinkio/shlink/issues/1001) Increased required MSI to 83%. ### Deprecated * [#1260](https://github.com/shlinkio/shlink/issues/1260) Deprecated `USE_HTTPS` env var that was added in previous release, in favor of the new `IS_HTTPS_ENABLED`. diff --git a/composer.json b/composer.json index 7a0a8542..e244fbfc 100644 --- a/composer.json +++ b/composer.json @@ -142,7 +142,7 @@ "test:api": "bin/test/run-api-tests.sh", "test:api:ci": "GENERATE_COVERAGE=yes composer test:api", "infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --only-covering-test-cases --skip-initial-tests", - "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=83", + "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=80", "infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json", "infect:ci:api": "@infect:ci:base --coverage=build/coverage-api --min-msi=80 --configuration=infection-api.json", "infect:ci": "@parallel infect:ci:unit infect:ci:db infect:ci:api", diff --git a/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php b/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php index 426b02bb..6ff79eb8 100644 --- a/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php +++ b/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php @@ -38,9 +38,8 @@ class NotifyVisitToRabbit } $visitId = $shortUrlLocated->visitId(); - - /** @var Visit|null $visit */ $visit = $this->em->find(Visit::class, $visitId); + if ($visit === null) { $this->logger->warning('Tried to notify RabbitMQ for visit with id "{visitId}", but it does not exist.', [ 'visitId' => $visitId, diff --git a/module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php b/module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php new file mode 100644 index 00000000..73a970fb --- /dev/null +++ b/module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php @@ -0,0 +1,178 @@ +channel = $this->prophesize(AMQPChannel::class); + + $this->connection = $this->prophesize(AMQPStreamConnection::class); + $this->connection->isConnected()->willReturn(false); + $this->connection->channel()->willReturn($this->channel->reveal()); + + $this->em = $this->prophesize(EntityManagerInterface::class); + $this->logger = $this->prophesize(LoggerInterface::class); + + $this->listener = new NotifyVisitToRabbit( + $this->connection->reveal(), + $this->em->reveal(), + $this->logger->reveal(), + new OrphanVisitDataTransformer(), + true, + ); + } + + /** @test */ + public function doesNothingWhenTheFeatureIsNotEnabled(): void + { + $listener = new NotifyVisitToRabbit( + $this->connection->reveal(), + $this->em->reveal(), + $this->logger->reveal(), + new OrphanVisitDataTransformer(), + false, + ); + + $listener(new VisitLocated('123')); + + $this->em->find(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->logger->warning(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->logger->debug(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->connection->isConnected()->shouldNotHaveBeenCalled(); + $this->connection->close()->shouldNotHaveBeenCalled(); + } + + /** @test */ + public function notificationsAreNotSentWhenVisitCannotBeFound(): void + { + $visitId = '123'; + $findVisit = $this->em->find(Visit::class, $visitId)->willReturn(null); + $logWarning = $this->logger->warning( + 'Tried to notify RabbitMQ for visit with id "{visitId}", but it does not exist.', + ['visitId' => $visitId], + ); + + ($this->listener)(new VisitLocated($visitId)); + + $findVisit->shouldHaveBeenCalledOnce(); + $logWarning->shouldHaveBeenCalledOnce(); + $this->logger->debug(Argument::cetera())->shouldNotHaveBeenCalled(); + $this->connection->isConnected()->shouldNotHaveBeenCalled(); + $this->connection->close()->shouldNotHaveBeenCalled(); + } + + /** + * @test + * @dataProvider provideVisits + */ + public function expectedChannelsAreNotifiedBasedOnTheVisitType(Visit $visit, array $expectedChannels): void + { + $visitId = '123'; + $findVisit = $this->em->find(Visit::class, $visitId)->willReturn($visit); + $argumentWithExpectedChannel = Argument::that(fn (string $channel) => contains($expectedChannels, $channel)); + + ($this->listener)(new VisitLocated($visitId)); + + $findVisit->shouldHaveBeenCalledOnce(); + $this->channel->exchange_declare($argumentWithExpectedChannel, Argument::cetera())->shouldHaveBeenCalledTimes( + count($expectedChannels), + ); + $this->channel->queue_declare($argumentWithExpectedChannel, Argument::cetera())->shouldHaveBeenCalledTimes( + count($expectedChannels), + ); + $this->channel->queue_bind( + $argumentWithExpectedChannel, + $argumentWithExpectedChannel, + )->shouldHaveBeenCalledTimes(count($expectedChannels)); + $this->channel->basic_publish(Argument::any(), $argumentWithExpectedChannel)->shouldHaveBeenCalledTimes( + count($expectedChannels), + ); + $this->channel->close()->shouldHaveBeenCalledOnce(); + $this->connection->reconnect()->shouldHaveBeenCalledOnce(); + $this->connection->close()->shouldHaveBeenCalledOnce(); + $this->logger->debug(Argument::cetera())->shouldNotHaveBeenCalled(); + } + + public function provideVisits(): iterable + { + $visitor = Visitor::emptyInstance(); + + yield 'orphan visit' => [Visit::forBasePath($visitor), ['https://shlink.io/new-orphan-visit']]; + yield 'non-orphan visit' => [ + Visit::forValidShortUrl( + ShortUrl::fromMeta(ShortUrlMeta::fromRawData([ + 'longUrl' => 'foo', + 'customSlug' => 'bar', + ])), + $visitor, + ), + ['https://shlink.io/new-visit', 'https://shlink.io/new-visit/bar'], + ]; + } + + /** + * @test + * @dataProvider provideExceptions + */ + public function printsDebugMessageInCaseOfError(Throwable $e): void + { + $visitId = '123'; + $findVisit = $this->em->find(Visit::class, $visitId)->willReturn(Visit::forBasePath(Visitor::emptyInstance())); + $channel = $this->connection->channel()->willThrow($e); + + ($this->listener)(new VisitLocated($visitId)); + + $this->logger->debug( + 'Error while trying to notify RabbitMQ with new visit. {e}', + ['e' => $e], + )->shouldHaveBeenCalledOnce(); + $this->connection->close()->shouldHaveBeenCalledOnce(); + $this->connection->reconnect()->shouldHaveBeenCalledOnce(); + $findVisit->shouldHaveBeenCalledOnce(); + $channel->shouldHaveBeenCalledOnce(); + $this->channel->close()->shouldNotHaveBeenCalled(); + } + + public function provideExceptions(): iterable + { + yield [new RuntimeException('RuntimeException Error')]; + yield [new Exception('Exception Error')]; + yield [new DomainException('DomainException Error')]; + } +} From 8e5730f37409312f7bbbc7424514e7f74b2c16d9 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Dec 2021 10:32:57 +0100 Subject: [PATCH 6/7] Renamed Rabbit instances to use RabbitMq --- README.md | 3 ++- config/autoload/rabbit.global.php | 14 +++++++------- config/autoload/rabbit.local.php.dist | 2 +- module/Core/config/event_dispatcher.config.php | 10 +++++----- ...VisitToRabbit.php => NotifyVisitToRabbitMq.php} | 2 +- ...abbitTest.php => NotifyVisitToRabbitMqTest.php} | 10 +++++----- 6 files changed, 21 insertions(+), 20 deletions(-) rename module/Core/src/EventDispatcher/{NotifyVisitToRabbit.php => NotifyVisitToRabbitMq.php} (99%) rename module/Core/test/EventDispatcher/{NotifyVisitToRabbitTest.php => NotifyVisitToRabbitMqTest.php} (96%) diff --git a/README.md b/README.md index 4fe5f5cf..9aad62d9 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,11 @@ The idea is that you can just generate a container using the image and provide t First, make sure the host where you are going to run shlink fulfills these requirements: -* PHP 8.0 +* PHP 8.0 or 8.1 * The next PHP extensions: json, curl, pdo, intl, gd and gmp. * apcu extension is recommended if you don't plan to use swoole or openswoole. * xml extension is required if you want to generate QR codes in svg format. + * sockets and bcmath extensions are required if you want to integrate with a RabbitMQ instance. * MySQL, MariaDB, PostgreSQL, Microsoft SQL Server or SQLite. * The web server of your choice with PHP integration (Apache or Nginx recommended). diff --git a/config/autoload/rabbit.global.php b/config/autoload/rabbit.global.php index a17e9887..b08dccf2 100644 --- a/config/autoload/rabbit.global.php +++ b/config/autoload/rabbit.global.php @@ -10,10 +10,10 @@ use function Shlinkio\Shlink\Common\env; return [ - 'rabbit' => [ + 'rabbitmq' => [ 'enabled' => (bool) env('RABBITMQ_ENABLED', false), 'host' => env('RABBITMQ_HOST'), - 'port' => env('RABBITMQ_PORT', '5672'), + 'port' => (int) env('RABBITMQ_PORT', '5672'), 'user' => env('RABBITMQ_USER'), 'password' => env('RABBITMQ_PASSWORD'), 'vhost' => env('RABBITMQ_VHOST', '/'), @@ -37,11 +37,11 @@ return [ ConfigAbstractFactory::class => [ AMQPStreamConnection::class => [ - 'config.rabbit.host', - 'config.rabbit.port', - 'config.rabbit.user', - 'config.rabbit.password', - 'config.rabbit.vhost', + 'config.rabbitmq.host', + 'config.rabbitmq.port', + 'config.rabbitmq.user', + 'config.rabbitmq.password', + 'config.rabbitmq.vhost', ], ], diff --git a/config/autoload/rabbit.local.php.dist b/config/autoload/rabbit.local.php.dist index 141b4b8b..83cd4a88 100644 --- a/config/autoload/rabbit.local.php.dist +++ b/config/autoload/rabbit.local.php.dist @@ -4,7 +4,7 @@ declare(strict_types=1); return [ - 'rabbit' => [ + 'rabbitmq' => [ 'enabled' => true, 'host' => 'shlink_rabbitmq', 'user' => 'rabbit', diff --git a/module/Core/config/event_dispatcher.config.php b/module/Core/config/event_dispatcher.config.php index a0f09beb..d47cc128 100644 --- a/module/Core/config/event_dispatcher.config.php +++ b/module/Core/config/event_dispatcher.config.php @@ -23,7 +23,7 @@ return [ 'async' => [ EventDispatcher\Event\VisitLocated::class => [ EventDispatcher\NotifyVisitToMercure::class, - EventDispatcher\NotifyVisitToRabbit::class, + EventDispatcher\NotifyVisitToRabbitMq::class, EventDispatcher\NotifyVisitToWebHooks::class, EventDispatcher\UpdateGeoLiteDb::class, ], @@ -35,7 +35,7 @@ return [ EventDispatcher\LocateVisit::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class, EventDispatcher\NotifyVisitToMercure::class => ConfigAbstractFactory::class, - EventDispatcher\NotifyVisitToRabbit::class => ConfigAbstractFactory::class, + EventDispatcher\NotifyVisitToRabbitMq::class => ConfigAbstractFactory::class, EventDispatcher\UpdateGeoLiteDb::class => ConfigAbstractFactory::class, ], @@ -43,7 +43,7 @@ return [ EventDispatcher\NotifyVisitToMercure::class => [ EventDispatcher\CloseDbConnectionEventListenerDelegator::class, ], - EventDispatcher\NotifyVisitToRabbit::class => [ + EventDispatcher\NotifyVisitToRabbitMq::class => [ EventDispatcher\CloseDbConnectionEventListenerDelegator::class, ], EventDispatcher\NotifyVisitToWebHooks::class => [ @@ -74,12 +74,12 @@ return [ 'em', 'Logger_Shlink', ], - EventDispatcher\NotifyVisitToRabbit::class => [ + EventDispatcher\NotifyVisitToRabbitMq::class => [ AMQPStreamConnection::class, 'em', 'Logger_Shlink', Visit\Transformer\OrphanVisitDataTransformer::class, - 'config.rabbit.enabled', + 'config.rabbitmq.enabled', ], EventDispatcher\UpdateGeoLiteDb::class => [GeolocationDbUpdater::class, 'Logger_Shlink'], ], diff --git a/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php b/module/Core/src/EventDispatcher/NotifyVisitToRabbitMq.php similarity index 99% rename from module/Core/src/EventDispatcher/NotifyVisitToRabbit.php rename to module/Core/src/EventDispatcher/NotifyVisitToRabbitMq.php index 6ff79eb8..f05ecf64 100644 --- a/module/Core/src/EventDispatcher/NotifyVisitToRabbit.php +++ b/module/Core/src/EventDispatcher/NotifyVisitToRabbitMq.php @@ -17,7 +17,7 @@ use Throwable; use function Shlinkio\Shlink\Common\json_encode; use function sprintf; -class NotifyVisitToRabbit +class NotifyVisitToRabbitMq { private const NEW_VISIT_QUEUE = 'https://shlink.io/new-visit'; private const NEW_ORPHAN_VISIT_QUEUE = 'https://shlink.io/new-orphan-visit'; diff --git a/module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php b/module/Core/test/EventDispatcher/NotifyVisitToRabbitMqTest.php similarity index 96% rename from module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php rename to module/Core/test/EventDispatcher/NotifyVisitToRabbitMqTest.php index 73a970fb..778da889 100644 --- a/module/Core/test/EventDispatcher/NotifyVisitToRabbitTest.php +++ b/module/Core/test/EventDispatcher/NotifyVisitToRabbitMqTest.php @@ -18,7 +18,7 @@ use RuntimeException; use Shlinkio\Shlink\Core\Entity\ShortUrl; use Shlinkio\Shlink\Core\Entity\Visit; use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated; -use Shlinkio\Shlink\Core\EventDispatcher\NotifyVisitToRabbit; +use Shlinkio\Shlink\Core\EventDispatcher\NotifyVisitToRabbitMq; use Shlinkio\Shlink\Core\Model\ShortUrlMeta; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Visit\Transformer\OrphanVisitDataTransformer; @@ -27,11 +27,11 @@ use Throwable; use function count; use function Functional\contains; -class NotifyVisitToRabbitTest extends TestCase +class NotifyVisitToRabbitMqTest extends TestCase { use ProphecyTrait; - private NotifyVisitToRabbit $listener; + private NotifyVisitToRabbitMq $listener; private ObjectProphecy $connection; private ObjectProphecy $em; private ObjectProphecy $logger; @@ -49,7 +49,7 @@ class NotifyVisitToRabbitTest extends TestCase $this->em = $this->prophesize(EntityManagerInterface::class); $this->logger = $this->prophesize(LoggerInterface::class); - $this->listener = new NotifyVisitToRabbit( + $this->listener = new NotifyVisitToRabbitMq( $this->connection->reveal(), $this->em->reveal(), $this->logger->reveal(), @@ -61,7 +61,7 @@ class NotifyVisitToRabbitTest extends TestCase /** @test */ public function doesNothingWhenTheFeatureIsNotEnabled(): void { - $listener = new NotifyVisitToRabbit( + $listener = new NotifyVisitToRabbitMq( $this->connection->reveal(), $this->em->reveal(), $this->logger->reveal(), From 54dcaaac0ceaf39e1510d7c46b9e7073174e70be Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sun, 12 Dec 2021 11:24:58 +0100 Subject: [PATCH 7/7] Updated to an installer version with support for RabbitMQ --- CHANGELOG.md | 6 ++++++ composer.json | 2 +- config/autoload/installer.global.php | 6 ++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f6d10c1..18802cc2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * The `GET /domains` endpoint includes a new `defaultRedirects` property in the response, with the default redirects set via config or env vars. * The `INVALID_SHORT_URL_REDIRECT_TO`, `REGULAR_404_REDIRECT_TO` and `BASE_URL_REDIRECT_TO` env vars are now deprecated, and should be replaced by `DEFAULT_INVALID_SHORT_URL_REDIRECT`, `DEFAULT_REGULAR_404_REDIRECT` and `DEFAULT_BASE_URL_REDIRECT` respectively. Deprecated ones will continue to work until v3.0.0, where they will be removed. +* [#868](https://github.com/shlinkio/shlink/issues/868) Added support to publish real-time updates in a RabbitMQ server. + + Shlink will create new exchanges and queues for every topic documented in the [Async API spec](https://api-spec.shlink.io/async-api/), meaning, you will have one queue for orphan visits, one for regular visits, and one queue for every short URL with its visits. + + The RabbitMQ server config can be provided via installer config options, or via environment variables. + * [#1204](https://github.com/shlinkio/shlink/issues/1204) Added support for `openswoole` and migrated official docker image to `openswoole`. * [#1242](https://github.com/shlinkio/shlink/issues/1242) Added support to import urls and visits from YOURLS. diff --git a/composer.json b/composer.json index e244fbfc..7064a2b3 100644 --- a/composer.json +++ b/composer.json @@ -52,7 +52,7 @@ "shlinkio/shlink-config": "^1.4", "shlinkio/shlink-event-dispatcher": "dev-main#3925299 as 2.3", "shlinkio/shlink-importer": "dev-main#d099072 as 2.5", - "shlinkio/shlink-installer": "dev-develop#7dd00fb as 6.3", + "shlinkio/shlink-installer": "^6.3", "shlinkio/shlink-ip-geolocation": "^2.2", "symfony/console": "^6.0 || ^5.4", "symfony/filesystem": "^6.0 || ^5.4", diff --git a/config/autoload/installer.global.php b/config/autoload/installer.global.php index def478af..238dea42 100644 --- a/config/autoload/installer.global.php +++ b/config/autoload/installer.global.php @@ -57,6 +57,12 @@ return [ Option\QrCode\DefaultFormatConfigOption::class, Option\QrCode\DefaultErrorCorrectionConfigOption::class, Option\QrCode\DefaultRoundBlockSizeConfigOption::class, + Option\RabbitMq\RabbitMqEnabledConfigOption::class, + Option\RabbitMq\RabbitMqHostConfigOption::class, + Option\RabbitMq\RabbitMqPortConfigOption::class, + Option\RabbitMq\RabbitMqUserConfigOption::class, + Option\RabbitMq\RabbitMqPasswordConfigOption::class, + Option\RabbitMq\RabbitMqVhostConfigOption::class, ], 'installation_commands' => [