diff --git a/CHANGELOG.md b/CHANGELOG.md index de5c2dc2..4f3d4b4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this * `after-date`: matches when current date and time is later than the defined threshold. * [#2513](https://github.com/shlinkio/shlink/issues/2513) Add support for redis connections via unix socket (e.g. `REDIS_SERVERS=unix:/path/to/redis.sock`). +* Visits generated in the command line can now be formatted in CSV, via `--format=csv`. ### Changed * [#2522](https://github.com/shlinkio/shlink/issues/2522) Shlink no longer tries to detect trusted proxies automatically, when resolving the visitor's IP address, as this is a potential security issue. diff --git a/composer.json b/composer.json index 6b560969..a205e391 100644 --- a/composer.json +++ b/composer.json @@ -33,6 +33,7 @@ "laminas/laminas-inputfilter": "^2.31", "laminas/laminas-servicemanager": "^3.23", "laminas/laminas-stdlib": "^3.20", + "league/csv": "^9.28", "matomo/matomo-php-tracker": "^3.3", "mezzio/mezzio": "^3.20", "mezzio/mezzio-fastroute": "^3.12", diff --git a/module/CLI/src/Command/Domain/GetDomainVisitsCommand.php b/module/CLI/src/Command/Domain/GetDomainVisitsCommand.php index c2b8d859..ac05ee7b 100644 --- a/module/CLI/src/Command/Domain/GetDomainVisitsCommand.php +++ b/module/CLI/src/Command/Domain/GetDomainVisitsCommand.php @@ -5,8 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Domain; use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils; -use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput; -use Shlinkio\Shlink\CLI\Util\ShlinkTable; +use Shlinkio\Shlink\CLI\Input\VisitsListInput; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface; use Shlinkio\Shlink\Core\Visit\Entity\Visit; use Shlinkio\Shlink\Core\Visit\Model\VisitsParams; @@ -34,12 +33,10 @@ class GetDomainVisitsCommand extends Command SymfonyStyle $io, #[Argument('The domain which visits we want to get'), Ask('For what domain do you want to get visits?')] string $domain, - #[MapInput] VisitsDateRangeInput $dateRangeInput, + #[MapInput] VisitsListInput $input, ): int { - $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($dateRangeInput->toDateRange())); - [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...)); - - ShlinkTable::default($io)->render($headers, $rows); + $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($input->dateRange())); + VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...)); return self::SUCCESS; } diff --git a/module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php b/module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php index 38d9e371..b0e4cce0 100644 --- a/module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php +++ b/module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php @@ -5,8 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\ShortUrl; use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils; -use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput; -use Shlinkio\Shlink\CLI\Util\ShlinkTable; +use Shlinkio\Shlink\CLI\Input\VisitsListInput; use Shlinkio\Shlink\Core\ShortUrl\Model\ShortUrlIdentifier; use Shlinkio\Shlink\Core\Visit\Model\VisitsParams; use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface; @@ -32,16 +31,15 @@ class GetShortUrlVisitsCommand extends Command SymfonyStyle $io, #[Argument('The short code which visits we want to get'), Ask('Which short code do you want to use?')] string $shortCode, - #[MapInput] VisitsDateRangeInput $dateRangeInput, + #[MapInput] VisitsListInput $input, #[Option('The domain for the short code', shortcut: 'd')] string|null $domain = null, ): int { $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain); - $dateRange = $dateRangeInput->toDateRange(); + $dateRange = $input->dateRange(); $paginator = $this->visitsHelper->visitsForShortUrl($identifier, new VisitsParams($dateRange)); - [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, static fn () => []); - ShlinkTable::default($io)->render($headers, $rows); + VisitsCommandUtils::renderOutput($io, $input, $paginator); return self::SUCCESS; } diff --git a/module/CLI/src/Command/Tag/GetTagVisitsCommand.php b/module/CLI/src/Command/Tag/GetTagVisitsCommand.php index 7db1b7e1..529fb536 100644 --- a/module/CLI/src/Command/Tag/GetTagVisitsCommand.php +++ b/module/CLI/src/Command/Tag/GetTagVisitsCommand.php @@ -5,8 +5,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Tag; use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils; -use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput; -use Shlinkio\Shlink\CLI\Util\ShlinkTable; +use Shlinkio\Shlink\CLI\Input\VisitsListInput; use Shlinkio\Shlink\Core\Domain\Entity\Domain; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface; use Shlinkio\Shlink\Core\Visit\Entity\Visit; @@ -35,7 +34,7 @@ class GetTagVisitsCommand extends Command public function __invoke( SymfonyStyle $io, #[Argument('The tag which visits we want to get'), Ask('For what tag do you want to get visits')] string $tag, - #[MapInput] VisitsDateRangeInput $dateRangeInput, + #[MapInput] VisitsListInput $input, #[Option( 'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits ' . 'in default domain', @@ -44,12 +43,11 @@ class GetTagVisitsCommand extends Command string|null $domain = null, ): int { $paginator = $this->visitsHelper->visitsForTag($tag, new WithDomainVisitsParams( - dateRange: $dateRangeInput->toDateRange(), + dateRange: $input->dateRange(), domain: $domain, )); - [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...)); - ShlinkTable::default($io)->render($headers, $rows); + VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...)); return self::SUCCESS; } diff --git a/module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php b/module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php index 6291d6db..2ff8aa52 100644 --- a/module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php +++ b/module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php @@ -4,8 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Visit; -use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput; -use Shlinkio\Shlink\CLI\Util\ShlinkTable; +use Shlinkio\Shlink\CLI\Input\VisitsListInput; use Shlinkio\Shlink\Core\Domain\Entity\Domain; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface; use Shlinkio\Shlink\Core\Visit\Entity\Visit; @@ -31,7 +30,7 @@ class GetNonOrphanVisitsCommand extends Command public function __invoke( SymfonyStyle $io, - #[MapInput] VisitsDateRangeInput $dateRangeInput, + #[MapInput] VisitsListInput $input, #[Option( 'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits ' . 'in default domain', @@ -40,12 +39,10 @@ class GetNonOrphanVisitsCommand extends Command string|null $domain = null, ): int { $paginator = $this->visitsHelper->nonOrphanVisits(new WithDomainVisitsParams( - dateRange: $dateRangeInput->toDateRange(), + dateRange: $input->dateRange(), domain: $domain, )); - [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...)); - - ShlinkTable::default($io)->render($headers, $rows); + VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...)); return self::SUCCESS; } diff --git a/module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php b/module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php index 5640361f..d1ce8d66 100644 --- a/module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php +++ b/module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php @@ -4,8 +4,7 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Visit; -use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput; -use Shlinkio\Shlink\CLI\Util\ShlinkTable; +use Shlinkio\Shlink\CLI\Input\VisitsListInput; use Shlinkio\Shlink\Core\Domain\Entity\Domain; use Shlinkio\Shlink\Core\Visit\Entity\Visit; use Shlinkio\Shlink\Core\Visit\Model\OrphanVisitsParams; @@ -29,7 +28,7 @@ class GetOrphanVisitsCommand extends Command public function __invoke( SymfonyStyle $io, - #[MapInput] VisitsDateRangeInput $dateRangeInput, + #[MapInput] VisitsListInput $input, #[Option( 'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits ' . 'in default domain', @@ -39,13 +38,11 @@ class GetOrphanVisitsCommand extends Command #[Option('Return visits only with this type', shortcut: 't')] OrphanVisitType|null $type = null, ): int { $paginator = $this->visitsHelper->orphanVisits(new OrphanVisitsParams( - dateRange: $dateRangeInput->toDateRange(), + dateRange: $input->dateRange(), domain: $domain, type: $type, )); - [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...)); - - ShlinkTable::default($io)->render($headers, $rows); + VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...)); return self::SUCCESS; } diff --git a/module/CLI/src/Command/Visit/VisitsCommandUtils.php b/module/CLI/src/Command/Visit/VisitsCommandUtils.php index 55f425c7..0eadc2c4 100644 --- a/module/CLI/src/Command/Visit/VisitsCommandUtils.php +++ b/module/CLI/src/Command/Visit/VisitsCommandUtils.php @@ -4,8 +4,14 @@ declare(strict_types=1); namespace Shlinkio\Shlink\CLI\Command\Visit; +use League\Csv\Writer; +use Shlinkio\Shlink\CLI\Input\VisitsListFormat; +use Shlinkio\Shlink\CLI\Input\VisitsListInput; +use Shlinkio\Shlink\CLI\Util\ShlinkTable; use Shlinkio\Shlink\Common\Paginator\Paginator; +use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtils; use Shlinkio\Shlink\Core\Visit\Entity\Visit; +use Symfony\Component\Console\Output\OutputInterface; use function array_keys; use function array_map; @@ -16,14 +22,86 @@ class VisitsCommandUtils { /** * @param Paginator $paginator - * @param callable(Visit $visits): array $mapExtraFields + * @param null|callable(Visit $visits): array $mapExtraFields */ - public static function resolveRowsAndHeaders(Paginator $paginator, callable $mapExtraFields): array + public static function renderOutput( + OutputInterface $output, + VisitsListInput $inputData, + Paginator $paginator, + callable|null $mapExtraFields = null, + ): void { + if ($inputData->format !== VisitsListFormat::FULL) { + // Avoid running out of memory by loading visits in chunks + $paginator->setMaxPerPage(1000); + } + + match ($inputData->format) { + VisitsListFormat::CSV => self::renderCSVOutput($output, $paginator, $mapExtraFields), + default => self::renderHumanFriendlyOutput($output, $paginator, $mapExtraFields), + }; + } + + /** + * @param Paginator $paginator + * @param null|callable(Visit $visits): array $mapExtraFields + */ + private static function renderCSVOutput( + OutputInterface $output, + Paginator $paginator, + callable|null $mapExtraFields, + ): void { + $page = 1; + do { + $paginator->setCurrentPage($page); + + [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields); + $csv = Writer::fromString(); + if ($page === 1) { + $csv->insertOne($headers); + } + + $csv->insertAll($rows); + $output->write($csv->toString()); + + $page++; + } while ($paginator->hasNextPage()); + } + + /** + * @param Paginator $paginator + * @param null|callable(Visit $visits): array $mapExtraFields + */ + private static function renderHumanFriendlyOutput( + OutputInterface $output, + Paginator $paginator, + callable|null $mapExtraFields, + ): void { + $page = 1; + do { + $paginator->setCurrentPage($page); + $page++; + + [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields); + ShlinkTable::default($output)->render( + $headers, + $rows, + footerTitle: PagerfantaUtils::formatCurrentPageMessage($paginator, 'Page %s of %s'), + ); + } while ($paginator->hasNextPage()); + } + + /** + * @param Paginator $paginator + * @param null|callable(Visit $visits): array $mapExtraFields + */ + private static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields): array { - $extraKeys = []; + $extraKeys = null; + $mapExtraFields ??= static fn (Visit $_) => []; + $rows = array_map(function (Visit $visit) use (&$extraKeys, $mapExtraFields) { $extraFields = $mapExtraFields($visit); - $extraKeys = array_keys($extraFields); + $extraKeys ??= array_keys($extraFields); $rowData = [ 'referer' => $visit->referer, @@ -38,7 +116,7 @@ class VisitsCommandUtils // Filter out unknown keys return select_keys($rowData, ['referer', 'date', 'userAgent', 'country', 'city', ...$extraKeys]); }, [...$paginator->getCurrentPageResults()]); - $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys); + $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys ?? []); return [ $rows, diff --git a/module/CLI/src/Input/VisitsListFormat.php b/module/CLI/src/Input/VisitsListFormat.php new file mode 100644 index 00000000..acb95301 --- /dev/null +++ b/module/CLI/src/Input/VisitsListFormat.php @@ -0,0 +1,18 @@ +value . '", "' . VisitsListFormat::PAGINATED->value . '" or "' + . VisitsListFormat::CSV->value . '")', + shortcut: 'f', + )] + public VisitsListFormat $format = VisitsListFormat::FULL; + + public function dateRange(): DateRange { return buildDateRange( startDate: normalizeOptionalDate($this->startDate), diff --git a/module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php b/module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php index ddf55283..d5abbb22 100644 --- a/module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php +++ b/module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php @@ -61,7 +61,7 @@ class GetDomainVisitsCommandTest extends TestCase | Referer | Date | User agent | Country | City | Short Url | +---------+---------------------------+------------+---------+--------+---------------+ | foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | the_short_url | - +---------+---------------------------+------------+---------+--------+---------------+ + +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+ OUTPUT, $output, diff --git a/module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php b/module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php index 3fd53c48..8e94b043 100644 --- a/module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php +++ b/module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php @@ -6,10 +6,12 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl; use Cake\Chronos\Chronos; use Pagerfanta\Adapter\ArrayAdapter; +use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; use Shlinkio\Shlink\CLI\Command\ShortUrl\GetShortUrlVisitsCommand; +use Shlinkio\Shlink\CLI\Input\VisitsListFormat; use Shlinkio\Shlink\Common\Paginator\Paginator; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\ShortUrl\Entity\ShortUrl; @@ -80,8 +82,11 @@ class GetShortUrlVisitsCommandTest extends TestCase ]); } - #[Test] - public function outputIsProperlyGenerated(): void + /** + * @param callable(Chronos $date): string $getExpectedOutput + */ + #[Test, DataProvider('provideOutput')] + public function outputIsProperlyGenerated(VisitsListFormat $format, callable $getExpectedOutput): void { $visit = Visit::forValidShortUrl(ShortUrl::createFake(), Visitor::fromParams('bar', 'foo', ''))->locate( VisitLocation::fromLocation(new Location('', 'Spain', '', 'Madrid', 0, 0, '')), @@ -92,19 +97,32 @@ class GetShortUrlVisitsCommandTest extends TestCase $this->anything(), )->willReturn(new Paginator(new ArrayAdapter([$visit]))); - $this->commandTester->execute(['short-code' => $shortCode]); + $this->commandTester->execute(['short-code' => $shortCode, '--format' => $format->value]); $output = $this->commandTester->getDisplay(); - self::assertEquals( - <<date->toAtomString()} | bar | Spain | Madrid | - +---------+---------------------------+------------+---------+--------+ + self::assertEquals($getExpectedOutput($visit->date), $output); + } - OUTPUT, - $output, - ); + public static function provideOutput(): iterable + { + yield 'regular' => [ + VisitsListFormat::FULL, + static fn (Chronos $date) => <<toAtomString()} | bar | Spain | Madrid | + +---------+------------------ Page 1 of 1 ---------+---------+--------+ + + OUTPUT, + ]; + yield 'CSV' => [ + VisitsListFormat::CSV, + static fn (Chronos $date) => <<toAtomString()},bar,Spain,Madrid + + OUTPUT, + ]; } } diff --git a/module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php b/module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php index 30580951..426ff29b 100644 --- a/module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php +++ b/module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php @@ -58,7 +58,7 @@ class GetTagVisitsCommandTest extends TestCase | Referer | Date | User agent | Country | City | Short Url | +---------+---------------------------+------------+---------+--------+---------------+ | foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | the_short_url | - +---------+---------------------------+------------+---------+--------+---------------+ + +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+ OUTPUT, $output, diff --git a/module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php b/module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php index f583d063..b6e973a9 100644 --- a/module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php @@ -57,7 +57,7 @@ class GetNonOrphanVisitsCommandTest extends TestCase | Referer | Date | User agent | Country | City | Short Url | +---------+---------------------------+------------+---------+--------+---------------+ | foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | the_short_url | - +---------+---------------------------+------------+---------+--------+---------------+ + +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+ OUTPUT, $output, diff --git a/module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php b/module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php index bf453c65..1633eee8 100644 --- a/module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php +++ b/module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php @@ -55,7 +55,7 @@ class GetOrphanVisitsCommandTest extends TestCase | Referer | Date | User agent | Country | City | Type | +---------+---------------------------+------------+---------+--------+----------+ | foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | base_url | - +---------+---------------------------+------------+---------+--------+----------+ + +---------+----------------------- Page 1 of 1 ----+---------+--------+----------+ OUTPUT, $output,