Merge pull request #2562 from acelaya-forks/visits-export

Allow exporting visits in CSV format
2026-02-28 04:03:12 +08:00 · 2025-12-29 11:07:03 +01:00
parent a65c5c3b56 248e8032e3
commit 0d964f0fde
15 changed files with 167 additions and 57 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
  * `after-date`: matches when current date and time is later than the defined threshold.

 * [#2513](https://github.com/shlinkio/shlink/issues/2513) Add support for redis connections via unix socket (e.g. `REDIS_SERVERS=unix:/path/to/redis.sock`).
+* Visits generated in the command line can now be formatted in CSV, via `--format=csv`.

 ### Changed
 * [#2522](https://github.com/shlinkio/shlink/issues/2522) Shlink no longer tries to detect trusted proxies automatically, when resolving the visitor's IP address, as this is a potential security issue.
--- a/composer.json
+++ b/composer.json
@@ -33,6 +33,7 @@
        "laminas/laminas-inputfilter": "^2.31",
        "laminas/laminas-servicemanager": "^3.23",
        "laminas/laminas-stdlib": "^3.20",
+        "league/csv": "^9.28",
        "matomo/matomo-php-tracker": "^3.3",
        "mezzio/mezzio": "^3.20",
        "mezzio/mezzio-fastroute": "^3.12",
--- a/module/CLI/src/Command/Domain/GetDomainVisitsCommand.php
+++ b/module/CLI/src/Command/Domain/GetDomainVisitsCommand.php
@@ -5,8 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\Domain;

 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
@@ -34,12 +33,10 @@ class GetDomainVisitsCommand extends Command
        SymfonyStyle $io,
        #[Argument('The domain which visits we want to get'), Ask('For what domain do you want to get visits?')]
        string $domain,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
    ): int {
-        $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($dateRangeInput->toDateRange()));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
+        $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($input->dateRange()));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));

        return self::SUCCESS;
    }
--- a/module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php
+++ b/module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php
@@ -5,8 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\ShortUrl;

 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
 use Shlinkio\Shlink\Core\ShortUrl\Model\ShortUrlIdentifier;
 use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
 use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface;
@@ -32,16 +31,15 @@ class GetShortUrlVisitsCommand extends Command
        SymfonyStyle $io,
        #[Argument('The short code which visits we want to get'), Ask('Which short code do you want to use?')]
        string $shortCode,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
        #[Option('The domain for the short code', shortcut: 'd')]
        string|null $domain = null,
    ): int {
        $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain);
-        $dateRange = $dateRangeInput->toDateRange();
+        $dateRange = $input->dateRange();
        $paginator = $this->visitsHelper->visitsForShortUrl($identifier, new VisitsParams($dateRange));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, static fn () => []);

-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator);

        return self::SUCCESS;
    }
--- a/module/CLI/src/Command/Tag/GetTagVisitsCommand.php
+++ b/module/CLI/src/Command/Tag/GetTagVisitsCommand.php
@@ -5,8 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\Tag;

 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -35,7 +34,7 @@ class GetTagVisitsCommand extends Command
    public function __invoke(
        SymfonyStyle $io,
        #[Argument('The tag which visits we want to get'), Ask('For what tag do you want to get visits')] string $tag,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
        #[Option(
            'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits '
            . 'in default domain',
@@ -44,12 +43,11 @@ class GetTagVisitsCommand extends Command
        string|null $domain = null,
    ): int {
        $paginator = $this->visitsHelper->visitsForTag($tag, new WithDomainVisitsParams(
-            dateRange: $dateRangeInput->toDateRange(),
+            dateRange: $input->dateRange(),
            domain: $domain,
        ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));

-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));

        return self::SUCCESS;
    }
--- a/module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php
+++ b/module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php
@@ -4,8 +4,7 @@ declare(strict_types=1);

 namespace Shlinkio\Shlink\CLI\Command\Visit;

-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -31,7 +30,7 @@ class GetNonOrphanVisitsCommand extends Command

    public function __invoke(
        SymfonyStyle $io,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
        #[Option(
            'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits '
            . 'in default domain',
@@ -40,12 +39,10 @@ class GetNonOrphanVisitsCommand extends Command
        string|null $domain = null,
    ): int {
        $paginator = $this->visitsHelper->nonOrphanVisits(new WithDomainVisitsParams(
-            dateRange: $dateRangeInput->toDateRange(),
+            dateRange: $input->dateRange(),
            domain: $domain,
        ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));

        return self::SUCCESS;
    }
--- a/module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php
+++ b/module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php
@@ -4,8 +4,7 @@ declare(strict_types=1);

 namespace Shlinkio\Shlink\CLI\Command\Visit;

-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\OrphanVisitsParams;
@@ -29,7 +28,7 @@ class GetOrphanVisitsCommand extends Command

    public function __invoke(
        SymfonyStyle $io,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
        #[Option(
            'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits '
            . 'in default domain',
@@ -39,13 +38,11 @@ class GetOrphanVisitsCommand extends Command
        #[Option('Return visits only with this type', shortcut: 't')] OrphanVisitType|null $type = null,
    ): int {
        $paginator = $this->visitsHelper->orphanVisits(new OrphanVisitsParams(
-            dateRange: $dateRangeInput->toDateRange(),
+            dateRange: $input->dateRange(),
            domain: $domain,
            type: $type,
        ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));

        return self::SUCCESS;
    }
--- a/module/CLI/src/Command/Visit/VisitsCommandUtils.php
+++ b/module/CLI/src/Command/Visit/VisitsCommandUtils.php
@@ -4,8 +4,14 @@ declare(strict_types=1);

 namespace Shlinkio\Shlink\CLI\Command\Visit;

+use League\Csv\Writer;
+use Shlinkio\Shlink\CLI\Input\VisitsListFormat;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
+use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
+use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtils;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
+use Symfony\Component\Console\Output\OutputInterface;

 use function array_keys;
 use function array_map;
@@ -16,14 +22,86 @@ class VisitsCommandUtils
 {
    /**
     * @param Paginator<Visit> $paginator
-     * @param callable(Visit $visits): array<string, string> $mapExtraFields
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
     */
-    public static function resolveRowsAndHeaders(Paginator $paginator, callable $mapExtraFields): array
+    public static function renderOutput(
+        OutputInterface $output,
+        VisitsListInput $inputData,
+        Paginator $paginator,
+        callable|null $mapExtraFields = null,
+    ): void {
+        if ($inputData->format !== VisitsListFormat::FULL) {
+            // Avoid running out of memory by loading visits in chunks
+            $paginator->setMaxPerPage(1000);
+        }
+
+        match ($inputData->format) {
+            VisitsListFormat::CSV => self::renderCSVOutput($output, $paginator, $mapExtraFields),
+            default => self::renderHumanFriendlyOutput($output, $paginator, $mapExtraFields),
+        };
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function renderCSVOutput(
+        OutputInterface $output,
+        Paginator $paginator,
+        callable|null $mapExtraFields,
+    ): void {
+        $page = 1;
+        do {
+            $paginator->setCurrentPage($page);
+
+            [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields);
+            $csv = Writer::fromString();
+            if ($page === 1) {
+                $csv->insertOne($headers);
+            }
+
+            $csv->insertAll($rows);
+            $output->write($csv->toString());
+
+            $page++;
+        } while ($paginator->hasNextPage());
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function renderHumanFriendlyOutput(
+        OutputInterface $output,
+        Paginator $paginator,
+        callable|null $mapExtraFields,
+    ): void {
+        $page = 1;
+        do {
+            $paginator->setCurrentPage($page);
+            $page++;
+
+            [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields);
+            ShlinkTable::default($output)->render(
+                $headers,
+                $rows,
+                footerTitle: PagerfantaUtils::formatCurrentPageMessage($paginator, 'Page %s of %s'),
+            );
+        } while ($paginator->hasNextPage());
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields): array
    {
-        $extraKeys = [];
+        $extraKeys = null;
+        $mapExtraFields ??= static fn (Visit $_) => [];
+
        $rows = array_map(function (Visit $visit) use (&$extraKeys, $mapExtraFields) {
            $extraFields = $mapExtraFields($visit);
-            $extraKeys = array_keys($extraFields);
+            $extraKeys ??= array_keys($extraFields);

            $rowData = [
                'referer' => $visit->referer,
@@ -38,7 +116,7 @@ class VisitsCommandUtils
            // Filter out unknown keys
            return select_keys($rowData, ['referer', 'date', 'userAgent', 'country', 'city', ...$extraKeys]);
        }, [...$paginator->getCurrentPageResults()]);
-        $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys);
+        $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys ?? []);

        return [
            $rows,
--- a/module/CLI/src/Input/VisitsListFormat.php
+++ b/module/CLI/src/Input/VisitsListFormat.php
@@ -0,0 +1,18 @@
+<?php
+
+namespace Shlinkio\Shlink\CLI\Input;
+
+enum VisitsListFormat: string
+{
+    /** Load and dump all visits at once, in a human-friendly format */
+    case FULL = 'full';
+
+    /**
+     * Load and dump visits in 1000-visit chunks, in a human-friendly format.
+     * This format is recommended over `default` for large number of visits, to avoid running out of memory.
+     */
+    case PAGINATED = 'paginated';
+
+    /** Load and dump visits in chunks, in CSV format */
+    case CSV = 'csv';
+}
--- a/module/CLI/src/Input/VisitsDateRangeInput.php
+++ b/module/CLI/src/Input/VisitsDateRangeInput.php
@@ -10,7 +10,7 @@ use Symfony\Component\Console\Attribute\Option;
 use function Shlinkio\Shlink\Common\buildDateRange;
 use function Shlinkio\Shlink\Core\normalizeOptionalDate;

-class VisitsDateRangeInput
+class VisitsListInput
 {
    #[Option('Only return visits older than this date', shortcut: 's')]
    public string|null $startDate = null;
@@ -18,7 +18,14 @@ class VisitsDateRangeInput
    #[Option('Only return visits newer than this date', shortcut: 'e')]
    public string|null $endDate = null;

-    public function toDateRange(): DateRange
+    #[Option(
+        'Output format ("' . VisitsListFormat::FULL->value . '", "' . VisitsListFormat::PAGINATED->value . '" or "'
+        . VisitsListFormat::CSV->value . '")',
+        shortcut: 'f',
+    )]
+    public VisitsListFormat $format = VisitsListFormat::FULL;
+
+    public function dateRange(): DateRange
    {
        return buildDateRange(
            startDate: normalizeOptionalDate($this->startDate),
--- a/module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php
+++ b/module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php
@@ -61,7 +61,7 @@ class GetDomainVisitsCommandTest extends TestCase
            | Referer | Date                      | User agent | Country | City   | Short Url     |
            +---------+---------------------------+------------+---------+--------+---------------+
            | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+

            OUTPUT,
            $output,
--- a/module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php
+++ b/module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php
@@ -6,10 +6,12 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl;

 use Cake\Chronos\Chronos;
 use Pagerfanta\Adapter\ArrayAdapter;
+use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Test;
 use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\GetShortUrlVisitsCommand;
+use Shlinkio\Shlink\CLI\Input\VisitsListFormat;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
 use Shlinkio\Shlink\Common\Util\DateRange;
 use Shlinkio\Shlink\Core\ShortUrl\Entity\ShortUrl;
@@ -80,8 +82,11 @@ class GetShortUrlVisitsCommandTest extends TestCase
        ]);
    }

-    #[Test]
-    public function outputIsProperlyGenerated(): void
+    /**
+     * @param callable(Chronos $date): string $getExpectedOutput
+     */
+    #[Test, DataProvider('provideOutput')]
+    public function outputIsProperlyGenerated(VisitsListFormat $format, callable $getExpectedOutput): void
    {
        $visit = Visit::forValidShortUrl(ShortUrl::createFake(), Visitor::fromParams('bar', 'foo', ''))->locate(
            VisitLocation::fromLocation(new Location('', 'Spain', '', 'Madrid', 0, 0, '')),
@@ -92,19 +97,32 @@ class GetShortUrlVisitsCommandTest extends TestCase
            $this->anything(),
        )->willReturn(new Paginator(new ArrayAdapter([$visit])));

-        $this->commandTester->execute(['short-code' => $shortCode]);
+        $this->commandTester->execute(['short-code' => $shortCode, '--format' => $format->value]);
        $output = $this->commandTester->getDisplay();

-        self::assertEquals(
-            <<<OUTPUT
-            +---------+---------------------------+------------+---------+--------+
-            | Referer | Date                      | User agent | Country | City   |
-            +---------+---------------------------+------------+---------+--------+
-            | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid |
-            +---------+---------------------------+------------+---------+--------+
+        self::assertEquals($getExpectedOutput($visit->date), $output);
+    }

-            OUTPUT,
-            $output,
-        );
+    public static function provideOutput(): iterable
+    {
+        yield 'regular' => [
+            VisitsListFormat::FULL,
+            static fn (Chronos $date) => <<<OUTPUT
+                +---------+---------------------------+------------+---------+--------+
+                | Referer | Date                      | User agent | Country | City   |
+                +---------+---------------------------+------------+---------+--------+
+                | foo     | {$date->toAtomString()} | bar        | Spain   | Madrid |
+                +---------+------------------ Page 1 of 1 ---------+---------+--------+
+
+                OUTPUT,
+        ];
+        yield 'CSV' => [
+            VisitsListFormat::CSV,
+            static fn (Chronos $date) => <<<OUTPUT
+                Referer,Date,"User agent",Country,City
+                foo,{$date->toAtomString()},bar,Spain,Madrid
+
+                OUTPUT,
+        ];
    }
 }
--- a/module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php
+++ b/module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php
@@ -58,7 +58,7 @@ class GetTagVisitsCommandTest extends TestCase
            | Referer | Date                      | User agent | Country | City   | Short Url     |
            +---------+---------------------------+------------+---------+--------+---------------+
            | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+

            OUTPUT,
            $output,
--- a/module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php
+++ b/module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php
@@ -57,7 +57,7 @@ class GetNonOrphanVisitsCommandTest extends TestCase
            | Referer | Date                      | User agent | Country | City   | Short Url     |
            +---------+---------------------------+------------+---------+--------+---------------+
            | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+

            OUTPUT,
            $output,
--- a/module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php
+++ b/module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php
@@ -55,7 +55,7 @@ class GetOrphanVisitsCommandTest extends TestCase
            | Referer | Date                      | User agent | Country | City   | Type     |
            +---------+---------------------------+------------+---------+--------+----------+
            | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | base_url |
-            +---------+---------------------------+------------+---------+--------+----------+
+            +---------+----------------------- Page 1 of 1 ----+---------+--------+----------+

            OUTPUT,
            $output,