Merge pull request #2562 from acelaya-forks/visits-export

Allow exporting visits in CSV format

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
   * `after-date`: matches when current date and time is later than the defined threshold.
 
 * [#2513](https://github.com/shlinkio/shlink/issues/2513) Add support for redis connections via unix socket (e.g. `REDIS_SERVERS=unix:/path/to/redis.sock`).
+* Visits generated in the command line can now be formatted in CSV, via `--format=csv`.
 
 ### Changed
 * [#2522](https://github.com/shlinkio/shlink/issues/2522) Shlink no longer tries to detect trusted proxies automatically, when resolving the visitor's IP address, as this is a potential security issue.

composer.json

@@ -33,6 +33,7 @@
         "laminas/laminas-inputfilter": "^2.31",
         "laminas/laminas-servicemanager": "^3.23",
         "laminas/laminas-stdlib": "^3.20",
+        "league/csv": "^9.28",
         "matomo/matomo-php-tracker": "^3.3",
         "mezzio/mezzio": "^3.20",
         "mezzio/mezzio-fastroute": "^3.12",

module/CLI/src/Command/Domain/GetDomainVisitsCommand.php

@@ -5,8 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\Domain;
 
 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
@@ -34,12 +33,10 @@ class GetDomainVisitsCommand extends Command
         SymfonyStyle $io,
         #[Argument('The domain which visits we want to get'), Ask('For what domain do you want to get visits?')]
         string $domain,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
     ): int {
-        $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($dateRangeInput->toDateRange()));
+        $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($input->dateRange()));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php

@@ -5,8 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\ShortUrl;
 
 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\ShortUrl\Model\ShortUrlIdentifier;
 use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
 use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface;
@@ -32,16 +31,15 @@ class GetShortUrlVisitsCommand extends Command
         SymfonyStyle $io,
         #[Argument('The short code which visits we want to get'), Ask('Which short code do you want to use?')]
         string $shortCode,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
         #[Option('The domain for the short code', shortcut: 'd')]
         string|null $domain = null,
     ): int {
         $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain);
-        $dateRange = $dateRangeInput->toDateRange();
+        $dateRange = $input->dateRange();
         $paginator = $this->visitsHelper->visitsForShortUrl($identifier, new VisitsParams($dateRange));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, static fn () => []);
 
-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Tag/GetTagVisitsCommand.php

@@ -5,8 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\Tag;
 
 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -35,7 +34,7 @@ class GetTagVisitsCommand extends Command
     public function __invoke(
         SymfonyStyle $io,
         #[Argument('The tag which visits we want to get'), Ask('For what tag do you want to get visits')] string $tag,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
         #[Option(
             'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits '
             . 'in default domain',
@@ -44,12 +43,11 @@ class GetTagVisitsCommand extends Command
         string|null $domain = null,
     ): int {
         $paginator = $this->visitsHelper->visitsForTag($tag, new WithDomainVisitsParams(
-            dateRange: $dateRangeInput->toDateRange(),
+            dateRange: $input->dateRange(),
             domain: $domain,
         ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
 
-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php

@@ -4,8 +4,7 @@ declare(strict_types=1);
 
 namespace Shlinkio\Shlink\CLI\Command\Visit;
 
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -31,7 +30,7 @@ class GetNonOrphanVisitsCommand extends Command
 
     public function __invoke(
         SymfonyStyle $io,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
         #[Option(
             'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits '
             . 'in default domain',
@@ -40,12 +39,10 @@ class GetNonOrphanVisitsCommand extends Command
         string|null $domain = null,
     ): int {
         $paginator = $this->visitsHelper->nonOrphanVisits(new WithDomainVisitsParams(
-            dateRange: $dateRangeInput->toDateRange(),
+            dateRange: $input->dateRange(),
             domain: $domain,
         ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php

@@ -4,8 +4,7 @@ declare(strict_types=1);
 
 namespace Shlinkio\Shlink\CLI\Command\Visit;
 
-use Shlinkio\Shlink\CLI\Input\VisitsDateRangeInput;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\OrphanVisitsParams;
@@ -29,7 +28,7 @@ class GetOrphanVisitsCommand extends Command
 
     public function __invoke(
         SymfonyStyle $io,
-        #[MapInput] VisitsDateRangeInput $dateRangeInput,
+        #[MapInput] VisitsListInput $input,
         #[Option(
             'Return visits that belong to this domain only. Use ' . Domain::DEFAULT_AUTHORITY . ' keyword for visits '
             . 'in default domain',
@@ -39,13 +38,11 @@ class GetOrphanVisitsCommand extends Command
         #[Option('Return visits only with this type', shortcut: 't')] OrphanVisitType|null $type = null,
     ): int {
         $paginator = $this->visitsHelper->orphanVisits(new OrphanVisitsParams(
-            dateRange: $dateRangeInput->toDateRange(),
+            dateRange: $input->dateRange(),
             domain: $domain,
             type: $type,
         ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Visit/VisitsCommandUtils.php

@@ -4,8 +4,14 @@ declare(strict_types=1);
 
 namespace Shlinkio\Shlink\CLI\Command\Visit;
 
+use League\Csv\Writer;
+use Shlinkio\Shlink\CLI\Input\VisitsListFormat;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
+use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
+use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtils;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
+use Symfony\Component\Console\Output\OutputInterface;
 
 use function array_keys;
 use function array_map;
@@ -16,14 +22,86 @@ class VisitsCommandUtils
 {
     /**
      * @param Paginator<Visit> $paginator
-     * @param callable(Visit $visits): array<string, string> $mapExtraFields
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
      */
-    public static function resolveRowsAndHeaders(Paginator $paginator, callable $mapExtraFields): array
+    public static function renderOutput(
+        OutputInterface $output,
+        VisitsListInput $inputData,
+        Paginator $paginator,
+        callable|null $mapExtraFields = null,
+    ): void {
+        if ($inputData->format !== VisitsListFormat::FULL) {
+            // Avoid running out of memory by loading visits in chunks
+            $paginator->setMaxPerPage(1000);
+        }
+
+        match ($inputData->format) {
+            VisitsListFormat::CSV => self::renderCSVOutput($output, $paginator, $mapExtraFields),
+            default => self::renderHumanFriendlyOutput($output, $paginator, $mapExtraFields),
+        };
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function renderCSVOutput(
+        OutputInterface $output,
+        Paginator $paginator,
+        callable|null $mapExtraFields,
+    ): void {
+        $page = 1;
+        do {
+            $paginator->setCurrentPage($page);
+
+            [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields);
+            $csv = Writer::fromString();
+            if ($page === 1) {
+                $csv->insertOne($headers);
+            }
+
+            $csv->insertAll($rows);
+            $output->write($csv->toString());
+
+            $page++;
+        } while ($paginator->hasNextPage());
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function renderHumanFriendlyOutput(
+        OutputInterface $output,
+        Paginator $paginator,
+        callable|null $mapExtraFields,
+    ): void {
+        $page = 1;
+        do {
+            $paginator->setCurrentPage($page);
+            $page++;
+
+            [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields);
+            ShlinkTable::default($output)->render(
+                $headers,
+                $rows,
+                footerTitle: PagerfantaUtils::formatCurrentPageMessage($paginator, 'Page %s of %s'),
+            );
+        } while ($paginator->hasNextPage());
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields): array
     {
-        $extraKeys = [];
+        $extraKeys = null;
+        $mapExtraFields ??= static fn (Visit $_) => [];
+
         $rows = array_map(function (Visit $visit) use (&$extraKeys, $mapExtraFields) {
             $extraFields = $mapExtraFields($visit);
-            $extraKeys = array_keys($extraFields);
+            $extraKeys ??= array_keys($extraFields);
 
             $rowData = [
                 'referer' => $visit->referer,
@@ -38,7 +116,7 @@ class VisitsCommandUtils
             // Filter out unknown keys
             return select_keys($rowData, ['referer', 'date', 'userAgent', 'country', 'city', ...$extraKeys]);
         }, [...$paginator->getCurrentPageResults()]);
-        $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys);
+        $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys ?? []);
 
         return [
             $rows,

module/CLI/src/Input/VisitsListFormat.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace Shlinkio\Shlink\CLI\Input;
+
+enum VisitsListFormat: string
+{
+    /** Load and dump all visits at once, in a human-friendly format */
+    case FULL = 'full';
+
+    /**
+     * Load and dump visits in 1000-visit chunks, in a human-friendly format.
+     * This format is recommended over `default` for large number of visits, to avoid running out of memory.
+     */
+    case PAGINATED = 'paginated';
+
+    /** Load and dump visits in chunks, in CSV format */
+    case CSV = 'csv';
+}

module/CLI/src/Input/VisitsListInput.php

@@ -10,7 +10,7 @@ use Symfony\Component\Console\Attribute\Option;
 use function Shlinkio\Shlink\Common\buildDateRange;
 use function Shlinkio\Shlink\Core\normalizeOptionalDate;
 
-class VisitsDateRangeInput
+class VisitsListInput
 {
     #[Option('Only return visits older than this date', shortcut: 's')]
     public string|null $startDate = null;
@@ -18,7 +18,14 @@ class VisitsDateRangeInput
     #[Option('Only return visits newer than this date', shortcut: 'e')]
     public string|null $endDate = null;
 
-    public function toDateRange(): DateRange
+    #[Option(
+        'Output format ("' . VisitsListFormat::FULL->value . '", "' . VisitsListFormat::PAGINATED->value . '" or "'
+        . VisitsListFormat::CSV->value . '")',
+        shortcut: 'f',
+    )]
+    public VisitsListFormat $format = VisitsListFormat::FULL;
+
+    public function dateRange(): DateRange
     {
         return buildDateRange(
             startDate: normalizeOptionalDate($this->startDate),

module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php

@@ -61,7 +61,7 @@ class GetDomainVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Short Url     |
             +---------+---------------------------+------------+---------+--------+---------------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php

@@ -6,10 +6,12 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl;
 
 use Cake\Chronos\Chronos;
 use Pagerfanta\Adapter\ArrayAdapter;
+use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Test;
 use PHPUnit\Framework\MockObject\MockObject;
 use PHPUnit\Framework\TestCase;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\GetShortUrlVisitsCommand;
+use Shlinkio\Shlink\CLI\Input\VisitsListFormat;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
 use Shlinkio\Shlink\Common\Util\DateRange;
 use Shlinkio\Shlink\Core\ShortUrl\Entity\ShortUrl;
@@ -80,8 +82,11 @@ class GetShortUrlVisitsCommandTest extends TestCase
         ]);
     }
 
-    #[Test]
+    /**
-    public function outputIsProperlyGenerated(): void
+     * @param callable(Chronos $date): string $getExpectedOutput
+     */
+    #[Test, DataProvider('provideOutput')]
+    public function outputIsProperlyGenerated(VisitsListFormat $format, callable $getExpectedOutput): void
     {
         $visit = Visit::forValidShortUrl(ShortUrl::createFake(), Visitor::fromParams('bar', 'foo', ''))->locate(
             VisitLocation::fromLocation(new Location('', 'Spain', '', 'Madrid', 0, 0, '')),
@@ -92,19 +97,32 @@ class GetShortUrlVisitsCommandTest extends TestCase
             $this->anything(),
         )->willReturn(new Paginator(new ArrayAdapter([$visit])));
 
-        $this->commandTester->execute(['short-code' => $shortCode]);
+        $this->commandTester->execute(['short-code' => $shortCode, '--format' => $format->value]);
         $output = $this->commandTester->getDisplay();
 
-        self::assertEquals(
+        self::assertEquals($getExpectedOutput($visit->date), $output);
-            <<<OUTPUT
+    }
-            +---------+---------------------------+------------+---------+--------+
-            | Referer | Date                      | User agent | Country | City   |
-            +---------+---------------------------+------------+---------+--------+
-            | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid |
-            +---------+---------------------------+------------+---------+--------+
 
-            OUTPUT,
+    public static function provideOutput(): iterable
-            $output,
+    {
-        );
+        yield 'regular' => [
+            VisitsListFormat::FULL,
+            static fn (Chronos $date) => <<<OUTPUT
+                +---------+---------------------------+------------+---------+--------+
+                | Referer | Date                      | User agent | Country | City   |
+                +---------+---------------------------+------------+---------+--------+
+                | foo     | {$date->toAtomString()} | bar        | Spain   | Madrid |
+                +---------+------------------ Page 1 of 1 ---------+---------+--------+
+
+                OUTPUT,
+        ];
+        yield 'CSV' => [
+            VisitsListFormat::CSV,
+            static fn (Chronos $date) => <<<OUTPUT
+                Referer,Date,"User agent",Country,City
+                foo,{$date->toAtomString()},bar,Spain,Madrid
+
+                OUTPUT,
+        ];
     }
 }

module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php

@@ -58,7 +58,7 @@ class GetTagVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Short Url     |
             +---------+---------------------------+------------+---------+--------+---------------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php

@@ -57,7 +57,7 @@ class GetNonOrphanVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Short Url     |
             +---------+---------------------------+------------+---------+--------+---------------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php

@@ -55,7 +55,7 @@ class GetOrphanVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Type     |
             +---------+---------------------------+------------+---------+--------+----------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | base_url |
-            +---------+---------------------------+------------+---------+--------+----------+
+            +---------+----------------------- Page 1 of 1 ----+---------+--------+----------+
 
             OUTPUT,
             $output,