Support paginating the output of visits commands to avoid out of memory errors

module/CLI/src/Command/Domain/GetDomainVisitsCommand.php

@@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\CLI\Command\Domain;
 
 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
 use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
@@ -37,9 +36,7 @@ class GetDomainVisitsCommand extends Command
         #[MapInput] VisitsListInput $input,
     ): int {
         $paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($input->dateRange()));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php

@@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\CLI\Command\ShortUrl;
 
 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
 use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\ShortUrl\Model\ShortUrlIdentifier;
 use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
 use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface;
@@ -39,9 +38,8 @@ class GetShortUrlVisitsCommand extends Command
         $identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain);
         $dateRange = $input->dateRange();
         $paginator = $this->visitsHelper->visitsForShortUrl($identifier, new VisitsParams($dateRange));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator);
 
-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Tag/GetTagVisitsCommand.php

@@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\CLI\Command\Tag;
 
 use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
 use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -47,9 +46,8 @@ class GetTagVisitsCommand extends Command
             dateRange: $input->dateRange(),
             domain: $domain,
         ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
 
-        ShlinkTable::default($io)->render($headers, $rows);
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php

@@ -5,7 +5,6 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\Visit;
 
 use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -43,9 +42,7 @@ class GetNonOrphanVisitsCommand extends Command
             dateRange: $input->dateRange(),
             domain: $domain,
         ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php

@@ -5,7 +5,6 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\CLI\Command\Visit;
 
 use Shlinkio\Shlink\CLI\Input\VisitsListInput;
-use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Core\Domain\Entity\Domain;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
 use Shlinkio\Shlink\Core\Visit\Model\OrphanVisitsParams;
@@ -43,9 +42,7 @@ class GetOrphanVisitsCommand extends Command
             domain: $domain,
             type: $type,
         ));
-        [$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
+        VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
-
-        ShlinkTable::default($io)->render($headers, $rows);
 
         return self::SUCCESS;
     }

module/CLI/src/Command/Visit/VisitsCommandUtils.php

@@ -4,8 +4,13 @@ declare(strict_types=1);
 
 namespace Shlinkio\Shlink\CLI\Command\Visit;
 
+use Shlinkio\Shlink\CLI\Input\VisitsListFormat;
+use Shlinkio\Shlink\CLI\Input\VisitsListInput;
+use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
+use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtils;
 use Shlinkio\Shlink\Core\Visit\Entity\Visit;
+use Symfony\Component\Console\Output\OutputInterface;
 
 use function array_keys;
 use function array_map;
@@ -18,14 +23,70 @@ class VisitsCommandUtils
      * @param Paginator<Visit> $paginator
      * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
      */
-    public static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields = null): array
+    public static function renderOutput(
+        OutputInterface $output,
+        VisitsListInput $inputData,
+        Paginator $paginator,
+        callable|null $mapExtraFields = null,
+    ): void {
+        if ($inputData->format !== VisitsListFormat::FULL) {
+            // Avoid running out of memory by loading visits in chunks
+            $paginator->setMaxPerPage(1000);
+        }
+
+        match ($inputData->format) {
+            VisitsListFormat::CSV => self::renderCSVOutput($output, $paginator, $mapExtraFields),
+            default => self::renderHumanFriendlyOutput($output, $paginator, $mapExtraFields),
+        };
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function renderCSVOutput(
+        OutputInterface $output,
+        Paginator $paginator,
+        callable|null $mapExtraFields,
+    ): void {
+        // TODO
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function renderHumanFriendlyOutput(
+        OutputInterface $output,
+        Paginator $paginator,
+        callable|null $mapExtraFields,
+    ): void {
+        $page = 1;
+        do {
+            $paginator->setCurrentPage($page);
+            $page++;
+
+            [$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields);
+            ShlinkTable::default($output)->render(
+                $headers,
+                $rows,
+                footerTitle: PagerfantaUtils::formatCurrentPageMessage($paginator, 'Page %s of %s'),
+            );
+        } while ($paginator->hasNextPage());
+    }
+
+    /**
+     * @param Paginator<Visit> $paginator
+     * @param null|callable(Visit $visits): array<string, string> $mapExtraFields
+     */
+    private static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields): array
     {
-        $extraKeys = [];
+        $extraKeys = null;
         $mapExtraFields ??= static fn (Visit $_) => [];
 
         $rows = array_map(function (Visit $visit) use (&$extraKeys, $mapExtraFields) {
             $extraFields = $mapExtraFields($visit);
-            $extraKeys = array_keys($extraFields);
+            $extraKeys ??= array_keys($extraFields);
 
             $rowData = [
                 'referer' => $visit->referer,
@@ -40,7 +101,7 @@ class VisitsCommandUtils
             // Filter out unknown keys
             return select_keys($rowData, ['referer', 'date', 'userAgent', 'country', 'city', ...$extraKeys]);
         }, [...$paginator->getCurrentPageResults()]);
-        $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys);
+        $extra = array_map(camelCaseToHumanFriendly(...), $extraKeys ?? []);
 
         return [
             $rows,

module/CLI/src/Input/VisitsListFormat.php

@@ -0,0 +1,18 @@
+<?php
+
+namespace Shlinkio\Shlink\CLI\Input;
+
+enum VisitsListFormat: string
+{
+    /** Load and dump all visits at once, in a human-friendly format */
+    case FULL = 'full';
+
+    /**
+     * Load and dump visits in 1000-visit chunks, in a human-friendly format.
+     * This format is recommended over `default` for large number of visits, to avoid running out of memory.
+     */
+    case PAGINATED = 'paginated';
+
+    /** Load and dump visits in chunks, in CSV format */
+    case CSV = 'csv';
+}

module/CLI/src/Input/VisitsListInput.php

@@ -18,6 +18,13 @@ class VisitsListInput
     #[Option('Only return visits newer than this date', shortcut: 'e')]
     public string|null $endDate = null;
 
+    #[Option(
+        'Output format ("' . VisitsListFormat::FULL->value . '", "' . VisitsListFormat::PAGINATED->value . '" or "'
+        . VisitsListFormat::CSV->value . '")',
+        shortcut: 'f',
+    )]
+    public VisitsListFormat $format = VisitsListFormat::FULL;
+
     public function dateRange(): DateRange
     {
         return buildDateRange(

module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php

@@ -61,7 +61,7 @@ class GetDomainVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Short Url     |
             +---------+---------------------------+------------+---------+--------+---------------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php

@@ -101,7 +101,7 @@ class GetShortUrlVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   |
             +---------+---------------------------+------------+---------+--------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid |
-            +---------+---------------------------+------------+---------+--------+
+            +---------+------------------ Page 1 of 1 ---------+---------+--------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php

@@ -58,7 +58,7 @@ class GetTagVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Short Url     |
             +---------+---------------------------+------------+---------+--------+---------------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php

@@ -57,7 +57,7 @@ class GetNonOrphanVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Short Url     |
             +---------+---------------------------+------------+---------+--------+---------------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | the_short_url |
-            +---------+---------------------------+------------+---------+--------+---------------+
+            +---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
 
             OUTPUT,
             $output,

module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php

@@ -55,7 +55,7 @@ class GetOrphanVisitsCommandTest extends TestCase
             | Referer | Date                      | User agent | Country | City   | Type     |
             +---------+---------------------------+------------+---------+--------+----------+
             | foo     | {$visit->date->toAtomString()} | bar        | Spain   | Madrid | base_url |
-            +---------+---------------------------+------------+---------+--------+----------+
+            +---------+----------------------- Page 1 of 1 ----+---------+--------+----------+
 
             OUTPUT,
             $output,