Mirrors/shlink
1
0
Fork 0
mirror of https://github.com/shlinkio/shlink.git synced 2026-02-27 19:53:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support paginating the output of visits commands to avoid out of memory errors

Browse Source
This commit is contained in:
Alejandro Celaya
2025-12-29 10:22:50 +01:00
parent faed7ae60b
commit c0edcd3cfd
13 changed files with 100 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
module/CLI/src/Command/Domain/GetDomainVisitsCommand.php
View File

@@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\CLI\Command\Domain;
use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
use Shlinkio\Shlink\CLI\Input\VisitsListInput;
use Shlinkio\Shlink\CLI\Util\ShlinkTable;
use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
use Shlinkio\Shlink\Core\Visit\Entity\Visit;
use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
@@ -37,9 +36,7 @@ class GetDomainVisitsCommand extends Command
#[MapInput] VisitsListInput $input,
): int {
$paginator = $this->visitsHelper->visitsForDomain($domain, new VisitsParams($input->dateRange()));
[$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
ShlinkTable::default($io)->render($headers, $rows);
VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
return self::SUCCESS;
}

4
module/CLI/src/Command/ShortUrl/GetShortUrlVisitsCommand.php
View File

@@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\CLI\Command\ShortUrl;
use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
use Shlinkio\Shlink\CLI\Input\VisitsListInput;
use Shlinkio\Shlink\CLI\Util\ShlinkTable;
use Shlinkio\Shlink\Core\ShortUrl\Model\ShortUrlIdentifier;
use Shlinkio\Shlink\Core\Visit\Model\VisitsParams;
use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface;
@@ -39,9 +38,8 @@ class GetShortUrlVisitsCommand extends Command
$identifier = ShortUrlIdentifier::fromShortCodeAndDomain($shortCode, $domain);
$dateRange = $input->dateRange();
$paginator = $this->visitsHelper->visitsForShortUrl($identifier, new VisitsParams($dateRange));
[$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator);
ShlinkTable::default($io)->render($headers, $rows);
VisitsCommandUtils::renderOutput($io, $input, $paginator);
return self::SUCCESS;
}

4
module/CLI/src/Command/Tag/GetTagVisitsCommand.php
View File

@@ -6,7 +6,6 @@ namespace Shlinkio\Shlink\CLI\Command\Tag;
use Shlinkio\Shlink\CLI\Command\Visit\VisitsCommandUtils;
use Shlinkio\Shlink\CLI\Input\VisitsListInput;
use Shlinkio\Shlink\CLI\Util\ShlinkTable;
use Shlinkio\Shlink\Core\Domain\Entity\Domain;
use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -47,9 +46,8 @@ class GetTagVisitsCommand extends Command
dateRange: $input->dateRange(),
domain: $domain,
));
[$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
ShlinkTable::default($io)->render($headers, $rows);
VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
return self::SUCCESS;
}

5
module/CLI/src/Command/Visit/GetNonOrphanVisitsCommand.php
View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace Shlinkio\Shlink\CLI\Command\Visit;
use Shlinkio\Shlink\CLI\Input\VisitsListInput;
use Shlinkio\Shlink\CLI\Util\ShlinkTable;
use Shlinkio\Shlink\Core\Domain\Entity\Domain;
use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
use Shlinkio\Shlink\Core\Visit\Entity\Visit;
@@ -43,9 +42,7 @@ class GetNonOrphanVisitsCommand extends Command
dateRange: $input->dateRange(),
domain: $domain,
));
[$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
ShlinkTable::default($io)->render($headers, $rows);
VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
return self::SUCCESS;
}

5
module/CLI/src/Command/Visit/GetOrphanVisitsCommand.php
View File

@@ -5,7 +5,6 @@ declare(strict_types=1);
namespace Shlinkio\Shlink\CLI\Command\Visit;
use Shlinkio\Shlink\CLI\Input\VisitsListInput;
use Shlinkio\Shlink\CLI\Util\ShlinkTable;
use Shlinkio\Shlink\Core\Domain\Entity\Domain;
use Shlinkio\Shlink\Core\Visit\Entity\Visit;
use Shlinkio\Shlink\Core\Visit\Model\OrphanVisitsParams;
@@ -43,9 +42,7 @@ class GetOrphanVisitsCommand extends Command
domain: $domain,
type: $type,
));
[$rows, $headers] = VisitsCommandUtils::resolveRowsAndHeaders($paginator, $this->mapExtraFields(...));
ShlinkTable::default($io)->render($headers, $rows);
VisitsCommandUtils::renderOutput($io, $input, $paginator, $this->mapExtraFields(...));
return self::SUCCESS;
}

69
module/CLI/src/Command/Visit/VisitsCommandUtils.php
View File

@@ -4,8 +4,13 @@ declare(strict_types=1);
namespace Shlinkio\Shlink\CLI\Command\Visit;
use Shlinkio\Shlink\CLI\Input\VisitsListFormat;
use Shlinkio\Shlink\CLI\Input\VisitsListInput;
use Shlinkio\Shlink\CLI\Util\ShlinkTable;
use Shlinkio\Shlink\Common\Paginator\Paginator;
use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtils;
use Shlinkio\Shlink\Core\Visit\Entity\Visit;
use Symfony\Component\Console\Output\OutputInterface;
use function array_keys;
use function array_map;
@@ -18,14 +23,70 @@ class VisitsCommandUtils
* @param Paginator<Visit> $paginator
* @param null|callable(Visit $visits): array<string, string> $mapExtraFields
*/
public static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields = null): array
public static function renderOutput(
OutputInterface $output,
VisitsListInput $inputData,
Paginator $paginator,
callable|null $mapExtraFields = null,
): void {
if ($inputData->format !== VisitsListFormat::FULL) {
// Avoid running out of memory by loading visits in chunks
$paginator->setMaxPerPage(1000);
}
match ($inputData->format) {
VisitsListFormat::CSV => self::renderCSVOutput($output, $paginator, $mapExtraFields),
default => self::renderHumanFriendlyOutput($output, $paginator, $mapExtraFields),
};
}
/**
* @param Paginator<Visit> $paginator
* @param null|callable(Visit $visits): array<string, string> $mapExtraFields
*/
private static function renderCSVOutput(
OutputInterface $output,
Paginator $paginator,
callable|null $mapExtraFields,
): void {
// TODO
}
/**
* @param Paginator<Visit> $paginator
* @param null|callable(Visit $visits): array<string, string> $mapExtraFields
*/
private static function renderHumanFriendlyOutput(
OutputInterface $output,
Paginator $paginator,
callable|null $mapExtraFields,
): void {
$page = 1;
do {
$paginator->setCurrentPage($page);
$page++;
[$rows, $headers] = self::resolveRowsAndHeaders($paginator, $mapExtraFields);
ShlinkTable::default($output)->render(
$headers,
$rows,
footerTitle: PagerfantaUtils::formatCurrentPageMessage($paginator, 'Page %s of %s'),
);
} while ($paginator->hasNextPage());
}
/**
* @param Paginator<Visit> $paginator
* @param null|callable(Visit $visits): array<string, string> $mapExtraFields
*/
private static function resolveRowsAndHeaders(Paginator $paginator, callable|null $mapExtraFields): array
{
$extraKeys = [];
$extraKeys = null;
$mapExtraFields ??= static fn (Visit $_) => [];
$rows = array_map(function (Visit $visit) use (&$extraKeys, $mapExtraFields) {
$extraFields = $mapExtraFields($visit);
$extraKeys = array_keys($extraFields);
$extraKeys ??= array_keys($extraFields);
$rowData = [
'referer' => $visit->referer,
@@ -40,7 +101,7 @@ class VisitsCommandUtils
// Filter out unknown keys
return select_keys($rowData, ['referer', 'date', 'userAgent', 'country', 'city', ...$extraKeys]);
}, [...$paginator->getCurrentPageResults()]);
$extra = array_map(camelCaseToHumanFriendly(...), $extraKeys);
$extra = array_map(camelCaseToHumanFriendly(...), $extraKeys ?? []);
return [
$rows,

18
module/CLI/src/Input/VisitsListFormat.php Normal file
View File

@@ -0,0 +1,18 @@
<?php
namespace Shlinkio\Shlink\CLI\Input;
enum VisitsListFormat: string
{
/** Load and dump all visits at once, in a human-friendly format */
case FULL = 'full';
/**
* Load and dump visits in 1000-visit chunks, in a human-friendly format.
* This format is recommended over `default` for large number of visits, to avoid running out of memory.
*/
case PAGINATED = 'paginated';
/** Load and dump visits in chunks, in CSV format */
case CSV = 'csv';
}

7
module/CLI/src/Input/VisitsListInput.php
View File

@@ -18,6 +18,13 @@ class VisitsListInput
#[Option('Only return visits newer than this date', shortcut: 'e')]
public string|null $endDate = null;
#[Option(
'Output format ("' . VisitsListFormat::FULL->value . '", "' . VisitsListFormat::PAGINATED->value . '" or "'
. VisitsListFormat::CSV->value . '")',
shortcut: 'f',
)]
public VisitsListFormat $format = VisitsListFormat::FULL;
public function dateRange(): DateRange
{
return buildDateRange(

2
module/CLI/test/Command/Domain/GetDomainVisitsCommandTest.php
View File

@@ -61,7 +61,7 @@ class GetDomainVisitsCommandTest extends TestCase
| Referer | Date | User agent | Country | City | Short Url |
+---------+---------------------------+------------+---------+--------+---------------+
| foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | the_short_url |
+---------+---------------------------+------------+---------+--------+---------------+
+---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
OUTPUT,
$output,

2
module/CLI/test/Command/ShortUrl/GetShortUrlVisitsCommandTest.php
View File

@@ -101,7 +101,7 @@ class GetShortUrlVisitsCommandTest extends TestCase
| Referer | Date | User agent | Country | City |
+---------+---------------------------+------------+---------+--------+
| foo | {$visit->date->toAtomString()} | bar | Spain | Madrid |
+---------+---------------------------+------------+---------+--------+
+---------+------------------ Page 1 of 1 ---------+---------+--------+
OUTPUT,
$output,

2
module/CLI/test/Command/Tag/GetTagVisitsCommandTest.php
View File

@@ -58,7 +58,7 @@ class GetTagVisitsCommandTest extends TestCase
| Referer | Date | User agent | Country | City | Short Url |
+---------+---------------------------+------------+---------+--------+---------------+
| foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | the_short_url |
+---------+---------------------------+------------+---------+--------+---------------+
+---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
OUTPUT,
$output,

2
module/CLI/test/Command/Visit/GetNonOrphanVisitsCommandTest.php
View File

@@ -57,7 +57,7 @@ class GetNonOrphanVisitsCommandTest extends TestCase
| Referer | Date | User agent | Country | City | Short Url |
+---------+---------------------------+------------+---------+--------+---------------+
| foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | the_short_url |
+---------+---------------------------+------------+---------+--------+---------------+
+---------+-------------------------- Page 1 of 1 -+---------+--------+---------------+
OUTPUT,
$output,

2
module/CLI/test/Command/Visit/GetOrphanVisitsCommandTest.php
View File

@@ -55,7 +55,7 @@ class GetOrphanVisitsCommandTest extends TestCase
| Referer | Date | User agent | Country | City | Type |
+---------+---------------------------+------------+---------+--------+----------+
| foo | {$visit->date->toAtomString()} | bar | Spain | Madrid | base_url |
+---------+---------------------------+------------+---------+--------+----------+
+---------+----------------------- Page 1 of 1 ----+---------+--------+----------+
OUTPUT,
$output,