diff --git a/CHANGELOG.md b/CHANGELOG.md index 12b5aae4..fb43ec7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,13 +15,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this This option effectively replaces the old `REDIRECT_APPEND_EXTRA_PATH` option, which is now deprecated and will be removed in Shlink 5.0.0 ### Changed -* * [#2281](https://github.com/shlinkio/shlink/issues/2281) Update docker image to PHP 8.4 +* [#2281](https://github.com/shlinkio/shlink/issues/2281) Update docker image to PHP 8.4 +* [#2124](https://github.com/shlinkio/shlink/issues/2124) Improve how Shlink decides if a GeoLite db file needs to be downloaded, and reduces the chances for API limits to be reached. + + Now Shlink tracks all download attempts, and knows which of them failed and succeeded. This lets it know when was the last error or success, how many consecutive errors have happened, etc. + + It also tracks now the reason for a download to be attempted, and the error that happened when one fails. ### Deprecated * *Nothing* ### Removed -* * [#2247](https://github.com/shlinkio/shlink/issues/2247) Drop support for PHP 8.2 +* [#2247](https://github.com/shlinkio/shlink/issues/2247) Drop support for PHP 8.2 ### Fixed * *Nothing* diff --git a/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php b/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php index b0a22c97..cd7c4ffb 100644 --- a/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php +++ b/module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php @@ -51,6 +51,16 @@ class DownloadGeoLiteDbCommand extends Command implements GeolocationDownloadPro return ExitCode::EXIT_WARNING; } + if ($result === GeolocationResult::MAX_ERRORS_REACHED) { + $this->io->warning('Max consecutive errors reached. Cannot retry for a couple of days.'); + return ExitCode::EXIT_WARNING; + } + + if ($result === GeolocationResult::UPDATE_IN_PROGRESS) { + $this->io->warning('A geolocation db is already being downloaded by another process.'); + return ExitCode::EXIT_WARNING; + } + if ($this->progressBar === null) { $this->io->info('GeoLite2 db file is up to date.'); } else { @@ -66,7 +76,7 @@ class DownloadGeoLiteDbCommand extends Command implements GeolocationDownloadPro private function processGeoLiteUpdateError(GeolocationDbUpdateFailedException $e, SymfonyStyle $io): int { - $olderDbExists = $e->olderDbExists(); + $olderDbExists = $e->olderDbExists; if ($olderDbExists) { $io->warning( diff --git a/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php b/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php index 7fa46a05..01322f0b 100644 --- a/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php +++ b/module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php @@ -6,6 +6,7 @@ namespace ShlinkioTest\Shlink\CLI\Command\Visit; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; +use PHPUnit\Framework\Attributes\TestWith; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; use Shlinkio\Shlink\CLI\Command\Visit\DownloadGeoLiteDbCommand; @@ -74,17 +75,18 @@ class DownloadGeoLiteDbCommandTest extends TestCase } #[Test] - public function warningIsPrintedWhenLicenseIsMissing(): void + #[TestWith([GeolocationResult::LICENSE_MISSING, 'It was not possible to download GeoLite2 db'])] + #[TestWith([GeolocationResult::MAX_ERRORS_REACHED, 'Max consecutive errors reached'])] + #[TestWith([GeolocationResult::UPDATE_IN_PROGRESS, 'A geolocation db is already being downloaded'])] + public function warningIsPrintedForSomeResults(GeolocationResult $result, string $expectedWarningMessage): void { - $this->dbUpdater->expects($this->once())->method('checkDbUpdate')->withAnyParameters()->willReturn( - GeolocationResult::LICENSE_MISSING, - ); + $this->dbUpdater->expects($this->once())->method('checkDbUpdate')->withAnyParameters()->willReturn($result); $this->commandTester->execute([]); $output = $this->commandTester->getDisplay(); $exitCode = $this->commandTester->getStatusCode(); - self::assertStringContainsString('[WARNING] It was not possible to download GeoLite2 db', $output); + self::assertStringContainsString('[WARNING] ' . $expectedWarningMessage, $output); self::assertSame(ExitCode::EXIT_WARNING, $exitCode); } diff --git a/module/CLI/test/Exception/GeolocationDbUpdateFailedExceptionTest.php b/module/CLI/test/Exception/GeolocationDbUpdateFailedExceptionTest.php index a1d6db65..86ed9548 100644 --- a/module/CLI/test/Exception/GeolocationDbUpdateFailedExceptionTest.php +++ b/module/CLI/test/Exception/GeolocationDbUpdateFailedExceptionTest.php @@ -19,7 +19,7 @@ class GeolocationDbUpdateFailedExceptionTest extends TestCase { $e = GeolocationDbUpdateFailedException::withOlderDb($prev); - self::assertTrue($e->olderDbExists()); + self::assertTrue($e->olderDbExists); self::assertEquals( 'An error occurred while updating geolocation database, but an older DB is already present.', $e->getMessage(), @@ -33,7 +33,7 @@ class GeolocationDbUpdateFailedExceptionTest extends TestCase { $e = GeolocationDbUpdateFailedException::withoutOlderDb($prev); - self::assertFalse($e->olderDbExists()); + self::assertFalse($e->olderDbExists); self::assertEquals( 'An error occurred while updating geolocation database, and an older version could not be found.', $e->getMessage(), @@ -48,16 +48,4 @@ class GeolocationDbUpdateFailedExceptionTest extends TestCase yield 'RuntimeException' => [new RuntimeException('prev')]; yield 'Exception' => [new Exception('prev')]; } - - #[Test] - public function withInvalidEpochInOldDbBuildsException(): void - { - $e = GeolocationDbUpdateFailedException::withInvalidEpochInOldDb('foobar'); - - self::assertTrue($e->olderDbExists()); - self::assertEquals( - 'Build epoch with value "foobar" from existing geolocation database, could not be parsed to integer.', - $e->getMessage(), - ); - } } diff --git a/module/Core/config/dependencies.config.php b/module/Core/config/dependencies.config.php index b16a4c5c..adc9ae2a 100644 --- a/module/Core/config/dependencies.config.php +++ b/module/Core/config/dependencies.config.php @@ -13,7 +13,6 @@ use Shlinkio\Shlink\Core\Geolocation\GeolocationDbUpdater; use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifier; use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface; use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater; -use Shlinkio\Shlink\IpGeolocation\GeoLite2\GeoLite2ReaderFactory; use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface; use Symfony\Component\Lock; @@ -247,9 +246,9 @@ return [ GeolocationDbUpdater::class => [ DbUpdater::class, - GeoLite2ReaderFactory::class, LOCAL_LOCK_FACTORY, Config\Options\TrackingOptions::class, + 'em', ], Geolocation\Middleware\IpGeolocationMiddleware::class => [ IpLocationResolverInterface::class, diff --git a/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Geolocation.Entity.GeolocationDbUpdate.php b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Geolocation.Entity.GeolocationDbUpdate.php new file mode 100644 index 00000000..7b60abdc --- /dev/null +++ b/module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Geolocation.Entity.GeolocationDbUpdate.php @@ -0,0 +1,63 @@ +setTable(determineTableName('geolocation_db_updates', $emConfig)); + + $builder->createField('id', Types::BIGINT) + ->columnName('id') + ->makePrimaryKey() + ->generatedValue('IDENTITY') + ->option('unsigned', true) + ->build(); + + $builder->createField('dateCreated', ChronosDateTimeType::CHRONOS_DATETIME) + ->columnName('date_created') + ->build(); + + $builder->createField('dateUpdated', ChronosDateTimeType::CHRONOS_DATETIME) + ->columnName('date_updated') + ->nullable() + ->build(); + + (new FieldBuilder($builder, [ + 'fieldName' => 'status', + 'type' => Types::STRING, + 'enumType' => GeolocationDbUpdateStatus::class, + ]))->columnName('status') + ->length(128) + ->build(); + + fieldWithUtf8Charset($builder->createField('error', Types::STRING), $emConfig) + ->columnName('error') + ->length(1024) + ->nullable() + ->build(); + + fieldWithUtf8Charset($builder->createField('reason', Types::STRING), $emConfig) + ->columnName('reason') + ->length(1024) + ->build(); + + fieldWithUtf8Charset($builder->createField('filesystemId', Types::STRING), $emConfig) + ->columnName('filesystem_id') + ->length(512) + ->build(); + + // Index on date_updated, as we'll usually sort the query by this field + $builder->addIndex(['date_updated'], 'IDX_geolocation_date_updated'); + // Index on filesystem_id, as we'll usually filter the query by this field + $builder->addIndex(['filesystem_id'], 'IDX_geolocation_status_filesystem'); +}; diff --git a/module/Core/migrations/Version20241212131058.php b/module/Core/migrations/Version20241212131058.php new file mode 100644 index 00000000..23e61803 --- /dev/null +++ b/module/Core/migrations/Version20241212131058.php @@ -0,0 +1,64 @@ +skipIf($schema->hasTable(self::TABLE_NAME)); + + $table = $schema->createTable(self::TABLE_NAME); + $table->addColumn('id', Types::BIGINT, [ + 'unsigned' => true, + 'autoincrement' => true, + 'notnull' => true, + ]); + $table->setPrimaryKey(['id']); + + $table->addColumn('date_created', ChronosDateTimeType::CHRONOS_DATETIME, ['default' => 'CURRENT_TIMESTAMP']); + $table->addColumn('date_updated', ChronosDateTimeType::CHRONOS_DATETIME, ['default' => 'CURRENT_TIMESTAMP']); + + $table->addColumn('status', Types::STRING, [ + 'length' => 128, + 'default' => 'in-progress', // in-progress, success, error + ]); + $table->addColumn('filesystem_id', Types::STRING, ['length' => 512]); + + $table->addColumn('reason', Types::STRING, ['length' => 1024]); + $table->addColumn('error', Types::STRING, [ + 'length' => 1024, + 'default' => null, + 'notnull' => false, + ]); + + // Index on date_updated, as we'll usually sort the query by this field + $table->addIndex(['date_updated'], 'IDX_geolocation_date_updated'); + // Index on filesystem_id, as we'll usually filter the query by this field + $table->addIndex(['filesystem_id'], 'IDX_geolocation_status_filesystem'); + } + + public function down(Schema $schema): void + { + $this->skipIf(! $schema->hasTable(self::TABLE_NAME)); + $schema->dropTable(self::TABLE_NAME); + } + + public function isTransactional(): bool + { + return ! ($this->connection->getDatabasePlatform() instanceof MySQLPlatform); + } +} diff --git a/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php b/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php index 87108279..8d288959 100644 --- a/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php +++ b/module/Core/src/EventDispatcher/UpdateGeoLiteDb.php @@ -14,6 +14,7 @@ use Throwable; use function sprintf; +/** @todo Rename to UpdateGeolocationDb */ readonly class UpdateGeoLiteDb { public function __construct( diff --git a/module/Core/src/Exception/GeolocationDbUpdateFailedException.php b/module/Core/src/Exception/GeolocationDbUpdateFailedException.php index f3c3f65f..a818b263 100644 --- a/module/Core/src/Exception/GeolocationDbUpdateFailedException.php +++ b/module/Core/src/Exception/GeolocationDbUpdateFailedException.php @@ -7,52 +7,28 @@ namespace Shlinkio\Shlink\Core\Exception; use RuntimeException; use Throwable; -use function sprintf; - class GeolocationDbUpdateFailedException extends RuntimeException implements ExceptionInterface { - private bool $olderDbExists; - - private function __construct(string $message, Throwable|null $previous = null) + private function __construct(string $message, public readonly bool $olderDbExists, Throwable|null $prev = null) { - parent::__construct($message, previous: $previous); + parent::__construct($message, previous: $prev); } public static function withOlderDb(Throwable|null $prev = null): self { - $e = new self( + return new self( 'An error occurred while updating geolocation database, but an older DB is already present.', - $prev, + olderDbExists: true, + prev: $prev, ); - $e->olderDbExists = true; - - return $e; } public static function withoutOlderDb(Throwable|null $prev = null): self { - $e = new self( + return new self( 'An error occurred while updating geolocation database, and an older version could not be found.', - $prev, + olderDbExists: false, + prev: $prev, ); - $e->olderDbExists = false; - - return $e; - } - - public static function withInvalidEpochInOldDb(mixed $buildEpoch): self - { - $e = new self(sprintf( - 'Build epoch with value "%s" from existing geolocation database, could not be parsed to integer.', - $buildEpoch, - )); - $e->olderDbExists = true; - - return $e; - } - - public function olderDbExists(): bool - { - return $this->olderDbExists; } } diff --git a/module/Core/src/Geolocation/Entity/GeolocationDbUpdate.php b/module/Core/src/Geolocation/Entity/GeolocationDbUpdate.php new file mode 100644 index 00000000..f3735a64 --- /dev/null +++ b/module/Core/src/Geolocation/Entity/GeolocationDbUpdate.php @@ -0,0 +1,77 @@ +dateUpdated = Chronos::now(); + $this->status = GeolocationDbUpdateStatus::SUCCESS; + return $this; + } + + public function finishWithError(string $error): self + { + $this->error = $error; + $this->dateUpdated = Chronos::now(); + $this->status = GeolocationDbUpdateStatus::ERROR; + return $this; + } + + /** + * @param positive-int $days + */ + public function isOlderThan(int $days): bool + { + return Chronos::now()->greaterThan($this->dateUpdated->addDays($days)); + } + + public function isInProgress(): bool + { + return $this->status === GeolocationDbUpdateStatus::IN_PROGRESS; + } + + public function isError(): bool + { + return $this->status === GeolocationDbUpdateStatus::ERROR; + } + + public function isSuccess(): bool + { + return $this->status === GeolocationDbUpdateStatus::SUCCESS; + } +} diff --git a/module/Core/src/Geolocation/Entity/GeolocationDbUpdateStatus.php b/module/Core/src/Geolocation/Entity/GeolocationDbUpdateStatus.php new file mode 100644 index 00000000..8216f2bd --- /dev/null +++ b/module/Core/src/Geolocation/Entity/GeolocationDbUpdateStatus.php @@ -0,0 +1,10 @@ +geoLiteDbReaderFactory = $geoLiteDbReaderFactory(...); } /** @@ -46,6 +39,7 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface return GeolocationResult::CHECK_SKIPPED; } + $lock = $this->locker->createLock(self::LOCK_NAME); $lock->acquire(blocking: true); @@ -62,43 +56,88 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface private function downloadIfNeeded( GeolocationDownloadProgressHandlerInterface|null $downloadProgressHandler, ): GeolocationResult { - if (! $this->dbUpdater->databaseFileExists()) { - return $this->downloadNewDb($downloadProgressHandler, olderDbExists: false); + $recentDownloads = $this->em->getRepository(GeolocationDbUpdate::class)->findBy( + criteria: ['filesystemId' => GeolocationDbUpdate::currentFilesystemId()], + orderBy: ['dateUpdated' => 'DESC'], + limit: $this->maxRecentAttemptsToCheck, + ); + $mostRecentDownload = $recentDownloads[0] ?? null; + + // If most recent attempt is in progress, skip check. + // This is a safety check in case the lock is released before the previous download has finished. + if ($mostRecentDownload?->isInProgress()) { + return GeolocationResult::UPDATE_IN_PROGRESS; } - $meta = ($this->geoLiteDbReaderFactory)()->metadata(); - if ($this->buildIsTooOld($meta)) { - return $this->downloadNewDb($downloadProgressHandler, olderDbExists: true); + $amountOfErrorsSinceLastSuccess = 0; + foreach ($recentDownloads as $recentDownload) { + // Count attempts until a success is found + if ($recentDownload->isSuccess()) { + break; + } + $amountOfErrorsSinceLastSuccess++; + } + + // If max amount of consecutive errors has been reached and the most recent one is not old enough, skip download + // for 2 days to avoid hitting potential API limits in geolocation services + $lastAttemptIsError = $mostRecentDownload !== null && $mostRecentDownload->isError(); + // FIXME Once max errors are reached there will be one attempt every 2 days, but it should be 15 attempts every + // 2 days. Leaving like this for simplicity for now. + $maxConsecutiveErrorsReached = $amountOfErrorsSinceLastSuccess === $this->maxRecentAttemptsToCheck; + if ($lastAttemptIsError && $maxConsecutiveErrorsReached && ! $mostRecentDownload->isOlderThan(days: 2)) { + return GeolocationResult::MAX_ERRORS_REACHED; + } + + // Try to download if: + // - There are no attempts tracked + // - The database file does not exist + // - Last update errored (and implicitly, the max amount of consecutive errors has not been reached) + // - Most recent attempt is older than 30 days (and implicitly, successful) + $reasonMatch = match (true) { + $mostRecentDownload === null => [false, 'No download attempts tracked for this instance'], + ! $this->dbUpdater->databaseFileExists() => [false, 'Geolocation db file does not exist'], + $lastAttemptIsError => [true, 'Max consecutive errors not reached'], + $mostRecentDownload->isOlderThan(days: 30) => [true, 'Last successful attempt is old enough'], + default => null, + }; + if ($reasonMatch !== null) { + [$olderDbExists, $reason] = $reasonMatch; + return $this->downloadAndTrackUpdate($downloadProgressHandler, $olderDbExists, $reason); } return GeolocationResult::DB_IS_UP_TO_DATE; } - private function buildIsTooOld(Metadata $meta): bool - { - $buildTimestamp = $this->resolveBuildTimestamp($meta); - $buildDate = Chronos::createFromTimestamp($buildTimestamp); + /** + * @throws GeolocationDbUpdateFailedException + */ + private function downloadAndTrackUpdate( + GeolocationDownloadProgressHandlerInterface|null $downloadProgressHandler, + bool $olderDbExists, + string $reason, + ): GeolocationResult { + $dbUpdate = GeolocationDbUpdate::withReason($reason); + $this->em->persist($dbUpdate); + $this->em->flush(); - return Chronos::now()->greaterThan($buildDate->addDays(35)); - } - - private function resolveBuildTimestamp(Metadata $meta): int - { - // In theory the buildEpoch should be an int, but it has been reported to come as a string. - // See https://github.com/shlinkio/shlink/issues/1002 for context - - /** @var int|string $buildEpoch */ - $buildEpoch = $meta->buildEpoch; - if (is_int($buildEpoch)) { - return $buildEpoch; + try { + $result = $this->downloadNewDb($downloadProgressHandler, $olderDbExists); + $dbUpdate->finishSuccessfully(); + return $result; + } catch (MissingLicenseException) { + $dbUpdate->finishWithError('Geolocation license key is missing'); + return GeolocationResult::LICENSE_MISSING; + } catch (GeolocationDbUpdateFailedException $e) { + $dbUpdate->finishWithError( + sprintf('%s Prev: %s', $e->getMessage(), $e->getPrevious()?->getMessage() ?? '-'), + ); + throw $e; + } catch (Throwable $e) { + $dbUpdate->finishWithError(sprintf('Unknown error: %s', $e->getMessage())); + throw $e; + } finally { + $this->em->flush(); } - - $intBuildEpoch = (int) $buildEpoch; - if ($buildEpoch === (string) $intBuildEpoch) { - return $intBuildEpoch; - } - - throw GeolocationDbUpdateFailedException::withInvalidEpochInOldDb($buildEpoch); } /** @@ -116,8 +155,6 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface => $downloadProgressHandler?->handleProgress($total, $downloaded, $olderDbExists), ); return $olderDbExists ? GeolocationResult::DB_UPDATED : GeolocationResult::DB_CREATED; - } catch (MissingLicenseException) { - return GeolocationResult::LICENSE_MISSING; } catch (DbUpdateException $e) { throw $olderDbExists ? GeolocationDbUpdateFailedException::withOlderDb($e) diff --git a/module/Core/src/Geolocation/GeolocationResult.php b/module/Core/src/Geolocation/GeolocationResult.php index 3b472d09..eb5b4a5c 100644 --- a/module/Core/src/Geolocation/GeolocationResult.php +++ b/module/Core/src/Geolocation/GeolocationResult.php @@ -4,9 +4,18 @@ namespace Shlinkio\Shlink\Core\Geolocation; enum GeolocationResult { + /** Geolocation is not relevant, so updates are skipped */ case CHECK_SKIPPED; + /** Update is skipped because max amount of consecutive errors was reached */ + case MAX_ERRORS_REACHED; + /** Update was skipped because a geolocation license key was not provided */ case LICENSE_MISSING; + /** A geolocation database didn't exist and has been created */ case DB_CREATED; + /** An outdated geolocation database existed and has been updated */ case DB_UPDATED; + /** Geolocation database does not need to be updated yet */ case DB_IS_UP_TO_DATE; + /** Geolocation db update is currently in progress */ + case UPDATE_IN_PROGRESS; } diff --git a/module/Core/test/Geolocation/GeolocationDbUpdaterTest.php b/module/Core/test/Geolocation/GeolocationDbUpdaterTest.php index 5c76747b..c2983030 100644 --- a/module/Core/test/Geolocation/GeolocationDbUpdaterTest.php +++ b/module/Core/test/Geolocation/GeolocationDbUpdaterTest.php @@ -6,14 +6,16 @@ namespace ShlinkioTest\Shlink\Core\Geolocation; use Cake\Chronos\Chronos; use Closure; -use GeoIp2\Database\Reader; -use MaxMind\Db\Reader\Metadata; +use Doctrine\ORM\EntityManagerInterface; +use Doctrine\ORM\EntityRepository; use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\Attributes\Test; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; +use RuntimeException; use Shlinkio\Shlink\Core\Config\Options\TrackingOptions; use Shlinkio\Shlink\Core\Exception\GeolocationDbUpdateFailedException; +use Shlinkio\Shlink\Core\Geolocation\Entity\GeolocationDbUpdate; use Shlinkio\Shlink\Core\Geolocation\GeolocationDbUpdater; use Shlinkio\Shlink\Core\Geolocation\GeolocationDownloadProgressHandlerInterface; use Shlinkio\Shlink\Core\Geolocation\GeolocationResult; @@ -29,17 +31,24 @@ use function range; class GeolocationDbUpdaterTest extends TestCase { private MockObject & DbUpdaterInterface $dbUpdater; - private MockObject & Reader $geoLiteDbReader; private MockObject & Lock\LockInterface $lock; + private MockObject & EntityManagerInterface $em; + /** @var MockObject&EntityRepository */ + private MockObject & EntityRepository $repo; /** @var GeolocationDownloadProgressHandlerInterface&object{beforeDownloadCalled: bool, handleProgressCalled: bool} */ private GeolocationDownloadProgressHandlerInterface $progressHandler; protected function setUp(): void { $this->dbUpdater = $this->createMock(DbUpdaterInterface::class); - $this->geoLiteDbReader = $this->createMock(Reader::class); + $this->lock = $this->createMock(Lock\SharedLockInterface::class); $this->lock->method('acquire')->with($this->isTrue())->willReturn(true); + + $this->em = $this->createMock(EntityManagerInterface::class); + $this->repo = $this->createMock(EntityRepository::class); + $this->em->method('getRepository')->willReturn($this->repo); + $this->progressHandler = new class implements GeolocationDownloadProgressHandlerInterface { public function __construct( public bool $beforeDownloadCalled = false, @@ -59,6 +68,32 @@ class GeolocationDbUpdaterTest extends TestCase }; } + #[Test] + public function properResultIsReturnedIfMostRecentUpdateIsInProgress(): void + { + $this->repo->expects($this->once())->method('findBy')->willReturn([GeolocationDbUpdate::withReason('')]); + $this->dbUpdater->expects($this->never())->method('databaseFileExists'); + + $result = $this->geolocationDbUpdater()->checkDbUpdate(); + + self::assertEquals(GeolocationResult::UPDATE_IN_PROGRESS, $result); + } + + #[Test] + public function properResultIsReturnedIfMaxConsecutiveErrorsAreReached(): void + { + $this->repo->expects($this->once())->method('findBy')->willReturn([ + GeolocationDbUpdate::withReason('')->finishWithError(''), + GeolocationDbUpdate::withReason('')->finishWithError(''), + GeolocationDbUpdate::withReason('')->finishWithError(''), + ]); + $this->dbUpdater->expects($this->never())->method('databaseFileExists'); + + $result = $this->geolocationDbUpdater()->checkDbUpdate(); + + self::assertEquals(GeolocationResult::MAX_ERRORS_REACHED, $result); + } + #[Test] public function properResultIsReturnedWhenLicenseIsMissing(): void { @@ -66,7 +101,9 @@ class GeolocationDbUpdaterTest extends TestCase $this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->willThrowException( new MissingLicenseException(''), ); - $this->geoLiteDbReader->expects($this->never())->method('metadata'); + $this->repo->expects($this->once())->method('findBy')->willReturn([ + GeolocationDbUpdate::withReason('')->finishSuccessfully(), + ]); $result = $this->geolocationDbUpdater()->checkDbUpdate($this->progressHandler); @@ -74,16 +111,19 @@ class GeolocationDbUpdaterTest extends TestCase self::assertEquals(GeolocationResult::LICENSE_MISSING, $result); } - #[Test] - public function exceptionIsThrownWhenOlderDbDoesNotExistAndDownloadFails(): void + #[Test, DataProvider('provideDbDoesNotExist')] + public function exceptionIsThrownWhenOlderDbDoesNotExistAndDownloadFails(Closure $setUp): void { $prev = new DbUpdateException(''); - $this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(false); + $expectedReason = $setUp($this); + $this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->with( $this->isInstanceOf(Closure::class), )->willThrowException($prev); - $this->geoLiteDbReader->expects($this->never())->method('metadata'); + $this->em->expects($this->once())->method('persist')->with($this->callback( + fn (GeolocationDbUpdate $newUpdate): bool => $newUpdate->reason === $expectedReason, + )); try { $this->geolocationDbUpdater()->checkDbUpdate($this->progressHandler); @@ -91,22 +131,36 @@ class GeolocationDbUpdaterTest extends TestCase } catch (Throwable $e) { self::assertInstanceOf(GeolocationDbUpdateFailedException::class, $e); self::assertSame($prev, $e->getPrevious()); - self::assertFalse($e->olderDbExists()); + self::assertFalse($e->olderDbExists); self::assertTrue($this->progressHandler->beforeDownloadCalled); } } + public static function provideDbDoesNotExist(): iterable + { + yield 'file does not exist' => [function (self $test): string { + $test->repo->expects($test->once())->method('findBy')->willReturn([ + GeolocationDbUpdate::withReason('')->finishSuccessfully(), + ]); + $test->dbUpdater->expects($test->once())->method('databaseFileExists')->willReturn(false); + return 'Geolocation db file does not exist'; + }]; + yield 'no attempts' => [function (self $test): string { + $test->repo->expects($test->once())->method('findBy')->willReturn([]); + $test->dbUpdater->expects($test->never())->method('databaseFileExists'); + return 'No download attempts tracked for this instance'; + }]; + } + #[Test, DataProvider('provideBigDays')] - public function exceptionIsThrownWhenOlderDbIsTooOldAndDownloadFails(int $days): void + public function exceptionIsThrownWhenOlderDbIsOldEnoughAndDownloadFails(int $days): void { $prev = new DbUpdateException(''); $this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(true); $this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->with( $this->isInstanceOf(Closure::class), )->willThrowException($prev); - $this->geoLiteDbReader->expects($this->once())->method('metadata')->with()->willReturn( - $this->buildMetaWithBuildEpoch(Chronos::now()->subDays($days)->getTimestamp()), - ); + $this->repo->expects($this->once())->method('findBy')->willReturn([self::createFinishedOldUpdate($days)]); try { $this->geolocationDbUpdater()->checkDbUpdate(); @@ -114,80 +168,115 @@ class GeolocationDbUpdaterTest extends TestCase } catch (Throwable $e) { self::assertInstanceOf(GeolocationDbUpdateFailedException::class, $e); self::assertSame($prev, $e->getPrevious()); - self::assertTrue($e->olderDbExists()); + self::assertTrue($e->olderDbExists); } } public static function provideBigDays(): iterable { - yield [36]; + yield [31]; yield [50]; yield [75]; yield [100]; } - #[Test, DataProvider('provideSmallDays')] - public function databaseIsNotUpdatedIfItIsNewEnough(string|int $buildEpoch): void + #[Test] + public function exceptionIsThrownWhenUnknownErrorHappens(): void + { + $this->dbUpdater->expects($this->once())->method('downloadFreshCopy')->with( + $this->isInstanceOf(Closure::class), + )->willThrowException(new RuntimeException('An error occurred')); + + $newUpdate = null; + $this->em->expects($this->once())->method('persist')->with($this->callback( + function (GeolocationDbUpdate $u) use (&$newUpdate): bool { + $newUpdate = $u; + return true; + }, + )); + + try { + $this->geolocationDbUpdater()->checkDbUpdate($this->progressHandler); + self::fail(); + } catch (Throwable) { + } + + self::assertTrue($this->progressHandler->beforeDownloadCalled); + self::assertNotNull($newUpdate); + self::assertTrue($newUpdate->isError()); + } + + #[Test, DataProvider('provideNotAldEnoughDays')] + public function databaseIsNotUpdatedIfItIsNewEnough(int $days): void { $this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(true); $this->dbUpdater->expects($this->never())->method('downloadFreshCopy'); - $this->geoLiteDbReader->expects($this->once())->method('metadata')->with()->willReturn( - $this->buildMetaWithBuildEpoch($buildEpoch), - ); + $this->repo->expects($this->once())->method('findBy')->willReturn([self::createFinishedOldUpdate($days)]); $result = $this->geolocationDbUpdater()->checkDbUpdate(); self::assertEquals(GeolocationResult::DB_IS_UP_TO_DATE, $result); } - public static function provideSmallDays(): iterable + public static function provideNotAldEnoughDays(): iterable { - $generateParamsWithTimestamp = static function (int $days) { - $timestamp = Chronos::now()->subDays($days)->getTimestamp(); - return [$days % 2 === 0 ? $timestamp : (string) $timestamp]; - }; - - return array_map($generateParamsWithTimestamp, range(0, 34)); + return array_map(static fn (int $value) => [$value], range(0, 29)); } - #[Test] - public function exceptionIsThrownWhenCheckingExistingDatabaseWithInvalidBuildEpoch(): void - { - $this->dbUpdater->expects($this->once())->method('databaseFileExists')->willReturn(true); - $this->dbUpdater->expects($this->never())->method('downloadFreshCopy'); - $this->geoLiteDbReader->expects($this->once())->method('metadata')->with()->willReturn( - $this->buildMetaWithBuildEpoch('invalid'), - ); + #[Test, DataProvider('provideUpdatesThatWillDownload')] + public function properResultIsReturnedWhenDownloadSucceeds( + array $updates, + GeolocationResult $expectedResult, + string $expectedReason, + ): void { + $this->repo->expects($this->once())->method('findBy')->willReturn($updates); + $this->dbUpdater->method('databaseFileExists')->willReturn(true); + $this->dbUpdater->expects($this->once())->method('downloadFreshCopy'); + $this->em->expects($this->once())->method('persist')->with($this->callback( + fn (GeolocationDbUpdate $newUpdate): bool => $newUpdate->reason === $expectedReason, + )); - $this->expectException(GeolocationDbUpdateFailedException::class); - $this->expectExceptionMessage( - 'Build epoch with value "invalid" from existing geolocation database, could not be parsed to integer.', - ); + $result = $this->geolocationDbUpdater()->checkDbUpdate(); - $this->geolocationDbUpdater()->checkDbUpdate(); + self::assertEquals($expectedResult, $result); } - private function buildMetaWithBuildEpoch(string|int $buildEpoch): Metadata + public static function provideUpdatesThatWillDownload(): iterable { - return new Metadata([ - 'binary_format_major_version' => '', - 'binary_format_minor_version' => '', - 'build_epoch' => $buildEpoch, - 'database_type' => '', - 'languages' => '', - 'description' => '', - 'ip_version' => '', - 'node_count' => 1, - 'record_size' => 4, - ]); + yield 'no updates' => [[], GeolocationResult::DB_CREATED, 'No download attempts tracked for this instance']; + yield 'old successful update' => [ + [self::createFinishedOldUpdate(days: 31)], + GeolocationResult::DB_UPDATED, + 'Last successful attempt is old enough', + ]; + yield 'not enough errors' => [ + [self::createFinishedOldUpdate(days: 3, successful: false)], + GeolocationResult::DB_UPDATED, + 'Max consecutive errors not reached', + ]; + } + + public static function createFinishedOldUpdate(int $days, bool $successful = true): GeolocationDbUpdate + { + Chronos::setTestNow(Chronos::now()->subDays($days)); + $update = GeolocationDbUpdate::withReason(''); + if ($successful) { + $update->finishSuccessfully(); + } else { + $update->finishWithError(''); + } + Chronos::setTestNow(); + + return $update; } #[Test, DataProvider('provideTrackingOptions')] public function downloadDbIsSkippedIfTrackingIsDisabled(TrackingOptions $options): void { - $result = $this->geolocationDbUpdater($options)->checkDbUpdate(); $this->dbUpdater->expects($this->never())->method('databaseFileExists'); - $this->geoLiteDbReader->expects($this->never())->method('metadata'); + $this->em->expects($this->never())->method('getRepository'); + + $result = $this->geolocationDbUpdater($options)->checkDbUpdate(); self::assertEquals(GeolocationResult::CHECK_SKIPPED, $result); } @@ -204,11 +293,6 @@ class GeolocationDbUpdaterTest extends TestCase $locker = $this->createMock(Lock\LockFactory::class); $locker->method('createLock')->with($this->isType('string'))->willReturn($this->lock); - return new GeolocationDbUpdater( - $this->dbUpdater, - fn () => $this->geoLiteDbReader, - $locker, - $options ?? new TrackingOptions(), - ); + return new GeolocationDbUpdater($this->dbUpdater, $locker, $options ?? new TrackingOptions(), $this->em, 3); } }