Mirrors/shlink
1
0
Fork 0
mirror of https://github.com/shlinkio/shlink.git synced 2026-03-11 09:43:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixed bug missing unprocessed visits while iterating and updating, while drastically improving the performance

Browse Source
This commit is contained in:
Alejandro Celaya
2019-02-23 09:58:02 +01:00
parent 62133c994f
commit d2fad0128f
5 changed files with 45 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
module/Core/src/Repository/VisitRepository.php
View File

@@ -11,22 +11,26 @@ use Shlinkio\Shlink\Core\Entity\Visit;
class VisitRepository extends EntityRepository implements VisitRepositoryInterface
{
/**
* This method will allow you to iterate the whole list of unlocated visits, but loading them into memory in
* smaller blocks of a specific size.
* This will have side effects if you update those rows while you iterate them.
* If you plan to do so, pass the first argument as false in order to disable applying offsets while slicing the
* dataset
*
* @return iterable|Visit[]
*/
public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable
public function findUnlocatedVisits(bool $applyOffset = true, int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable
{
$dql = <<<DQL
SELECT v FROM Shlinkio\Shlink\Core\Entity\Visit AS v WHERE v.visitLocation IS NULL
DQL;
$query = $this->getEntityManager()->createQuery($dql);
$query = $this->getEntityManager()->createQuery($dql)
->setMaxResults($blockSize);
$remainingVisitsToProcess = $this->count(['visitLocation' => null]);
$offset = 0;
while ($remainingVisitsToProcess > 0) {
$iterator = $query->setMaxResults($blockSize)
->setFirstResult($offset)
->iterate();
$iterator = $query->setFirstResult($applyOffset ? $offset : null)->iterate();
foreach ($iterator as $key => [$value]) {
yield $key => $value;
}

8
module/Core/src/Repository/VisitRepositoryInterface.php
View File

@@ -12,9 +12,15 @@ interface VisitRepositoryInterface extends ObjectRepository
public const DEFAULT_BLOCK_SIZE = 10000;
/**
* This method will allow you to iterate the whole list of unlocated visits, but loading them into memory in
* smaller blocks of a specific size.
* This will have side effects if you update those rows while you iterate them.
* If you plan to do so, pass the first argument as false in order to disable applying offsets while slicing the
* dataset
*
* @return iterable|Visit[]
*/
public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable;
public function findUnlocatedVisits(bool $applyOffset = true, int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable;
/**
* @return Visit[]

18
module/Core/src/Service/VisitService.php
View File

@@ -24,9 +24,12 @@ class VisitService implements VisitServiceInterface
{
/** @var VisitRepository $repo */
$repo = $this->em->getRepository(Visit::class);
$results = $repo->findUnlocatedVisits();
$results = $repo->findUnlocatedVisits(false);
$count = 0;
$persistBlock = 200;
foreach ($results as $visit) {
$count++;
try {
/** @var Location $location */
$location = $geolocateVisit($visit);
@@ -37,20 +40,25 @@ class VisitService implements VisitServiceInterface
$location = new VisitLocation($location);
$this->locateVisit($visit, $location, $notifyVisitWithLocation);
// Flush and clear after X iterations
if ($count % $persistBlock === 0) {
$this->em->flush();
$this->em->clear();
}
}
$this->em->flush();
$this->em->clear();
}
private function locateVisit(Visit $visit, VisitLocation $location, ?callable $notifyVisitWithLocation): void
{
$visit->locate($location);
$this->em->persist($visit);
$this->em->flush();
if ($notifyVisitWithLocation !== null) {
$notifyVisitWithLocation($location, $visit);
}
$this->em->clear();
}
}