From 292937b9623117315addb03e487bdcd9fb747183 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 22 Feb 2019 19:31:03 +0100 Subject: [PATCH 1/6] Updated VisitRepository::findUnlocatedVisits methods so that it paginates the amount of elements loaded in memory --- .../Adapter/PaginableQueryAdapter.php | 34 +++++++++++++++++ .../src/Paginator/ImplicitLoopPaginator.php | 38 +++++++++++++++++++ .../Core/src/Repository/VisitRepository.php | 21 ++++++++-- .../Repository/VisitRepositoryInterface.php | 7 +++- module/Core/src/Service/VisitService.php | 2 +- .../Repository/VisitRepositoryTest.php | 18 +++++++-- module/Core/test/Service/VisitServiceTest.php | 6 +-- 7 files changed, 115 insertions(+), 11 deletions(-) create mode 100644 module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php create mode 100644 module/Common/src/Paginator/ImplicitLoopPaginator.php diff --git a/module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php b/module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php new file mode 100644 index 00000000..d6de1ada --- /dev/null +++ b/module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php @@ -0,0 +1,34 @@ +query = $query; + $this->totalItems = $totalItems; + } + + public function getItems($offset, $itemCountPerPage): iterable + { + return $this->query + ->setMaxResults($itemCountPerPage) + ->setFirstResult($offset) + ->iterate(); + } + + public function count(): int + { + return $this->totalItems; + } +} diff --git a/module/Common/src/Paginator/ImplicitLoopPaginator.php b/module/Common/src/Paginator/ImplicitLoopPaginator.php new file mode 100644 index 00000000..9019f248 --- /dev/null +++ b/module/Common/src/Paginator/ImplicitLoopPaginator.php @@ -0,0 +1,38 @@ +paginator = $paginator; + $this->valueParser = $valueParser ?? function ($value) { + return $value; + }; + } + + public function getIterator(): iterable + { + $totalPages = $this->paginator->count(); + $processedPages = 0; + + do { + $processedPages++; + $this->paginator->setCurrentPageNumber($processedPages); + + foreach ($this->paginator as $key => $value) { + yield $key => ($this->valueParser)($value); + } + } while ($processedPages < $totalPages); + } +} diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 781df089..dfa5498f 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -5,17 +5,32 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\QueryBuilder; +use Shlinkio\Shlink\Common\Paginator\Adapter\PaginableQueryAdapter; +use Shlinkio\Shlink\Common\Paginator\ImplicitLoopPaginator; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Visit; +use Zend\Paginator\Paginator; +use function array_shift; class VisitRepository extends EntityRepository implements VisitRepositoryInterface { - public function findUnlocatedVisits(): iterable + /** + * @return iterable|Visit[] + */ + public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable { - $dql = 'SELECT v FROM Shlinkio\Shlink\Core\Entity\Visit AS v WHERE v.visitLocation IS NULL'; + $count = $this->count(['visitLocation' => null]); + $dql = <<getEntityManager()->createQuery($dql); - return $query->iterate(); + $paginator = new Paginator(new PaginableQueryAdapter($query, $count)); + $paginator->setItemCountPerPage($blockSize); + + return new ImplicitLoopPaginator($paginator, function (array $value) { + return array_shift($value); + }); } /** diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index 4de9024e..2307946a 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -9,7 +9,12 @@ use Shlinkio\Shlink\Core\Entity\Visit; interface VisitRepositoryInterface extends ObjectRepository { - public function findUnlocatedVisits(): iterable; + public const DEFAULT_BLOCK_SIZE = 10000; + + /** + * @return iterable|Visit[] + */ + public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable; /** * @return Visit[] diff --git a/module/Core/src/Service/VisitService.php b/module/Core/src/Service/VisitService.php index 705dab59..78c0eddd 100644 --- a/module/Core/src/Service/VisitService.php +++ b/module/Core/src/Service/VisitService.php @@ -26,7 +26,7 @@ class VisitService implements VisitServiceInterface $repo = $this->em->getRepository(Visit::class); $results = $repo->findUnlocatedVisits(); - foreach ($results as [$visit]) { + foreach ($results as $visit) { try { /** @var Location $location */ $location = $geolocateVisit($visit); diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index db797920..64702fcd 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -12,6 +12,8 @@ use Shlinkio\Shlink\Core\Entity\VisitLocation; use Shlinkio\Shlink\Core\Model\Visitor; use Shlinkio\Shlink\Core\Repository\VisitRepository; use ShlinkioTest\Shlink\Common\DbTest\DatabaseTestCase; +use function Functional\map; +use function range; use function sprintf; class VisitRepositoryTest extends DatabaseTestCase @@ -30,8 +32,11 @@ class VisitRepositoryTest extends DatabaseTestCase $this->repo = $this->getEntityManager()->getRepository(Visit::class); } - /** @test */ - public function findUnlocatedVisitsReturnsProperVisits(): void + /** + * @test + * @dataProvider provideBlockSize + */ + public function findUnlocatedVisitsReturnsProperVisits(int $blockSize): void { $shortUrl = new ShortUrl(''); $this->getEntityManager()->persist($shortUrl); @@ -50,7 +55,7 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); $resultsCount = 0; - $results = $this->repo->findUnlocatedVisits(); + $results = $this->repo->findUnlocatedVisits($blockSize); foreach ($results as $value) { $resultsCount++; } @@ -58,6 +63,13 @@ class VisitRepositoryTest extends DatabaseTestCase $this->assertEquals(3, $resultsCount); } + public function provideBlockSize(): iterable + { + return map(range(1, 5), function (int $value) { + return [$value]; + }); + } + /** @test */ public function findVisitsByShortCodeReturnsProperData(): void { diff --git a/module/Core/test/Service/VisitServiceTest.php b/module/Core/test/Service/VisitServiceTest.php index 618e4781..4f249b23 100644 --- a/module/Core/test/Service/VisitServiceTest.php +++ b/module/Core/test/Service/VisitServiceTest.php @@ -36,8 +36,8 @@ class VisitServiceTest extends TestCase public function locateVisitsIteratesAndLocatesUnlocatedVisits(): void { $unlocatedVisits = [ - [new Visit(new ShortUrl('foo'), Visitor::emptyInstance())], - [new Visit(new ShortUrl('bar'), Visitor::emptyInstance())], + new Visit(new ShortUrl('foo'), Visitor::emptyInstance()), + new Visit(new ShortUrl('bar'), Visitor::emptyInstance()), ]; $repo = $this->prophesize(VisitRepository::class); @@ -71,7 +71,7 @@ class VisitServiceTest extends TestCase public function visitsWhichCannotBeLocatedAreIgnored() { $unlocatedVisits = [ - [new Visit(new ShortUrl('foo'), Visitor::emptyInstance())], + new Visit(new ShortUrl('foo'), Visitor::emptyInstance()), ]; $repo = $this->prophesize(VisitRepository::class); From 7d4de590e5b0c72ae2c77c2fa4f3ced8e6a5ac9e Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 22 Feb 2019 19:53:10 +0100 Subject: [PATCH 2/6] Created ImplicitLoopPaginatorTest --- .../PaginableRepositoryAdapterTest.php | 2 +- .../Paginator/ImplicitLoopPaginatorTest.php | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) rename module/Common/test/Paginator/{ => Adapter}/PaginableRepositoryAdapterTest.php (95%) create mode 100644 module/Common/test/Paginator/ImplicitLoopPaginatorTest.php diff --git a/module/Common/test/Paginator/PaginableRepositoryAdapterTest.php b/module/Common/test/Paginator/Adapter/PaginableRepositoryAdapterTest.php similarity index 95% rename from module/Common/test/Paginator/PaginableRepositoryAdapterTest.php rename to module/Common/test/Paginator/Adapter/PaginableRepositoryAdapterTest.php index 4e63ba78..4f5afb23 100644 --- a/module/Common/test/Paginator/PaginableRepositoryAdapterTest.php +++ b/module/Common/test/Paginator/Adapter/PaginableRepositoryAdapterTest.php @@ -1,7 +1,7 @@ paginator = new Paginator(new ArrayAdapter(range(1, self::TOTAL_ITEMS))); + } + + /** + * @test + * @dataProvider provideItemsPerPage + */ + public function allElementsAreIteratedRegardlessThePageSize(int $itemsPerPage): void + { + $this->paginator->setItemCountPerPage($itemsPerPage); + $implicitLoopPaginator = new ImplicitLoopPaginator($this->paginator); + + $iteratedItems = 0; + foreach ($implicitLoopPaginator as $item) { + $iteratedItems++; + } + + $this->assertEquals(self::TOTAL_ITEMS, $iteratedItems); + } + + public function provideItemsPerPage(): iterable + { + return map(range(1, 20), function (int $i) { + return [$i]; + }); + } + + /** @test */ + public function valuesWrappedInPaginatorAreProperlyParsed(): void + { + $valueParser = function (int $item) { + return $item * 3; + }; + $this->paginator->setItemCountPerPage(5); + $implicitLoopPaginator = new ImplicitLoopPaginator($this->paginator, $valueParser); + + $items = []; + foreach ($implicitLoopPaginator as $item) { + $items[] = $item; + } + + $this->assertEquals([3, 6, 9, 12, 15, 18, 21, 24, 27, 30], $items); + } +} From 955ae00036f551b2b6cf7bc85e9a8672d441d68c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Fri, 22 Feb 2019 19:54:23 +0100 Subject: [PATCH 3/6] Updated changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5f6254b..b97580cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this #### Fixed * [#317](https://github.com/shlinkio/shlink/issues/317) Fixed error while trying to generate previews because of global config file being deleted by mistake by build script. +* [#307](https://github.com/shlinkio/shlink/issues/307) Fixed memory leak while trying to process huge amounts of visits due to the query not being properly paginated. ## 1.15.1 - 2018-12-16 From 091ea974eb483abddedb3aa87475c751766e6fa8 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 23 Feb 2019 07:29:07 +0100 Subject: [PATCH 4/6] Simplified implementation iterating unlocated visits --- .../Adapter/PaginableQueryAdapter.php | 34 ---------- .../src/Paginator/ImplicitLoopPaginator.php | 38 ----------- .../Paginator/ImplicitLoopPaginatorTest.php | 65 ------------------- .../Core/src/Repository/VisitRepository.php | 23 ++++--- 4 files changed, 13 insertions(+), 147 deletions(-) delete mode 100644 module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php delete mode 100644 module/Common/src/Paginator/ImplicitLoopPaginator.php delete mode 100644 module/Common/test/Paginator/ImplicitLoopPaginatorTest.php diff --git a/module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php b/module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php deleted file mode 100644 index d6de1ada..00000000 --- a/module/Common/src/Paginator/Adapter/PaginableQueryAdapter.php +++ /dev/null @@ -1,34 +0,0 @@ -query = $query; - $this->totalItems = $totalItems; - } - - public function getItems($offset, $itemCountPerPage): iterable - { - return $this->query - ->setMaxResults($itemCountPerPage) - ->setFirstResult($offset) - ->iterate(); - } - - public function count(): int - { - return $this->totalItems; - } -} diff --git a/module/Common/src/Paginator/ImplicitLoopPaginator.php b/module/Common/src/Paginator/ImplicitLoopPaginator.php deleted file mode 100644 index 9019f248..00000000 --- a/module/Common/src/Paginator/ImplicitLoopPaginator.php +++ /dev/null @@ -1,38 +0,0 @@ -paginator = $paginator; - $this->valueParser = $valueParser ?? function ($value) { - return $value; - }; - } - - public function getIterator(): iterable - { - $totalPages = $this->paginator->count(); - $processedPages = 0; - - do { - $processedPages++; - $this->paginator->setCurrentPageNumber($processedPages); - - foreach ($this->paginator as $key => $value) { - yield $key => ($this->valueParser)($value); - } - } while ($processedPages < $totalPages); - } -} diff --git a/module/Common/test/Paginator/ImplicitLoopPaginatorTest.php b/module/Common/test/Paginator/ImplicitLoopPaginatorTest.php deleted file mode 100644 index 0e494afb..00000000 --- a/module/Common/test/Paginator/ImplicitLoopPaginatorTest.php +++ /dev/null @@ -1,65 +0,0 @@ -paginator = new Paginator(new ArrayAdapter(range(1, self::TOTAL_ITEMS))); - } - - /** - * @test - * @dataProvider provideItemsPerPage - */ - public function allElementsAreIteratedRegardlessThePageSize(int $itemsPerPage): void - { - $this->paginator->setItemCountPerPage($itemsPerPage); - $implicitLoopPaginator = new ImplicitLoopPaginator($this->paginator); - - $iteratedItems = 0; - foreach ($implicitLoopPaginator as $item) { - $iteratedItems++; - } - - $this->assertEquals(self::TOTAL_ITEMS, $iteratedItems); - } - - public function provideItemsPerPage(): iterable - { - return map(range(1, 20), function (int $i) { - return [$i]; - }); - } - - /** @test */ - public function valuesWrappedInPaginatorAreProperlyParsed(): void - { - $valueParser = function (int $item) { - return $item * 3; - }; - $this->paginator->setItemCountPerPage(5); - $implicitLoopPaginator = new ImplicitLoopPaginator($this->paginator, $valueParser); - - $items = []; - foreach ($implicitLoopPaginator as $item) { - $items[] = $item; - } - - $this->assertEquals([3, 6, 9, 12, 15, 18, 21, 24, 27, 30], $items); - } -} diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index dfa5498f..9c65877c 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -5,12 +5,8 @@ namespace Shlinkio\Shlink\Core\Repository; use Doctrine\ORM\EntityRepository; use Doctrine\ORM\QueryBuilder; -use Shlinkio\Shlink\Common\Paginator\Adapter\PaginableQueryAdapter; -use Shlinkio\Shlink\Common\Paginator\ImplicitLoopPaginator; use Shlinkio\Shlink\Common\Util\DateRange; use Shlinkio\Shlink\Core\Entity\Visit; -use Zend\Paginator\Paginator; -use function array_shift; class VisitRepository extends EntityRepository implements VisitRepositoryInterface { @@ -19,18 +15,25 @@ class VisitRepository extends EntityRepository implements VisitRepositoryInterfa */ public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable { - $count = $this->count(['visitLocation' => null]); $dql = <<getEntityManager()->createQuery($dql); + $remainingVisitsToProcess = $this->count(['visitLocation' => null]); + $offset = 0; - $paginator = new Paginator(new PaginableQueryAdapter($query, $count)); - $paginator->setItemCountPerPage($blockSize); + while ($remainingVisitsToProcess > 0) { + $iterator = $query->setMaxResults($blockSize) + ->setFirstResult($offset) + ->iterate(); - return new ImplicitLoopPaginator($paginator, function (array $value) { - return array_shift($value); - }); + foreach ($iterator as $key => [$value]) { + yield $key => $value; + } + + $remainingVisitsToProcess -= $blockSize; + $offset += $blockSize; + } } /** From 62133c994ffa5002445cc18be06cdc2bb77fbf3c Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 23 Feb 2019 08:30:35 +0100 Subject: [PATCH 5/6] Tagged v1.16 in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b97580cb..4c252ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org). -## [Unreleased] +## 1.16.0 - 2019-02-23 #### Added From d2fad0128fb5ea338e078c6b3878189c1edde654 Mon Sep 17 00:00:00 2001 From: Alejandro Celaya Date: Sat, 23 Feb 2019 09:58:02 +0100 Subject: [PATCH 6/6] Fixed bug missing unprocessed visits while iterating and updating, while drastically improving the performance --- .../Core/src/Repository/VisitRepository.php | 16 +++++++----- .../Repository/VisitRepositoryInterface.php | 8 +++++- module/Core/src/Service/VisitService.php | 18 +++++++++---- .../Repository/VisitRepositoryTest.php | 2 +- module/Core/test/Service/VisitServiceTest.php | 25 +++++++++++-------- 5 files changed, 45 insertions(+), 24 deletions(-) diff --git a/module/Core/src/Repository/VisitRepository.php b/module/Core/src/Repository/VisitRepository.php index 9c65877c..06c7f252 100644 --- a/module/Core/src/Repository/VisitRepository.php +++ b/module/Core/src/Repository/VisitRepository.php @@ -11,22 +11,26 @@ use Shlinkio\Shlink\Core\Entity\Visit; class VisitRepository extends EntityRepository implements VisitRepositoryInterface { /** + * This method will allow you to iterate the whole list of unlocated visits, but loading them into memory in + * smaller blocks of a specific size. + * This will have side effects if you update those rows while you iterate them. + * If you plan to do so, pass the first argument as false in order to disable applying offsets while slicing the + * dataset + * * @return iterable|Visit[] */ - public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable + public function findUnlocatedVisits(bool $applyOffset = true, int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable { $dql = <<getEntityManager()->createQuery($dql); + $query = $this->getEntityManager()->createQuery($dql) + ->setMaxResults($blockSize); $remainingVisitsToProcess = $this->count(['visitLocation' => null]); $offset = 0; while ($remainingVisitsToProcess > 0) { - $iterator = $query->setMaxResults($blockSize) - ->setFirstResult($offset) - ->iterate(); - + $iterator = $query->setFirstResult($applyOffset ? $offset : null)->iterate(); foreach ($iterator as $key => [$value]) { yield $key => $value; } diff --git a/module/Core/src/Repository/VisitRepositoryInterface.php b/module/Core/src/Repository/VisitRepositoryInterface.php index 2307946a..585bb3ef 100644 --- a/module/Core/src/Repository/VisitRepositoryInterface.php +++ b/module/Core/src/Repository/VisitRepositoryInterface.php @@ -12,9 +12,15 @@ interface VisitRepositoryInterface extends ObjectRepository public const DEFAULT_BLOCK_SIZE = 10000; /** + * This method will allow you to iterate the whole list of unlocated visits, but loading them into memory in + * smaller blocks of a specific size. + * This will have side effects if you update those rows while you iterate them. + * If you plan to do so, pass the first argument as false in order to disable applying offsets while slicing the + * dataset + * * @return iterable|Visit[] */ - public function findUnlocatedVisits(int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable; + public function findUnlocatedVisits(bool $applyOffset = true, int $blockSize = self::DEFAULT_BLOCK_SIZE): iterable; /** * @return Visit[] diff --git a/module/Core/src/Service/VisitService.php b/module/Core/src/Service/VisitService.php index 78c0eddd..4abbf4a2 100644 --- a/module/Core/src/Service/VisitService.php +++ b/module/Core/src/Service/VisitService.php @@ -24,9 +24,12 @@ class VisitService implements VisitServiceInterface { /** @var VisitRepository $repo */ $repo = $this->em->getRepository(Visit::class); - $results = $repo->findUnlocatedVisits(); + $results = $repo->findUnlocatedVisits(false); + $count = 0; + $persistBlock = 200; foreach ($results as $visit) { + $count++; try { /** @var Location $location */ $location = $geolocateVisit($visit); @@ -37,20 +40,25 @@ class VisitService implements VisitServiceInterface $location = new VisitLocation($location); $this->locateVisit($visit, $location, $notifyVisitWithLocation); + + // Flush and clear after X iterations + if ($count % $persistBlock === 0) { + $this->em->flush(); + $this->em->clear(); + } } + + $this->em->flush(); + $this->em->clear(); } private function locateVisit(Visit $visit, VisitLocation $location, ?callable $notifyVisitWithLocation): void { $visit->locate($location); - $this->em->persist($visit); - $this->em->flush(); if ($notifyVisitWithLocation !== null) { $notifyVisitWithLocation($location, $visit); } - - $this->em->clear(); } } diff --git a/module/Core/test-db/Repository/VisitRepositoryTest.php b/module/Core/test-db/Repository/VisitRepositoryTest.php index 64702fcd..46bda817 100644 --- a/module/Core/test-db/Repository/VisitRepositoryTest.php +++ b/module/Core/test-db/Repository/VisitRepositoryTest.php @@ -55,7 +55,7 @@ class VisitRepositoryTest extends DatabaseTestCase $this->getEntityManager()->flush(); $resultsCount = 0; - $results = $this->repo->findUnlocatedVisits($blockSize); + $results = $this->repo->findUnlocatedVisits(true, $blockSize); foreach ($results as $value) { $resultsCount++; } diff --git a/module/Core/test/Service/VisitServiceTest.php b/module/Core/test/Service/VisitServiceTest.php index 4f249b23..be2bf8ba 100644 --- a/module/Core/test/Service/VisitServiceTest.php +++ b/module/Core/test/Service/VisitServiceTest.php @@ -17,7 +17,11 @@ use Shlinkio\Shlink\Core\Repository\VisitRepository; use Shlinkio\Shlink\Core\Service\VisitService; use function array_shift; use function count; +use function floor; use function func_get_args; +use function Functional\map; +use function range; +use function sprintf; class VisitServiceTest extends TestCase { @@ -35,13 +39,12 @@ class VisitServiceTest extends TestCase /** @test */ public function locateVisitsIteratesAndLocatesUnlocatedVisits(): void { - $unlocatedVisits = [ - new Visit(new ShortUrl('foo'), Visitor::emptyInstance()), - new Visit(new ShortUrl('bar'), Visitor::emptyInstance()), - ]; + $unlocatedVisits = map(range(1, 200), function (int $i) { + return new Visit(new ShortUrl(sprintf('short_code_%s', $i)), Visitor::emptyInstance()); + }); $repo = $this->prophesize(VisitRepository::class); - $findUnlocatedVisits = $repo->findUnlocatedVisits()->willReturn($unlocatedVisits); + $findUnlocatedVisits = $repo->findUnlocatedVisits(false)->willReturn($unlocatedVisits); $getRepo = $this->em->getRepository(Visit::class)->willReturn($repo->reveal()); $persist = $this->em->persist(Argument::type(Visit::class))->will(function () { @@ -63,19 +66,19 @@ class VisitServiceTest extends TestCase $findUnlocatedVisits->shouldHaveBeenCalledOnce(); $getRepo->shouldHaveBeenCalledOnce(); $persist->shouldHaveBeenCalledTimes(count($unlocatedVisits)); - $flush->shouldHaveBeenCalledTimes(count($unlocatedVisits)); - $clear->shouldHaveBeenCalledTimes(count($unlocatedVisits)); + $flush->shouldHaveBeenCalledTimes(floor(count($unlocatedVisits) / 200) + 1); + $clear->shouldHaveBeenCalledTimes(floor(count($unlocatedVisits) / 200) + 1); } /** @test */ - public function visitsWhichCannotBeLocatedAreIgnored() + public function visitsWhichCannotBeLocatedAreIgnored(): void { $unlocatedVisits = [ new Visit(new ShortUrl('foo'), Visitor::emptyInstance()), ]; $repo = $this->prophesize(VisitRepository::class); - $findUnlocatedVisits = $repo->findUnlocatedVisits()->willReturn($unlocatedVisits); + $findUnlocatedVisits = $repo->findUnlocatedVisits(false)->willReturn($unlocatedVisits); $getRepo = $this->em->getRepository(Visit::class)->willReturn($repo->reveal()); $persist = $this->em->persist(Argument::type(Visit::class))->will(function () { @@ -92,7 +95,7 @@ class VisitServiceTest extends TestCase $findUnlocatedVisits->shouldHaveBeenCalledOnce(); $getRepo->shouldHaveBeenCalledOnce(); $persist->shouldNotHaveBeenCalled(); - $flush->shouldNotHaveBeenCalled(); - $clear->shouldNotHaveBeenCalled(); + $flush->shouldHaveBeenCalledOnce(); + $clear->shouldHaveBeenCalledOnce(); } }