Added v2.7.1 to changelog

Feature/disable geolite download

.dockerignore

@@ -5,7 +5,7 @@ data/log/*
 data/locks/*
 data/proxies/*
 data/migrations_template.txt
-data/GeoLite2-City.*
+data/GeoLite2-City*
 data/database.sqlite
 data/shlink-tests.db
 CHANGELOG.md

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        php-version: ['7.4']
+        php-version: ['8.0']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -21,7 +21,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: none
       - run: composer install --no-interaction --prefer-dist
       - run: composer cs
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        php-version: ['7.4']
+        php-version: ['8.0']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -39,7 +39,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: none
       - run: composer install --no-interaction --prefer-dist
       - run: composer stan
@@ -57,7 +57,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: pcov
           ini-values: pcov.directory=module
       - run: composer install --no-interaction --prefer-dist
@@ -83,7 +83,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: pcov
           ini-values: pcov.directory=module
       - run: composer install --no-interaction --prefer-dist
@@ -111,7 +111,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: none
       - run: composer install --no-interaction --prefer-dist
       - run: composer test:db:mysql
@@ -131,7 +131,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: none
       - run: composer install --no-interaction --prefer-dist
       - run: composer test:db:maria
@@ -151,7 +151,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: none
       - run: composer install --no-interaction --prefer-dist
       - run: composer test:db:postgres
@@ -173,7 +173,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3, pdo_sqlsrv-5.9.0
+          extensions: swoole-4.6.7, pdo_sqlsrv-5.9.0
           coverage: none
       - run: composer install --no-interaction --prefer-dist
       - name: Create test database
@@ -195,7 +195,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: pcov
           ini-values: pcov.directory=module
       - run: composer install --no-interaction --prefer-dist
@@ -217,6 +217,7 @@ jobs:
     strategy:
       matrix:
         php-version: ['7.4', '8.0']
+        test-group: ['unit', 'db']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
@@ -225,14 +226,14 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
           coverage: pcov
           ini-values: pcov.directory=module
       - run: composer install --no-interaction --prefer-dist
       - uses: actions/download-artifact@v2
         with:
           path: build
-      - run: composer infect:ci
+      - run: composer infect:ci:${{ matrix.test-group }}
 
   upload-coverage:
     needs:
@@ -242,7 +243,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        php-version: ['7.4']
+        php-version: ['8.0']
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

.github/workflows/publish-release.yml

@@ -20,7 +20,7 @@ jobs:
         with:
           php-version: ${{ matrix.php-version }}
           tools: composer
-          extensions: swoole-4.6.3
+          extensions: swoole-4.6.7
       - if: ${{ matrix.swoole == 'yes' }}
         run: ./build.sh ${GITHUB_REF#refs/tags/v}
       - if: ${{ matrix.swoole == 'no' }}

.gitignore

@@ -6,8 +6,7 @@ composer.phar
 vendor/
 data/database.sqlite
 data/shlink-tests.db
-data/GeoLite2-City.mmdb
-data/GeoLite2-City.mmdb.*
+data/GeoLite2-City.*
 docs/swagger-ui*
 docs/mercure.html
 docker-compose.override.yml

CHANGELOG.md

@@ -4,6 +4,98 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com), and this project adheres to [Semantic Versioning](https://semver.org).
 
+## [2.7.1] - 2021-05-30
+### Added
+* *Nothing*
+
+### Changed
+* *Nothing*
+
+### Deprecated
+* *Nothing*
+
+### Removed
+* *Nothing*
+
+### Fixed
+* [#1100](https://github.com/shlinkio/shlink/issues/1100) Fixed Shlink trying to download GeoLite2 db files even when tracking has been disabled.
+
+
+## [2.7.0] - 2021-05-23
+### Added
+* [#1044](https://github.com/shlinkio/shlink/issues/1044) Added ability to set names on API keys, which helps to identify them when the list grows.
+* [#819](https://github.com/shlinkio/shlink/issues/819) Visits are now always located in real time, even when not using swoole.
+
+    The only side effect is that a GeoLite2 db file is now installed when the docker image starts or during shlink installation or update.
+
+    Also, when using swoole, the file is now updated **after** tracking a visit, which means it will not apply until the next one.
+
+* [#1059](https://github.com/shlinkio/shlink/issues/1059) Added ability to optionally display author API key and its name when listing short URLs from the command line.
+* [#1066](https://github.com/shlinkio/shlink/issues/1066) Added support to import short URLs and their visits from another Shlink instance using its API.
+* [#898](https://github.com/shlinkio/shlink/issues/898) Improved tracking granularity, allowing to disable visits tracking completely, or just parts of it.
+
+    In order to achieve it, Shlink now supports 4 new tracking-related options, that can be customized via env vars for docker, or via installer:
+
+    * `disable_tracking`: If true, visits will not be tracked at all.
+    * `disable_ip_tracking`: If true, visits will be tracked, but neither the IP address, nor the location will be resolved.
+    * `disable_referrer_tracking`: If true, the referrer will not be tracked.
+    * `disable_ua_tracking`: If true, the user agent will not be tracked.
+
+* [#955](https://github.com/shlinkio/shlink/issues/955) Added new option to set short URLs as crawlable, making them be listed in the robots.txt as Allowed.
+* [#900](https://github.com/shlinkio/shlink/issues/900) Shlink now tries to detect if the visit is coming from a potential bot or crawler, and allows to exclude those visits from visits lists if desired.
+
+### Changed
+* [#1036](https://github.com/shlinkio/shlink/issues/1036) Updated to `happyr/doctrine-specification` 2.0.
+* [#1039](https://github.com/shlinkio/shlink/issues/1039) Updated to `endroid/qr-code` 4.0.
+* [#1008](https://github.com/shlinkio/shlink/issues/1008) Ensured all logs are sent to the filesystem while running API tests, which helps debugging the reason for tests to fail.
+
+### Deprecated
+* *Nothing*
+
+### Removed
+* *Nothing*
+
+### Fixed
+* [#1041](https://github.com/shlinkio/shlink/issues/1041) Ensured the default value for the version while building the docker image is `latest`.
+* [#1067](https://github.com/shlinkio/shlink/issues/1067) Fixed exception when persisting multiple short URLs in one batch which include the same new tags/domains. This can potentially happen when importing URLs.
+
+
+## [2.6.2] - 2021-03-12
+### Added
+* *Nothing*
+
+### Changed
+* *Nothing*
+
+### Deprecated
+* *Nothing*
+
+### Removed
+* *Nothing*
+
+### Fixed
+* [#1047](https://github.com/shlinkio/shlink/issues/1047) Fixed error in migrations when doing a fresh installation using PHP8 and MySQL/Mariadb databases.
+
+
+## [2.6.1] - 2021-02-22
+### Added
+* *Nothing*
+
+### Changed
+* [#1026](https://github.com/shlinkio/shlink/issues/1026) Removed non-inclusive terms from source code.
+
+### Deprecated
+* *Nothing*
+
+### Removed
+* *Nothing*
+
+### Fixed
+* [#1024](https://github.com/shlinkio/shlink/issues/1024) Fixed migration that is incorrectly skipped due to the wrong condition being used to check it.
+* [#1031](https://github.com/shlinkio/shlink/issues/1031) Fixed shortening of twitter URLs with URL validation enabled.
+* [#1034](https://github.com/shlinkio/shlink/issues/1034) Fixed warning displayed when shlink is stopped while running it with swoole.
+
+
 ## [2.6.0] - 2021-02-13
 ### Added
 * [#856](https://github.com/shlinkio/shlink/issues/856) Added PHP 8.0 support.
@@ -22,8 +114,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com), and this
 
     This new orphan visits can be consumed in these ways:
 
-      * The `https://shlink.io/new-orphan-visit` mercure topic, which gets notified when an orphan visit occurs.
-      * The `GET /visits/orphan` REST endpoint, which behaves like the short URL visits and tags visits endpoints, but returns only orphan visits.
+    * The `https://shlink.io/new-orphan-visit` mercure topic, which gets notified when an orphan visit occurs.
+    * The `GET /visits/orphan` REST endpoint, which behaves like the short URL visits and tags visits endpoints, but returns only orphan visits.
 
 ### Changed
 * [#977](https://github.com/shlinkio/shlink/issues/977) Migrated from `laminas/laminas-paginator` to `pagerfanta/core` to handle pagination.

CONTRIBUTING.md

@@ -96,11 +96,13 @@ In order to ensure stability and no regressions are introduced while developing
 
     The project provides some tooling to run them against any of the supported database engines.
 
-* **API tests**: These are E2E tests that spin up an instance of the app and test it from the outside, by interacting with the REST API.
+* **API tests**: These are E2E tests that spin up an instance of the app with swoole, and test it from the outside by interacting with the REST API.
 
     These are the best tests to catch regressions, and to verify everything behaves as expected.
 
-    They use MySQL as the database engine, and include some fixtures that ensure the same data exists at the beginning of the execution.
+    They use Postgres as the database engine, and include some fixtures that ensure the same data exists at the beginning of the execution.
+
+    Since the app instance is run on a process different from the one running the tests, when a test fails it might not be obvious why. To help debugging that, the app will dump all its logs inside `data/log/api-tests`, where you will find the `shlink.log` and `access.log` files.
 
 * **CLI tests**: *TBD. Once included, its purpose will be the same as API tests, but running through the command line*
 
@@ -118,7 +120,7 @@ Depending on the kind of contribution, maybe not all kinds of tests are needed,
     
     For example, `test:db:postgres`.
 
-* Run `./indocker composer test:api` to run API E2E tests. For these, the MySQL database engine is used.
+* Run `./indocker composer test:api` to run API E2E tests. For these, the Postgres database engine is used.
 * Run `./indocker composer infect:test` ti run both unit and database tests (over sqlite) and then apply mutations to them with [infection](https://infection.github.io/).
 * Run `./indocker composer ci` to run all previous commands together. This command is run during the project's continuous integration.
 * Run `./indocker composer ci:parallel` to do the same as in previous case, but parallelizing non-conflicting tasks as much as possible.

Dockerfile

@@ -1,9 +1,10 @@
-FROM php:8.0.2-alpine3.13 as base
+FROM php:8.0.6-alpine3.13 as base
 
-ARG SHLINK_VERSION=2.5.2
+ARG SHLINK_VERSION=latest
 ENV SHLINK_VERSION ${SHLINK_VERSION}
-ENV SWOOLE_VERSION 4.6.3
+ENV SWOOLE_VERSION 4.6.7
 ENV PDO_SQLSRV_VERSION 5.9.0
+ENV MS_ODBC_SQL_VERSION 17.5.2.1
 ENV LC_ALL "C"
 
 WORKDIR /etc/shlink
@@ -30,13 +31,13 @@ RUN \
 
 # Install sqlsrv driver
 RUN if [ $(uname -m) == "x86_64" ]; then \
-      wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \
-      apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \
+      wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \
+      apk add --allow-untrusted msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \
       apk add --no-cache --virtual .phpize-deps ${PHPIZE_DEPS} unixodbc-dev && \
       pecl install pdo_sqlsrv-${PDO_SQLSRV_VERSION} && \
       docker-php-ext-enable pdo_sqlsrv && \
       apk del .phpize-deps && \
-      rm msodbcsql17_17.5.1.1-1_amd64.apk ; \
+      rm msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk ; \
     fi
 
 # Install swoole
@@ -69,6 +70,8 @@ EXPOSE 8080
 
 # Expose params config dir, since the user is expected to provide custom config from there
 VOLUME /etc/shlink/config/params
+# Expose data dir to allow persistent runtime data and SQLite db
+VOLUME /etc/shlink/data
 
 # Copy config specific for the image
 COPY docker/docker-entrypoint.sh docker-entrypoint.sh

README.md

@@ -33,7 +33,8 @@ The idea is that you can just generate a container using the image and provide t
 
 First, make sure the host where you are going to run shlink fulfills these requirements:
 
-* PHP 7.4 with JSON, curl, PDO, intl and gd extensions enabled (PHP 8.0 support is coming).
+* PHP 7.4 or 8.0
+* The next PHP extensions: json, curl, pdo, intl, gd and gmp.
     * apcu extension is recommended if you don't plan to use swoole.
     * xml extension is required if you want to generate QR codes in svg format.
 * MySQL, MariaDB, PostgreSQL, Microsoft SQL Server or SQLite.

bin/test/run-api-tests.sh

@@ -3,6 +3,8 @@ export APP_ENV=test
 export DB_DRIVER=postgres
 export TEST_ENV=api
 
+rm -rf data/log/api-tests
+
 # Try to stop server just in case it hanged in last execution
 vendor/bin/laminas mezzio:swoole:stop
 

composer.json

@@ -19,12 +19,13 @@
         "cakephp/chronos": "^2.0",
         "cocur/slugify": "^4.0",
         "doctrine/cache": "^1.9",
-        "doctrine/migrations": "^3.0.2",
-        "doctrine/orm": "^2.8",
-        "endroid/qr-code": "dev-master#0f1613a as 3.10",
+        "doctrine/migrations": "^3.1.1",
+        "doctrine/orm": "^2.8.4",
+        "endroid/qr-code": "^4.0",
         "geoip2/geoip2": "^2.9",
         "guzzlehttp/guzzle": "^7.0",
-        "happyr/doctrine-specification": "2.0.x-dev#cb116d3 as 2.0",
+        "happyr/doctrine-specification": "^2.0",
+        "jaybizzle/crawler-detect": "^1.2",
         "laminas/laminas-config": "^3.3",
         "laminas/laminas-config-aggregator": "^1.1",
         "laminas/laminas-diactoros": "^2.1.3",
@@ -33,11 +34,11 @@
         "laminas/laminas-stdlib": "^3.2",
         "lcobucci/jwt": "^4.0",
         "league/uri": "^6.2",
-        "lstrojny/functional-php": "^1.15",
+        "lstrojny/functional-php": "^1.17",
         "mezzio/mezzio": "^3.3",
         "mezzio/mezzio-fastroute": "^3.1",
         "mezzio/mezzio-problem-details": "^1.3",
-        "mezzio/mezzio-swoole": "^3.1",
+        "mezzio/mezzio-swoole": "^3.3",
         "monolog/monolog": "^2.0",
         "nikolaposa/monolog-factory": "^3.1",
         "ocramius/proxy-manager": "^2.11",
@@ -46,16 +47,16 @@
         "predis/predis": "^1.1",
         "pugx/shortid-php": "^0.7",
         "ramsey/uuid": "^3.9",
-        "shlinkio/shlink-common": "^3.5",
+        "shlinkio/shlink-common": "^3.7",
         "shlinkio/shlink-config": "^1.0",
         "shlinkio/shlink-event-dispatcher": "^2.1",
-        "shlinkio/shlink-importer": "^2.2",
-        "shlinkio/shlink-installer": "^5.4",
-        "shlinkio/shlink-ip-geolocation": "^1.5",
+        "shlinkio/shlink-importer": "^2.3",
+        "shlinkio/shlink-installer": "^6.0",
+        "shlinkio/shlink-ip-geolocation": "^2.0",
         "symfony/console": "^5.1",
         "symfony/filesystem": "^5.1",
         "symfony/lock": "^5.1",
-        "symfony/mercure": "^0.4.1",
+        "symfony/mercure": "^0.5.1",
         "symfony/process": "^5.1",
         "symfony/string": "^5.1"
     },
@@ -70,7 +71,7 @@
         "phpunit/phpunit": "^9.5",
         "roave/security-advisories": "dev-master",
         "shlinkio/php-coding-standard": "~2.1.1",
-        "shlinkio/shlink-test-utils": "^2.0",
+        "shlinkio/shlink-test-utils": "^2.1",
         "symfony/var-dumper": "^5.2",
         "veewee/composer-run-parallel": "^0.1.0"
     },
@@ -124,6 +125,7 @@
         ],
         "test:unit": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-php build/coverage-unit.cov --testdox",
         "test:unit:ci": "@test:unit --coverage-xml=build/coverage-unit/coverage-xml --log-junit=build/coverage-unit/junit.xml",
+        "test:unit:pretty": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-html build/coverage-unit-html",
         "test:db": "@parallel test:db:sqlite:ci test:db:mysql test:db:maria test:db:postgres test:db:ms",
         "test:db:sqlite": "APP_ENV=test php vendor/bin/phpunit --order-by=random --colors=always --testdox -c phpunit-db.xml",
         "test:db:sqlite:ci": "@test:db:sqlite --coverage-php build/coverage-db.cov --coverage-xml=build/coverage-db/coverage-xml --log-junit=build/coverage-db/junit.xml",
@@ -132,7 +134,6 @@
         "test:db:postgres": "DB_DRIVER=postgres composer test:db:sqlite",
         "test:db:ms": "DB_DRIVER=mssql composer test:db:sqlite",
         "test:api": "bin/test/run-api-tests.sh",
-        "test:unit:pretty": "@php vendor/bin/phpunit --order-by=random --colors=always --coverage-html build/coverage-unit-html",
         "infect:ci:base": "infection --threads=4 --log-verbosity=default --only-covered --skip-initial-tests",
         "infect:ci:unit": "@infect:ci:base --coverage=build/coverage-unit --min-msi=80",
         "infect:ci:db": "@infect:ci:base --coverage=build/coverage-db --min-msi=95 --configuration=infection-db.json",

config/autoload/app_options.global.php

@@ -7,7 +7,6 @@ return [
     'app_options' => [
         'name' => 'Shlink',
         'version' => '%SHLINK_VERSION%',
-        'disable_track_param' => null,
     ],
 
 ];

config/autoload/entity-manager.global.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace Shlinkio\Shlink\Common;
 
-use Happyr\DoctrineSpecification\EntitySpecificationRepository;
+use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository;
 
 return [
 

config/autoload/entity-manager.local.php.dist

@@ -2,14 +2,6 @@
 
 declare(strict_types=1);
 
-use function Shlinkio\Shlink\Common\env;
-
-// When running tests, any mysql-specific option can interfere with other drivers
-$driverOptions = env('APP_ENV') === 'test' ? [] : [
-    PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8',
-    PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true,
-];
-
 return [
 
     'entity_manager' => [
@@ -18,7 +10,6 @@ return [
             'password' => 'root',
             'driver' => 'pdo_mysql',
             'host' => 'shlink_db',
-            'driverOptions' => $driverOptions,
         ],
     ],
 

config/autoload/installer.global.php

@@ -2,7 +2,10 @@
 
 declare(strict_types=1);
 
+namespace Shlinkio\Shlink\CLI;
+
 use Shlinkio\Shlink\Installer\Config\Option;
+use Shlinkio\Shlink\Installer\Util\InstallationCommand;
 
 return [
 
@@ -24,7 +27,6 @@ return [
             Option\Redirect\BaseUrlRedirectConfigOption::class,
             Option\Redirect\InvalidShortUrlRedirectConfigOption::class,
             Option\Redirect\Regular404RedirectConfigOption::class,
-            Option\DisableTrackParamConfigOption::class,
             Option\Visit\CheckVisitsThresholdConfigOption::class,
             Option\Visit\VisitsThresholdConfigOption::class,
             Option\BasePathConfigOption::class,
@@ -37,19 +39,27 @@ return [
             Option\Mercure\MercureInternalUrlConfigOption::class,
             Option\Mercure\MercureJwtSecretConfigOption::class,
             Option\UrlShortener\GeoLiteLicenseKeyConfigOption::class,
-            Option\UrlShortener\IpAnonymizationConfigOption::class,
             Option\UrlShortener\RedirectStatusCodeConfigOption::class,
             Option\UrlShortener\RedirectCacheLifeTimeConfigOption::class,
             Option\UrlShortener\AutoResolveTitlesConfigOption::class,
-            Option\UrlShortener\OrphanVisitsTrackingConfigOption::class,
+            Option\Tracking\IpAnonymizationConfigOption::class,
+            Option\Tracking\OrphanVisitsTrackingConfigOption::class,
+            Option\Tracking\DisableTrackParamConfigOption::class,
+            Option\Tracking\DisableTrackingConfigOption::class,
+            Option\Tracking\DisableIpTrackingConfigOption::class,
+            Option\Tracking\DisableReferrerTrackingConfigOption::class,
+            Option\Tracking\DisableUaTrackingConfigOption::class,
         ],
 
         'installation_commands' => [
-            'db_create_schema' => [
-                'command' => 'bin/cli db:create',
+            InstallationCommand::DB_CREATE_SCHEMA => [
+                'command' => 'bin/cli ' . Command\Db\CreateDatabaseCommand::NAME,
             ],
-            'db_migrate' => [
-                'command' => 'bin/cli db:migrate',
+            InstallationCommand::DB_MIGRATE => [
+                'command' => 'bin/cli ' . Command\Db\MigrateDatabaseCommand::NAME,
+            ],
+            InstallationCommand::GEOLITE_DOWNLOAD_DB => [
+                'command' => 'bin/cli ' . Command\Visit\DownloadGeoLiteDbCommand::NAME,
             ],
         ],
     ],

config/autoload/mercure.global.php

@@ -4,8 +4,8 @@ declare(strict_types=1);
 
 use Laminas\ServiceManager\Proxy\LazyServiceFactory;
 use Shlinkio\Shlink\Common\Mercure\LcobucciJwtProvider;
-use Symfony\Component\Mercure\Publisher;
-use Symfony\Component\Mercure\PublisherInterface;
+use Symfony\Component\Mercure\Hub;
+use Symfony\Component\Mercure\HubInterface;
 
 return [
 
@@ -21,14 +21,14 @@ return [
             LcobucciJwtProvider::class => [
                 LazyServiceFactory::class,
             ],
-            Publisher::class => [
+            Hub::class => [
                 LazyServiceFactory::class,
             ],
         ],
         'lazy_services' => [
             'class_map' => [
                 LcobucciJwtProvider::class => LcobucciJwtProvider::class,
-                Publisher::class => PublisherInterface::class,
+                Hub::class => HubInterface::class,
             ],
         ],
     ],

config/autoload/tracking.global.php

@@ -0,0 +1,31 @@
+<?php
+
+declare(strict_types=1);
+
+return [
+
+    'tracking' => [
+        // Tells if IP addresses should be anonymized before persisting, to fulfil data protection regulations
+        // This applies only if IP address tracking is enabled
+        'anonymize_remote_addr' => true,
+
+        // Tells if visits to not-found URLs should be tracked. The disable_tracking option takes precedence
+        'track_orphan_visits' => true,
+
+        // A query param that, if provided, will disable tracking of one particular visit. Always takes precedence
+        'disable_track_param' => null,
+
+        // If true, visits will not be tracked at all
+        'disable_tracking' => false,
+
+        // If true, visits will be tracked, but neither the IP address, nor the location will be resolved
+        'disable_ip_tracking' => false,
+
+        // If true, the referrer will not be tracked
+        'disable_referrer_tracking' => false,
+
+        // If true, the user agent will not be tracked
+        'disable_ua_tracking' => false,
+    ],
+
+];

config/autoload/url-shortener.global.php

@@ -14,13 +14,11 @@ return [
             'hostname' => '',
         ],
         'validate_url' => false, // Deprecated
-        'anonymize_remote_addr' => true,
         'visits_webhooks' => [],
         'default_short_codes_length' => DEFAULT_SHORT_CODES_LENGTH,
         'redirect_status_code' => DEFAULT_REDIRECT_STATUS_CODE,
         'redirect_cache_lifetime' => DEFAULT_REDIRECT_CACHE_LIFETIME,
         'auto_resolve_titles' => false,
-        'track_orphan_visits' => true,
     ],
 
 ];

config/test/test_config.global.php

@@ -9,7 +9,8 @@ use Laminas\ConfigAggregator\ConfigAggregator;
 use Laminas\Diactoros\Response\EmptyResponse;
 use Laminas\ServiceManager\Factory\InvokableFactory;
 use Laminas\Stdlib\Glob;
-use PDO;
+use Monolog\Handler\StreamHandler;
+use Monolog\Logger;
 use PHPUnit\Runner\Version;
 use SebastianBergmann\CodeCoverage\CodeCoverage;
 use SebastianBergmann\CodeCoverage\Driver\Selector;
@@ -53,10 +54,6 @@ $buildDbConnection = function (): array {
             'password' => 'root',
             'dbname' => 'shlink_test',
             'charset' => 'utf8',
-            'driverOptions' => [
-                PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8',
-                PDO::MYSQL_ATTR_USE_BUFFERED_QUERY => true,
-            ],
         ],
         'postgres' => [
             'driver' => 'pdo_pgsql',
@@ -80,6 +77,18 @@ $buildDbConnection = function (): array {
     return $driverConfigMap[$driver] ?? [];
 };
 
+$buildTestLoggerConfig = fn (string $handlerName, string $filename) => [
+    'handlers' => [
+        $handlerName => [
+            'name' => StreamHandler::class,
+            'params' => [
+                'level' => Logger::DEBUG,
+                'stream' => sprintf('data/log/api-tests/%s', $filename),
+            ],
+        ],
+    ],
+];
+
 return [
 
     'debug' => true,
@@ -163,4 +172,9 @@ return [
         ],
     ],
 
+    'logger' => [
+        'Shlink' => $buildTestLoggerConfig('shlink_handler', 'shlink.log'),
+        'Access' => $buildTestLoggerConfig('access_handler', 'access.log'),
+    ],
+
 ];

data/infra/php.Dockerfile

@@ -1,8 +1,9 @@
-FROM php:8.0.2-fpm-alpine3.13
+FROM php:8.0.6-fpm-alpine3.13
 MAINTAINER Alejandro Celaya <alejandro@alejandrocelaya.com>
 
 ENV APCU_VERSION 5.1.19
 ENV PDO_SQLSRV_VERSION 5.9.0
+ENV MS_ODBC_SQL_VERSION 17.5.2.1
 
 RUN apk update
 
@@ -44,13 +45,13 @@ RUN mkdir -p /usr/src/php/ext/apcu \
   && echo extension=apcu.so > /usr/local/etc/php/conf.d/20-php-ext-apcu.ini
 
 # Install pcov and sqlsrv driver
-RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \
-    apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \
+RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \
+    apk add --allow-untrusted msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \
     apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \
     pecl install pdo_sqlsrv-${PDO_SQLSRV_VERSION} pcov && \
     docker-php-ext-enable pdo_sqlsrv pcov && \
     apk del .phpize-deps && \
-    rm msodbcsql17_17.5.1.1-1_amd64.apk
+    rm msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk
 
 # Install composer
 COPY --from=composer:2 /usr/bin/composer /usr/local/bin/composer

data/infra/swoole.Dockerfile

@@ -1,10 +1,11 @@
-FROM php:8.0.2-alpine3.13
+FROM php:8.0.6-alpine3.13
 MAINTAINER Alejandro Celaya <alejandro@alejandrocelaya.com>
 
 ENV APCU_VERSION 5.1.19
 ENV PDO_SQLSRV_VERSION 5.9.0
 ENV INOTIFY_VERSION 3.0.0
-ENV SWOOLE_VERSION 4.6.3
+ENV SWOOLE_VERSION 4.6.7
+ENV MS_ODBC_SQL_VERSION 17.5.2.1
 
 RUN apk update
 
@@ -54,13 +55,13 @@ RUN mkdir -p /usr/src/php/ext/inotify \
   && rm /tmp/inotify.tar.gz
 
 # Install swoole, pcov and mssql driver
-RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_17.5.1.1-1_amd64.apk && \
-    apk add --allow-untrusted msodbcsql17_17.5.1.1-1_amd64.apk && \
+RUN wget https://download.microsoft.com/download/e/4/e/e4e67866-dffd-428c-aac7-8d28ddafb39b/msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \
+    apk add --allow-untrusted msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk && \
     apk add --no-cache --virtual .phpize-deps $PHPIZE_DEPS unixodbc-dev && \
     pecl install swoole-${SWOOLE_VERSION} pdo_sqlsrv-${PDO_SQLSRV_VERSION} pcov && \
     docker-php-ext-enable swoole pdo_sqlsrv pcov && \
     apk del .phpize-deps && \
-    rm msodbcsql17_17.5.1.1-1_amd64.apk
+    rm msodbcsql17_${MS_ODBC_SQL_VERSION}-1_amd64.apk
 
 # Install composer
 COPY --from=composer:2 /usr/bin/composer /usr/local/bin/composer

data/migrations/Version20160819142757.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace ShlinkMigrations;
 
-use Doctrine\DBAL\DBALException;
+use Doctrine\DBAL\Exception;
 use Doctrine\DBAL\Schema\Schema;
 use Doctrine\DBAL\Schema\SchemaException;
 use Doctrine\Migrations\AbstractMigration;
@@ -18,7 +18,7 @@ class Version20160819142757 extends AbstractMigration
     private const SQLITE = 'sqlite';
 
     /**
-     * @throws DBALException
+     * @throws Exception
      * @throws SchemaException
      */
     public function up(Schema $schema): void
@@ -35,7 +35,7 @@ class Version20160819142757 extends AbstractMigration
     }
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function down(Schema $schema): void
     {

data/migrations/Version20180913205455.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace ShlinkMigrations;
 
-use Doctrine\DBAL\DBALException;
+use Doctrine\DBAL\Exception;
 use Doctrine\DBAL\Schema\Schema;
 use Doctrine\Migrations\AbstractMigration;
 use PDO;
@@ -16,15 +16,13 @@ use Shlinkio\Shlink\Common\Util\IpAddress;
  */
 final class Version20180913205455 extends AbstractMigration
 {
-    /**
-     */
     public function up(Schema $schema): void
     {
         // Nothing to create
     }
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function postUp(Schema $schema): void
     {
@@ -64,8 +62,6 @@ final class Version20180913205455 extends AbstractMigration
         }
     }
 
-    /**
-     */
     public function down(Schema $schema): void
     {
         // Nothing to rollback

data/migrations/Version20181020060559.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace ShlinkMigrations;
 
-use Doctrine\DBAL\DBALException;
+use Doctrine\DBAL\Exception;
 use Doctrine\DBAL\Schema\Schema;
 use Doctrine\DBAL\Schema\SchemaException;
 use Doctrine\DBAL\Schema\Table;
@@ -42,7 +42,7 @@ final class Version20181020060559 extends AbstractMigration
 
     /**
      * @throws SchemaException
-     * @throws DBALException
+     * @throws Exception
      */
     public function postUp(Schema $schema): void
     {

data/migrations/Version20200105165647.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace ShlinkMigrations;
 
-use Doctrine\DBAL\DBALException;
+use Doctrine\DBAL\Exception;
 use Doctrine\DBAL\Schema\Schema;
 use Doctrine\DBAL\Types\Types;
 use Doctrine\Migrations\AbstractMigration;
@@ -16,7 +16,7 @@ final class Version20200105165647 extends AbstractMigration
     private const COLUMNS = ['lat' => 'latitude', 'lon' => 'longitude'];
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function preUp(Schema $schema): void
     {
@@ -43,7 +43,7 @@ final class Version20200105165647 extends AbstractMigration
     }
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function up(Schema $schema): void
     {
@@ -57,7 +57,7 @@ final class Version20200105165647 extends AbstractMigration
     }
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function postUp(Schema $schema): void
     {
@@ -83,7 +83,7 @@ final class Version20200105165647 extends AbstractMigration
     }
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function down(Schema $schema): void
     {

data/migrations/Version20200106215144.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace ShlinkMigrations;
 
-use Doctrine\DBAL\DBALException;
+use Doctrine\DBAL\Exception;
 use Doctrine\DBAL\Schema\Schema;
 use Doctrine\DBAL\Types\Types;
 use Doctrine\Migrations\AbstractMigration;
@@ -16,7 +16,7 @@ final class Version20200106215144 extends AbstractMigration
     private const COLUMNS = ['latitude', 'longitude'];
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function up(Schema $schema): void
     {
@@ -32,7 +32,7 @@ final class Version20200106215144 extends AbstractMigration
     }
 
     /**
-     * @throws DBALException
+     * @throws Exception
      */
     public function down(Schema $schema): void
     {

data/migrations/Version20210202181026.php

@@ -33,12 +33,4 @@ final class Version20210202181026 extends AbstractMigration
         $shortUrls->dropColumn(self::TITLE);
         $shortUrls->dropColumn('title_was_auto_resolved');
     }
-
-    /**
-     * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104
-     */
-    public function isTransactional(): bool
-    {
-        return false;
-    }
 }

data/migrations/Version20210207100807.php

@@ -15,10 +15,9 @@ final class Version20210207100807 extends AbstractMigration
     public function up(Schema $schema): void
     {
         $visits = $schema->getTable('visits');
+        $this->skipIf($visits->hasColumn('visited_url'));
+
         $shortUrlId = $visits->getColumn('short_url_id');
-
-        $this->skipIf(! $shortUrlId->getNotnull());
-
         $shortUrlId->setNotnull(false);
 
         $visits->addColumn('visited_url', Types::STRING, [
@@ -34,20 +33,11 @@ final class Version20210207100807 extends AbstractMigration
     public function down(Schema $schema): void
     {
         $visits = $schema->getTable('visits');
+        $this->skipIf(! $visits->hasColumn('visited_url'));
+
         $shortUrlId = $visits->getColumn('short_url_id');
-
-        $this->skipIf($shortUrlId->getNotnull());
-
         $shortUrlId->setNotnull(true);
         $visits->dropColumn('visited_url');
         $visits->dropColumn('type');
     }
-
-    /**
-     * @fixme Workaround for https://github.com/doctrine/migrations/issues/1104
-     */
-    public function isTransactional(): bool
-    {
-        return false;
-    }
 }

data/migrations/Version20210306165711.php

@@ -0,0 +1,37 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Types;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20210306165711 extends AbstractMigration
+{
+    private const TABLE = 'api_keys';
+    private const COLUMN = 'name';
+
+    public function up(Schema $schema): void
+    {
+        $apiKeys = $schema->getTable(self::TABLE);
+        $this->skipIf($apiKeys->hasColumn(self::COLUMN));
+
+        $apiKeys->addColumn(
+            self::COLUMN,
+            Types::STRING,
+            [
+                'notnull' => false,
+            ],
+        );
+    }
+
+    public function down(Schema $schema): void
+    {
+        $apiKeys = $schema->getTable(self::TABLE);
+        $this->skipIf(! $apiKeys->hasColumn(self::COLUMN));
+
+        $apiKeys->dropColumn(self::COLUMN);
+    }
+}

data/migrations/Version20210522051601.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Types;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20210522051601 extends AbstractMigration
+{
+    public function up(Schema $schema): void
+    {
+        $shortUrls = $schema->getTable('short_urls');
+        $this->skipIf($shortUrls->hasColumn('crawlable'));
+        $shortUrls->addColumn('crawlable', Types::BOOLEAN, ['default' => false]);
+    }
+
+    public function down(Schema $schema): void
+    {
+        $shortUrls = $schema->getTable('short_urls');
+        $this->skipIf(! $shortUrls->hasColumn('crawlable'));
+        $shortUrls->dropColumn('crawlable');
+    }
+}

data/migrations/Version20210522124633.php

@@ -0,0 +1,28 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\DBAL\Types\Types;
+use Doctrine\Migrations\AbstractMigration;
+
+final class Version20210522124633 extends AbstractMigration
+{
+    private const POTENTIAL_BOT_COLUMN = 'potential_bot';
+
+    public function up(Schema $schema): void
+    {
+        $visits = $schema->getTable('visits');
+        $this->skipIf($visits->hasColumn(self::POTENTIAL_BOT_COLUMN));
+        $visits->addColumn(self::POTENTIAL_BOT_COLUMN, Types::BOOLEAN, ['default' => false]);
+    }
+
+    public function down(Schema $schema): void
+    {
+        $visits = $schema->getTable('visits');
+        $this->skipIf(! $visits->hasColumn(self::POTENTIAL_BOT_COLUMN));
+        $visits->dropColumn(self::POTENTIAL_BOT_COLUMN);
+    }
+}

docker/config/shlink_in_docker.local.php

@@ -42,12 +42,6 @@ $helper = new class {
             ];
         }
 
-        $driverOptions = ! $isMysql ? [] : [
-            // 1002 -> PDO::MYSQL_ATTR_INIT_COMMAND
-            1002 => 'SET NAMES utf8',
-            // 1000 -> PDO::MYSQL_ATTR_USE_BUFFERED_QUERY
-            1000 => true,
-        ];
         return [
             'driver' => self::DB_DRIVERS_MAP[$driver],
             'dbname' => env('DB_NAME', 'shlink'),
@@ -55,7 +49,6 @@ $helper = new class {
             'password' => env('DB_PASSWORD'),
             'host' => env('DB_HOST', $driver === 'postgres' ? env('DB_UNIX_SOCKET') : null),
             'port' => env('DB_PORT', self::DB_PORTS_MAP[$driver]),
-            'driverOptions' => $driverOptions,
             'unix_socket' => $isMysql ? env('DB_UNIX_SOCKET') : null,
         ];
     }
@@ -101,10 +94,6 @@ $helper = new class {
 
 return [
 
-    'app_options' => [
-        'disable_track_param' => env('DISABLE_TRACK_PARAM'),
-    ],
-
     'delete_short_urls' => [
         'check_visits_threshold' => true,
         'visits_threshold' => (int) env('DELETE_SHORT_URL_THRESHOLD', DEFAULT_DELETE_SHORT_URL_THRESHOLD),
@@ -120,13 +109,21 @@ return [
             'hostname' => env('SHORT_DOMAIN_HOST', ''),
         ],
         'validate_url' => (bool) env('VALIDATE_URLS', false),
-        'anonymize_remote_addr' => (bool) env('ANONYMIZE_REMOTE_ADDR', true),
         'visits_webhooks' => $helper->getVisitsWebhooks(),
         'default_short_codes_length' => $helper->getDefaultShortCodesLength(),
         'redirect_status_code' => (int) env('REDIRECT_STATUS_CODE', DEFAULT_REDIRECT_STATUS_CODE),
         'redirect_cache_lifetime' => (int) env('REDIRECT_CACHE_LIFETIME', DEFAULT_REDIRECT_CACHE_LIFETIME),
         'auto_resolve_titles' => (bool) env('AUTO_RESOLVE_TITLES', false),
+    ],
+
+    'tracking' => [
+        'anonymize_remote_addr' => (bool) env('ANONYMIZE_REMOTE_ADDR', true),
         'track_orphan_visits' => (bool) env('TRACK_ORPHAN_VISITS', true),
+        'disable_track_param' => env('DISABLE_TRACK_PARAM'),
+        'disable_tracking' => (bool) env('DISABLE_TRACKING', false),
+        'disable_ip_tracking' => (bool) env('DISABLE_IP_TRACKING', false),
+        'disable_referrer_tracking' => (bool) env('DISABLE_REFERRER_TRACKING', false),
+        'disable_ua_tracking' => (bool) env('DISABLE_UA_TRACKING', false),
     ],
 
     'not_found_redirects' => $helper->getNotFoundRedirectsConfig(),
@@ -170,7 +167,7 @@ return [
     ],
 
     'geolite2' => [
-        'license_key' => env('GEOLITE_LICENSE_KEY', 'G4Lm0C60yJsnkdPi'),
+        'license_key' => env('GEOLITE_LICENSE_KEY', 'G4Lm0C60yJsnkdPi'), // Deprecated. Remove hardcoded license on v3
     ],
 
     'mercure' => $helper->getMercureConfig(),

docker/docker-entrypoint.sh

@@ -15,6 +15,12 @@ php vendor/doctrine/orm/bin/doctrine.php orm:generate-proxies -n -q
 echo "Clearing entities cache..."
 php vendor/doctrine/orm/bin/doctrine.php orm:clear-cache:metadata -n -q
 
+# Try to download GeoLite2 db file only if the license key env var was defined
+if [ ! -z "${GEOLITE_LICENSE_KEY}" ]; then
+  echo "Downloading GeoLite2 db file..."
+  php bin/cli visit:download-db -n -q
+fi
+
 # When restarting the container, swoole might think it is already in execution
 # This forces the app to be started every second until the exit code is 0
 until php vendor/bin/laminas mezzio:swoole:start; do sleep 1 ; done

docs/async-api/async-api.json

@@ -116,6 +116,15 @@
                     "domain": {
                         "type": "string",
                         "description": "The domain in which the short URL was created. Null if it belongs to default domain."
+                    },
+                    "title": {
+                        "type": "string",
+                        "nullable": true,
+                        "description": "A descriptive title of the short URL."
+                    },
+                    "crawlable": {
+                        "type": "boolean",
+                        "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
                     }
                 },
                 "example": {
@@ -133,7 +142,9 @@
                         "validUntil": null,
                         "maxVisits": 100
                     },
-                    "domain": "example.com"
+                    "domain": "example.com",
+                    "title": "The title",
+                    "crawlable": false
                 }
             },
             "ShortUrlMeta": {
@@ -179,6 +190,10 @@
                     },
                     "visitLocation": {
                         "$ref": "#/components/schemas/VisitLocation"
+                    },
+                    "potentialBot": {
+                        "type": "boolean",
+                        "description": "Tells if Shlink thinks this visit comes potentially from a bot or crawler"
                     }
                 },
                 "example": {
@@ -193,7 +208,8 @@
                         "longitude": -122.0946,
                         "regionName": "California",
                         "timezone": "America/Los_Angeles"
-                    }
+                    },
+                    "potentialBot": false
                 }
             },
             "OrphanVisit": {
@@ -232,6 +248,7 @@
                         "regionName": "California",
                         "timezone": "America/Los_Angeles"
                     },
+                    "potentialBot": false,
                     "visitedUrl": "https://doma.in",
                     "type": "base_url"
                 }

docs/swagger/definitions/ShortUrl.json

@@ -41,6 +41,10 @@
             "type": "string",
             "nullable": true,
             "description": "A descriptive title of the short URL."
+        },
+        "crawlable": {
+            "type": "boolean",
+            "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
         }
     }
 }

docs/swagger/definitions/Visit.json

@@ -17,6 +17,10 @@
         },
         "visitLocation": {
             "$ref": "./VisitLocation.json"
+        },
+        "potentialBot": {
+            "type": "boolean",
+            "description": "Tells if Shlink thinks this visit comes potentially from a bot or crawler"
         }
     }
 }

docs/swagger/paths/v1_short-urls.json

@@ -140,7 +140,8 @@
                                         "maxVisits": 100
                                     },
                                     "domain": null,
-                                    "title": "Welcome to Steam"
+                                    "title": "Welcome to Steam",
+                                    "crawlable": false
                                 },
                                 {
                                     "shortCode": "12Kb3",
@@ -157,7 +158,8 @@
                                         "maxVisits": null
                                     },
                                     "domain":  null,
-                                    "title": null
+                                    "title": null,
+                                    "crawlable": false
                                 },
                                 {
                                     "shortCode": "123bA",
@@ -172,7 +174,8 @@
                                         "maxVisits": null
                                     },
                                     "domain":  "example.com",
-                                    "title": null
+                                    "title": null,
+                                    "crawlable": false
                                 }
                             ],
                             "pagination": {
@@ -273,6 +276,10 @@
                             "title": {
                                 "type": "string",
                                 "description": "A descriptive title of the short URL."
+                            },
+                            "crawlable": {
+                                "type": "boolean",
+                                "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
                             }
                         }
                     }
@@ -305,7 +312,9 @@
                             "validUntil": null,
                             "maxVisits": 500
                         },
-                        "domain": null
+                        "domain": null,
+                        "title": null,
+                        "crawlable": false
                     }
                 }
             },

docs/swagger/paths/v1_short-urls_shorten.json

@@ -74,7 +74,8 @@
                             "maxVisits": 100
                         },
                         "domain": null,
-                        "title": null
+                        "title": null,
+                        "crawlable": false
                     },
                     "text/plain": "https://doma.in/abc123"
                 }

docs/swagger/paths/v1_short-urls_{shortCode}.json

@@ -54,7 +54,8 @@
                             "maxVisits": 100
                         },
                         "domain": null,
-                        "title": null
+                        "title": null,
+                        "crawlable": false
                     }
                 }
             },
@@ -147,6 +148,10 @@
                                 "type": "string",
                                 "description": "A descriptive title of the short URL.",
                                 "nullable": true
+                            },
+                            "crawlable": {
+                                "type": "boolean",
+                                "description": "Tells if this URL will be included as 'Allow' in Shlink's robots.txt."
                             }
                         }
                     }
@@ -184,7 +189,8 @@
                             "maxVisits": 100
                         },
                         "domain": null,
-                        "title": "Shlink - The URL shortener"
+                        "title": "Shlink - The URL shortener",
+                        "crawlable": false
                     }
                 }
             },

docs/swagger/paths/v1_short-urls_{shortCode}_visits.json

@@ -57,6 +57,16 @@
                 "schema": {
                     "type": "number"
                 }
+            },
+            {
+                "name": "excludeBots",
+                "in": "query",
+                "description": "Tells if visits from potential bots should be excluded from the result set",
+                "required": false,
+                "schema": {
+                    "type": "string",
+                    "enum": ["true"]
+                }
             }
         ],
         "security": [
@@ -98,7 +108,8 @@
                                     "referer": "https://twitter.com",
                                     "date": "2015-08-20T05:05:03+04:00",
                                     "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0",
-                                    "visitLocation": null
+                                    "visitLocation": null,
+                                    "potentialBot": false
                                 },
                                 {
                                     "referer": "https://t.co",
@@ -112,13 +123,15 @@
                                         "longitude": -122.0946,
                                         "regionName": "California",
                                         "timezone": "America/Los_Angeles"
-                                    }
+                                    },
+                                    "potentialBot": false
                                 },
                                 {
                                     "referer": null,
                                     "date": "2015-08-20T05:05:03+04:00",
                                     "userAgent": "some_web_crawler/1.4",
-                                    "visitLocation": null
+                                    "visitLocation": null,
+                                    "potentialBot": true
                                 }
                             ],
                             "pagination": {

docs/swagger/paths/v2_tags_{tag}_visits.json

@@ -54,6 +54,16 @@
                 "schema": {
                     "type": "number"
                 }
+            },
+            {
+                "name": "excludeBots",
+                "in": "query",
+                "description": "Tells if visits from potential bots should be excluded from the result set",
+                "required": false,
+                "schema": {
+                    "type": "string",
+                    "enum": ["true"]
+                }
             }
         ],
         "security": [
@@ -95,7 +105,8 @@
                                     "referer": "https://twitter.com",
                                     "date": "2015-08-20T05:05:03+04:00",
                                     "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0",
-                                    "visitLocation": null
+                                    "visitLocation": null,
+                                    "potentialBot": false
                                 },
                                 {
                                     "referer": "https://t.co",
@@ -109,13 +120,15 @@
                                         "longitude": -122.0946,
                                         "regionName": "California",
                                         "timezone": "America/Los_Angeles"
-                                    }
+                                    },
+                                    "potentialBot": false
                                 },
                                 {
                                     "referer": null,
                                     "date": "2015-08-20T05:05:03+04:00",
                                     "userAgent": "some_web_crawler/1.4",
-                                    "visitLocation": null
+                                    "visitLocation": null,
+                                    "potentialBot": true
                                 }
                             ],
                             "pagination": {

docs/swagger/paths/v2_visits_orphan.json

@@ -45,6 +45,16 @@
                 "schema": {
                     "type": "number"
                 }
+            },
+            {
+                "name": "excludeBots",
+                "in": "query",
+                "description": "Tells if visits from potential bots should be excluded from the result set",
+                "required": false,
+                "schema": {
+                    "type": "string",
+                    "enum": ["true"]
+                }
             }
         ],
         "security": [
@@ -87,6 +97,7 @@
                                     "date": "2015-08-20T05:05:03+04:00",
                                     "userAgent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0 Mozilla/5.0 (Macintosh; Intel Mac OS X x.y; rv:42.0) Gecko/20100101 Firefox/42.0",
                                     "visitLocation": null,
+                                    "potentialBot": false,
                                     "visitedUrl": "https://doma.in",
                                     "type": "base_url"
                                 },
@@ -103,6 +114,7 @@
                                         "regionName": "California",
                                         "timezone": "America/Los_Angeles"
                                     },
+                                    "potentialBot": false,
                                     "visitedUrl": "https://doma.in/foo",
                                     "type": "invalid_short_url"
                                 },
@@ -111,6 +123,7 @@
                                     "date": "2015-08-20T05:05:03+04:00",
                                     "userAgent": "some_web_crawler/1.4",
                                     "visitLocation": null,
+                                    "potentialBot": true,
                                     "visitedUrl": "https://doma.in/foo/bar/baz",
                                     "type": "regular_404"
                                 }

module/CLI/config/cli.config.php

@@ -15,6 +15,7 @@ return [
             Command\ShortUrl\DeleteShortUrlCommand::NAME => Command\ShortUrl\DeleteShortUrlCommand::class,
 
             Command\Visit\LocateVisitsCommand::NAME => Command\Visit\LocateVisitsCommand::class,
+            Command\Visit\DownloadGeoLiteDbCommand::NAME => Command\Visit\DownloadGeoLiteDbCommand::class,
 
             Command\Api\GenerateKeyCommand::NAME => Command\Api\GenerateKeyCommand::class,
             Command\Api\DisableKeyCommand::NAME => Command\Api\DisableKeyCommand::class,

module/CLI/config/dependencies.config.php

@@ -10,6 +10,7 @@ use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory;
 use Laminas\ServiceManager\Factory\InvokableFactory;
 use Shlinkio\Shlink\Common\Doctrine\NoDbNameConnectionFactory;
 use Shlinkio\Shlink\Core\Domain\DomainService;
+use Shlinkio\Shlink\Core\Options\TrackingOptions;
 use Shlinkio\Shlink\Core\Service;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifier;
 use Shlinkio\Shlink\Core\ShortUrl\Transformer\ShortUrlDataTransformer;
@@ -44,6 +45,7 @@ return [
             Command\ShortUrl\GetVisitsCommand::class => ConfigAbstractFactory::class,
             Command\ShortUrl\DeleteShortUrlCommand::class => ConfigAbstractFactory::class,
 
+            Command\Visit\DownloadGeoLiteDbCommand::class => ConfigAbstractFactory::class,
             Command\Visit\LocateVisitsCommand::class => ConfigAbstractFactory::class,
 
             Command\Api\GenerateKeyCommand::class => ConfigAbstractFactory::class,
@@ -63,7 +65,12 @@ return [
     ],
 
     ConfigAbstractFactory::class => [
-        Util\GeolocationDbUpdater::class => [DbUpdater::class, Reader::class, LOCAL_LOCK_FACTORY],
+        Util\GeolocationDbUpdater::class => [
+            DbUpdater::class,
+            Reader::class,
+            LOCAL_LOCK_FACTORY,
+            TrackingOptions::class,
+        ],
         Util\ProcessRunner::class => [SymfonyCli\Helper\ProcessHelper::class],
         ApiKey\RoleResolver::class => [DomainService::class],
 
@@ -80,11 +87,11 @@ return [
         Command\ShortUrl\GetVisitsCommand::class => [Visit\VisitsStatsHelper::class],
         Command\ShortUrl\DeleteShortUrlCommand::class => [Service\ShortUrl\DeleteShortUrlService::class],
 
+        Command\Visit\DownloadGeoLiteDbCommand::class => [Util\GeolocationDbUpdater::class],
         Command\Visit\LocateVisitsCommand::class => [
             Visit\VisitLocator::class,
             IpLocationResolverInterface::class,
             LockFactory::class,
-            Util\GeolocationDbUpdater::class,
         ],
 
         Command\Api\GenerateKeyCommand::class => [ApiKeyService::class, ApiKey\RoleResolver::class],

module/CLI/src/Command/Api/GenerateKeyCommand.php

@@ -42,6 +42,10 @@ class GenerateKeyCommand extends BaseCommand
 
             <info>%command.full_name%</info>
 
+        You can optionally set its name for tracking purposes with <comment>--name</comment> or <comment>-m</comment>:
+
+            <info>%command.full_name% --name Alice</info>
+
         You can optionally set its expiration date with <comment>--expiration-date</comment> or <comment>-e</comment>:
 
             <info>%command.full_name% --expiration-date 2020-01-01</info>
@@ -56,6 +60,12 @@ class GenerateKeyCommand extends BaseCommand
         $this
             ->setName(self::NAME)
             ->setDescription('Generates a new valid API key.')
+            ->addOption(
+                'name',
+                'm',
+                InputOption::VALUE_REQUIRED,
+                'The name by which this API key will be known.',
+            )
             ->addOptionWithDeprecatedFallback(
                 'expiration-date',
                 'e',
@@ -82,6 +92,7 @@ class GenerateKeyCommand extends BaseCommand
         $expirationDate = $this->getOptionWithDeprecatedFallback($input, 'expiration-date');
         $apiKey = $this->apiKeyService->create(
             isset($expirationDate) ? Chronos::parse($expirationDate) : null,
+            $input->getOption('name'),
             ...$this->roleResolver->determineRoles($input),
         );
 

module/CLI/src/Command/Api/ListKeysCommand.php

@@ -57,7 +57,7 @@ class ListKeysCommand extends BaseCommand
             $messagePattern = $this->determineMessagePattern($apiKey);
 
             // Set columns for this row
-            $rowData = [sprintf($messagePattern, $apiKey)];
+            $rowData = [sprintf($messagePattern, $apiKey), sprintf($messagePattern, $apiKey->name() ?? '-')];
             if (! $enabledOnly) {
                 $rowData[] = sprintf($messagePattern, $this->getEnabledSymbol($apiKey));
             }
@@ -74,10 +74,12 @@ class ListKeysCommand extends BaseCommand
 
         ShlinkTable::fromOutput($output)->render(array_filter([
             'Key',
+            'Name',
             ! $enabledOnly ? 'Is enabled' : null,
             'Expiration date',
             'Roles',
         ]), $rows);
+
         return ExitCodes::EXIT_SUCCESS;
     }
 

module/CLI/src/Command/ShortUrl/ListShortUrlsCommand.php

@@ -10,6 +10,7 @@ use Shlinkio\Shlink\CLI\Util\ShlinkTable;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
 use Shlinkio\Shlink\Common\Paginator\Util\PagerfantaUtilsTrait;
 use Shlinkio\Shlink\Common\Rest\DataTransformerInterface;
+use Shlinkio\Shlink\Core\Entity\ShortUrl;
 use Shlinkio\Shlink\Core\Model\ShortUrlsOrdering;
 use Shlinkio\Shlink\Core\Model\ShortUrlsParams;
 use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface;
@@ -19,6 +20,7 @@ use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Style\SymfonyStyle;
 
+use function array_keys;
 use function array_pad;
 use function explode;
 use function Functional\map;
@@ -30,18 +32,6 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand
     use PagerfantaUtilsTrait;
 
     public const NAME = 'short-url:list';
-    private const COLUMNS_TO_SHOW = [
-        'shortCode',
-        'title',
-        'shortUrl',
-        'longUrl',
-        'dateCreated',
-        'visitsCount',
-    ];
-    private const COLUMNS_TO_SHOW_WITH_TAGS = [
-        ...self::COLUMNS_TO_SHOW,
-        'tags',
-    ];
 
     private ShortUrlServiceInterface $shortUrlService;
     private DataTransformerInterface $transformer;
@@ -90,6 +80,18 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand
                 InputOption::VALUE_NONE,
                 'Whether to display the tags or not.',
             )
+            ->addOption(
+                'show-api-key',
+                'k',
+                InputOption::VALUE_NONE,
+                'Whether to display the API key from which the URL was generated or not.',
+            )
+            ->addOption(
+                'show-api-key-name',
+                'm',
+                InputOption::VALUE_NONE,
+                'Whether to display the API key name from which the URL was generated or not.',
+            )
             ->addOption(
                 'all',
                 'a',
@@ -117,11 +119,11 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand
         $searchTerm = $this->getOptionWithDeprecatedFallback($input, 'search-term');
         $tags = $input->getOption('tags');
         $tags = ! empty($tags) ? explode(',', $tags) : [];
-        $showTags = $this->getOptionWithDeprecatedFallback($input, 'show-tags');
         $all = $input->getOption('all');
         $startDate = $this->getStartDateOption($input, $output);
         $endDate = $this->getEndDateOption($input, $output);
         $orderBy = $this->processOrderBy($input);
+        $columnsMap = $this->resolveColumnsMap($input);
 
         $data = [
             ShortUrlsParamsInputFilter::SEARCH_TERM => $searchTerm,
@@ -137,7 +139,7 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand
 
         do {
             $data[ShortUrlsParamsInputFilter::PAGE] = $page;
-            $result = $this->renderPage($output, $showTags, ShortUrlsParams::fromRawData($data), $all);
+            $result = $this->renderPage($output, $columnsMap, ShortUrlsParams::fromRawData($data), $all);
             $page++;
 
             $continue = $result->hasNextPage() && $io->confirm(
@@ -152,32 +154,26 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand
         return ExitCodes::EXIT_SUCCESS;
     }
 
-    private function renderPage(OutputInterface $output, bool $showTags, ShortUrlsParams $params, bool $all): Paginator
-    {
-        $result = $this->shortUrlService->listShortUrls($params);
+    private function renderPage(
+        OutputInterface $output,
+        array $columnsMap,
+        ShortUrlsParams $params,
+        bool $all
+    ): Paginator {
+        $shortUrls = $this->shortUrlService->listShortUrls($params);
 
-        $headers = ['Short code', 'Title', 'Short URL', 'Long URL', 'Date created', 'Visits count'];
-        if ($showTags) {
-            $headers[] = 'Tags';
-        }
+        $rows = map($shortUrls, function (ShortUrl $shortUrl) use ($columnsMap) {
+            $rawShortUrl = $this->transformer->transform($shortUrl);
+            return map($columnsMap, fn (callable $call) => $call($rawShortUrl, $shortUrl));
+        });
 
-        $rows = [];
-        foreach ($result as $row) {
-            $columnsToShow = $showTags ? self::COLUMNS_TO_SHOW_WITH_TAGS : self::COLUMNS_TO_SHOW;
-            $shortUrl = $this->transformer->transform($row);
-            if ($showTags) {
-                $shortUrl['tags'] = implode(', ', $shortUrl['tags']);
-            }
+        ShlinkTable::fromOutput($output)->render(
+            array_keys($columnsMap),
+            $rows,
+            $all ? null : $this->formatCurrentPageMessage($shortUrls, 'Page %s of %s'),
+        );
 
-            $rows[] = map($columnsToShow, fn (string $prop) => $shortUrl[$prop]);
-        }
-
-        ShlinkTable::fromOutput($output)->render($headers, $rows, $all ? null : $this->formatCurrentPageMessage(
-            $result,
-            'Page %s of %s',
-        ));
-
-        return $result;
+        return $shortUrls;
     }
 
     private function processOrderBy(InputInterface $input): ?string
@@ -190,4 +186,33 @@ class ListShortUrlsCommand extends AbstractWithDateRangeCommand
         [$field, $dir] = array_pad(explode(',', $orderBy), 2, null);
         return $dir === null ? $field : sprintf('%s-%s', $field, $dir);
     }
+
+    private function resolveColumnsMap(InputInterface $input): array
+    {
+        $pickProp = static fn (string $prop): callable => static fn (array $shortUrl) => $shortUrl[$prop];
+        $columnsMap = [
+            'Short Code' => $pickProp('shortCode'),
+            'Title' => $pickProp('title'),
+            'Short URL' => $pickProp('shortUrl'),
+            'Long URL' => $pickProp('longUrl'),
+            'Date created' => $pickProp('dateCreated'),
+            'Visits count' => $pickProp('visitsCount'),
+        ];
+        if ($this->getOptionWithDeprecatedFallback($input, 'show-tags')) {
+            $columnsMap['Tags'] = static fn (array $shortUrl): string => implode(', ', $shortUrl['tags']);
+        }
+        if ($input->getOption('show-api-key')) {
+            $columnsMap['API Key'] = static fn (array $_, ShortUrl $shortUrl): string =>
+            (string) $shortUrl->authorApiKey();
+        }
+        if ($input->getOption('show-api-key-name')) {
+            $columnsMap['API Key Name'] = static function (array $_, ShortUrl $shortUrl): ?string {
+                $apiKey = $shortUrl->authorApiKey();
+
+                return $apiKey !== null ? $apiKey->name() : null;
+            };
+        }
+
+        return $columnsMap;
+    }
 }

module/CLI/src/Command/Visit/DownloadGeoLiteDbCommand.php

@@ -0,0 +1,80 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\CLI\Command\Visit;
+
+use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
+use Shlinkio\Shlink\CLI\Util\ExitCodes;
+use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+use function sprintf;
+
+class DownloadGeoLiteDbCommand extends Command
+{
+    public const NAME = 'visit:download-db';
+
+    private GeolocationDbUpdaterInterface $dbUpdater;
+    private ?ProgressBar $progressBar = null;
+
+    public function __construct(GeolocationDbUpdaterInterface $dbUpdater)
+    {
+        parent::__construct();
+        $this->dbUpdater = $dbUpdater;
+    }
+
+    protected function configure(): void
+    {
+        $this
+            ->setName(self::NAME)
+            ->setDescription(
+                'Checks if the GeoLite2 db file is too old or it does not exist, and tries to download an up-to-date '
+                . 'copy if so.',
+            );
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): ?int
+    {
+        $io = new SymfonyStyle($input, $output);
+
+        try {
+            $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists) use ($io): void {
+                $io->text(sprintf('<fg=blue>%s GeoLite2 db file...</>', $olderDbExists ? 'Updating' : 'Downloading'));
+                $this->progressBar = new ProgressBar($io);
+            }, function (int $total, int $downloaded): void {
+                $this->progressBar->setMaxSteps($total);
+                $this->progressBar->setProgress($downloaded);
+            });
+
+            if ($this->progressBar === null) {
+                $io->info('GeoLite2 db file is up to date.');
+            } else {
+                $this->progressBar->finish();
+                $io->success('GeoLite2 db file properly downloaded.');
+            }
+
+            return ExitCodes::EXIT_SUCCESS;
+        } catch (GeolocationDbUpdateFailedException $e) {
+            $olderDbExists = $e->olderDbExists();
+
+            if ($olderDbExists) {
+                $io->warning(
+                    'GeoLite2 db file update failed. Visits will continue to be located with the old version.',
+                );
+            } else {
+                $io->error('GeoLite2 db file download failed. It will not be possible to locate visits.');
+            }
+
+            if ($io->isVerbose()) {
+                $this->getApplication()->renderThrowable($e, $io);
+            }
+
+            return $olderDbExists ? ExitCodes::EXIT_WARNING : ExitCodes::EXIT_FAILURE;
+        }
+    }
+}

module/CLI/src/Command/Visit/LocateVisitsCommand.php

@@ -6,9 +6,7 @@ namespace Shlinkio\Shlink\CLI\Command\Visit;
 
 use Shlinkio\Shlink\CLI\Command\Util\AbstractLockedCommand;
 use Shlinkio\Shlink\CLI\Command\Util\LockedCommandConfig;
-use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
 use Shlinkio\Shlink\CLI\Util\ExitCodes;
-use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface;
 use Shlinkio\Shlink\Common\Util\IpAddress;
 use Shlinkio\Shlink\Core\Entity\Visit;
 use Shlinkio\Shlink\Core\Entity\VisitLocation;
@@ -19,7 +17,6 @@ use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException;
 use Shlinkio\Shlink\IpGeolocation\Model\Location;
 use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface;
 use Symfony\Component\Console\Exception\RuntimeException;
-use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -35,28 +32,26 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat
 
     private VisitLocatorInterface $visitLocator;
     private IpLocationResolverInterface $ipLocationResolver;
-    private GeolocationDbUpdaterInterface $dbUpdater;
 
     private SymfonyStyle $io;
-    private ?ProgressBar $progressBar = null;
 
     public function __construct(
         VisitLocatorInterface $visitLocator,
         IpLocationResolverInterface $ipLocationResolver,
-        LockFactory $locker,
-        GeolocationDbUpdaterInterface $dbUpdater
+        LockFactory $locker
     ) {
         parent::__construct($locker);
         $this->visitLocator = $visitLocator;
         $this->ipLocationResolver = $ipLocationResolver;
-        $this->dbUpdater = $dbUpdater;
     }
 
     protected function configure(): void
     {
         $this
             ->setName(self::NAME)
-            ->setDescription('Resolves visits origin locations.')
+            ->setDescription(
+                'Resolves visits origin locations. It implicitly downloads/updates the GeoLite2 db file if needed.',
+            )
             ->addOption(
                 'retry',
                 'r',
@@ -90,12 +85,12 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat
             );
         }
 
-        if ($all && $retry && ! $this->warnAndVerifyContinue()) {
+        if ($all && $retry && ! $this->warnAndVerifyContinue($input)) {
             throw new RuntimeException('Execution aborted');
         }
     }
 
-    private function warnAndVerifyContinue(): bool
+    private function warnAndVerifyContinue(InputInterface $input): bool
     {
         $this->io->warning([
             'You are about to process the location of all existing visits your short URLs received.',
@@ -113,7 +108,7 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat
         $all = $retry && $input->getOption('all');
 
         try {
-            $this->checkDbUpdate();
+            $this->checkDbUpdate($input);
 
             if ($all) {
                 $this->visitLocator->locateAllVisits($this);
@@ -128,7 +123,7 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat
             return ExitCodes::EXIT_SUCCESS;
         } catch (Throwable $e) {
             $this->io->error($e->getMessage());
-            if ($e instanceof Throwable && $this->io->isVerbose()) {
+            if ($this->io->isVerbose()) {
                 $this->getApplication()->renderThrowable($e, $this->io);
             }
 
@@ -176,33 +171,13 @@ class LocateVisitsCommand extends AbstractLockedCommand implements VisitGeolocat
         $this->io->writeln($message);
     }
 
-    private function checkDbUpdate(): void
+    private function checkDbUpdate(InputInterface $input): void
     {
-        try {
-            $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void {
-                $this->io->writeln(
-                    sprintf('<fg=blue>%s GeoLite2 database...</>', $olderDbExists ? 'Updating' : 'Downloading'),
-                );
-                $this->progressBar = new ProgressBar($this->io);
-            }, function (int $total, int $downloaded): void {
-                $this->progressBar->setMaxSteps($total);
-                $this->progressBar->setProgress($downloaded);
-            });
+        $downloadDbCommand = $this->getApplication()->find(DownloadGeoLiteDbCommand::NAME);
+        $exitCode = $downloadDbCommand->run($input, $this->io);
 
-            if ($this->progressBar !== null) {
-                $this->progressBar->finish();
-                $this->io->newLine();
-            }
-        } catch (GeolocationDbUpdateFailedException $e) {
-            if (! $e->olderDbExists()) {
-                $this->io->error('GeoLite2 database download failed. It is not possible to locate visits.');
-                throw $e;
-            }
-
-            $this->io->newLine();
-            $this->io->writeln(
-                '<fg=yellow;options=bold>[Warning] GeoLite2 database update failed. Proceeding with old version.</>',
-            );
+        if ($exitCode === ExitCodes::EXIT_FAILURE) {
+            throw new RuntimeException('It is not possible to locate visits without a GeoLite2 db file.');
         }
     }
 

module/CLI/src/Exception/GeolocationDbUpdateFailedException.php

@@ -13,6 +13,11 @@ class GeolocationDbUpdateFailedException extends RuntimeException implements Exc
 {
     private bool $olderDbExists;
 
+    private function __construct(string $message, int $code = 0, ?Throwable $previous = null)
+    {
+        parent::__construct($message, $code, $previous);
+    }
+
     public static function withOlderDb(?Throwable $prev = null): self
     {
         $e = new self(

module/CLI/src/Util/GeolocationDbUpdater.php

@@ -8,6 +8,7 @@ use Cake\Chronos\Chronos;
 use GeoIp2\Database\Reader;
 use MaxMind\Db\Reader\Metadata;
 use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
+use Shlinkio\Shlink\Core\Options\TrackingOptions;
 use Shlinkio\Shlink\IpGeolocation\Exception\RuntimeException;
 use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface;
 use Symfony\Component\Lock\LockFactory;
@@ -21,24 +22,34 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
     private DbUpdaterInterface $dbUpdater;
     private Reader $geoLiteDbReader;
     private LockFactory $locker;
+    private TrackingOptions $trackingOptions;
 
-    public function __construct(DbUpdaterInterface $dbUpdater, Reader $geoLiteDbReader, LockFactory $locker)
-    {
+    public function __construct(
+        DbUpdaterInterface $dbUpdater,
+        Reader $geoLiteDbReader,
+        LockFactory $locker,
+        TrackingOptions $trackingOptions
+    ) {
         $this->dbUpdater = $dbUpdater;
         $this->geoLiteDbReader = $geoLiteDbReader;
         $this->locker = $locker;
+        $this->trackingOptions = $trackingOptions;
     }
 
     /**
      * @throws GeolocationDbUpdateFailedException
      */
-    public function checkDbUpdate(?callable $mustBeUpdated = null, ?callable $handleProgress = null): void
+    public function checkDbUpdate(?callable $beforeDownload = null, ?callable $handleProgress = null): void
     {
+        if ($this->trackingOptions->disableTracking() || $this->trackingOptions->disableIpTracking()) {
+            return;
+        }
+
         $lock = $this->locker->createLock(self::LOCK_NAME);
         $lock->acquire(true); // Block until lock is released
 
         try {
-            $this->downloadIfNeeded($mustBeUpdated, $handleProgress);
+            $this->downloadIfNeeded($beforeDownload, $handleProgress);
         } finally {
             $lock->release();
         }
@@ -47,34 +58,16 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
     /**
      * @throws GeolocationDbUpdateFailedException
      */
-    private function downloadIfNeeded(?callable $mustBeUpdated, ?callable $handleProgress): void
+    private function downloadIfNeeded(?callable $beforeDownload, ?callable $handleProgress): void
     {
         if (! $this->dbUpdater->databaseFileExists()) {
-            $this->downloadNewDb(false, $mustBeUpdated, $handleProgress);
+            $this->downloadNewDb(false, $beforeDownload, $handleProgress);
             return;
         }
 
         $meta = $this->geoLiteDbReader->metadata();
         if ($this->buildIsTooOld($meta)) {
-            $this->downloadNewDb(true, $mustBeUpdated, $handleProgress);
-        }
-    }
-
-    /**
-     * @throws GeolocationDbUpdateFailedException
-     */
-    private function downloadNewDb(bool $olderDbExists, ?callable $mustBeUpdated, ?callable $handleProgress): void
-    {
-        if ($mustBeUpdated !== null) {
-            $mustBeUpdated($olderDbExists);
-        }
-
-        try {
-            $this->dbUpdater->downloadFreshCopy($handleProgress);
-        } catch (RuntimeException $e) {
-            throw $olderDbExists
-                ? GeolocationDbUpdateFailedException::withOlderDb($e)
-                : GeolocationDbUpdateFailedException::withoutOlderDb($e);
+            $this->downloadNewDb(true, $beforeDownload, $handleProgress);
         }
     }
 
@@ -105,4 +98,31 @@ class GeolocationDbUpdater implements GeolocationDbUpdaterInterface
 
         throw GeolocationDbUpdateFailedException::withInvalidEpochInOldDb($buildEpoch);
     }
+
+    /**
+     * @throws GeolocationDbUpdateFailedException
+     */
+    private function downloadNewDb(bool $olderDbExists, ?callable $beforeDownload, ?callable $handleProgress): void
+    {
+        if ($beforeDownload !== null) {
+            $beforeDownload($olderDbExists);
+        }
+
+        try {
+            $this->dbUpdater->downloadFreshCopy($this->wrapHandleProgressCallback($handleProgress, $olderDbExists));
+        } catch (RuntimeException $e) {
+            throw $olderDbExists
+                ? GeolocationDbUpdateFailedException::withOlderDb($e)
+                : GeolocationDbUpdateFailedException::withoutOlderDb($e);
+        }
+    }
+
+    private function wrapHandleProgressCallback(?callable $handleProgress, bool $olderDbExists): ?callable
+    {
+        if ($handleProgress === null) {
+            return null;
+        }
+
+        return fn (int $total, int $downloaded) => $handleProgress($total, $downloaded, $olderDbExists);
+    }
 }

module/CLI/src/Util/GeolocationDbUpdaterInterface.php

@@ -11,5 +11,5 @@ interface GeolocationDbUpdaterInterface
     /**
      * @throws GeolocationDbUpdateFailedException
      */
-    public function checkDbUpdate(?callable $mustBeUpdated = null, ?callable $handleProgress = null): void;
+    public function checkDbUpdate(?callable $beforeDownload = null, ?callable $handleProgress = null): void;
 }

module/CLI/test/CliTestUtilsTrait.php

@@ -0,0 +1,44 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioTest\Shlink\CLI;
+
+use Prophecy\Argument;
+use Prophecy\PhpUnit\ProphecyTrait;
+use Prophecy\Prophecy\ObjectProphecy;
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Tester\CommandTester;
+
+trait CliTestUtilsTrait
+{
+    use ProphecyTrait;
+
+    /**
+     * @return ObjectProphecy|Command
+     */
+    private function createCommandMock(string $name): ObjectProphecy
+    {
+        $command = $this->prophesize(Command::class);
+        $command->getName()->willReturn($name);
+        $command->getDefinition()->willReturn($name);
+        $command->isEnabled()->willReturn(true);
+        $command->getAliases()->willReturn([]);
+        $command->setApplication(Argument::type(Application::class))->willReturn(function (): void {
+        });
+
+        return $command;
+    }
+
+    private function testerForCommand(Command $mainCommand, Command ...$extraCommands): CommandTester
+    {
+        $app = new Application();
+        $app->add($mainCommand);
+        foreach ($extraCommands as $command) {
+            $app->add($command);
+        }
+
+        return new CommandTester($mainCommand);
+    }
+}

module/CLI/test/Command/Api/DisableKeyCommandTest.php

@@ -5,17 +5,16 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\Api;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Api\DisableKeyCommand;
 use Shlinkio\Shlink\Common\Exception\InvalidArgumentException;
 use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class DisableKeyCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $apiKeyService;
@@ -23,10 +22,7 @@ class DisableKeyCommandTest extends TestCase
     public function setUp(): void
     {
         $this->apiKeyService = $this->prophesize(ApiKeyServiceInterface::class);
-        $command = new DisableKeyCommand($this->apiKeyService->reveal());
-        $app = new Application();
-        $app->add($command);
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new DisableKeyCommand($this->apiKeyService->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/Api/GenerateKeyCommandTest.php

@@ -7,55 +7,64 @@ namespace ShlinkioTest\Shlink\CLI\Command\Api;
 use Cake\Chronos\Chronos;
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\ApiKey\RoleResolverInterface;
 use Shlinkio\Shlink\CLI\Command\Api\GenerateKeyCommand;
 use Shlinkio\Shlink\Rest\Entity\ApiKey;
 use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class GenerateKeyCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $apiKeyService;
-    private ObjectProphecy $roleResolver;
 
     public function setUp(): void
     {
         $this->apiKeyService = $this->prophesize(ApiKeyServiceInterface::class);
-        $this->roleResolver = $this->prophesize(RoleResolverInterface::class);
-        $this->roleResolver->determineRoles(Argument::type(InputInterface::class))->willReturn([]);
+        $roleResolver = $this->prophesize(RoleResolverInterface::class);
+        $roleResolver->determineRoles(Argument::type(InputInterface::class))->willReturn([]);
 
-        $command = new GenerateKeyCommand($this->apiKeyService->reveal(), $this->roleResolver->reveal());
-        $app = new Application();
-        $app->add($command);
-        $this->commandTester = new CommandTester($command);
+        $command = new GenerateKeyCommand($this->apiKeyService->reveal(), $roleResolver->reveal());
+        $this->commandTester = $this->testerForCommand($command);
     }
 
     /** @test */
     public function noExpirationDateIsDefinedIfNotProvided(): void
     {
-        $create = $this->apiKeyService->create(null)->willReturn(new ApiKey());
+        $this->apiKeyService->create(null, null)->shouldBeCalledOnce()->willReturn(ApiKey::create());
 
         $this->commandTester->execute([]);
         $output = $this->commandTester->getDisplay();
 
         self::assertStringContainsString('Generated API key: ', $output);
-        $create->shouldHaveBeenCalledOnce();
     }
 
     /** @test */
     public function expirationDateIsDefinedIfProvided(): void
     {
-        $this->apiKeyService->create(Argument::type(Chronos::class))->shouldBeCalledOnce()
-                                                                    ->willReturn(new ApiKey());
+        $this->apiKeyService->create(Argument::type(Chronos::class), null)->shouldBeCalledOnce()->willReturn(
+            ApiKey::create(),
+        );
+
         $this->commandTester->execute([
             '--expiration-date' => '2016-01-01',
         ]);
     }
+
+    /** @test */
+    public function nameIsDefinedIfProvided(): void
+    {
+        $this->apiKeyService->create(null, Argument::type('string'))->shouldBeCalledOnce()->willReturn(
+            ApiKey::create(),
+        );
+
+        $this->commandTester->execute([
+            '--name' => 'Alice',
+        ]);
+    }
 }

module/CLI/test/Command/Api/ListKeysCommandTest.php

@@ -5,19 +5,19 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\Api;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Api\ListKeysCommand;
 use Shlinkio\Shlink\Core\Entity\Domain;
+use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta;
 use Shlinkio\Shlink\Rest\ApiKey\Model\RoleDefinition;
 use Shlinkio\Shlink\Rest\Entity\ApiKey;
 use Shlinkio\Shlink\Rest\Service\ApiKeyServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class ListKeysCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $apiKeyService;
@@ -25,10 +25,7 @@ class ListKeysCommandTest extends TestCase
     public function setUp(): void
     {
         $this->apiKeyService = $this->prophesize(ApiKeyServiceInterface::class);
-        $command = new ListKeysCommand($this->apiKeyService->reveal());
-        $app = new Application();
-        $app->add($command);
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new ListKeysCommand($this->apiKeyService->reveal()));
     }
 
     /**
@@ -49,65 +46,85 @@ class ListKeysCommandTest extends TestCase
     public function provideKeysAndOutputs(): iterable
     {
         yield 'all keys' => [
-            [ApiKey::withKey('foo'), ApiKey::withKey('bar'), ApiKey::withKey('baz')],
+            [$apiKey1 = ApiKey::create(), $apiKey2 = ApiKey::create(), $apiKey3 = ApiKey::create()],
             false,
             <<<OUTPUT
-            +-----+------------+-----------------+-------+
-            | Key | Is enabled | Expiration date | Roles |
-            +-----+------------+-----------------+-------+
-            | foo | +++        | -               | Admin |
-            | bar | +++        | -               | Admin |
-            | baz | +++        | -               | Admin |
-            +-----+------------+-----------------+-------+
+            +--------------------------------------+------+------------+-----------------+-------+
+            | Key                                  | Name | Is enabled | Expiration date | Roles |
+            +--------------------------------------+------+------------+-----------------+-------+
+            | {$apiKey1} | -    | +++        | -               | Admin |
+            | {$apiKey2} | -    | +++        | -               | Admin |
+            | {$apiKey3} | -    | +++        | -               | Admin |
+            +--------------------------------------+------+------------+-----------------+-------+
 
             OUTPUT,
         ];
         yield 'enabled keys' => [
-            [ApiKey::withKey('foo')->disable(), ApiKey::withKey('bar')],
+            [$apiKey1 = ApiKey::create()->disable(), $apiKey2 = ApiKey::create()],
             true,
             <<<OUTPUT
-            +-----+-----------------+-------+
-            | Key | Expiration date | Roles |
-            +-----+-----------------+-------+
-            | foo | -               | Admin |
-            | bar | -               | Admin |
-            +-----+-----------------+-------+
+            +--------------------------------------+------+-----------------+-------+
+            | Key                                  | Name | Expiration date | Roles |
+            +--------------------------------------+------+-----------------+-------+
+            | {$apiKey1} | -    | -               | Admin |
+            | {$apiKey2} | -    | -               | Admin |
+            +--------------------------------------+------+-----------------+-------+
 
             OUTPUT,
         ];
         yield 'with roles' => [
             [
-                ApiKey::withKey('foo'),
-                $this->apiKeyWithRoles('bar', [RoleDefinition::forAuthoredShortUrls()]),
-                $this->apiKeyWithRoles('baz', [RoleDefinition::forDomain((new Domain('example.com'))->setId('1'))]),
-                ApiKey::withKey('foo2'),
-                $this->apiKeyWithRoles('baz2', [
+                $apiKey1 = ApiKey::create(),
+                $apiKey2 = $this->apiKeyWithRoles([RoleDefinition::forAuthoredShortUrls()]),
+                $apiKey3 = $this->apiKeyWithRoles([RoleDefinition::forDomain((new Domain('example.com'))->setId('1'))]),
+                $apiKey4 = ApiKey::create(),
+                $apiKey5 = $this->apiKeyWithRoles([
                     RoleDefinition::forAuthoredShortUrls(),
                     RoleDefinition::forDomain((new Domain('example.com'))->setId('1')),
                 ]),
-                ApiKey::withKey('foo3'),
+                $apiKey6 = ApiKey::create(),
             ],
             true,
             <<<OUTPUT
-            +------+-----------------+--------------------------+
-            | Key  | Expiration date | Roles                    |
-            +------+-----------------+--------------------------+
-            | foo  | -               | Admin                    |
-            | bar  | -               | Author only              |
-            | baz  | -               | Domain only: example.com |
-            | foo2 | -               | Admin                    |
-            | baz2 | -               | Author only              |
-            |      |                 | Domain only: example.com |
-            | foo3 | -               | Admin                    |
-            +------+-----------------+--------------------------+
+            +--------------------------------------+------+-----------------+--------------------------+
+            | Key                                  | Name | Expiration date | Roles                    |
+            +--------------------------------------+------+-----------------+--------------------------+
+            | {$apiKey1} | -    | -               | Admin                    |
+            | {$apiKey2} | -    | -               | Author only              |
+            | {$apiKey3} | -    | -               | Domain only: example.com |
+            | {$apiKey4} | -    | -               | Admin                    |
+            | {$apiKey5} | -    | -               | Author only              |
+            |                                      |      |                 | Domain only: example.com |
+            | {$apiKey6} | -    | -               | Admin                    |
+            +--------------------------------------+------+-----------------+--------------------------+
+
+            OUTPUT,
+        ];
+        yield 'with names' => [
+            [
+                $apiKey1 = ApiKey::fromMeta(ApiKeyMeta::withName('Alice')),
+                $apiKey2 = ApiKey::fromMeta(ApiKeyMeta::withName('Alice and Bob')),
+                $apiKey3 = ApiKey::fromMeta(ApiKeyMeta::withName('')),
+                $apiKey4 = ApiKey::create(),
+            ],
+            true,
+            <<<OUTPUT
+            +--------------------------------------+---------------+-----------------+-------+
+            | Key                                  | Name          | Expiration date | Roles |
+            +--------------------------------------+---------------+-----------------+-------+
+            | {$apiKey1} | Alice         | -               | Admin |
+            | {$apiKey2} | Alice and Bob | -               | Admin |
+            | {$apiKey3} |               | -               | Admin |
+            | {$apiKey4} | -             | -               | Admin |
+            +--------------------------------------+---------------+-----------------+-------+
 
             OUTPUT,
         ];
     }
 
-    private function apiKeyWithRoles(string $key, array $roles): ApiKey
+    private function apiKeyWithRoles(array $roles): ApiKey
     {
-        $apiKey = ApiKey::withKey($key);
+        $apiKey = ApiKey::create();
         foreach ($roles as $role) {
             $apiKey->registerRole($role);
         }

module/CLI/test/Command/Db/CreateDatabaseCommandTest.php

@@ -9,11 +9,10 @@ use Doctrine\DBAL\Platforms\AbstractPlatform;
 use Doctrine\DBAL\Schema\AbstractSchemaManager;
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Db\CreateDatabaseCommand;
 use Shlinkio\Shlink\CLI\Util\ProcessRunnerInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Tester\CommandTester;
 use Symfony\Component\Lock\LockFactory;
@@ -22,7 +21,7 @@ use Symfony\Component\Process\PhpExecutableFinder;
 
 class CreateDatabaseCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $processHelper;
@@ -59,10 +58,8 @@ class CreateDatabaseCommandTest extends TestCase
             $this->regularConn->reveal(),
             $noDbNameConn->reveal(),
         );
-        $app = new Application();
-        $app->add($command);
 
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand($command);
     }
 
     /** @test */

module/CLI/test/Command/Db/MigrateDatabaseCommandTest.php

@@ -6,11 +6,10 @@ namespace ShlinkioTest\Shlink\CLI\Command\Db;
 
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Db\MigrateDatabaseCommand;
 use Shlinkio\Shlink\CLI\Util\ProcessRunnerInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Tester\CommandTester;
 use Symfony\Component\Lock\LockFactory;
@@ -19,7 +18,7 @@ use Symfony\Component\Process\PhpExecutableFinder;
 
 class MigrateDatabaseCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $processHelper;
@@ -43,10 +42,7 @@ class MigrateDatabaseCommandTest extends TestCase
             $this->processHelper->reveal(),
             $phpExecutableFinder->reveal(),
         );
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand($command);
     }
 
     /** @test */

module/CLI/test/Command/Domain/ListDomainsCommandTest.php

@@ -5,18 +5,17 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\Domain;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Domain\ListDomainsCommand;
 use Shlinkio\Shlink\CLI\Util\ExitCodes;
 use Shlinkio\Shlink\Core\Domain\DomainServiceInterface;
 use Shlinkio\Shlink\Core\Domain\Model\DomainItem;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class ListDomainsCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $domainService;
@@ -24,12 +23,7 @@ class ListDomainsCommandTest extends TestCase
     public function setUp(): void
     {
         $this->domainService = $this->prophesize(DomainServiceInterface::class);
-
-        $command = new ListDomainsCommand($this->domainService->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new ListDomainsCommand($this->domainService->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/ShortUrl/DeleteShortUrlCommandTest.php

@@ -6,13 +6,12 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl;
 
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\DeleteShortUrlCommand;
 use Shlinkio\Shlink\Core\Exception;
 use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier;
 use Shlinkio\Shlink\Core\Service\ShortUrl\DeleteShortUrlServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 use function array_pop;
@@ -22,7 +21,7 @@ use const PHP_EOL;
 
 class DeleteShortUrlCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $service;
@@ -30,12 +29,7 @@ class DeleteShortUrlCommandTest extends TestCase
     public function setUp(): void
     {
         $this->service = $this->prophesize(DeleteShortUrlServiceInterface::class);
-
-        $command = new DeleteShortUrlCommand($this->service->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new DeleteShortUrlCommand($this->service->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/ShortUrl/GenerateShortUrlCommandTest.php

@@ -7,7 +7,6 @@ namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl;
 use PHPUnit\Framework\Assert;
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\GenerateShortUrlCommand;
 use Shlinkio\Shlink\CLI\Util\ExitCodes;
@@ -17,12 +16,12 @@ use Shlinkio\Shlink\Core\Exception\NonUniqueSlugException;
 use Shlinkio\Shlink\Core\Model\ShortUrlMeta;
 use Shlinkio\Shlink\Core\Service\UrlShortener;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifierInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class GenerateShortUrlCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $urlShortener;
@@ -35,9 +34,7 @@ class GenerateShortUrlCommandTest extends TestCase
         $this->stringifier->stringify(Argument::type(ShortUrl::class))->willReturn('');
 
         $command = new GenerateShortUrlCommand($this->urlShortener->reveal(), $this->stringifier->reveal(), 5);
-        $app = new Application();
-        $app->add($command);
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand($command);
     }
 
     /** @test */

module/CLI/test/Command/ShortUrl/GetVisitsCommandTest.php

@@ -8,7 +8,6 @@ use Cake\Chronos\Chronos;
 use Pagerfanta\Adapter\ArrayAdapter;
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\GetVisitsCommand;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
@@ -21,14 +20,14 @@ use Shlinkio\Shlink\Core\Model\Visitor;
 use Shlinkio\Shlink\Core\Model\VisitsParams;
 use Shlinkio\Shlink\Core\Visit\VisitsStatsHelperInterface;
 use Shlinkio\Shlink\IpGeolocation\Model\Location;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 use function sprintf;
 
 class GetVisitsCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $visitsHelper;
@@ -37,9 +36,7 @@ class GetVisitsCommandTest extends TestCase
     {
         $this->visitsHelper = $this->prophesize(VisitsStatsHelperInterface::class);
         $command = new GetVisitsCommand($this->visitsHelper->reveal());
-        $app = new Application();
-        $app->add($command);
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand($command);
     }
 
     /** @test */
@@ -106,7 +103,7 @@ class GetVisitsCommandTest extends TestCase
         $this->visitsHelper->visitsForShortUrl(new ShortUrlIdentifier($shortCode), Argument::any())->willReturn(
             new Paginator(new ArrayAdapter([
                 Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('bar', 'foo', '', ''))->locate(
-                    new VisitLocation(new Location('', 'Spain', '', '', 0, 0, '')),
+                    VisitLocation::fromGeolocation(new Location('', 'Spain', '', '', 0, 0, '')),
                 ),
             ])),
         )->shouldBeCalledOnce();

module/CLI/test/Command/ShortUrl/ListShortUrlsCommandTest.php

@@ -8,23 +8,26 @@ use Cake\Chronos\Chronos;
 use Pagerfanta\Adapter\ArrayAdapter;
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\ListShortUrlsCommand;
 use Shlinkio\Shlink\Common\Paginator\Paginator;
 use Shlinkio\Shlink\Core\Entity\ShortUrl;
+use Shlinkio\Shlink\Core\Model\ShortUrlMeta;
 use Shlinkio\Shlink\Core\Model\ShortUrlsParams;
 use Shlinkio\Shlink\Core\Service\ShortUrlServiceInterface;
 use Shlinkio\Shlink\Core\ShortUrl\Helper\ShortUrlStringifier;
 use Shlinkio\Shlink\Core\ShortUrl\Transformer\ShortUrlDataTransformer;
-use Symfony\Component\Console\Application;
+use Shlinkio\Shlink\Rest\ApiKey\Model\ApiKeyMeta;
+use Shlinkio\Shlink\Rest\Entity\ApiKey;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
+use function count;
 use function explode;
 
 class ListShortUrlsCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $shortUrlService;
@@ -32,12 +35,10 @@ class ListShortUrlsCommandTest extends TestCase
     public function setUp(): void
     {
         $this->shortUrlService = $this->prophesize(ShortUrlServiceInterface::class);
-        $app = new Application();
         $command = new ListShortUrlsCommand($this->shortUrlService->reveal(), new ShortUrlDataTransformer(
             new ShortUrlStringifier([]),
         ));
-        $app->add($command);
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand($command);
     }
 
     /** @test */
@@ -101,17 +102,77 @@ class ListShortUrlsCommandTest extends TestCase
         $this->commandTester->execute(['--page' => $page]);
     }
 
-    /** @test */
-    public function ifTagsFlagIsProvidedTagsColumnIsIncluded(): void
-    {
+    /**
+     * @test
+     * @dataProvider provideOptionalFlags
+     */
+    public function provideOptionalFlagsMakesNewColumnsToBeIncluded(
+        array $input,
+        array $expectedContents,
+        array $notExpectedContents,
+        ApiKey $apiKey
+    ): void {
         $this->shortUrlService->listShortUrls(ShortUrlsParams::emptyInstance())
-            ->willReturn(new Paginator(new ArrayAdapter([])))
+            ->willReturn(new Paginator(new ArrayAdapter([
+                ShortUrl::fromMeta(ShortUrlMeta::fromRawData([
+                    'longUrl' => 'foo.com',
+                    'tags' => ['foo', 'bar', 'baz'],
+                    'apiKey' => $apiKey,
+                ])),
+            ])))
             ->shouldBeCalledOnce();
 
         $this->commandTester->setInputs(['y']);
-        $this->commandTester->execute(['--show-tags' => true]);
+        $this->commandTester->execute($input);
         $output = $this->commandTester->getDisplay();
-        self::assertStringContainsString('Tags', $output);
+
+        if (count($expectedContents) === 0 && count($notExpectedContents) === 0) {
+            self::fail('No expectations were run');
+        }
+
+        foreach ($expectedContents as $column) {
+            self::assertStringContainsString($column, $output);
+        }
+        foreach ($notExpectedContents as $column) {
+            self::assertStringNotContainsString($column, $output);
+        }
+    }
+
+    public function provideOptionalFlags(): iterable
+    {
+        $apiKey = ApiKey::fromMeta(ApiKeyMeta::withName('my api key'));
+        $key = $apiKey->toString();
+
+        yield 'tags only' => [
+            ['--show-tags' => true],
+            ['| Tags    ', '| foo, bar, baz'],
+            ['| API Key    ', '| API Key Name |', $key, '| my api key'],
+            $apiKey,
+        ];
+        yield 'api key only' => [
+            ['--show-api-key' => true],
+            ['| API Key    ', $key],
+            ['| Tags    ', '| foo, bar, baz', '| API Key Name |', '| my api key'],
+            $apiKey,
+        ];
+        yield 'api key name only' => [
+            ['--show-api-key-name' => true],
+            ['| API Key Name |', '| my api key'],
+            ['| Tags    ', '| foo, bar, baz', '| API Key    ', $key],
+            $apiKey,
+        ];
+        yield 'tags and api key' => [
+            ['--show-tags' => true, '--show-api-key' => true],
+            ['| API Key    ', '| Tags    ', '| foo, bar, baz', $key],
+            ['| API Key Name |', '| my api key'],
+            $apiKey,
+        ];
+        yield 'all' => [
+            ['--show-tags' => true, '--show-api-key' => true, '--show-api-key-name' => true],
+            ['| API Key    ', '| Tags    ', '| API Key Name |', '| foo, bar, baz', $key, '| my api key'],
+            [],
+            $apiKey,
+        ];
     }
 
     /**

module/CLI/test/Command/ShortUrl/ResolveUrlCommandTest.php

@@ -5,14 +5,13 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\ShortUrl;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\ShortUrl\ResolveUrlCommand;
 use Shlinkio\Shlink\Core\Entity\ShortUrl;
 use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException;
 use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier;
 use Shlinkio\Shlink\Core\Service\ShortUrl\ShortUrlResolverInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 use function sprintf;
@@ -21,7 +20,7 @@ use const PHP_EOL;
 
 class ResolveUrlCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $urlResolver;
@@ -29,11 +28,7 @@ class ResolveUrlCommandTest extends TestCase
     public function setUp(): void
     {
         $this->urlResolver = $this->prophesize(ShortUrlResolverInterface::class);
-        $command = new ResolveUrlCommand($this->urlResolver->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new ResolveUrlCommand($this->urlResolver->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/Tag/CreateTagCommandTest.php

@@ -6,16 +6,15 @@ namespace ShlinkioTest\Shlink\CLI\Command\Tag;
 
 use Doctrine\Common\Collections\ArrayCollection;
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Tag\CreateTagCommand;
 use Shlinkio\Shlink\Core\Tag\TagServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class CreateTagCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $tagService;
@@ -23,12 +22,7 @@ class CreateTagCommandTest extends TestCase
     public function setUp(): void
     {
         $this->tagService = $this->prophesize(TagServiceInterface::class);
-
-        $command = new CreateTagCommand($this->tagService->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new CreateTagCommand($this->tagService->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/Tag/DeleteTagsCommandTest.php

@@ -5,16 +5,15 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\Tag;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Tag\DeleteTagsCommand;
 use Shlinkio\Shlink\Core\Tag\TagServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class DeleteTagsCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $tagService;
@@ -22,12 +21,7 @@ class DeleteTagsCommandTest extends TestCase
     public function setUp(): void
     {
         $this->tagService = $this->prophesize(TagServiceInterface::class);
-
-        $command = new DeleteTagsCommand($this->tagService->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new DeleteTagsCommand($this->tagService->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/Tag/ListTagsCommandTest.php

@@ -5,18 +5,17 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\Tag;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Tag\ListTagsCommand;
 use Shlinkio\Shlink\Core\Entity\Tag;
 use Shlinkio\Shlink\Core\Tag\Model\TagInfo;
 use Shlinkio\Shlink\Core\Tag\TagServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class ListTagsCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $tagService;
@@ -24,12 +23,7 @@ class ListTagsCommandTest extends TestCase
     public function setUp(): void
     {
         $this->tagService = $this->prophesize(TagServiceInterface::class);
-
-        $command = new ListTagsCommand($this->tagService->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new ListTagsCommand($this->tagService->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/Tag/RenameTagCommandTest.php

@@ -5,19 +5,18 @@ declare(strict_types=1);
 namespace ShlinkioTest\Shlink\CLI\Command\Tag;
 
 use PHPUnit\Framework\TestCase;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Command\Tag\RenameTagCommand;
 use Shlinkio\Shlink\Core\Entity\Tag;
 use Shlinkio\Shlink\Core\Exception\TagNotFoundException;
 use Shlinkio\Shlink\Core\Tag\Model\TagRenaming;
 use Shlinkio\Shlink\Core\Tag\TagServiceInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Tester\CommandTester;
 
 class RenameTagCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $tagService;
@@ -25,12 +24,7 @@ class RenameTagCommandTest extends TestCase
     public function setUp(): void
     {
         $this->tagService = $this->prophesize(TagServiceInterface::class);
-
-        $command = new RenameTagCommand($this->tagService->reveal());
-        $app = new Application();
-        $app->add($command);
-
-        $this->commandTester = new CommandTester($command);
+        $this->commandTester = $this->testerForCommand(new RenameTagCommand($this->tagService->reveal()));
     }
 
     /** @test */

module/CLI/test/Command/Visit/DownloadGeoLiteDbCommandTest.php

@@ -0,0 +1,107 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioTest\Shlink\CLI\Command\Visit;
+
+use PHPUnit\Framework\TestCase;
+use Prophecy\Argument;
+use Prophecy\Prophecy\ObjectProphecy;
+use Shlinkio\Shlink\CLI\Command\Visit\DownloadGeoLiteDbCommand;
+use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
+use Shlinkio\Shlink\CLI\Util\ExitCodes;
+use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
+use Symfony\Component\Console\Tester\CommandTester;
+
+use function sprintf;
+
+class DownloadGeoLiteDbCommandTest extends TestCase
+{
+    use CliTestUtilsTrait;
+
+    private CommandTester $commandTester;
+    private ObjectProphecy $dbUpdater;
+
+    protected function setUp(): void
+    {
+        $this->dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class);
+        $this->commandTester = $this->testerForCommand(new DownloadGeoLiteDbCommand($this->dbUpdater->reveal()));
+    }
+
+    /**
+     * @test
+     * @dataProvider provideFailureParams
+     */
+    public function showsProperMessageWhenGeoLiteUpdateFails(
+        bool $olderDbExists,
+        string $expectedMessage,
+        int $expectedExitCode
+    ): void {
+        $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will(
+            function (array $args) use ($olderDbExists): void {
+                [$beforeDownload, $handleProgress] = $args;
+
+                $beforeDownload($olderDbExists);
+                $handleProgress(100, 50);
+
+                throw $olderDbExists
+                    ? GeolocationDbUpdateFailedException::withOlderDb()
+                    : GeolocationDbUpdateFailedException::withoutOlderDb();
+            },
+        );
+
+        $this->commandTester->execute([]);
+        $output = $this->commandTester->getDisplay();
+        $exitCode = $this->commandTester->getStatusCode();
+
+        self::assertStringContainsString(
+            sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'),
+            $output,
+        );
+        self::assertStringContainsString($expectedMessage, $output);
+        self::assertSame($expectedExitCode, $exitCode);
+        $checkDbUpdate->shouldHaveBeenCalledOnce();
+    }
+
+    public function provideFailureParams(): iterable
+    {
+        yield 'existing db' => [
+            true,
+            '[WARNING] GeoLite2 db file update failed. Visits will continue to be located',
+            ExitCodes::EXIT_WARNING,
+        ];
+        yield 'not existing db' => [
+            false,
+            '[ERROR] GeoLite2 db file download failed. It will not be possible to locate',
+            ExitCodes::EXIT_FAILURE,
+        ];
+    }
+
+    /**
+     * @test
+     * @dataProvider provideSuccessParams
+     */
+    public function printsExpectedMessageWhenNoErrorOccurs(callable $checkUpdateBehavior, string $expectedMessage): void
+    {
+        $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will($checkUpdateBehavior);
+
+        $this->commandTester->execute([]);
+        $output = $this->commandTester->getDisplay();
+        $exitCode = $this->commandTester->getStatusCode();
+
+        self::assertStringContainsString($expectedMessage, $output);
+        self::assertSame(ExitCodes::EXIT_SUCCESS, $exitCode);
+        $checkDbUpdate->shouldHaveBeenCalledOnce();
+    }
+
+    public function provideSuccessParams(): iterable
+    {
+        yield 'up to date db' => [function (): void {
+        }, '[INFO] GeoLite2 db file is up to date.'];
+        yield 'outdated db' => [function (array $args): void {
+            [$beforeDownload] = $args;
+            $beforeDownload(true);
+        }, '[OK] GeoLite2 db file properly downloaded.'];
+    }
+}

module/CLI/test/Command/Visit/LocateVisitsCommandTest.php

@@ -6,11 +6,10 @@ namespace ShlinkioTest\Shlink\CLI\Command\Visit;
 
 use PHPUnit\Framework\TestCase;
 use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
+use Shlinkio\Shlink\CLI\Command\Visit\DownloadGeoLiteDbCommand;
 use Shlinkio\Shlink\CLI\Command\Visit\LocateVisitsCommand;
-use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
-use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface;
+use Shlinkio\Shlink\CLI\Util\ExitCodes;
 use Shlinkio\Shlink\Common\Util\IpAddress;
 use Shlinkio\Shlink\Core\Entity\ShortUrl;
 use Shlinkio\Shlink\Core\Entity\Visit;
@@ -21,7 +20,7 @@ use Shlinkio\Shlink\Core\Visit\VisitLocator;
 use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException;
 use Shlinkio\Shlink\IpGeolocation\Model\Location;
 use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface;
-use Symfony\Component\Console\Application;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 use Symfony\Component\Console\Exception\RuntimeException;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Tester\CommandTester;
@@ -33,19 +32,18 @@ use const PHP_EOL;
 
 class LocateVisitsCommandTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private CommandTester $commandTester;
     private ObjectProphecy $visitService;
     private ObjectProphecy $ipResolver;
     private ObjectProphecy $lock;
-    private ObjectProphecy $dbUpdater;
+    private ObjectProphecy $downloadDbCommand;
 
     public function setUp(): void
     {
         $this->visitService = $this->prophesize(VisitLocator::class);
         $this->ipResolver = $this->prophesize(IpLocationResolverInterface::class);
-        $this->dbUpdater = $this->prophesize(GeolocationDbUpdaterInterface::class);
 
         $locker = $this->prophesize(Lock\LockFactory::class);
         $this->lock = $this->prophesize(Lock\LockInterface::class);
@@ -58,12 +56,12 @@ class LocateVisitsCommandTest extends TestCase
             $this->visitService->reveal(),
             $this->ipResolver->reveal(),
             $locker->reveal(),
-            $this->dbUpdater->reveal(),
         );
-        $app = new Application();
-        $app->add($command);
 
-        $this->commandTester = new CommandTester($command);
+        $this->downloadDbCommand = $this->createCommandMock(DownloadGeoLiteDbCommand::NAME);
+        $this->downloadDbCommand->run(Argument::cetera())->willReturn(ExitCodes::EXIT_SUCCESS);
+
+        $this->commandTester = $this->testerForCommand($command, $this->downloadDbCommand->reveal());
     }
 
     /**
@@ -78,7 +76,7 @@ class LocateVisitsCommandTest extends TestCase
         array $args
     ): void {
         $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', '1.2.3.4', ''));
-        $location = new VisitLocation(Location::emptyInstance());
+        $location = VisitLocation::fromGeolocation(Location::emptyInstance());
         $mockMethodBehavior = $this->invokeHelperMethods($visit, $location);
 
         $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will($mockMethodBehavior);
@@ -122,7 +120,7 @@ class LocateVisitsCommandTest extends TestCase
     public function localhostAndEmptyAddressesAreIgnored(?string $address, string $message): void
     {
         $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', $address, ''));
-        $location = new VisitLocation(Location::emptyInstance());
+        $location = VisitLocation::fromGeolocation(Location::emptyInstance());
 
         $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will(
             $this->invokeHelperMethods($visit, $location),
@@ -155,7 +153,7 @@ class LocateVisitsCommandTest extends TestCase
     public function errorWhileLocatingIpIsDisplayed(): void
     {
         $visit = Visit::forValidShortUrl(ShortUrl::createEmpty(), new Visitor('', '', '1.2.3.4', ''));
-        $location = new VisitLocation(Location::emptyInstance());
+        $location = VisitLocation::fromGeolocation(Location::emptyInstance());
 
         $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will(
             $this->invokeHelperMethods($visit, $location),
@@ -202,43 +200,16 @@ class LocateVisitsCommandTest extends TestCase
         $resolveIpLocation->shouldNotHaveBeenCalled();
     }
 
-    /**
-     * @test
-     * @dataProvider provideParams
-     */
-    public function showsProperMessageWhenGeoLiteUpdateFails(bool $olderDbExists, string $expectedMessage): void
+    /** @test */
+    public function showsProperMessageWhenGeoLiteUpdateFails(): void
     {
-        $locateVisits = $this->visitService->locateUnlocatedVisits(Argument::cetera())->will(function (): void {
-        });
-        $checkDbUpdate = $this->dbUpdater->checkDbUpdate(Argument::cetera())->will(
-            function (array $args) use ($olderDbExists): void {
-                [$mustBeUpdated, $handleProgress] = $args;
-
-                $mustBeUpdated($olderDbExists);
-                $handleProgress(100, 50);
-
-                throw $olderDbExists
-                    ? GeolocationDbUpdateFailedException::withOlderDb()
-                    : GeolocationDbUpdateFailedException::withoutOlderDb();
-            },
-        );
+        $this->downloadDbCommand->run(Argument::cetera())->willReturn(ExitCodes::EXIT_FAILURE);
 
         $this->commandTester->execute([]);
         $output = $this->commandTester->getDisplay();
 
-        self::assertStringContainsString(
-            sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading'),
-            $output,
-        );
-        self::assertStringContainsString($expectedMessage, $output);
-        $locateVisits->shouldHaveBeenCalledTimes((int) $olderDbExists);
-        $checkDbUpdate->shouldHaveBeenCalledOnce();
-    }
-
-    public function provideParams(): iterable
-    {
-        yield [true, '[Warning] GeoLite2 database update failed. Proceeding with old version.'];
-        yield [false, 'GeoLite2 database download failed. It is not possible to locate visits.'];
+        self::assertStringContainsString('It is not possible to locate visits without a GeoLite2 db file.', $output);
+        $this->visitService->locateUnlocatedVisits(Argument::cetera())->shouldNotHaveBeenCalled();
     }
 
     /** @test */

module/CLI/test/Factory/ApplicationFactoryTest.php

@@ -6,17 +6,13 @@ namespace ShlinkioTest\Shlink\CLI\Factory;
 
 use Laminas\ServiceManager\ServiceManager;
 use PHPUnit\Framework\TestCase;
-use Prophecy\Argument;
-use Prophecy\PhpUnit\ProphecyTrait;
-use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Factory\ApplicationFactory;
 use Shlinkio\Shlink\Core\Options\AppOptions;
-use Symfony\Component\Console\Application;
-use Symfony\Component\Console\Command\Command;
+use ShlinkioTest\Shlink\CLI\CliTestUtilsTrait;
 
 class ApplicationFactoryTest extends TestCase
 {
-    use ProphecyTrait;
+    use CliTestUtilsTrait;
 
     private ApplicationFactory $factory;
 
@@ -54,17 +50,4 @@ class ApplicationFactoryTest extends TestCase
             AppOptions::class => new AppOptions(),
         ]]);
     }
-
-    private function createCommandMock(string $name): ObjectProphecy
-    {
-        $command = $this->prophesize(Command::class);
-        $command->getName()->willReturn($name);
-        $command->getDefinition()->willReturn($name);
-        $command->isEnabled()->willReturn(true);
-        $command->getAliases()->willReturn([]);
-        $command->setApplication(Argument::type(Application::class))->willReturn(function (): void {
-        });
-
-        return $command;
-    }
 }

module/CLI/test/Util/GeolocationDbUpdaterTest.php

@@ -13,6 +13,7 @@ use Prophecy\PhpUnit\ProphecyTrait;
 use Prophecy\Prophecy\ObjectProphecy;
 use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
 use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater;
+use Shlinkio\Shlink\Core\Options\TrackingOptions;
 use Shlinkio\Shlink\IpGeolocation\Exception\RuntimeException;
 use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface;
 use Symfony\Component\Lock;
@@ -28,11 +29,13 @@ class GeolocationDbUpdaterTest extends TestCase
     private GeolocationDbUpdater $geolocationDbUpdater;
     private ObjectProphecy $dbUpdater;
     private ObjectProphecy $geoLiteDbReader;
+    private TrackingOptions $trackingOptions;
 
     public function setUp(): void
     {
         $this->dbUpdater = $this->prophesize(DbUpdaterInterface::class);
         $this->geoLiteDbReader = $this->prophesize(Reader::class);
+        $this->trackingOptions = new TrackingOptions();
 
         $locker = $this->prophesize(Lock\LockFactory::class);
         $lock = $this->prophesize(Lock\LockInterface::class);
@@ -45,6 +48,7 @@ class GeolocationDbUpdaterTest extends TestCase
             $this->dbUpdater->reveal(),
             $this->geoLiteDbReader->reveal(),
             $locker->reveal(),
+            $this->trackingOptions,
         );
     }
 
@@ -174,4 +178,27 @@ class GeolocationDbUpdaterTest extends TestCase
             'record_size' => 4,
         ]);
     }
+
+    /**
+     * @test
+     * @dataProvider provideTrackingOptions
+     */
+    public function downloadDbIsSkippedIfTrackingIsDisabled(array $props): void
+    {
+        foreach ($props as $prop) {
+            $this->trackingOptions->{$prop} = true;
+        }
+
+        $this->geolocationDbUpdater->checkDbUpdate();
+
+        $this->dbUpdater->databaseFileExists(Argument::cetera())->shouldNotHaveBeenCalled();
+        $this->geoLiteDbReader->metadata(Argument::cetera())->shouldNotHaveBeenCalled();
+    }
+
+    public function provideTrackingOptions(): iterable
+    {
+        yield 'disableTracking' => [['disableTracking']];
+        yield 'disableIpTracking' => [['disableIpTracking']];
+        yield 'both' => [['disableTracking', 'disableIpTracking']];
+    }
 }

module/Core/config/dependencies.config.php

@@ -24,6 +24,7 @@ return [
             Options\DeleteShortUrlsOptions::class => ConfigAbstractFactory::class,
             Options\NotFoundRedirectOptions::class => ConfigAbstractFactory::class,
             Options\UrlShortenerOptions::class => ConfigAbstractFactory::class,
+            Options\TrackingOptions::class => ConfigAbstractFactory::class,
 
             Service\UrlShortener::class => ConfigAbstractFactory::class,
             Service\ShortUrlService::class => ConfigAbstractFactory::class,
@@ -47,6 +48,7 @@ return [
             Action\RedirectAction::class => ConfigAbstractFactory::class,
             Action\PixelAction::class => ConfigAbstractFactory::class,
             Action\QrCodeAction::class => ConfigAbstractFactory::class,
+            Action\RobotsAction::class => ConfigAbstractFactory::class,
 
             ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ConfigAbstractFactory::class,
             ShortUrl\Helper\ShortUrlStringifier::class => ConfigAbstractFactory::class,
@@ -56,6 +58,8 @@ return [
             Mercure\MercureUpdatesGenerator::class => ConfigAbstractFactory::class,
 
             Importer\ImportedLinksProcessor::class => ConfigAbstractFactory::class,
+
+            Crawling\CrawlingHelper::class => ConfigAbstractFactory::class,
         ],
 
         'aliases' => [
@@ -75,6 +79,7 @@ return [
         Options\DeleteShortUrlsOptions::class => ['config.delete_short_urls'],
         Options\NotFoundRedirectOptions::class => ['config.not_found_redirects'],
         Options\UrlShortenerOptions::class => ['config.url_shortener'],
+        Options\TrackingOptions::class => ['config.tracking'],
 
         Service\UrlShortener::class => [
             ShortUrl\Helper\ShortUrlTitleResolutionHelper::class,
@@ -85,7 +90,7 @@ return [
         Visit\VisitsTracker::class => [
             'em',
             EventDispatcherInterface::class,
-            Options\UrlShortenerOptions::class,
+            Options\TrackingOptions::class,
         ],
         Service\ShortUrlService::class => [
             'em',
@@ -112,14 +117,14 @@ return [
         Action\RedirectAction::class => [
             Service\ShortUrl\ShortUrlResolver::class,
             Visit\VisitsTracker::class,
-            Options\AppOptions::class,
+            Options\TrackingOptions::class,
             Util\RedirectResponseHelper::class,
             'Logger_Shlink',
         ],
         Action\PixelAction::class => [
             Service\ShortUrl\ShortUrlResolver::class,
             Visit\VisitsTracker::class,
-            Options\AppOptions::class,
+            Options\TrackingOptions::class,
             'Logger_Shlink',
         ],
         Action\QrCodeAction::class => [
@@ -127,6 +132,7 @@ return [
             ShortUrl\Helper\ShortUrlStringifier::class,
             'Logger_Shlink',
         ],
+        Action\RobotsAction::class => [Crawling\CrawlingHelper::class],
 
         ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ['em'],
         ShortUrl\Helper\ShortUrlStringifier::class => ['config.url_shortener.domain', 'config.router.base_path'],
@@ -144,6 +150,8 @@ return [
             Service\ShortUrl\ShortCodeHelper::class,
             Util\DoctrineBatchHelper::class,
         ],
+
+        Crawling\CrawlingHelper::class => ['em'],
     ],
 
 ];

module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.ShortUrl.php

@@ -95,4 +95,9 @@ return static function (ClassMetadata $metadata, array $emConfig): void {
             ->columnName('title_was_auto_resolved')
             ->option('default', false)
             ->build();
+
+    $builder->createField('crawlable', Types::BOOLEAN)
+            ->columnName('crawlable')
+            ->option('default', false)
+            ->build();
 };

module/Core/config/entities-mappings/Shlinkio.Shlink.Core.Entity.Visit.php

@@ -65,4 +65,9 @@ return static function (ClassMetadata $metadata, array $emConfig): void {
             ->columnName('type')
             ->length(255)
             ->build();
+
+    $builder->createField('potentialBot', Types::BOOLEAN)
+            ->columnName('potential_bot')
+            ->option('default', false)
+            ->build();
 };

module/Core/config/event_dispatcher.config.php

@@ -7,21 +7,23 @@ namespace Shlinkio\Shlink\Core;
 use Laminas\ServiceManager\AbstractFactory\ConfigAbstractFactory;
 use Psr\EventDispatcher\EventDispatcherInterface;
 use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdater;
+use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdater;
 use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface;
-use Symfony\Component\Mercure\Publisher;
+use Symfony\Component\Mercure\Hub;
 
 return [
 
     'events' => [
         'regular' => [
-            EventDispatcher\Event\VisitLocated::class => [
-                EventDispatcher\NotifyVisitToMercure::class,
-                EventDispatcher\NotifyVisitToWebHooks::class,
+            EventDispatcher\Event\UrlVisited::class => [
+                EventDispatcher\LocateVisit::class,
             ],
         ],
         'async' => [
-            EventDispatcher\Event\UrlVisited::class => [
-                EventDispatcher\LocateVisit::class,
+            EventDispatcher\Event\VisitLocated::class => [
+                EventDispatcher\NotifyVisitToMercure::class,
+                EventDispatcher\NotifyVisitToWebHooks::class,
+                EventDispatcher\UpdateGeoLiteDb::class,
             ],
         ],
     ],
@@ -31,10 +33,14 @@ return [
             EventDispatcher\LocateVisit::class => ConfigAbstractFactory::class,
             EventDispatcher\NotifyVisitToWebHooks::class => ConfigAbstractFactory::class,
             EventDispatcher\NotifyVisitToMercure::class => ConfigAbstractFactory::class,
+            EventDispatcher\UpdateGeoLiteDb::class => ConfigAbstractFactory::class,
         ],
 
         'delegators' => [
-            EventDispatcher\LocateVisit::class => [
+            EventDispatcher\NotifyVisitToMercure::class => [
+                EventDispatcher\CloseDbConnectionEventListenerDelegator::class,
+            ],
+            EventDispatcher\NotifyVisitToWebHooks::class => [
                 EventDispatcher\CloseDbConnectionEventListenerDelegator::class,
             ],
         ],
@@ -45,7 +51,7 @@ return [
             IpLocationResolverInterface::class,
             'em',
             'Logger_Shlink',
-            GeolocationDbUpdater::class,
+            DbUpdater::class,
             EventDispatcherInterface::class,
         ],
         EventDispatcher\NotifyVisitToWebHooks::class => [
@@ -57,11 +63,12 @@ return [
             Options\AppOptions::class,
         ],
         EventDispatcher\NotifyVisitToMercure::class => [
-            Publisher::class,
+            Hub::class,
             Mercure\MercureUpdatesGenerator::class,
             'em',
             'Logger_Shlink',
         ],
+        EventDispatcher\UpdateGeoLiteDb::class => [GeolocationDbUpdater::class, 'Logger_Shlink'],
     ],
 
 ];

module/Core/config/routes.config.php

@@ -9,6 +9,14 @@ use Shlinkio\Shlink\Core\Action;
 return [
 
     'routes' => [
+        [
+            'name' => Action\RobotsAction::class,
+            'path' => '/robots.txt',
+            'middleware' => [
+                Action\RobotsAction::class,
+            ],
+            'allowed_methods' => [RequestMethod::METHOD_GET],
+        ],
         [
             'name' => Action\RedirectAction::class,
             'path' => '/{shortCode}',

module/Core/functions/functions.php

@@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core;
 use Cake\Chronos\Chronos;
 use DateTimeInterface;
 use Fig\Http\Message\StatusCodeInterface;
+use Jaybizzle\CrawlerDetect\CrawlerDetect;
 use Laminas\InputFilter\InputFilter;
 use PUGX\Shortid\Factory as ShortIdFactory;
 use Shlinkio\Shlink\Common\Util\DateRange;
@@ -50,6 +51,7 @@ function parseDateRangeFromQuery(array $query, string $startDateName, string $en
     $startDate = parseDateFromQuery($query, $startDateName);
     $endDate = parseDateFromQuery($query, $endDateName);
 
+    // TODO Use match expression when migrating to PHP8
     if ($startDate === null && $endDate === null) {
         return DateRange::emptyInstance();
     }
@@ -127,3 +129,13 @@ function kebabCaseToCamelCase(string $name): string
 {
     return lcfirst(str_replace(' ', '', ucwords(str_replace('-', ' ', $name))));
 }
+
+function isCrawler(string $userAgent): bool
+{
+    static $detector;
+    if ($detector === null) {
+        $detector = new CrawlerDetect();
+    }
+
+    return $detector->isCrawler($userAgent);
+}

module/Core/src/Action/AbstractTrackingAction.php

@@ -18,7 +18,7 @@ use Shlinkio\Shlink\Core\Entity\ShortUrl;
 use Shlinkio\Shlink\Core\Exception\ShortUrlNotFoundException;
 use Shlinkio\Shlink\Core\Model\ShortUrlIdentifier;
 use Shlinkio\Shlink\Core\Model\Visitor;
-use Shlinkio\Shlink\Core\Options\AppOptions;
+use Shlinkio\Shlink\Core\Options\TrackingOptions;
 use Shlinkio\Shlink\Core\Service\ShortUrl\ShortUrlResolverInterface;
 use Shlinkio\Shlink\Core\Visit\VisitsTrackerInterface;
 
@@ -29,18 +29,18 @@ abstract class AbstractTrackingAction implements MiddlewareInterface, RequestMet
 {
     private ShortUrlResolverInterface $urlResolver;
     private VisitsTrackerInterface $visitTracker;
-    private AppOptions $appOptions;
+    private TrackingOptions $trackingOptions;
     private LoggerInterface $logger;
 
     public function __construct(
         ShortUrlResolverInterface $urlResolver,
         VisitsTrackerInterface $visitTracker,
-        AppOptions $appOptions,
+        TrackingOptions $trackingOptions,
         ?LoggerInterface $logger = null
     ) {
         $this->urlResolver = $urlResolver;
         $this->visitTracker = $visitTracker;
-        $this->appOptions = $appOptions;
+        $this->trackingOptions = $trackingOptions;
         $this->logger = $logger ?? new NullLogger();
     }
 
@@ -48,7 +48,7 @@ abstract class AbstractTrackingAction implements MiddlewareInterface, RequestMet
     {
         $identifier = ShortUrlIdentifier::fromRedirectRequest($request);
         $query = $request->getQueryParams();
-        $disableTrackParam = $this->appOptions->getDisableTrackParam();
+        $disableTrackParam = $this->trackingOptions->getDisableTrackParam();
 
         try {
             $shortUrl = $this->urlResolver->resolveEnabledShortUrl($identifier);

module/Core/src/Action/QrCodeAction.php

@@ -4,7 +4,7 @@ declare(strict_types=1);
 
 namespace Shlinkio\Shlink\Core\Action;
 
-use Endroid\QrCode\QrCode;
+use Endroid\QrCode\Builder\Builder;
 use Endroid\QrCode\Writer\SvgWriter;
 use Psr\Http\Message\ResponseInterface as Response;
 use Psr\Http\Message\ServerRequestInterface as Request;
@@ -50,16 +50,17 @@ class QrCodeAction implements MiddlewareInterface
         }
 
         $query = $request->getQueryParams();
-        $qrCode = new QrCode($this->stringifier->stringify($shortUrl));
-        $qrCode->setSize($this->resolveSize($request, $query));
-        $qrCode->setMargin($this->resolveMargin($query));
+        $qrCode = Builder::create()
+            ->data($this->stringifier->stringify($shortUrl))
+            ->size($this->resolveSize($request, $query))
+            ->margin($this->resolveMargin($query));
 
         $format = $query['format'] ?? 'png';
         if ($format === 'svg') {
-            $qrCode->setWriter(new SvgWriter());
+            $qrCode->writer(new SvgWriter());
         }
 
-        return new QrCodeResponse($qrCode);
+        return new QrCodeResponse($qrCode->build());
     }
 
     private function resolveSize(Request $request, array $query): int

module/Core/src/Action/RedirectAction.php

@@ -21,11 +21,11 @@ class RedirectAction extends AbstractTrackingAction implements StatusCodeInterfa
     public function __construct(
         ShortUrlResolverInterface $urlResolver,
         VisitsTrackerInterface $visitTracker,
-        Options\AppOptions $appOptions,
+        Options\TrackingOptions $trackingOptions,
         RedirectResponseHelperInterface $redirectResponseHelper,
         ?LoggerInterface $logger = null
     ) {
-        parent::__construct($urlResolver, $visitTracker, $appOptions, $logger);
+        parent::__construct($urlResolver, $visitTracker, $trackingOptions, $logger);
         $this->redirectResponseHelper = $redirectResponseHelper;
     }
 

module/Core/src/Action/RobotsAction.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Action;
+
+use Fig\Http\Message\StatusCodeInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use Psr\Http\Server\RequestHandlerInterface;
+use Shlinkio\Shlink\Core\Crawling\CrawlingHelperInterface;
+
+use function sprintf;
+
+use const PHP_EOL;
+
+class RobotsAction implements RequestHandlerInterface, StatusCodeInterface
+{
+    private CrawlingHelperInterface $crawlingHelper;
+
+    public function __construct(CrawlingHelperInterface $crawlingHelper)
+    {
+        $this->crawlingHelper = $crawlingHelper;
+    }
+
+    public function handle(ServerRequestInterface $request): ResponseInterface
+    {
+        return new Response(self::STATUS_OK, ['Content-type' => 'text/plain'], $this->buildRobots());
+    }
+
+    private function buildRobots(): iterable
+    {
+        yield <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+
+        ROBOTS;
+
+        $shortCodes = $this->crawlingHelper->listCrawlableShortCodes();
+        foreach ($shortCodes as $shortCode) {
+            yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL);
+        }
+
+        yield 'Disallow: /';
+    }
+}

module/Core/src/Config/DeprecatedConfigParser.php

@@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Config;
 
 use function Functional\compose;
 
+/** @deprecated */
 class DeprecatedConfigParser
 {
     public function __invoke(array $config): array

module/Core/src/Config/SimplifiedConfigParser.php

@@ -19,7 +19,7 @@ use function uksort;
 class SimplifiedConfigParser
 {
     private const SIMPLIFIED_CONFIG_MAPPING = [
-        'disable_track_param' => ['app_options', 'disable_track_param'],
+        'disable_track_param' => ['tracking', 'disable_track_param'],
         'short_domain_schema' => ['url_shortener', 'domain', 'schema'],
         'short_domain_host' => ['url_shortener', 'domain', 'hostname'],
         'validate_url' => ['url_shortener', 'validate_url'],
@@ -38,7 +38,7 @@ class SimplifiedConfigParser
         'mercure_public_hub_url' => ['mercure', 'public_hub_url'],
         'mercure_internal_hub_url' => ['mercure', 'internal_hub_url'],
         'mercure_jwt_secret' => ['mercure', 'jwt_secret'],
-        'anonymize_remote_addr' => ['url_shortener', 'anonymize_remote_addr'],
+        'anonymize_remote_addr' => ['tracking', 'anonymize_remote_addr'],
         'redirect_status_code' => ['url_shortener', 'redirect_status_code'],
         'redirect_cache_lifetime' => ['url_shortener', 'redirect_cache_lifetime'],
         'port' => ['mezzio-swoole', 'swoole-http-server', 'port'],

module/Core/src/Crawling/CrawlingHelper.php

@@ -0,0 +1,26 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Crawling;
+
+use Doctrine\ORM\EntityManagerInterface;
+use Shlinkio\Shlink\Core\Entity\ShortUrl;
+use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface;
+
+class CrawlingHelper implements CrawlingHelperInterface
+{
+    private EntityManagerInterface $em;
+
+    public function __construct(EntityManagerInterface $em)
+    {
+        $this->em = $em;
+    }
+
+    public function listCrawlableShortCodes(): iterable
+    {
+        /** @var ShortUrlRepositoryInterface $repo */
+        $repo = $this->em->getRepository(ShortUrl::class);
+        yield from $repo->findCrawlableShortCodes();
+    }
+}

module/Core/src/Crawling/CrawlingHelperInterface.php

@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Crawling;
+
+interface CrawlingHelperInterface
+{
+    /**
+     * @return string[]|iterable
+     */
+    public function listCrawlableShortCodes(): iterable;
+}

module/Core/src/Domain/Repository/DomainRepository.php

@@ -5,7 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\Core\Domain\Repository;
 
 use Doctrine\ORM\Query\Expr\Join;
-use Happyr\DoctrineSpecification\EntitySpecificationRepository;
+use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepository;
 use Shlinkio\Shlink\Core\Entity\Domain;
 use Shlinkio\Shlink\Core\Entity\ShortUrl;
 use Shlinkio\Shlink\Rest\Entity\ApiKey;

module/Core/src/Domain/Repository/DomainRepositoryInterface.php

@@ -5,7 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\Core\Domain\Repository;
 
 use Doctrine\Persistence\ObjectRepository;
-use Happyr\DoctrineSpecification\EntitySpecificationRepositoryInterface;
+use Happyr\DoctrineSpecification\Repository\EntitySpecificationRepositoryInterface;
 use Shlinkio\Shlink\Core\Entity\Domain;
 use Shlinkio\Shlink\Rest\Entity\ApiKey;
 

module/Core/src/Entity/ShortUrl.php

@@ -7,6 +7,8 @@ namespace Shlinkio\Shlink\Core\Entity;
 use Cake\Chronos\Chronos;
 use Doctrine\Common\Collections\ArrayCollection;
 use Doctrine\Common\Collections\Collection;
+use Doctrine\Common\Collections\Criteria;
+use Doctrine\Common\Collections\Selectable;
 use Shlinkio\Shlink\Common\Entity\AbstractEntity;
 use Shlinkio\Shlink\Core\Exception\ShortCodeCannotBeRegeneratedException;
 use Shlinkio\Shlink\Core\Model\ShortUrlEdit;
@@ -40,6 +42,7 @@ class ShortUrl extends AbstractEntity
     private ?ApiKey $authorApiKey = null;
     private ?string $title = null;
     private bool $titleWasAutoResolved = false;
+    private bool $crawlable = false;
 
     private function __construct()
     {
@@ -76,6 +79,7 @@ class ShortUrl extends AbstractEntity
         $instance->authorApiKey = $meta->getApiKey();
         $instance->title = $meta->getTitle();
         $instance->titleWasAutoResolved = $meta->titleWasAutoResolved();
+        $instance->crawlable = $meta->isCrawlable();
 
         return $instance;
     }
@@ -86,17 +90,29 @@ class ShortUrl extends AbstractEntity
         ?ShortUrlRelationResolverInterface $relationResolver = null
     ): self {
         $meta = [
+            ShortUrlInputFilter::VALIDATE_URL => false,
             ShortUrlInputFilter::LONG_URL => $url->longUrl(),
             ShortUrlInputFilter::DOMAIN => $url->domain(),
             ShortUrlInputFilter::TAGS => $url->tags(),
             ShortUrlInputFilter::TITLE => $url->title(),
-            ShortUrlInputFilter::VALIDATE_URL => false,
+            ShortUrlInputFilter::MAX_VISITS => $url->meta()->maxVisits(),
         ];
         if ($importShortCode) {
             $meta[ShortUrlInputFilter::CUSTOM_SLUG] = $url->shortCode();
         }
 
         $instance = self::fromMeta(ShortUrlMeta::fromRawData($meta), $relationResolver);
+
+        $validSince = $url->meta()->validSince();
+        if ($validSince !== null) {
+            $instance->validSince = Chronos::instance($validSince);
+        }
+
+        $validUntil = $url->meta()->validUntil();
+        if ($validUntil !== null) {
+            $instance->validUntil = Chronos::instance($validUntil);
+        }
+
         $instance->importSource = $url->source();
         $instance->importOriginalShortCode = $url->shortCode();
         $instance->dateCreated = Chronos::instance($url->createdAt());
@@ -132,6 +148,11 @@ class ShortUrl extends AbstractEntity
         return $this->tags;
     }
 
+    public function authorApiKey(): ?ApiKey
+    {
+        return $this->authorApiKey;
+    }
+
     public function getValidSince(): ?Chronos
     {
         return $this->validSince;
@@ -147,6 +168,20 @@ class ShortUrl extends AbstractEntity
         return count($this->visits);
     }
 
+    public function mostRecentImportedVisitDate(): ?Chronos
+    {
+        /** @var Selectable $visits */
+        $visits = $this->visits;
+        $criteria = Criteria::create()->where(Criteria::expr()->eq('type', Visit::TYPE_IMPORTED))
+                                      ->orderBy(['id' => 'DESC'])
+                                      ->setMaxResults(1);
+
+        /** @var Visit|false $visit */
+        $visit = $visits->matching($criteria)->last();
+
+        return $visit === false ? null : $visit->getDate();
+    }
+
     /**
      * @param Collection|Visit[] $visits
      * @internal
@@ -162,11 +197,16 @@ class ShortUrl extends AbstractEntity
         return $this->maxVisits;
     }
 
-    public function getTitle(): ?string
+    public function title(): ?string
     {
         return $this->title;
     }
 
+    public function crawlable(): bool
+    {
+        return $this->crawlable;
+    }
+
     public function update(
         ShortUrlEdit $shortUrlEdit,
         ?ShortUrlRelationResolverInterface $relationResolver = null
@@ -187,6 +227,9 @@ class ShortUrl extends AbstractEntity
             $relationResolver = $relationResolver ?? new SimpleShortUrlRelationResolver();
             $this->tags = $relationResolver->resolveTags($shortUrlEdit->tags());
         }
+        if ($shortUrlEdit->crawlableWasProvided()) {
+            $this->crawlable = $shortUrlEdit->crawlable();
+        }
         if (
             $this->title === null
             || $shortUrlEdit->titleWasProvided()

module/Core/src/Entity/Visit.php

@@ -11,32 +11,88 @@ use Shlinkio\Shlink\Common\Exception\InvalidArgumentException;
 use Shlinkio\Shlink\Common\Util\IpAddress;
 use Shlinkio\Shlink\Core\Model\Visitor;
 use Shlinkio\Shlink\Core\Visit\Model\VisitLocationInterface;
+use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit;
+
+use function Shlinkio\Shlink\Core\isCrawler;
 
 class Visit extends AbstractEntity implements JsonSerializable
 {
     public const TYPE_VALID_SHORT_URL = 'valid_short_url';
+    public const TYPE_IMPORTED = 'imported';
     public const TYPE_INVALID_SHORT_URL = 'invalid_short_url';
     public const TYPE_BASE_URL = 'base_url';
     public const TYPE_REGULAR_404 = 'regular_404';
 
     private string $referer;
     private Chronos $date;
-    private ?string $remoteAddr;
-    private ?string $visitedUrl;
+    private ?string $remoteAddr = null;
+    private ?string $visitedUrl = null;
     private string $userAgent;
     private string $type;
     private ?ShortUrl $shortUrl;
     private ?VisitLocation $visitLocation = null;
+    private bool $potentialBot;
 
-    private function __construct(?ShortUrl $shortUrl, Visitor $visitor, string $type, bool $anonymize = true)
+    private function __construct(?ShortUrl $shortUrl, string $type)
     {
         $this->shortUrl = $shortUrl;
         $this->date = Chronos::now();
+        $this->type = $type;
+    }
+
+    public static function forValidShortUrl(ShortUrl $shortUrl, Visitor $visitor, bool $anonymize = true): self
+    {
+        $instance = new self($shortUrl, self::TYPE_VALID_SHORT_URL);
+        $instance->hydrateFromVisitor($visitor, $anonymize);
+
+        return $instance;
+    }
+
+    public static function fromImport(ShortUrl $shortUrl, ImportedShlinkVisit $importedVisit): self
+    {
+        $instance = new self($shortUrl, self::TYPE_IMPORTED);
+        $instance->userAgent = $importedVisit->userAgent();
+        $instance->potentialBot = isCrawler($instance->userAgent);
+        $instance->referer = $importedVisit->referer();
+        $instance->date = Chronos::instance($importedVisit->date());
+
+        $importedLocation = $importedVisit->location();
+        $instance->visitLocation = $importedLocation !== null ? VisitLocation::fromImport($importedLocation) : null;
+
+        return $instance;
+    }
+
+    public static function forBasePath(Visitor $visitor, bool $anonymize = true): self
+    {
+        $instance = new self(null, self::TYPE_BASE_URL);
+        $instance->hydrateFromVisitor($visitor, $anonymize);
+
+        return $instance;
+    }
+
+    public static function forInvalidShortUrl(Visitor $visitor, bool $anonymize = true): self
+    {
+        $instance = new self(null, self::TYPE_INVALID_SHORT_URL);
+        $instance->hydrateFromVisitor($visitor, $anonymize);
+
+        return $instance;
+    }
+
+    public static function forRegularNotFound(Visitor $visitor, bool $anonymize = true): self
+    {
+        $instance = new self(null, self::TYPE_REGULAR_404);
+        $instance->hydrateFromVisitor($visitor, $anonymize);
+
+        return $instance;
+    }
+
+    private function hydrateFromVisitor(Visitor $visitor, bool $anonymize = true): void
+    {
         $this->userAgent = $visitor->getUserAgent();
         $this->referer = $visitor->getReferer();
         $this->remoteAddr = $this->processAddress($anonymize, $visitor->getRemoteAddress());
         $this->visitedUrl = $visitor->getVisitedUrl();
-        $this->type = $type;
+        $this->potentialBot = $visitor->isPotentialBot();
     }
 
     private function processAddress(bool $anonymize, ?string $address): ?string
@@ -53,26 +109,6 @@ class Visit extends AbstractEntity implements JsonSerializable
         }
     }
 
-    public static function forValidShortUrl(ShortUrl $shortUrl, Visitor $visitor, bool $anonymize = true): self
-    {
-        return new self($shortUrl, $visitor, self::TYPE_VALID_SHORT_URL, $anonymize);
-    }
-
-    public static function forBasePath(Visitor $visitor, bool $anonymize = true): self
-    {
-        return new self(null, $visitor, self::TYPE_BASE_URL, $anonymize);
-    }
-
-    public static function forInvalidShortUrl(Visitor $visitor, bool $anonymize = true): self
-    {
-        return new self(null, $visitor, self::TYPE_INVALID_SHORT_URL, $anonymize);
-    }
-
-    public static function forRegularNotFound(Visitor $visitor, bool $anonymize = true): self
-    {
-        return new self(null, $visitor, self::TYPE_REGULAR_404, $anonymize);
-    }
-
     public function getRemoteAddr(): ?string
     {
         return $this->remoteAddr;
@@ -119,6 +155,15 @@ class Visit extends AbstractEntity implements JsonSerializable
         return $this->type;
     }
 
+    /**
+     * Needed only for ArrayCollections to be able to apply criteria filtering
+     * @internal
+     */
+    public function getType(): string
+    {
+        return $this->type();
+    }
+
     public function jsonSerialize(): array
     {
         return [
@@ -126,6 +171,7 @@ class Visit extends AbstractEntity implements JsonSerializable
             'date' => $this->date->toAtomString(),
             'userAgent' => $this->userAgent,
             'visitLocation' => $this->visitLocation,
+            'potentialBot' => $this->potentialBot,
         ];
     }
 

module/Core/src/Entity/VisitLocation.php

@@ -6,6 +6,7 @@ namespace Shlinkio\Shlink\Core\Entity;
 
 use Shlinkio\Shlink\Common\Entity\AbstractEntity;
 use Shlinkio\Shlink\Core\Visit\Model\VisitLocationInterface;
+use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisitLocation;
 use Shlinkio\Shlink\IpGeolocation\Model\Location;
 
 class VisitLocation extends AbstractEntity implements VisitLocationInterface
@@ -19,9 +20,53 @@ class VisitLocation extends AbstractEntity implements VisitLocationInterface
     private string $timezone;
     private bool $isEmpty;
 
-    public function __construct(Location $location)
+    private function __construct()
     {
-        $this->exchangeLocationInfo($location);
+    }
+
+    public static function fromGeolocation(Location $location): self
+    {
+        $instance = new self();
+
+        $instance->countryCode = $location->countryCode();
+        $instance->countryName = $location->countryName();
+        $instance->regionName = $location->regionName();
+        $instance->cityName = $location->city();
+        $instance->latitude = $location->latitude();
+        $instance->longitude = $location->longitude();
+        $instance->timezone = $location->timeZone();
+        $instance->computeIsEmpty();
+
+        return $instance;
+    }
+
+    public static function fromImport(ImportedShlinkVisitLocation $location): self
+    {
+        $instance = new self();
+
+        $instance->countryCode = $location->countryCode();
+        $instance->countryName = $location->countryName();
+        $instance->regionName = $location->regionName();
+        $instance->cityName = $location->cityName();
+        $instance->latitude = $location->latitude();
+        $instance->longitude = $location->longitude();
+        $instance->timezone = $location->timeZone();
+        $instance->computeIsEmpty();
+
+        return $instance;
+    }
+
+    private function computeIsEmpty(): void
+    {
+        $this->isEmpty = (
+            $this->countryCode === '' &&
+            $this->countryName === '' &&
+            $this->regionName === '' &&
+            $this->cityName === '' &&
+            $this->latitude === 0.0 &&
+            $this->longitude === 0.0 &&
+            $this->timezone === ''
+        );
     }
 
     public function getCountryName(): string
@@ -49,26 +94,6 @@ class VisitLocation extends AbstractEntity implements VisitLocationInterface
         return $this->isEmpty;
     }
 
-    private function exchangeLocationInfo(Location $info): void
-    {
-        $this->countryCode = $info->countryCode();
-        $this->countryName = $info->countryName();
-        $this->regionName = $info->regionName();
-        $this->cityName = $info->city();
-        $this->latitude = $info->latitude();
-        $this->longitude = $info->longitude();
-        $this->timezone = $info->timeZone();
-        $this->isEmpty = (
-            $this->countryCode === '' &&
-            $this->countryName === '' &&
-            $this->regionName === '' &&
-            $this->cityName === '' &&
-            $this->latitude === 0.0 &&
-            $this->longitude === 0.0 &&
-            $this->timezone === ''
-        );
-    }
-
     public function jsonSerialize(): array
     {
         return [

module/Core/src/EventDispatcher/LocateVisit.php

@@ -7,31 +7,29 @@ namespace Shlinkio\Shlink\Core\EventDispatcher;
 use Doctrine\ORM\EntityManagerInterface;
 use Psr\EventDispatcher\EventDispatcherInterface;
 use Psr\Log\LoggerInterface;
-use Shlinkio\Shlink\CLI\Exception\GeolocationDbUpdateFailedException;
-use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface;
 use Shlinkio\Shlink\Core\Entity\Visit;
 use Shlinkio\Shlink\Core\Entity\VisitLocation;
 use Shlinkio\Shlink\Core\EventDispatcher\Event\UrlVisited;
 use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated;
 use Shlinkio\Shlink\IpGeolocation\Exception\WrongIpException;
+use Shlinkio\Shlink\IpGeolocation\GeoLite2\DbUpdaterInterface;
 use Shlinkio\Shlink\IpGeolocation\Model\Location;
 use Shlinkio\Shlink\IpGeolocation\Resolver\IpLocationResolverInterface;
-
-use function sprintf;
+use Throwable;
 
 class LocateVisit
 {
     private IpLocationResolverInterface $ipLocationResolver;
     private EntityManagerInterface $em;
     private LoggerInterface $logger;
-    private GeolocationDbUpdaterInterface $dbUpdater;
+    private DbUpdaterInterface $dbUpdater;
     private EventDispatcherInterface $eventDispatcher;
 
     public function __construct(
         IpLocationResolverInterface $ipLocationResolver,
         EntityManagerInterface $em,
         LoggerInterface $logger,
-        GeolocationDbUpdaterInterface $dbUpdater,
+        DbUpdaterInterface $dbUpdater,
         EventDispatcherInterface $eventDispatcher
     ) {
         $this->ipLocationResolver = $ipLocationResolver;
@@ -54,49 +52,37 @@ class LocateVisit
             return;
         }
 
-        if ($this->downloadOrUpdateGeoLiteDb($visitId)) {
-            $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit);
-        }
-
+        $this->locateVisit($visitId, $shortUrlVisited->originalIpAddress(), $visit);
         $this->eventDispatcher->dispatch(new VisitLocated($visitId));
     }
 
-    private function downloadOrUpdateGeoLiteDb(string $visitId): bool
-    {
-        try {
-            $this->dbUpdater->checkDbUpdate(function (bool $olderDbExists): void {
-                $this->logger->notice(sprintf('%s GeoLite2 database...', $olderDbExists ? 'Updating' : 'Downloading'));
-            });
-        } catch (GeolocationDbUpdateFailedException $e) {
-            if (! $e->olderDbExists()) {
-                $this->logger->error(
-                    'GeoLite2 database download failed. It is not possible to locate visit with id {visitId}. {e}',
-                    ['e' => $e, 'visitId' => $visitId],
-                );
-                return false;
-            }
-
-            $this->logger->warning('GeoLite2 database update failed. Proceeding with old version. {e}', ['e' => $e]);
-        }
-
-        return true;
-    }
-
     private function locateVisit(string $visitId, ?string $originalIpAddress, Visit $visit): void
     {
+        if (! $this->dbUpdater->databaseFileExists()) {
+            $this->logger->warning('Tried to locate visit with id "{visitId}", but a GeoLite2 db was not found.', [
+                'visitId' => $visitId,
+            ]);
+            return;
+        }
+
         $isLocatable = $originalIpAddress !== null || $visit->isLocatable();
         $addr = $originalIpAddress ?? $visit->getRemoteAddr();
 
         try {
             $location = $isLocatable ? $this->ipLocationResolver->resolveIpLocation($addr) : Location::emptyInstance();
 
-            $visit->locate(new VisitLocation($location));
+            $visit->locate(VisitLocation::fromGeolocation($location));
             $this->em->flush();
         } catch (WrongIpException $e) {
             $this->logger->warning(
                 'Tried to locate visit with id "{visitId}", but its address seems to be wrong. {e}',
                 ['e' => $e, 'visitId' => $visitId],
             );
+        } catch (Throwable $e) {
+            $this->logger->error(
+                'An unexpected error occurred while trying to locate visit with id "{visitId}". {e}',
+                ['e' => $e, 'visitId' => $visitId],
+            );
         }
     }
 }

module/Core/src/EventDispatcher/NotifyVisitToMercure.php

@@ -9,7 +9,7 @@ use Psr\Log\LoggerInterface;
 use Shlinkio\Shlink\Core\Entity\Visit;
 use Shlinkio\Shlink\Core\EventDispatcher\Event\VisitLocated;
 use Shlinkio\Shlink\Core\Mercure\MercureUpdatesGeneratorInterface;
-use Symfony\Component\Mercure\PublisherInterface;
+use Symfony\Component\Mercure\HubInterface;
 use Symfony\Component\Mercure\Update;
 use Throwable;
 
@@ -17,18 +17,18 @@ use function Functional\each;
 
 class NotifyVisitToMercure
 {
-    private PublisherInterface $publisher;
+    private HubInterface $hub;
     private MercureUpdatesGeneratorInterface $updatesGenerator;
     private EntityManagerInterface $em;
     private LoggerInterface $logger;
 
     public function __construct(
-        PublisherInterface $publisher,
+        HubInterface $hub,
         MercureUpdatesGeneratorInterface $updatesGenerator,
         EntityManagerInterface $em,
         LoggerInterface $logger
     ) {
-        $this->publisher = $publisher;
+        $this->hub = $hub;
         $this->em = $em;
         $this->logger = $logger;
         $this->updatesGenerator = $updatesGenerator;
@@ -48,7 +48,7 @@ class NotifyVisitToMercure
         }
 
         try {
-            each($this->determineUpdatesForVisit($visit), fn (Update $update) => ($this->publisher)($update));
+            each($this->determineUpdatesForVisit($visit), fn (Update $update) => $this->hub->publish($update));
         } catch (Throwable $e) {
             $this->logger->debug('Error while trying to notify mercure hub with new visit. {e}', [
                 'e' => $e,

module/Core/src/EventDispatcher/UpdateGeoLiteDb.php

@@ -0,0 +1,45 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\EventDispatcher;
+
+use Psr\Log\LoggerInterface;
+use Shlinkio\Shlink\CLI\Util\GeolocationDbUpdaterInterface;
+use Throwable;
+
+use function sprintf;
+
+class UpdateGeoLiteDb
+{
+    private GeolocationDbUpdaterInterface $dbUpdater;
+    private LoggerInterface $logger;
+
+    public function __construct(GeolocationDbUpdaterInterface $dbUpdater, LoggerInterface $logger)
+    {
+        $this->dbUpdater = $dbUpdater;
+        $this->logger = $logger;
+    }
+
+    public function __invoke(): void
+    {
+        $beforeDownload = fn (bool $olderDbExists) => $this->logger->notice(
+            sprintf('%s GeoLite2 db file...', $olderDbExists ? 'Updating' : 'Downloading'),
+        );
+        $messageLogged = false;
+        $handleProgress = function (int $total, int $downloaded, bool $olderDbExists) use (&$messageLogged): void {
+            if ($messageLogged || $total > $downloaded) {
+                return;
+            }
+
+            $messageLogged = true;
+            $this->logger->notice(sprintf('Finished %s GeoLite2 db file', $olderDbExists ? 'updating' : 'downloading'));
+        };
+
+        try {
+            $this->dbUpdater->checkDbUpdate($beforeDownload, $handleProgress);
+        } catch (Throwable $e) {
+            $this->logger->error('GeoLite2 database download failed. {e}', ['e' => $e]);
+        }
+    }
+}

module/Core/src/Exception/NonUniqueSlugException.php

@@ -7,6 +7,7 @@ namespace Shlinkio\Shlink\Core\Exception;
 use Fig\Http\Message\StatusCodeInterface;
 use Mezzio\ProblemDetails\Exception\CommonProblemDetailsExceptionTrait;
 use Mezzio\ProblemDetails\Exception\ProblemDetailsExceptionInterface;
+use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
 
 use function sprintf;
 
@@ -34,4 +35,9 @@ class NonUniqueSlugException extends InvalidArgumentException implements Problem
 
         return $e;
     }
+
+    public static function fromImport(ImportedShlinkUrl $importedUrl): self
+    {
+        return self::fromSlug($importedUrl->shortCode(), $importedUrl->domain());
+    }
 }

module/Core/src/Importer/ImportedLinksProcessor.php

@@ -6,12 +6,14 @@ namespace Shlinkio\Shlink\Core\Importer;
 
 use Doctrine\ORM\EntityManagerInterface;
 use Shlinkio\Shlink\Core\Entity\ShortUrl;
+use Shlinkio\Shlink\Core\Exception\NonUniqueSlugException;
 use Shlinkio\Shlink\Core\Repository\ShortUrlRepositoryInterface;
 use Shlinkio\Shlink\Core\Service\ShortUrl\ShortCodeHelperInterface;
 use Shlinkio\Shlink\Core\ShortUrl\Resolver\ShortUrlRelationResolverInterface;
 use Shlinkio\Shlink\Core\Util\DoctrineBatchHelperInterface;
 use Shlinkio\Shlink\Importer\ImportedLinksProcessorInterface;
 use Shlinkio\Shlink\Importer\Model\ImportedShlinkUrl;
+use Shlinkio\Shlink\Importer\Sources\ImportSources;
 use Symfony\Component\Console\Style\StyleInterface;
 
 use function sprintf;
@@ -22,6 +24,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface
     private ShortUrlRelationResolverInterface $relationResolver;
     private ShortCodeHelperInterface $shortCodeHelper;
     private DoctrineBatchHelperInterface $batchHelper;
+    private ShortUrlRepositoryInterface $shortUrlRepo;
 
     public function __construct(
         EntityManagerInterface $em,
@@ -33,6 +36,7 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface
         $this->relationResolver = $relationResolver;
         $this->shortCodeHelper = $shortCodeHelper;
         $this->batchHelper = $batchHelper;
+        $this->shortUrlRepo = $this->em->getRepository(ShortUrl::class); // @phpstan-ignore-line
     }
 
     /**
@@ -40,51 +44,65 @@ class ImportedLinksProcessor implements ImportedLinksProcessorInterface
      */
     public function process(StyleInterface $io, iterable $shlinkUrls, array $params): void
     {
-        /** @var ShortUrlRepositoryInterface $shortUrlRepo */
-        $shortUrlRepo = $this->em->getRepository(ShortUrl::class);
         $importShortCodes = $params['import_short_codes'];
-        $iterable = $this->batchHelper->wrapIterable($shlinkUrls, 100);
+        $source = $params['source'];
+        $iterable = $this->batchHelper->wrapIterable($shlinkUrls, $source === ImportSources::SHLINK ? 10 : 100);
 
-        /** @var ImportedShlinkUrl $url */
-        foreach ($iterable as $url) {
-            $longUrl = $url->longUrl();
+        /** @var ImportedShlinkUrl $importedUrl */
+        foreach ($iterable as $importedUrl) {
+            $skipOnShortCodeConflict = static function () use ($io, $importedUrl): bool {
+                $action = $io->choice(sprintf(
+                    'Failed to import URL "%s" because its short-code "%s" is already in use. Do you want to generate '
+                    . 'a new one or skip it?',
+                    $importedUrl->longUrl(),
+                    $importedUrl->shortCode(),
+                ), ['Generate new short-code', 'Skip'], 1);
 
-            // Skip already imported URLs
-            if ($shortUrlRepo->importedUrlExists($url)) {
-                $io->text(sprintf('%s: <comment>Skipped</comment>', $longUrl));
+                return $action === 'Skip';
+            };
+            $longUrl = $importedUrl->longUrl();
+
+            try {
+                $shortUrlImporting = $this->resolveShortUrl($importedUrl, $importShortCodes, $skipOnShortCodeConflict);
+            } catch (NonUniqueSlugException $e) {
+                $io->text(sprintf('%s: <fg=red>Error</>', $longUrl));
                 continue;
             }
 
-            $shortUrl = ShortUrl::fromImport($url, $importShortCodes, $this->relationResolver);
-            if (! $this->handleShortCodeUniqueness($url, $shortUrl, $io, $importShortCodes)) {
-                continue;
-            }
-
-            $this->em->persist($shortUrl);
-            $io->text(sprintf('%s: <info>Imported</info>', $longUrl));
+            $resultMessage = $shortUrlImporting->importVisits($importedUrl->visits(), $this->em);
+            $io->text(sprintf('%s: %s', $longUrl, $resultMessage));
         }
     }
 
+    private function resolveShortUrl(
+        ImportedShlinkUrl $importedUrl,
+        bool $importShortCodes,
+        callable $skipOnShortCodeConflict
+    ): ShortUrlImporting {
+        $alreadyImportedShortUrl = $this->shortUrlRepo->findOneByImportedUrl($importedUrl);
+        if ($alreadyImportedShortUrl !== null) {
+            return ShortUrlImporting::fromExistingShortUrl($alreadyImportedShortUrl);
+        }
+
+        $shortUrl = ShortUrl::fromImport($importedUrl, $importShortCodes, $this->relationResolver);
+        if (! $this->handleShortCodeUniqueness($shortUrl, $importShortCodes, $skipOnShortCodeConflict)) {
+            throw NonUniqueSlugException::fromImport($importedUrl);
+        }
+
+        $this->em->persist($shortUrl);
+        return ShortUrlImporting::fromNewShortUrl($shortUrl);
+    }
+
     private function handleShortCodeUniqueness(
-        ImportedShlinkUrl $url,
         ShortUrl $shortUrl,
-        StyleInterface $io,
-        bool $importShortCodes
+        bool $importShortCodes,
+        callable $skipOnShortCodeConflict
     ): bool {
         if ($this->shortCodeHelper->ensureShortCodeUniqueness($shortUrl, $importShortCodes)) {
             return true;
         }
 
-        $longUrl = $url->longUrl();
-        $action = $io->choice(sprintf(
-            'Failed to import URL "%s" because its short-code "%s" is already in use. Do you want to generate a new '
-            . 'one or skip it?',
-            $longUrl,
-            $url->shortCode(),
-        ), ['Generate new short-code', 'Skip'], 1);
-
-        if ($action === 'Skip') {
-            $io->text(sprintf('%s: <comment>Skipped</comment>', $longUrl));
+        if ($skipOnShortCodeConflict()) {
             return false;
         }
 

module/Core/src/Importer/ShortUrlImporting.php

@@ -0,0 +1,65 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Importer;
+
+use Cake\Chronos\Chronos;
+use Doctrine\ORM\EntityManagerInterface;
+use Shlinkio\Shlink\Core\Entity\ShortUrl;
+use Shlinkio\Shlink\Core\Entity\Visit;
+use Shlinkio\Shlink\Importer\Model\ImportedShlinkVisit;
+
+use function sprintf;
+
+final class ShortUrlImporting
+{
+    private ShortUrl $shortUrl;
+    private bool $isNew;
+
+    private function __construct(ShortUrl $shortUrl, bool $isNew)
+    {
+        $this->shortUrl = $shortUrl;
+        $this->isNew = $isNew;
+    }
+
+    public static function fromExistingShortUrl(ShortUrl $shortUrl): self
+    {
+        return new self($shortUrl, false);
+    }
+
+    public static function fromNewShortUrl(ShortUrl $shortUrl): self
+    {
+        return new self($shortUrl, true);
+    }
+
+    /**
+     * @param iterable|ImportedShlinkVisit[] $visits
+     */
+    public function importVisits(iterable $visits, EntityManagerInterface $em): string
+    {
+        $mostRecentImportedDate = $this->shortUrl->mostRecentImportedVisitDate();
+
+        $importedVisits = 0;
+        foreach ($visits as $importedVisit) {
+            // Skip visits which are older than the most recent already imported visit's date
+            if (
+                $mostRecentImportedDate !== null
+                && $mostRecentImportedDate->gte(Chronos::instance($importedVisit->date()))
+            ) {
+                continue;
+            }
+
+            $em->persist(Visit::fromImport($this->shortUrl, $importedVisit));
+            $importedVisits++;
+        }
+
+        if ($importedVisits === 0) {
+            return $this->isNew ? '<info>Imported</info>' : '<comment>Skipped</comment>';
+        }
+
+        return $this->isNew
+            ? sprintf('<info>Imported</info> with <info>%s</info> visits', $importedVisits)
+            : sprintf('<comment>Skipped</comment>. Imported <info>%s</info> visits', $importedVisits);
+    }
+}

module/Core/src/Model/ShortUrlEdit.php

@@ -30,6 +30,8 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
     private ?string $title = null;
     private bool $titleWasAutoResolved = false;
     private ?bool $validateUrl = null;
+    private bool $crawlablePropWasProvided = false;
+    private bool $crawlable = false;
 
     private function __construct()
     {
@@ -61,6 +63,7 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
         $this->maxVisitsPropWasProvided = array_key_exists(ShortUrlInputFilter::MAX_VISITS, $data);
         $this->tagsPropWasProvided = array_key_exists(ShortUrlInputFilter::TAGS, $data);
         $this->titlePropWasProvided = array_key_exists(ShortUrlInputFilter::TITLE, $data);
+        $this->crawlablePropWasProvided = array_key_exists(ShortUrlInputFilter::CRAWLABLE, $data);
 
         $this->longUrl = $inputFilter->getValue(ShortUrlInputFilter::LONG_URL);
         $this->validSince = parseDateField($inputFilter->getValue(ShortUrlInputFilter::VALID_SINCE));
@@ -69,6 +72,7 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
         $this->validateUrl = getOptionalBoolFromInputFilter($inputFilter, ShortUrlInputFilter::VALIDATE_URL);
         $this->tags = $inputFilter->getValue(ShortUrlInputFilter::TAGS);
         $this->title = $inputFilter->getValue(ShortUrlInputFilter::TITLE);
+        $this->crawlable = $inputFilter->getValue(ShortUrlInputFilter::CRAWLABLE);
     }
 
     public function longUrl(): ?string
@@ -162,4 +166,14 @@ final class ShortUrlEdit implements TitleResolutionModelInterface
     {
         return $this->validateUrl;
     }
+
+    public function crawlable(): bool
+    {
+        return $this->crawlable;
+    }
+
+    public function crawlableWasProvided(): bool
+    {
+        return $this->crawlablePropWasProvided;
+    }
 }

module/Core/src/Model/ShortUrlIdentifier.php

@@ -5,6 +5,7 @@ declare(strict_types=1);
 namespace Shlinkio\Shlink\Core\Model;
 
 use Psr\Http\Message\ServerRequestInterface;
+use Shlinkio\Shlink\Core\Entity\ShortUrl;
 use Symfony\Component\Console\Input\InputInterface;
 
 final class ShortUrlIdentifier
@@ -42,6 +43,19 @@ final class ShortUrlIdentifier
         return new self($shortCode, $domain);
     }
 
+    public static function fromShortUrl(ShortUrl $shortUrl): self
+    {
+        $domain = $shortUrl->getDomain();
+        $domainAuthority = $domain !== null ? $domain->getAuthority() : null;
+
+        return new self($shortUrl->getShortCode(), $domainAuthority);
+    }
+
+    public static function fromShortCodeAndDomain(string $shortCode, ?string $domain = null): self
+    {
+        return new self($shortCode, $domain);
+    }
+
     public function shortCode(): string
     {
         return $this->shortCode;

module/Core/src/Model/ShortUrlMeta.php

@@ -31,6 +31,7 @@ final class ShortUrlMeta implements TitleResolutionModelInterface
     private array $tags = [];
     private ?string $title = null;
     private bool $titleWasAutoResolved = false;
+    private bool $crawlable = false;
 
     private function __construct()
     {
@@ -80,6 +81,7 @@ final class ShortUrlMeta implements TitleResolutionModelInterface
         $this->apiKey = $inputFilter->getValue(ShortUrlInputFilter::API_KEY);
         $this->tags = $inputFilter->getValue(ShortUrlInputFilter::TAGS);
         $this->title = $inputFilter->getValue(ShortUrlInputFilter::TITLE);
+        $this->crawlable = $inputFilter->getValue(ShortUrlInputFilter::CRAWLABLE);
     }
 
     public function getLongUrl(): string
@@ -188,4 +190,9 @@ final class ShortUrlMeta implements TitleResolutionModelInterface
 
         return $copy;
     }
+
+    public function isCrawlable(): bool
+    {
+        return $this->crawlable;
+    }
 }

module/Core/src/Model/Visitor.php

@@ -6,7 +6,9 @@ namespace Shlinkio\Shlink\Core\Model;
 
 use Psr\Http\Message\ServerRequestInterface;
 use Shlinkio\Shlink\Common\Middleware\IpAddressMiddlewareFactory;
+use Shlinkio\Shlink\Core\Options\TrackingOptions;
 
+use function Shlinkio\Shlink\Core\isCrawler;
 use function substr;
 
 final class Visitor
@@ -20,6 +22,7 @@ final class Visitor
     private string $referer;
     private string $visitedUrl;
     private ?string $remoteAddress;
+    private bool $potentialBot;
 
     public function __construct(string $userAgent, string $referer, ?string $remoteAddress, string $visitedUrl)
     {
@@ -27,6 +30,7 @@ final class Visitor
         $this->referer = $this->cropToLength($referer, self::REFERER_MAX_LENGTH);
         $this->visitedUrl = $this->cropToLength($visitedUrl, self::VISITED_URL_MAX_LENGTH);
         $this->remoteAddress = $this->cropToLength($remoteAddress, self::REMOTE_ADDRESS_MAX_LENGTH);
+        $this->potentialBot = isCrawler($userAgent);
     }
 
     private function cropToLength(?string $value, int $length): ?string
@@ -49,6 +53,11 @@ final class Visitor
         return new self('', '', null, '');
     }
 
+    public static function botInstance(): self
+    {
+        return new self('cf-facebook', '', null, '');
+    }
+
     public function getUserAgent(): string
     {
         return $this->userAgent;
@@ -68,4 +77,24 @@ final class Visitor
     {
         return $this->visitedUrl;
     }
+
+    public function isPotentialBot(): bool
+    {
+        return $this->potentialBot;
+    }
+
+    public function normalizeForTrackingOptions(TrackingOptions $options): self
+    {
+        $instance = new self(
+            $options->disableUaTracking() ? '' : $this->userAgent,
+            $options->disableReferrerTracking() ? '' : $this->referer,
+            $options->disableIpTracking() ? null : $this->remoteAddress,
+            $this->visitedUrl,
+        );
+
+        // Keep the fact that the visit was a potential bot, even if we no longer save the user agent
+        $instance->potentialBot = $this->potentialBot;
+
+        return $instance;
+    }
 }