diff --git a/.env.dist b/.env.dist index d7168e55c..5b7015f81 100644 --- a/.env.dist +++ b/.env.dist @@ -24,7 +24,7 @@ DATABASE_URL=sqlite:///%kernel.project_dir%/var/data/bolt.sqlite #DATABASE_URL=mysql://db_user:"db_password"@localhost:3306/db_name # Postgres -#DATABASE_URL=postgresql://db_user:"db_password"@localhost:5432/db_name?serverVersion=11" +#DATABASE_URL=postgresql://db_user:"db_password"@localhost:5432/db_name?serverVersion=11&charset=utf8" # MYSQL / MariaDB (additional settings, needed for Docker) #DATABASE_USER=db_user diff --git a/.gitignore b/.gitignore index 69889c194..a14c6e815 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,7 @@ appveyor.yml ###> phpunit/phpunit ### /phpunit.xml ###< phpunit/phpunit ### + +###> .preload dev ### +/src/.preload.php +###< .preload dev ### diff --git a/README.md b/README.md index 6c10adfb9..58557d301 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,19 @@ bin/console doctrine:fixtures:load -n Alternatively, run `make db-reset`, on a UNIX-like system. -4 Run the prototype +4 How to build assets +------------------- + +To set up initially, run `npm install` to get the required dependencies / +`node_modules`. Alternatively give the path to the python executable (`npm install --python="/usr/local/bin/python3.7"`) Then: + - Prepare directory structure `mkdir -p node_modules/node-sass/vendor` + - Rebuild npm environment for current OS `npm rebuild node-sass` + - Run `npm run start` (alternatively `npm run start --python="/usr/local/bin/python3.7"`) + +See the other options by running `npm run`. +(Note: as I'm testing this as well remotely, I copied all assets from the released composer installation by `cp -r ../www_backup/public/assets/* public/assets/`) + +5 Run the prototype ------------------- - Using the Symfony CLI tool, just run `symfony server:start`. @@ -134,17 +146,6 @@ You can log on, using the default user & pass: - pass: `admin%1` -How to build assets -------------------- - -To set up initially, run `npm install` to get the required dependencies / -`node_modules`. Then: - - - Run `npm run start` - -See the other options by running `npm run`. - - Code Style checking / Static Analysis ---------------------------- diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 853b28ee2..4275d408a 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -32,7 +32,8 @@ doctrine: dql: string_functions: JSON_EXTRACT: Bolt\Doctrine\Functions\JsonExtract - CAST: DoctrineExtensions\Query\Mysql\Cast + JSON_GET_TEXT: Scienta\DoctrineJsonFunctions\Query\AST\Functions\Postgresql\JsonGetText + CAST: Bolt\Doctrine\Query\Cast numeric_functions: RAND: Bolt\Doctrine\Functions\Rand diff --git a/src/Doctrine/Functions/Rand.php b/src/Doctrine/Functions/Rand.php index 6637b85b6..200b01433 100644 --- a/src/Doctrine/Functions/Rand.php +++ b/src/Doctrine/Functions/Rand.php @@ -19,6 +19,10 @@ public function getSql(\Doctrine\ORM\Query\SqlWalker $sqlWalker) if (property_exists($this->expression, 'value') && $this->expression->value === '1') { return 'random()'; } + // value is two if PostgreSQL. See Bolt\Storage\Directive\RandomDirectiveHandler + if (property_exists($this->expression, 'value') && $this->expression->value === '2') { + return 'RANDOM()'; + } return 'RAND()'; } diff --git a/src/Doctrine/JsonHelper.php b/src/Doctrine/JsonHelper.php index d11de3ac3..0210b217f 100644 --- a/src/Doctrine/JsonHelper.php +++ b/src/Doctrine/JsonHelper.php @@ -25,7 +25,14 @@ public static function wrapJsonFunction(?string $where = null, ?string $slug = n $version = new Version($connection); if ($version->hasJson()) { - $resultWhere = 'JSON_EXTRACT(' . $where . ", '$[0]')"; + //PostgreSQL handles JSON differently than MySQL + if ($version->getPlatform()['driver_name'] === 'pgsql') { + // PostgreSQL + $resultWhere = 'JSON_GET_TEXT(' . $where . ', 0)'; + } else { + // MySQL and SQLite + $resultWhere = 'JSON_EXTRACT(' . $where . ", '$[0]')"; + } $resultSlug = $slug; } else { $resultWhere = $where; diff --git a/src/Doctrine/Query/Cast.php b/src/Doctrine/Query/Cast.php new file mode 100644 index 000000000..79b49caed --- /dev/null +++ b/src/Doctrine/Query/Cast.php @@ -0,0 +1,54 @@ +getConnection()->getDriver()->getName(); + + // test if we are using MySQL + if (mb_strpos($backend_driver, 'mysql') !== false) { + // YES we are using MySQL + // how do we know what type $this->first is? For now hardcoding + // type(t.value) = JSON for MySQL. JSONB for others. + // alternatively, test if true: $this->first->dispatch($sqlWalker)==='b2_.value', + // b4_.value for /bolt/new/showcases + if ($this->first->dispatch($sqlWalker) === 'b2_.value' || + $this->first->dispatch($sqlWalker) === 'b4_.value') { + return $this->first->dispatch($sqlWalker); + } + } + + return sprintf('CAST(%s AS %s)', + $this->first->dispatch($sqlWalker), + $this->second + ); + } + + public function parse(Parser $parser): void + { + $parser->match(Lexer::T_IDENTIFIER); + $parser->match(Lexer::T_OPEN_PARENTHESIS); + $this->first = $parser->ArithmeticPrimary(); + $parser->match(Lexer::T_AS); + $parser->match(Lexer::T_IDENTIFIER); + $this->second = $parser->getLexer()->token['value']; + $parser->match(Lexer::T_CLOSE_PARENTHESIS); + } +} diff --git a/src/Doctrine/Version.php b/src/Doctrine/Version.php index 9fb914125..fd7e37146 100644 --- a/src/Doctrine/Version.php +++ b/src/Doctrine/Version.php @@ -9,6 +9,7 @@ use Doctrine\DBAL\Driver\PDOConnection; use Doctrine\DBAL\Platforms\MariaDb1027Platform; use Doctrine\DBAL\Platforms\MySQL57Platform; +use Doctrine\DBAL\Platforms\PostgreSQL92Platform; use Doctrine\DBAL\Platforms\SqlitePlatform; class Version @@ -16,6 +17,12 @@ class Version /** * We're g̶u̶e̶s̶s̶i̶n̶g̶ doing empirical research on which versions of SQLite * support JSON. So far, tests indicate: + * https://www.sqlite.org/json1.html --> JSON since SQLite version 3.9.0 (2015-10-14) + * Docker uses an outdated version of SQLite/PHP combi with wrong implementation of JSON1 extensions: + * https://www.talvbansal.me/blog/unit-tests-with-json-columns-in-sqlite/ + * This explains why BOLT is working in stand-alone production/dev envs. but not in unit tests using Docker + * We need to replace this with a proper function test, instead of a guestimate. + * - 3.32.2 - OK (Wytse's FBSD 12.1 \w PHP 7.2) * - 3.20.1 - Not OK (Travis PHP 7.2) * - 3.27.2 - OK (Bob's Raspberry Pi, running PHP 7.3.11 on Raspbian) * - 3.28.0 - OK (Travis PHP 7.3) @@ -23,8 +30,10 @@ class Version * - 3.29.0 - OK (MacOS Mojave) * - 3.30.1 - OK (MacOS Catalina) */ + // JSON supported since SQLite version 3.9.0 public const SQLITE_WITH_JSON = '3.27.0'; - public const PHP_WITH_SQLITE = '7.3.0'; + // PHP supports SQLite since version 5.3.0, but not always bundled with JSON support! + public const PHP_WITH_SQLITE_JSON_SUPPORT = '7.3.0'; /** @var Connection */ private $connection; @@ -79,7 +88,11 @@ public function hasJson(): bool $platform = $this->connection->getDatabasePlatform(); if ($platform instanceof SqlitePlatform) { - return $this->checkSqliteVersion(); + // new method to test for JSON support + return $this->hasSQLiteJSONSupport(); + + // temporarily leave this in, until above method is fully tested + // return $this->checkSqliteVersion(); } // MySQL80Platform is implicitly included with MySQL57Platform @@ -87,6 +100,11 @@ public function hasJson(): bool return true; } + // PostgreSQL supports JSON from v9.2 and above, later versions are implicitly included + if ($platform instanceof PostgreSQL92Platform) { + return true; + } + return false; } @@ -104,19 +122,40 @@ public function hasCast(): bool return true; } - private function checkSqliteVersion(): bool + /* leave until alternative method fully tested */ + // private function checkSqliteVersion(): bool + // { + // /** @var PDOConnection */ + // $wrapped = $this->connection->getWrappedConnection(); + + // // If the wrapper doesn't have `getAttribute`, we bail… + // if (! method_exists($wrapped, 'getAttribute')) { + // return false; + // } + + // [$client_version] = explode(' - ', $wrapped->getAttribute(\PDO::ATTR_CLIENT_VERSION)); + + // return (version_compare($client_version, self::SQLITE_WITH_JSON) > 0) && + // (version_compare(PHP_VERSION, self::PHP_WITH_SQLITE_JSON_SUPPORT) > 0); + //} + + private function hasSQLiteJSONSupport(): bool { - /** @var PDOConnection $wrapped */ - $wrapped = $this->connection->getWrappedConnection(); + // Here we can test SQLite for JSON support + // This should also work for MySQL + // For PostgreSQL a different query is required, but this should be easy to expand (check for driver + adjust query) + // Query = "SELECT JSON_EXTRACT('{"jsonfunctionalitytest":["succes"]}', '$.jsonfunctionalitytest') as value"; + // Should return value ["succes"], and throw an error if JSON unsupported. - // If the wrapper doesn't have `getAttribute`, we bail… - if (! method_exists($wrapped, 'getAttribute')) { + try { + $query = $this->connection->createQueryBuilder(); + $query + ->select('JSON_EXTRACT(\'{"jsonfunctionalitytest":["succes"]}\', \'$.jsonfunctionalitytest\') as value'); + $query->execute(); + } catch (\Throwable $e) { return false; } - [$client_version] = explode(' - ', $wrapped->getAttribute(\PDO::ATTR_CLIENT_VERSION)); - - return (version_compare($client_version, self::SQLITE_WITH_JSON) > 0) && - (version_compare(PHP_VERSION, self::PHP_WITH_SQLITE) > 0); + return true; } } diff --git a/src/Entity/FieldTranslation.php b/src/Entity/FieldTranslation.php index dc7aad5e6..15d250ea8 100644 --- a/src/Entity/FieldTranslation.php +++ b/src/Entity/FieldTranslation.php @@ -21,7 +21,7 @@ class FieldTranslation implements TranslationInterface */ private $id; - /** @ORM\Column(type="json") */ + /** @ORM\Column(type="json", options={"jsonb": true}) */ protected $value = []; public function getId(): ?int diff --git a/src/Entity/Log.php b/src/Entity/Log.php index b209a76b6..0862e4940 100644 --- a/src/Entity/Log.php +++ b/src/Entity/Log.php @@ -38,7 +38,7 @@ class Log /** @ORM\Column(name="extra", type="array", nullable=true) */ private $extra; - /** @ORM\Column(name="user", type="array", nullable=true) */ + /** @ORM\Column(name="`user`", type="array", nullable=true) */ private $user; /** @ORM\Column(type="content", type="integer", nullable=true) */ diff --git a/src/Repository/ContentRepository.php b/src/Repository/ContentRepository.php index 5c1f51a34..3ef5ca72a 100644 --- a/src/Repository/ContentRepository.php +++ b/src/Repository/ContentRepository.php @@ -94,10 +94,14 @@ public function searchNaive(string $searchTerm, int $page, int $amountPerPage, C $qb = $this->getQueryBuilder() ->select('partial content.{id}'); + // proper JSON wrapping solves a lot of problems (added PostgreSQL compatibility) + $connection = $qb->getEntityManager()->getConnection(); + [$where] = JsonHelper::wrapJsonFunction('t.value', $searchTerm, $connection); + $qb->addSelect('f') ->innerJoin('content.fields', 'f') ->innerJoin('f.translations', 't') - ->andWhere($qb->expr()->like('t.value', ':search')) + ->andWhere($qb->expr()->like($where, ':search')) ->setParameter('search', '%' . $searchTerm . '%'); // These are the ID's of content we need. diff --git a/src/Storage/Directive/OrderDirective.php b/src/Storage/Directive/OrderDirective.php index 6fcfcc08b..a92b3727b 100644 --- a/src/Storage/Directive/OrderDirective.php +++ b/src/Storage/Directive/OrderDirective.php @@ -115,9 +115,10 @@ private function setOrderBy(QueryInterface $query, string $order, string $direct } else { // Note the `lower()` in the `addOrderBy()`. It is essential to sorting the // results correctly. See also https://github.com/bolt/core/issues/1190 + // again: lower breaks postgresql jsonb compatibility, first cast as txt $query ->getQueryBuilder() - ->addOrderBy('lower(' . $translationsAlias . '.value)', $direction); + ->addOrderBy('lower(CAST(' . $translationsAlias . '.value as TEXT))', $direction); } $query->incrementIndex(); } else { diff --git a/src/Storage/Directive/RandomDirectiveHandler.php b/src/Storage/Directive/RandomDirectiveHandler.php index b2a2bbeb5..8eb7b679e 100644 --- a/src/Storage/Directive/RandomDirectiveHandler.php +++ b/src/Storage/Directive/RandomDirectiveHandler.php @@ -31,6 +31,11 @@ public function __invoke(QueryInterface $query, $value, &$directives): void return; } + if ($this->version->getPlatform()['driver_name'] === 'pgsql') { + $query->getQueryBuilder()->addSelect('RAND(2) as HIDDEN rand')->orderBy('rand'); + + return; + } $query->getQueryBuilder()->addSelect('RAND(0) as HIDDEN rand')->orderBy('rand'); } diff --git a/src/Storage/SelectQuery.php b/src/Storage/SelectQuery.php index 9a71431d4..505ddcef8 100644 --- a/src/Storage/SelectQuery.php +++ b/src/Storage/SelectQuery.php @@ -537,7 +537,9 @@ private function getRegularFieldExpression(Filter $filter, EntityManagerInterfac $originalLeftExpression = 'content.' . $filter->getKey(); // LOWER() added to query to enable case insensitive search of JSON values. Used in conjunction with converting $params of setParameter() to lowercase. - $newLeftExpression = JsonHelper::wrapJsonFunction('LOWER(' . $valueAlias . ')', null, $em->getConnection()); + // BUG SQLSTATE[42883]: Undefined function: 7 ERROR: function lower(jsonb) does not exist + // We want to be able to search case-insensitive, database-agnostic, have to think of a good way.. + $newLeftExpression = JsonHelper::wrapJsonFunction($valueAlias, null, $em->getConnection()); $valueWhere = $filter->getExpression(); $valueWhere = str_replace($originalLeftExpression, $newLeftExpression, $valueWhere); $expr->add($valueWhere); diff --git a/templates/helpers/_field_blocks.twig b/templates/helpers/_field_blocks.twig index b6b71e0a1..4cc308340 100644 --- a/templates/helpers/_field_blocks.twig +++ b/templates/helpers/_field_blocks.twig @@ -43,11 +43,14 @@ {% block extended_fields %} {# Special case for 'select' fields: if it's a multiple select, the field is an array. #} + {# I dont know what is going on here, but field.selectedIds for pgsql showcase/mr-jean-feeney (and others) #} + {# yield field values (strings containing full name!), NOT integer IDs.. #} + {# TODO: FIX IT, bypassing for now by selecting on 'value' (string) and not 'id' (int) #} {% if type == "select" and field is not empty %}

{{ field|label }}: