From 45d526890b18ce486a2fd71aa4b5c7041131c6aa Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Sun, 15 Jun 2025 15:16:03 +0800 Subject: [PATCH 1/9] chore: refactor part of the code --- .clang-tidy | 2 +- .editorconfig | 4 + .git-blame-ignore-revs | 3 + .gitattributes | 1 + .gitignore | 58 +++-- .readthedocs.yml | 27 -- .yamllint | 15 +- CMakeLists.txt | 15 +- SECURITY.md | 43 +++- base/poco/Crypto/src/OpenSSLInitializer.cpp | 10 +- base/poco/Foundation/src/ErrorHandler.cpp | 4 +- base/poco/Net/src/TCPServer.cpp | 14 +- base/poco/Net/src/TCPServerDispatcher.cpp | 10 +- cmake/linux/toolchain-loongarch64.cmake | 23 ++ cmake/sanitize.cmake | 3 +- cmake/target.cmake | 2 +- cmake/xray_instrumentation.cmake | 20 ++ packages/clickhouse-library-bridge.yaml | 35 +++ packages/clickhouse-odbc-bridge.yaml | 35 +++ packages/clickhouse-server.init | 5 +- programs/client/Client.cpp | 2 +- .../local}/ColumnVectorHelper.h | 0 programs/local/LocalServer.cpp | 15 +- programs/local/LocalServer.h | 41 +-- programs/local/PythonSource.cpp | 2 +- programs/local/chdb.cpp | 2 +- programs/server/Server.cpp | 239 +++++++++--------- src/Access/Common/SSLCertificateSubjects.cpp | 1 + src/Analyzer/Resolve/QueryAnalyzer.cpp | 2 +- src/Client/ClientApplicationBase.cpp | 3 +- src/Columns/ColumnString.h | 3 +- src/Common/CgroupsMemoryUsageObserver.cpp | 3 +- src/Common/Coverage.cpp | 1 + .../tests/gtest_fields_binary_enciding.cpp | 1 + .../Serializations/SerializationObject.cpp | 1 + src/Interpreters/DatabaseCatalog.cpp | 14 +- src/Interpreters/DatabaseCatalog.h | 4 +- src/Interpreters/HashJoin/HashJoinMethods.h | 1 + src/Interpreters/InterpreterSetQuery.cpp | 2 +- src/Interpreters/InterpreterUseQuery.cpp | 3 - src/Parsers/ASTObjectTypeArgument.cpp | 2 + src/Parsers/ASTObjectTypeArgument.h | 1 + src/Parsers/ASTStatisticsDeclaration.cpp | 1 + src/Parsers/Kusto/KQL_ReleaseNote.md | 6 +- .../Transforms/PlanSquashingTransform.h | 1 + src/Storages/registerStorages.cpp | 2 +- src/TableFunctions/CMakeLists.txt | 29 --- 47 files changed, 403 insertions(+), 308 deletions(-) delete mode 100644 .readthedocs.yml create mode 100644 cmake/linux/toolchain-loongarch64.cmake create mode 100644 cmake/xray_instrumentation.cmake create mode 100644 packages/clickhouse-library-bridge.yaml create mode 100644 packages/clickhouse-odbc-bridge.yaml rename {src/Columns => programs/local}/ColumnVectorHelper.h (100%) diff --git a/.clang-tidy b/.clang-tidy index d0e7b7d1371..bb63bf2eea6 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -173,4 +173,4 @@ CheckOptions: performance-move-const-arg.CheckTriviallyCopyableMove: false # Workaround clang-tidy bug: https://github.com/llvm/llvm-project/issues/46097 readability-identifier-naming.TypeTemplateParameterIgnoredRegexp: expr-type - cppcoreguidelines-avoid-do-while.IgnoreMacros: true \ No newline at end of file + cppcoreguidelines-avoid-do-while.IgnoreMacros: true diff --git a/.editorconfig b/.editorconfig index 8ac06debb5a..8ecaf9b0267 100644 --- a/.editorconfig +++ b/.editorconfig @@ -19,3 +19,7 @@ charset = utf-8 indent_style = space indent_size = 4 trim_trailing_whitespace = true + +# Some SQL results have trailing whitespace which is removed by IDEs +[tests/queries/**.reference] +trim_trailing_whitespace = false diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 06e893fabb3..5088bf90ede 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -13,3 +13,6 @@ # dbms/ → src/ # (though it is unlikely that you will see it in blame) 06446b4f08a142d6f1bc30664c47ded88ab51782 + +# Applied Black formatter for Python code +e6f5a3f98b21ba99cf274a9833797889e020a2b3 diff --git a/.gitattributes b/.gitattributes index 56d6fecf4b8..dd94a48f8e7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,4 @@ contrib/* linguist-vendored *.h linguist-language=C++ tests/queries/0_stateless/data_json/* binary tests/queries/0_stateless/*.reference -crlf +src/Core/SettingsChangesHistory.cpp merge=union diff --git a/.gitignore b/.gitignore index 0d3a4ecfbbf..80424daf075 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,6 @@ *~ *\# .tramp_history -*.parquet -*.arrow # vim cache files *.swp @@ -11,32 +9,21 @@ # auto generated files *.logrt -/python_pkg/ -minitest/ -/tmps -/bak -*state_tmp_* -/chdb-0.*/ -*.strings -/arrow1100 -test_main -/build* -/bench +/build +/build_* +/build-* /tests/venv -/tests/data /obj-x86_64-linux-gnu/ -/examples/chdbStub -/examples/chdbSimple -/examples/chdbDlopen -libchdb.so - # logs *.log *.debuglog *.stderr *.stdout +# llvm-xray logs +xray-log.* + /docs/build /docs/publish /docs/edit @@ -99,9 +86,6 @@ vgcore* *.deb *.tar.zst -*.gz -*.bz2 -*.tgz *.build *.upload *.changes @@ -186,16 +170,15 @@ tests/integration/**/_gen # pytest --pdb history .pdb_history -/examples/ml-25m # rust /rust/**/target* # It is autogenerated from *.in /rust/**/.cargo/config.toml /rust/**/vendor + +# chDB: add additional files for chDB # Temporary and binary files -*~ *.py[cod] -*.so *.cfg !.isort.cfg !setup.cfg @@ -206,7 +189,6 @@ __pycache__/* .cache/* .*.swp */.ipynb_checkpoints/* -.DS_Store # Project files .ropeproject @@ -246,3 +228,27 @@ MANIFEST .venv*/ .conda*/ .python-version + +*.parquet +*.arrow +.cursorignore +/python_pkg/ +minitest/ +/tmps +/bak +*state_tmp_* +/chdb-0.*/ +*.strings +/arrow1100 +test_main +/build* +/bench +/tests/data +/examples/chdbStub +/examples/chdbSimple +/examples/chdbDlopen +libchdb.so +*.gz +*.bz2 +*.tgz +/examples/ml-25m diff --git a/.readthedocs.yml b/.readthedocs.yml deleted file mode 100644 index a2bcab30fa4..00000000000 --- a/.readthedocs.yml +++ /dev/null @@ -1,27 +0,0 @@ -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: docs/conf.py - -# Build documentation with MkDocs -#mkdocs: -# configuration: mkdocs.yml - -# Optionally build your docs in additional formats such as PDF -formats: - - pdf - -build: - os: ubuntu-22.04 - tools: - python: "3.11" - -python: - install: - - requirements: docs/requirements.txt - - {path: ., method: pip} diff --git a/.yamllint b/.yamllint index 9d6550ac960..b8f7c93e246 100644 --- a/.yamllint +++ b/.yamllint @@ -5,13 +5,12 @@ rules: indentation: level: warning indent-sequences: consistent - line-length: - # there are: - # - bash -c "", so this is OK - # - yaml in tests - max: 1000 - level: warning comments: min-spaces-from-content: 1 - document-start: - present: false + document-start: disable + colons: disable + indentation: disable + line-length: disable + trailing-spaces: disable + truthy: disable + new-line-at-end-of-file: disable diff --git a/CMakeLists.txt b/CMakeLists.txt index 45e4d5259d1..134bd386dab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.20) project(ClickHouse LANGUAGES C CXX ASM) -# If turned off (e.g. when ENABLE_FOO is ON, but FOO tool was not found) the CMake will continue. +# If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue. option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" ON) @@ -127,6 +127,8 @@ add_library(global-libs INTERFACE) include (cmake/sanitize.cmake) +include (cmake/xray_instrumentation.cmake) + option(ENABLE_COLORED_BUILD "Enable colors in compiler output" ON) set (CMAKE_COLOR_MAKEFILE ${ENABLE_COLORED_BUILD}) # works only for the makefile generator @@ -190,14 +192,6 @@ else () set(NO_WHOLE_ARCHIVE --no-whole-archive) endif () -if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE") - # Can be lld or ld-lld or lld-13 or /path/to/lld. - if (LINKER_NAME MATCHES "lld") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index") - message (STATUS "Adding .gdb-index via --gdb-index linker option.") - endif () -endif() - if (NOT (SANITIZE_COVERAGE OR WITH_COVERAGE) AND (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" @@ -213,8 +207,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS "Do not generate debugger info for heavy modules (ClickHouse functions and dictionaries, some contrib)" ${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT}) -option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS}) - option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF) if (NOT BUILD_STANDALONE_KEEPER) option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" ON) @@ -326,7 +318,6 @@ set (COMPILER_FLAGS "${COMPILER_FLAGS} -ffp-contract=off") set (COMPILER_FLAGS "${COMPILER_FLAGS} -fPIC") -# Our built-in unwinder only supports DWARF version up to 4. set (DEBUG_INFO_FLAGS "-g") # Disable omit frame pointer compiler optimization using -fno-omit-frame-pointer diff --git a/SECURITY.md b/SECURITY.md index 14c39129db9..8930dc96f8a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,22 +2,29 @@ the file is autogenerated by utils/security-generator/generate_security.py --> -# Security Policy +# ClickHouse Security Vulnerability Response Policy -## Security Announcements -Security fixes will be announced by posting them in the [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/). +## Security Change Log and Support -## Scope and Supported Versions +Details regarding security fixes are publicly reported in our [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/). A summary of known security vulnerabilities is shown at the bottom of this page. -The following versions of ClickHouse server are currently being supported with security updates: +Vulnerability notifications pre-release or during embargo periods are available to open source users and support customers registered for vulnerability alerts. Refer to our [Embargo Policy](#embargo-policy) below. + +The following versions of ClickHouse server are currently supported with security updates: | Version | Supported | |:-|:-| -| 24.4 | ✔️ | +| 24.7 | ✔️ | +| 24.6 | ✔️ | +| 24.5 | ✔️ | +| 24.4 | ❌ | | 24.3 | ✔️ | -| 24.2 | ✔️ | +| 24.2 | ❌ | | 24.1 | ❌ | -| 23.* | ❌ | +| 23.12 | ❌ | +| 23.11 | ❌ | +| 23.10 | ❌ | +| 23.9 | ❌ | | 23.8 | ✔️ | | 23.7 | ❌ | | 23.6 | ❌ | @@ -37,7 +44,7 @@ The following versions of ClickHouse server are currently being supported with s We're extremely grateful for security researchers and users that report vulnerabilities to the ClickHouse Open Source Community. All reports are thoroughly investigated by developers. -To report a potential vulnerability in ClickHouse please send the details about it to [security@clickhouse.com](mailto:security@clickhouse.com). We do not offer any financial rewards for reporting issues to us using this method. Alternatively, you can also submit your findings through our public bug bounty program hosted by [Bugcrowd](https://bugcrowd.com/clickhouse) and be rewarded for it as per the program scope and rules of engagement. +To report a potential vulnerability in ClickHouse please send the details about it through our public bug bounty program hosted by [Bugcrowd](https://bugcrowd.com/clickhouse) and be rewarded for it as per the program scope and rules of engagement. ### When Should I Report a Vulnerability? @@ -59,3 +66,21 @@ As the security issue moves from triage, to identified fix, to release planning A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect the report date to disclosure date to be on the order of 7 days. +## Embargo Policy + +Open source users and support customers may subscribe to receive alerts during the embargo period by visiting [https://trust.clickhouse.com/?product=clickhouseoss](https://trust.clickhouse.com/?product=clickhouseoss), requesting access and subscribing for alerts. Subscribers agree not to make these notifications public, issue communications, share this information with others, or issue public patches before the disclosure date. Accidental disclosures must be reported immediately to trust@clickhouse.com. Failure to follow this policy or repeated leaks may result in removal from the subscriber list. + +Participation criteria: +1. Be a current open source user or support customer with a valid corporate email domain (no @gmail.com, @azure.com, etc.). +1. Sign up to the ClickHouse OSS Trust Center at [https://trust.clickhouse.com](https://trust.clickhouse.com). +1. Accept the ClickHouse Security Vulnerability Response Policy as outlined above. +1. Subscribe to ClickHouse OSS Trust Center alerts. + +Removal criteria: +1. Members may be removed for failure to follow this policy or repeated leaks. +1. Members may be removed for bounced messages (mail delivery failure). +1. Members may unsubscribe at any time. + +Notification process: +ClickHouse will post notifications within our OSS Trust Center and notify subscribers. Subscribers must log in to the Trust Center to download the notification. The notification will include the timeframe for public disclosure. + diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp index fcc27e6c3f0..31798e8dd7e 100644 --- a/base/poco/Crypto/src/OpenSSLInitializer.cpp +++ b/base/poco/Crypto/src/OpenSSLInitializer.cpp @@ -67,12 +67,12 @@ void OpenSSLInitializer::initialize() SSL_library_init(); SSL_load_error_strings(); OpenSSL_add_all_algorithms(); - + char seed[SEEDSIZE]; RandomInputStream rnd; rnd.read(seed, sizeof(seed)); RAND_seed(seed, SEEDSIZE); - + int nMutexes = CRYPTO_num_locks(); _mutexes = new Poco::FastMutex[nMutexes]; CRYPTO_set_locking_callback(&OpenSSLInitializer::lock); @@ -80,8 +80,8 @@ void OpenSSLInitializer::initialize() // https://sourceforge.net/p/poco/bugs/110/ // // From http://www.openssl.org/docs/crypto/threads.html : -// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(), -// then a default implementation is used - on Windows and BeOS this uses the system's +// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(), +// then a default implementation is used - on Windows and BeOS this uses the system's // default thread identifying APIs" CRYPTO_set_id_callback(&OpenSSLInitializer::id); CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate); @@ -100,7 +100,7 @@ void OpenSSLInitializer::uninitialize() CRYPTO_set_locking_callback(0); CRYPTO_set_id_callback(0); delete [] _mutexes; - + CONF_modules_free(); } } diff --git a/base/poco/Foundation/src/ErrorHandler.cpp b/base/poco/Foundation/src/ErrorHandler.cpp index b8a0fb76236..1d0a41b77ec 100644 --- a/base/poco/Foundation/src/ErrorHandler.cpp +++ b/base/poco/Foundation/src/ErrorHandler.cpp @@ -38,7 +38,7 @@ void ErrorHandler::exception(const Exception& exc) poco_debugger_msg(exc.what()); } - + void ErrorHandler::exception(const std::exception& exc) { poco_debugger_msg(exc.what()); @@ -63,7 +63,7 @@ void ErrorHandler::handle(const Exception& exc) } } - + void ErrorHandler::handle(const std::exception& exc) { FastMutex::ScopedLock lock(_mutex); diff --git a/base/poco/Net/src/TCPServer.cpp b/base/poco/Net/src/TCPServer.cpp index f53a2d7e8e1..b957829fb7d 100644 --- a/base/poco/Net/src/TCPServer.cpp +++ b/base/poco/Net/src/TCPServer.cpp @@ -47,7 +47,7 @@ TCPServer::TCPServer(TCPServerConnectionFactory::Ptr pFactory, Poco::UInt16 port _socket(ServerSocket(portNumber)), _thread(threadName(_socket)), _stopped(true) -{ +{ Poco::ThreadPool& pool = Poco::ThreadPool::defaultPool(); if (pParams) { @@ -55,7 +55,7 @@ TCPServer::TCPServer(TCPServerConnectionFactory::Ptr pFactory, Poco::UInt16 port if (toAdd > 0) pool.addCapacity(toAdd); } _pDispatcher = new TCPServerDispatcher(pFactory, pool, pParams); - + } @@ -111,7 +111,7 @@ void TCPServer::start() _thread.start(*this); } - + void TCPServer::stop() { if (!_stopped) @@ -135,7 +135,7 @@ void TCPServer::run() try { StreamSocket ss = _socket.acceptConnection(); - + if (!_pConnectionFilter || _pConnectionFilter->accept(ss)) { // enable nodelay per default: OSX really needs that @@ -171,7 +171,7 @@ void TCPServer::run() ErrorHandler::handle(exc); // possibly a resource issue since poll() failed; // give some time to recover before trying again - Poco::Thread::sleep(50); + Poco::Thread::sleep(50); } } } @@ -188,7 +188,7 @@ int TCPServer::maxThreads() const return _pDispatcher->maxThreads(); } - + int TCPServer::totalConnections() const { return _pDispatcher->totalConnections(); @@ -206,7 +206,7 @@ int TCPServer::maxConcurrentConnections() const return _pDispatcher->maxConcurrentConnections(); } - + int TCPServer::queuedConnections() const { return _pDispatcher->queuedConnections(); diff --git a/base/poco/Net/src/TCPServerDispatcher.cpp b/base/poco/Net/src/TCPServerDispatcher.cpp index 686e103c8e3..9ca7c271e63 100644 --- a/base/poco/Net/src/TCPServerDispatcher.cpp +++ b/base/poco/Net/src/TCPServerDispatcher.cpp @@ -37,11 +37,11 @@ class TCPConnectionNotification: public Notification _socket(socket) { } - + ~TCPConnectionNotification() { } - + const StreamSocket& socket() const { return _socket; @@ -68,7 +68,7 @@ TCPServerDispatcher::TCPServerDispatcher(TCPServerConnectionFactory::Ptr pFactor if (!_pParams) _pParams = new TCPServerParams; - + if (_pParams->getMaxThreads() == 0) _pParams->setMaxThreads(threadPool.capacity()); } @@ -138,7 +138,7 @@ namespace static const std::string threadName("TCPServerConnection"); } - + void TCPServerDispatcher::enqueue(const StreamSocket& socket) { FastMutex::ScopedLock lock(_mutex); @@ -210,7 +210,7 @@ int TCPServerDispatcher::currentThreads() const int TCPServerDispatcher::maxThreads() const { FastMutex::ScopedLock lock(_mutex); - + return _threadPool.capacity(); } diff --git a/cmake/linux/toolchain-loongarch64.cmake b/cmake/linux/toolchain-loongarch64.cmake new file mode 100644 index 00000000000..bf1ca261183 --- /dev/null +++ b/cmake/linux/toolchain-loongarch64.cmake @@ -0,0 +1,23 @@ +# See linux/toolchain-x86_64.cmake for details about multiple load of toolchain file. +include_guard(GLOBAL) + +set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) + +set (CMAKE_SYSTEM_NAME "Linux") +set (CMAKE_SYSTEM_PROCESSOR "loongarch64") +set (CMAKE_C_COMPILER_TARGET "loongarch64-linux-gnu") +set (CMAKE_CXX_COMPILER_TARGET "loongarch64-linux-gnu") +set (CMAKE_ASM_COMPILER_TARGET "loongarch64-linux-gnu") + +# Adding `-mcmodel=extreme` is to handle the link error: +# relocation R_LARCH_B26 out of range: 194148892 is not in [-134217728, 134217727] +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mcmodel=extreme") +set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -mcmodel=extreme") + +set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-loongarch64") + +set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-loongarch64/usr") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") +set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}") diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake index 08716c1196b..091e20e1387 100644 --- a/cmake/sanitize.cmake +++ b/cmake/sanitize.cmake @@ -57,7 +57,8 @@ option(WITH_COVERAGE "Instrumentation for code coverage with default implementat if (WITH_COVERAGE) message (STATUS "Enabled instrumentation for code coverage") - set(COVERAGE_FLAGS "-fprofile-instr-generate -fcoverage-mapping") + set (COVERAGE_FLAGS -fprofile-instr-generate -fcoverage-mapping) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping") endif() option (SANITIZE_COVERAGE "Instrumentation for code coverage with custom callbacks" OFF) diff --git a/cmake/target.cmake b/cmake/target.cmake index d6c497955f6..3d0ecd032f9 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -84,5 +84,5 @@ if (CMAKE_CROSSCOMPILING) message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!") endif () - message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}") + message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILER_TARGET}") endif () diff --git a/cmake/xray_instrumentation.cmake b/cmake/xray_instrumentation.cmake new file mode 100644 index 00000000000..661c0575e54 --- /dev/null +++ b/cmake/xray_instrumentation.cmake @@ -0,0 +1,20 @@ +# https://llvm.org/docs/XRay.html + +option (ENABLE_XRAY "Enable LLVM XRay" OFF) + +if (NOT ENABLE_XRAY) + message (STATUS "Not using LLVM XRay") + return() +endif() + +if (NOT (ARCH_AMD64 AND OS_LINUX)) + message (STATUS "Not using LLVM XRay, only amd64 Linux or FreeBSD are supported") + return() +endif() + +# The target clang must support xray, otherwise it should error on invalid option +set (XRAY_FLAGS "-fxray-instrument -DUSE_XRAY") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${XRAY_FLAGS}") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${XRAY_FLAGS}") + +message (STATUS "Using LLVM XRay") diff --git a/packages/clickhouse-library-bridge.yaml b/packages/clickhouse-library-bridge.yaml new file mode 100644 index 00000000000..d041e7a26db --- /dev/null +++ b/packages/clickhouse-library-bridge.yaml @@ -0,0 +1,35 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-library-bridge" +description: | + ClickHouse Library Bridge - is a separate process for loading libraries for the 'library' dictionary sources and the CatBoost library. + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + +# Common packages config +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +deb: + fields: + Source: clickhouse + +# Package specific content +contents: +- src: root/usr/bin/clickhouse-library-bridge + dst: /usr/bin/clickhouse-library-bridge +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-library-bridge/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-library-bridge/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-library-bridge/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-library-bridge/README.md diff --git a/packages/clickhouse-odbc-bridge.yaml b/packages/clickhouse-odbc-bridge.yaml new file mode 100644 index 00000000000..98c459c8c26 --- /dev/null +++ b/packages/clickhouse-odbc-bridge.yaml @@ -0,0 +1,35 @@ +# package sources should be placed in ${PWD}/root +# nfpm should run from the same directory with a config +name: "clickhouse-odbc-bridge" +description: | + ClickHouse ODBC Bridge - is a separate process for loading ODBC drivers and interacting with external databases using the ODBC protocol. + ClickHouse is a column-oriented database management system + that allows generating analytical data reports in real time. + +# Common packages config +arch: "${DEB_ARCH}" # amd64, arm64 +platform: "linux" +version: "${CLICKHOUSE_VERSION_STRING}" +vendor: "ClickHouse Inc." +homepage: "https://clickhouse.com" +license: "Apache" +section: "database" +priority: "optional" +maintainer: "ClickHouse Dev Team " +deb: + fields: + Source: clickhouse + +# Package specific content +contents: +- src: root/usr/bin/clickhouse-odbc-bridge + dst: /usr/bin/clickhouse-odbc-bridge +# docs +- src: ../AUTHORS + dst: /usr/share/doc/clickhouse-odbc-bridge/AUTHORS +- src: ../CHANGELOG.md + dst: /usr/share/doc/clickhouse-odbc-bridge/CHANGELOG.md +- src: ../LICENSE + dst: /usr/share/doc/clickhouse-odbc-bridge/LICENSE +- src: ../README.md + dst: /usr/share/doc/clickhouse-odbc-bridge/README.md diff --git a/packages/clickhouse-server.init b/packages/clickhouse-server.init index f215e52b6f3..0ac9cf7ae1f 100755 --- a/packages/clickhouse-server.init +++ b/packages/clickhouse-server.init @@ -1,10 +1,11 @@ #!/bin/sh ### BEGIN INIT INFO # Provides: clickhouse-server +# Required-Start: $network +# Required-Stop: $network +# Should-Start: $time # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 -# Should-Start: $time $network -# Should-Stop: $network # Short-Description: clickhouse-server daemon ### END INIT INFO # diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 02928fd6c28..25c94c56aa6 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1063,7 +1063,7 @@ void Client::processOptions(const OptionsDescription & options_description, send_external_tables = true; - shared_context = Context::createSharedHolder(); + shared_context = Context::createShared(); global_context = Context::createGlobal(shared_context.get()); global_context->makeGlobalContext(); diff --git a/src/Columns/ColumnVectorHelper.h b/programs/local/ColumnVectorHelper.h similarity index 100% rename from src/Columns/ColumnVectorHelper.h rename to programs/local/ColumnVectorHelper.h diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index be96e2d17f8..fa345d4cbee 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -1,6 +1,6 @@ #include "LocalServer.h" -#include "chdb-internal.h" +#include "chdb-internal.h" #if USE_PYTHON #include "TableFunctionPython.h" #include @@ -8,7 +8,6 @@ #endif #include -#include "Common/Logger.h" #include #include #include @@ -50,7 +49,6 @@ #include #include #include -#include #include #include #include @@ -63,7 +61,6 @@ #include #include #include -#include #include "config.h" @@ -71,10 +68,12 @@ # include #endif + namespace fs = std::filesystem; + namespace CurrentMetrics { -extern const Metric MemoryTracking; + extern const Metric MemoryTracking; } namespace DB @@ -83,9 +82,9 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int UNKNOWN_FORMAT; extern const int CANNOT_LOAD_CONFIG; extern const int FILE_ALREADY_EXISTS; + extern const int UNKNOWN_FORMAT; } void applySettingsOverridesForLocal(ContextMutablePtr context) @@ -174,7 +173,9 @@ void LocalServer::initialize(Poco::Util::Application & self) #endif getIOThreadPool().initialize( - server_settings.max_io_thread_pool_size, server_settings.max_io_thread_pool_free_size, server_settings.io_thread_pool_queue_size); + server_settings.max_io_thread_pool_size, + server_settings.max_io_thread_pool_free_size, + server_settings.io_thread_pool_queue_size); const size_t active_parts_loading_threads = server_settings.max_active_parts_loading_thread_pool_size; getActivePartsLoadingThreadPool().initialize( diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 7c3d3b6be68..a11810920d1 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -30,24 +30,11 @@ class LocalServer : public ClientApplicationBase, public Loggers int main(const std::vector & /*args*/) override; - void cleanup(); - - void connect() override; - - size_t getStorgaeRowsRead() const - { - auto * local_connection = static_cast(connection.get()); - return local_connection->getCHDBProgress().read_rows; - } - size_t getStorageBytesRead() const - { - auto * local_connection = static_cast(connection.get()); - return local_connection->getCHDBProgress().read_bytes; - } - protected: Poco::Util::LayeredConfiguration & getClientConfiguration() override; + void connect() override; + void processError(const String & query) const override; String getName() const override { return "local"; } @@ -73,20 +60,40 @@ class LocalServer : public ClientApplicationBase, public Loggers void tryInitPath(); void setupUsers(); + void cleanup(); void applyCmdOptions(ContextMutablePtr context); void applyCmdSettings(ContextMutablePtr context); void createClientContext(); - void cleanStreamingQuery(); - ServerSettings server_settings; std::optional status; std::optional temporary_directory_to_delete; std::unique_ptr input; + +/// chDB: add new interfaces for chDB +public: + size_t getStorgaeRowsRead() const + { + auto * local_connection = static_cast(connection.get()); + return local_connection->getCHDBProgress().read_rows; + } + size_t getStorageBytesRead() const + { + auto * local_connection = static_cast(connection.get()); + return local_connection->getCHDBProgress().read_bytes; + } + + void chdbCleanup() + { + cleanup(); + } + +private: + void cleanStreamingQuery(); }; } diff --git a/programs/local/PythonSource.cpp b/programs/local/PythonSource.cpp index d1a76e13712..bce02bdc922 100644 --- a/programs/local/PythonSource.cpp +++ b/programs/local/PythonSource.cpp @@ -1,4 +1,5 @@ #include "PythonSource.h" +#include "ColumnVectorHelper.h" #include "ListScan.h" #include "PandasScan.h" #include "StoragePython.h" @@ -11,7 +12,6 @@ #include #include #include -#include #include #include #include diff --git a/programs/local/chdb.cpp b/programs/local/chdb.cpp index 5aa0ea22483..0bb1c3b04ab 100644 --- a/programs/local/chdb.cpp +++ b/programs/local/chdb.cpp @@ -560,7 +560,7 @@ chdb_conn ** connect_chdb(int argc, char ** argv) { try { - server->cleanup(); + server->chdbCleanup(); delete server; } catch (...) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f341754813f..196af80f2cd 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1,115 +1,115 @@ #include "Server.h" -#include #include -#include +#include +#include +#include #include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include #include +#include #include #include -#include -#include +#include #include #include #include #include -#include #include -#include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include #include -#include #include -#include +#include +#include +#include +#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include -#include -#include +#include +#include +#include +#include "MetricsTransmitter.h" #include -#include +#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include "MetricsTransmitter.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include "config.h" #include @@ -609,18 +609,14 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, Contex auto condition_write_buffer = WriteBufferFromOwnString(); LOG_DEBUG(log, "Checking startup query condition `{}`", condition); - executeQuery( - condition_read_buffer, condition_write_buffer, true, context, callback, QueryFlags{.internal = true}, std::nullopt, {}); + executeQuery(condition_read_buffer, condition_write_buffer, true, context, callback, QueryFlags{ .internal = true }, std::nullopt, {}); auto result = condition_write_buffer.str(); if (result != "1\n" && result != "true\n") { if (result != "0\n" && result != "false\n") - context->addWarningMessage(fmt::format( - "The condition query returned `{}`, which can't be interpreted as a boolean (`0`, `false`, `1`, `true`). Will " - "skip this query.", - result)); + context->addWarningMessage(fmt::format("The condition query returned `{}`, which can't be interpreted as a boolean (`0`, `false`, `1`, `true`). Will skip this query.", result)); continue; } @@ -633,7 +629,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, Contex auto write_buffer = WriteBufferFromOwnString(); LOG_DEBUG(log, "Executing query `{}`", query); - executeQuery(read_buffer, write_buffer, true, context, callback, QueryFlags{.internal = true}, std::nullopt, {}); + executeQuery(read_buffer, write_buffer, true, context, callback, QueryFlags{ .internal = true }, std::nullopt, {}); } } catch (...) @@ -686,18 +682,18 @@ static std::vector getSanitizerNames() { std::vector names; -# if defined(ADDRESS_SANITIZER) +#if defined(ADDRESS_SANITIZER) names.push_back("address"); -# endif -# if defined(THREAD_SANITIZER) +#endif +#if defined(THREAD_SANITIZER) names.push_back("thread"); -# endif -# if defined(MEMORY_SANITIZER) +#endif +#if defined(MEMORY_SANITIZER) names.push_back("memory"); -# endif -# if defined(UNDEFINED_BEHAVIOR_SANITIZER) +#endif +#if defined(UNDEFINED_BEHAVIOR_SANITIZER) names.push_back("undefined behavior"); -# endif +#endif return names; } @@ -763,9 +759,7 @@ try if (auto total_numa_memory = getNumaNodesTotalMemory(); total_numa_memory.has_value()) { LOG_INFO( - log, - "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", - ReadableSize(*total_numa_memory)); + log, "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory)); } registerInterpreters(); @@ -820,13 +814,11 @@ try const size_t physical_server_memory = getMemoryAmount(); - LOG_INFO( - log, - "Available RAM: {}; logical cores: {}; used cores: {}.", + LOG_INFO(log, "Available RAM: {}; logical cores: {}; used cores: {}.", formatReadableSizeWithBinarySuffix(physical_server_memory), std::thread::hardware_concurrency(), - getNumberOfPhysicalCPUCores() // on ARM processors it can show only enabled at current moment cores - ); + getNumberOfPhysicalCPUCores() // on ARM processors it can show only enabled at current moment cores + ); #if defined(__x86_64__) String cpu_info; @@ -858,16 +850,12 @@ try #endif #if defined(SANITIZER) - LOG_INFO( - log, - "Query Profiler is disabled because it cannot work under sanitizers" + LOG_INFO(log, "Query Profiler is disabled because it cannot work under sanitizers" " when two different stack unwinding methods will interfere with each other."); #endif if (!hasPHDRCache()) - LOG_INFO( - log, - "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" + LOG_INFO(log, "Query Profiler and TraceCollector are disabled because they require PHDR cache to be created" " (otherwise the function 'dl_iterate_phdr' is not lock free and not async-signal safe)."); // Initialize global thread pool. Do it before we fetch configs from zookeeper @@ -899,10 +887,10 @@ try } Poco::ThreadPool server_pool( - /* minCapacity */ 3, - /* maxCapacity */ server_settings.max_connections, - /* idleTime */ 60, - /* stackSize */ POCO_THREAD_STACK_SIZE, + /* minCapacity */3, + /* maxCapacity */server_settings.max_connections, + /* idleTime */60, + /* stackSize */POCO_THREAD_STACK_SIZE, server_settings.global_profiler_real_time_period_ns, server_settings.global_profiler_cpu_time_period_ns); @@ -936,7 +924,8 @@ try for (const auto & server : servers) metrics.emplace_back(ProtocolServerMetrics{server.getPortName(), server.currentThreads(), server.refusedConnections()}); return metrics; - }); + } + ); /// NOTE: global context should be destroyed *before* GlobalThreadPool::shutdown() /// Otherwise GlobalThreadPool::shutdown() will hang, since Context holds some threads. @@ -1078,7 +1067,7 @@ try fs::path path = path_str; /// Check that the process user id matches the owner of the data. - assertProcessUserMatchesDataOwner(path_str, [&](const std::string & message) { global_context->addWarningMessage(message); }); + assertProcessUserMatchesDataOwner(path_str, [&](const std::string & message){ global_context->addWarningMessage(message); }); global_context->setPath(path_str); @@ -2758,7 +2747,9 @@ void Server::createInterserverServers( } } -void Server::stopServers(std::vector & servers, const ServerType & server_type) const +void Server::stopServers( + std::vector & servers, + const ServerType & server_type) const { LoggerRawPtr log = &logger(); diff --git a/src/Access/Common/SSLCertificateSubjects.cpp b/src/Access/Common/SSLCertificateSubjects.cpp index 6faa0b28cf8..ca7001a31a2 100644 --- a/src/Access/Common/SSLCertificateSubjects.cpp +++ b/src/Access/Common/SSLCertificateSubjects.cpp @@ -92,3 +92,4 @@ bool operator==(const SSLCertificateSubjects & lhs, const SSLCertificateSubjects } } + diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 65b5d4f1a51..0fe64c3d38a 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -137,7 +137,7 @@ void QueryAnalyzer::resolve(QueryTreeNodePtr & node, const QueryTreeNodePtr & ta throw Exception(ErrorCodes::LOGICAL_ERROR, "For query analysis table expression must be empty"); - // chdb todo: this is a hack to reload UDFs when the query is re-analyzed + // chDB(todo): this is a hack to reload UDFs when the query is re-analyzed // the root cause is for chdb, the ClientBase and Server might have different Contexts // the hacking might impact the performance when running stateful query(with arg "path" specified) auto global_context = Context::getGlobalContextInstance(); diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp index 74c80f53d08..2599cf04a42 100644 --- a/src/Client/ClientApplicationBase.cpp +++ b/src/Client/ClientApplicationBase.cpp @@ -11,7 +11,6 @@ #include #include "config.h" -#include #include #include #include @@ -380,7 +379,7 @@ void ClientApplicationBase::init(int argc, char ** argv) fatal_channel_ptr->addChannel(fatal_file_channel_ptr); } - // Create loggers for once + /// chDB: create loggers for once static std::once_flag once; std::call_once(once, [this]() { diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h index cccbdb2b822..ec0563b3f00 100644 --- a/src/Columns/ColumnString.h +++ b/src/Columns/ColumnString.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include #include @@ -181,6 +179,7 @@ class ColumnString final : public COWHelper, ColumnS { const size_t old_size = chars.size(); const size_t new_size = old_size + length + 1; + chars.resize(new_size); if (length) memcpy(chars.data() + old_size, pos, length); diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp index 710cd049dbe..0b3d588838e 100644 --- a/src/Common/CgroupsMemoryUsageObserver.cpp +++ b/src/Common/CgroupsMemoryUsageObserver.cpp @@ -80,7 +80,6 @@ struct CgroupsV1Reader : ICgroupsReader { explicit CgroupsV1Reader(const fs::path & stat_file_dir) : buf(stat_file_dir / "memory.stat") { } - uint64_t readMemoryUsage() override { std::lock_guard lock(mutex); @@ -101,7 +100,7 @@ struct CgroupsV1Reader : ICgroupsReader }; struct CgroupsV2Reader : ICgroupsReader - { +{ explicit CgroupsV2Reader(const fs::path & stat_file_dir) : current_buf(stat_file_dir / "memory.current"), stat_buf(stat_file_dir / "memory.stat") { diff --git a/src/Common/Coverage.cpp b/src/Common/Coverage.cpp index 5f3056439ea..a21efe62fb6 100644 --- a/src/Common/Coverage.cpp +++ b/src/Common/Coverage.cpp @@ -62,3 +62,4 @@ void dumpCoverage() } } #endif + diff --git a/src/Core/tests/gtest_fields_binary_enciding.cpp b/src/Core/tests/gtest_fields_binary_enciding.cpp index 4898b4b55d4..087caf746bb 100644 --- a/src/Core/tests/gtest_fields_binary_enciding.cpp +++ b/src/Core/tests/gtest_fields_binary_enciding.cpp @@ -62,3 +62,4 @@ GTEST_TEST(FieldBinaryEncoding, EncodeAndDecode) Tuple({Field(UInt64(43)), Map({Tuple{Field(UInt64(43)), Field(String("str_43"))}, Tuple{Field(UInt64(44)), Field(String("str_44"))}}), Field(UUID(43)), Field(String("Hello, World 2!"))}) })); } + diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 803b50db5ac..760f6ce750d 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -790,3 +790,4 @@ SerializationPtr SerializationObject::TypedPathSubcolumnCreator::create(const DB } } + diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index 336637b46a6..c9840c79ee3 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -774,6 +774,8 @@ void DatabaseCatalog::addUUIDMapping(const UUID & uuid, const DatabasePtr & data throw Exception(ErrorCodes::LOGICAL_ERROR, "Mapping for table with UUID={} already exists", uuid); /// Normally this should never happen, but it's possible when the same UUIDs are explicitly specified in different CREATE queries, /// so it's not LOGICAL_ERROR + + /// chDB: No exception if table uuid exist for stateful query // throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Mapping for table with UUID={} already exists. It happened due to UUID collision, " // "most likely because some not random UUIDs were manually specified in CREATE queries.", uuid); } @@ -1062,12 +1064,6 @@ String DatabaseCatalog::getPathForMetadata(const StorageID & table_id) const return metadata_path + escapeForFileName(table_id.getTableName()) + ".sql"; } -//chdb session query need to fix the path -void DatabaseCatalog::fixPath(const String & path) -{ - getContext()->setPath(path); -} - void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr table, String dropped_metadata_path, bool ignore_delay) { assert(table_id.hasUUID()); @@ -1809,6 +1805,12 @@ void DatabaseCatalog::triggerReloadDisksTask(const Strings & new_added_disks) (*reload_disks_task)->schedule(); } +/// chdb: chdb session query need to fix the path. +void DatabaseCatalog::fixPath(const String & path) +{ + getContext()->setPath(path); +} + static void maybeUnlockUUID(UUID uuid) { if (uuid == UUIDHelpers::Nil) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 4707a259458..51bdebb3214 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -224,7 +224,6 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext String getPathForDroppedMetadata(const StorageID & table_id) const; String getPathForMetadata(const StorageID & table_id) const; - void fixPath(const String & path); void enqueueDroppedTableCleanup(StorageID table_id, StoragePtr table, String dropped_metadata_path, bool ignore_delay = false); void undropTable(StorageID table_id); @@ -267,6 +266,9 @@ class DatabaseCatalog : boost::noncopyable, WithMutableContext void triggerReloadDisksTask(const Strings & new_added_disks); + /// chdb: chdb session query need to fix the path + void fixPath(const String & path); + private: // The global instance of database catalog. unique_ptr is to allow // deferred initialization. Thought I'd use std::optional, but I can't diff --git a/src/Interpreters/HashJoin/HashJoinMethods.h b/src/Interpreters/HashJoin/HashJoinMethods.h index aa769590e41..3b7a67467e3 100644 --- a/src/Interpreters/HashJoin/HashJoinMethods.h +++ b/src/Interpreters/HashJoin/HashJoinMethods.h @@ -198,3 +198,4 @@ extern template class HashJoinMethods; extern template class HashJoinMethods; } + diff --git a/src/Interpreters/InterpreterSetQuery.cpp b/src/Interpreters/InterpreterSetQuery.cpp index 314edbee301..7e68fc5c4c1 100644 --- a/src/Interpreters/InterpreterSetQuery.cpp +++ b/src/Interpreters/InterpreterSetQuery.cpp @@ -12,7 +12,7 @@ namespace DB { - + BlockIO InterpreterSetQuery::execute() { const auto & ast = query_ptr->as(); diff --git a/src/Interpreters/InterpreterUseQuery.cpp b/src/Interpreters/InterpreterUseQuery.cpp index d3235744722..58be12927b9 100644 --- a/src/Interpreters/InterpreterUseQuery.cpp +++ b/src/Interpreters/InterpreterUseQuery.cpp @@ -3,10 +3,8 @@ #include #include #include -#include #include -#include namespace DB { @@ -16,7 +14,6 @@ BlockIO InterpreterUseQuery::execute() const String & new_database = query_ptr->as().getDatabase(); getContext()->checkAccess(AccessType::SHOW_DATABASES, new_database); getContext()->getSessionContext()->setCurrentDatabase(new_database); - return {}; } diff --git a/src/Parsers/ASTObjectTypeArgument.cpp b/src/Parsers/ASTObjectTypeArgument.cpp index b327657f4e5..975f0389505 100644 --- a/src/Parsers/ASTObjectTypeArgument.cpp +++ b/src/Parsers/ASTObjectTypeArgument.cpp @@ -60,3 +60,5 @@ void ASTObjectTypeArgument::formatImpl(const FormatSettings & settings, FormatSt } } + + diff --git a/src/Parsers/ASTObjectTypeArgument.h b/src/Parsers/ASTObjectTypeArgument.h index 8971c7ab059..ab18d00d770 100644 --- a/src/Parsers/ASTObjectTypeArgument.h +++ b/src/Parsers/ASTObjectTypeArgument.h @@ -30,3 +30,4 @@ class ASTObjectTypeArgument : public IAST } + diff --git a/src/Parsers/ASTStatisticsDeclaration.cpp b/src/Parsers/ASTStatisticsDeclaration.cpp index 72cd3db876f..f9b7a9e29db 100644 --- a/src/Parsers/ASTStatisticsDeclaration.cpp +++ b/src/Parsers/ASTStatisticsDeclaration.cpp @@ -57,3 +57,4 @@ void ASTStatisticsDeclaration::formatImpl(const FormatSettings & s, FormatState } } + diff --git a/src/Parsers/Kusto/KQL_ReleaseNote.md b/src/Parsers/Kusto/KQL_ReleaseNote.md index bc52e7154f3..440d0c73803 100644 --- a/src/Parsers/Kusto/KQL_ReleaseNote.md +++ b/src/Parsers/Kusto/KQL_ReleaseNote.md @@ -853,7 +853,7 @@ Please note that the functions listed below only take constant parameters for no ## KQL() function - create table - `CREATE TABLE kql_table4 ENGINE = Memory AS select *, now() as new_column From kql($$Customers | project LastName,Age$$);` + `CREATE TABLE kql_table4 ENGINE = Memory AS select *, now() as new_column From kql($$Customers | project LastName,Age$$);` verify the content of `kql_table` `select * from kql_table` @@ -867,11 +867,11 @@ Please note that the functions listed below only take constant parameters for no Age Nullable(UInt8) ) ENGINE = Memory; ``` - `INSERT INTO temp select * from kql($$Customers|project FirstName,LastName,Age$$);` + `INSERT INTO temp select * from kql($$Customers|project FirstName,LastName,Age$$);` verify the content of `temp` `select * from temp` - - Select from kql(...) + - Select from kql(...) `Select * from kql($$Customers|project FirstName$$)` ## KQL operators: diff --git a/src/Processors/Transforms/PlanSquashingTransform.h b/src/Processors/Transforms/PlanSquashingTransform.h index 4e011599329..e6db245499e 100644 --- a/src/Processors/Transforms/PlanSquashingTransform.h +++ b/src/Processors/Transforms/PlanSquashingTransform.h @@ -25,3 +25,4 @@ class PlanSquashingTransform : public IInflatingTransform Chunk squashed_chunk; }; } + diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 13436be5769..4ed74763810 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -133,7 +133,7 @@ void registerStorages() registerStorageTimeSeries(factory); #if USE_RAPIDJSON || USE_SIMDJSON - registerStorageFuzzJSON(factory); + registerStorageFuzzJSON(factory); #endif #if USE_AZURE_BLOB_STORAGE diff --git a/src/TableFunctions/CMakeLists.txt b/src/TableFunctions/CMakeLists.txt index 80d82e09cc7..770990cc405 100644 --- a/src/TableFunctions/CMakeLists.txt +++ b/src/TableFunctions/CMakeLists.txt @@ -21,30 +21,6 @@ add_library(clickhouse_table_functions ${clickhouse_table_functions_headers} ${c target_link_libraries(clickhouse_table_functions PRIVATE clickhouse_parsers clickhouse_storages_system dbms) -# if (USE_PYTHON) -# # Include path from shell cmd "python3 -m pybind11 --includes" -# execute_process(COMMAND python3 -m pybind11 --includes -# OUTPUT_VARIABLE PYBIND11_INCLUDES -# OUTPUT_STRIP_TRAILING_WHITESPACE -# ) - -# # Extract and set include directories specifically for source using pybind11 -# string(REGEX MATCHALL "-I([^ ]+)" INCLUDE_DIRS_MATCHES ${PYBIND11_INCLUDES}) -# set(PYTHON_INCLUDE_DIRS "") -# foreach(INCLUDE_DIR_MATCH ${INCLUDE_DIRS_MATCHES}) -# string(REGEX REPLACE "-I" "" INCLUDE_DIR_MATCH ${INCLUDE_DIR_MATCH}) -# # Accumulate all include directories -# set(PYTHON_INCLUDE_DIRS "${PYTHON_INCLUDE_DIRS};${INCLUDE_DIR_MATCH}") -# endforeach() - -# # get python version, something like python3.x -# execute_process(COMMAND python3 -c "import sys; print('python3.'+str(sys.version_info[1]))" -# OUTPUT_VARIABLE PYTHON_VERSION -# OUTPUT_STRIP_TRAILING_WHITESPACE -# ) - -# endif() - if (TARGET ch_contrib::hivemetastore) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::hivemetastore ch_contrib::hdfs ch_contrib::parquet) endif () @@ -57,11 +33,6 @@ if (TARGET ch_contrib::simdjson) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::simdjson) endif () -if (TARGET ch_contrib::utf8proc) - target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::utf8proc) - target_include_directories(clickhouse_table_functions PRIVATE ${UTF8PROC_INCLUDE_DIR}) -endif () - if (TARGET ch_contrib::rapidjson) target_link_libraries(clickhouse_table_functions PRIVATE ch_contrib::rapidjson) endif () From ee98528616c2e0ce045f65637338fb61dd9885e2 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 16 Jun 2025 01:36:27 +0800 Subject: [PATCH 2/9] feat: upgrade ClickHouse to v25.5.2.47 --- .clang-format | 1 + .clang-tidy | 32 +- .clangd | 3 + .git-blame-ignore-revs | 3 + .gitignore | 1 + .gitmodules | 68 +- CMakeLists.txt | 121 +- PreLoad.cmake | 15 +- SECURITY.md | 32 +- base/README.md | 10 + base/base/AlignedUnion.h | 17 + base/base/BFloat16.h | 339 + base/base/CMakeLists.txt | 7 +- base/base/DecomposedFloat.h | 55 +- base/base/EnumReflection.h | 5 +- base/base/GitHash.cpp.in | 3 + base/base/IPv4andIPv6.cpp | 35 + base/base/IPv4andIPv6.h | 29 +- base/base/JSON.cpp | 35 +- base/base/JSON.h | 6 +- {src/Common => base/base}/MemorySanitizer.h | 0 base/base/StringRef.h | 50 +- base/base/TypeLists.h | 5 +- base/base/TypeName.h | 1 + base/base/argsToConfig.cpp | 7 +- base/base/bit_cast.h | 4 +- base/base/cgroupsv2.cpp | 70 +- base/base/cgroupsv2.h | 17 +- base/base/chrono_io.h | 47 +- base/base/coverage.h | 1 + base/base/defines.h | 2 + base/base/extended_types.h | 16 +- base/base/find_symbols.h | 83 +- base/base/getMemoryAmount.cpp | 6 +- base/base/itoa.cpp | 4 +- base/base/libcpp_verbose_abort.cpp | 13 + {src/Common => base/base}/memcmpSmall.h | 7 +- base/base/openpty.h | 48 + base/base/preciseExp10.cpp | 5 +- base/base/unit.h | 6 +- base/base/wide_integer_impl.h | 206 +- base/base/wide_integer_to_string.cpp | 57 + base/base/wide_integer_to_string.h | 54 +- base/glibc-compatibility/CMakeLists.txt | 5 +- base/glibc-compatibility/musl/getauxval.c | 7 +- base/harmful/harmful.c | 17 +- base/poco/CMakeLists.txt | 4 - base/poco/Crypto/CMakeLists.txt | 35 - base/poco/Crypto/include/Poco/Crypto/Cipher.h | 141 - .../include/Poco/Crypto/CipherFactory.h | 78 - .../Crypto/include/Poco/Crypto/CipherImpl.h | 72 - .../Crypto/include/Poco/Crypto/CipherKey.h | 203 - .../include/Poco/Crypto/CipherKeyImpl.h | 168 - base/poco/Crypto/include/Poco/Crypto/Crypto.h | 116 - .../include/Poco/Crypto/CryptoException.h | 59 - .../Crypto/include/Poco/Crypto/CryptoStream.h | 195 - .../include/Poco/Crypto/CryptoTransform.h | 88 - .../Crypto/include/Poco/Crypto/DigestEngine.h | 83 - .../include/Poco/Crypto/ECDSADigestEngine.h | 103 - base/poco/Crypto/include/Poco/Crypto/ECKey.h | 138 - .../Crypto/include/Poco/Crypto/ECKeyImpl.h | 143 - .../poco/Crypto/include/Poco/Crypto/EVPPKey.h | 369 - .../poco/Crypto/include/Poco/Crypto/KeyPair.h | 103 - .../Crypto/include/Poco/Crypto/KeyPairImpl.h | 94 - .../include/Poco/Crypto/OpenSSLInitializer.h | 117 - .../include/Poco/Crypto/PKCS12Container.h | 162 - .../include/Poco/Crypto/RSACipherImpl.h | 80 - .../include/Poco/Crypto/RSADigestEngine.h | 114 - base/poco/Crypto/include/Poco/Crypto/RSAKey.h | 124 - .../Crypto/include/Poco/Crypto/RSAKeyImpl.h | 128 - .../include/Poco/Crypto/X509Certificate.h | 248 - base/poco/Crypto/src/Cipher.cpp | 140 - base/poco/Crypto/src/CipherFactory.cpp | 65 - base/poco/Crypto/src/CipherImpl.cpp | 272 - base/poco/Crypto/src/CipherKey.cpp | 49 - base/poco/Crypto/src/CipherKeyImpl.cpp | 222 - base/poco/Crypto/src/CryptoException.cpp | 108 - base/poco/Crypto/src/CryptoStream.cpp | 355 - base/poco/Crypto/src/CryptoTransform.cpp | 38 - base/poco/Crypto/src/DigestEngine.cpp | 80 - base/poco/Crypto/src/ECDSADigestEngine.cpp | 100 - base/poco/Crypto/src/ECKey.cpp | 75 - base/poco/Crypto/src/ECKeyImpl.cpp | 258 - base/poco/Crypto/src/EVPPKey.cpp | 321 - base/poco/Crypto/src/KeyPair.cpp | 34 - base/poco/Crypto/src/KeyPairImpl.cpp | 35 - base/poco/Crypto/src/OpenSSLInitializer.cpp | 162 - base/poco/Crypto/src/PKCS12Container.cpp | 191 - base/poco/Crypto/src/RSACipherImpl.cpp | 342 - base/poco/Crypto/src/RSADigestEngine.cpp | 96 - base/poco/Crypto/src/RSAKey.cpp | 87 - base/poco/Crypto/src/RSAKeyImpl.cpp | 289 - base/poco/Crypto/src/X509Certificate.cpp | 382 - base/poco/Data/CMakeLists.txt | 20 - base/poco/Data/ODBC/CMakeLists.txt | 52 - .../Data/ODBC/include/Poco/Data/ODBC/Binder.h | 1534 - .../include/Poco/Data/ODBC/ConnectionHandle.h | 98 - .../ODBC/include/Poco/Data/ODBC/Connector.h | 101 - .../ODBC/include/Poco/Data/ODBC/Diagnostics.h | 217 - .../Poco/Data/ODBC/EnvironmentHandle.h | 96 - .../Data/ODBC/include/Poco/Data/ODBC/Error.h | 124 - .../ODBC/include/Poco/Data/ODBC/Extractor.h | 748 - .../Data/ODBC/include/Poco/Data/ODBC/Handle.h | 111 - .../Data/ODBC/include/Poco/Data/ODBC/ODBC.h | 53 - .../include/Poco/Data/ODBC/ODBCException.h | 147 - .../include/Poco/Data/ODBC/ODBCMetaColumn.h | 94 - .../Poco/Data/ODBC/ODBCStatementImpl.h | 208 - .../ODBC/include/Poco/Data/ODBC/Parameter.h | 113 - .../ODBC/include/Poco/Data/ODBC/Preparator.h | 1256 - .../ODBC/include/Poco/Data/ODBC/SessionImpl.h | 294 - .../ODBC/include/Poco/Data/ODBC/TypeInfo.h | 121 - .../ODBC/include/Poco/Data/ODBC/Unicode.h | 791 - .../include/Poco/Data/ODBC/Unicode_UNIXODBC.h | 56 - .../ODBC/include/Poco/Data/ODBC/Utility.h | 224 - base/poco/Data/ODBC/src/Binder.cpp | 529 - base/poco/Data/ODBC/src/ConnectionHandle.cpp | 57 - base/poco/Data/ODBC/src/Connector.cpp | 64 - base/poco/Data/ODBC/src/EnvironmentHandle.cpp | 54 - base/poco/Data/ODBC/src/Extractor.cpp | 1312 - base/poco/Data/ODBC/src/ODBCException.cpp | 30 - base/poco/Data/ODBC/src/ODBCMetaColumn.cpp | 168 - base/poco/Data/ODBC/src/ODBCStatementImpl.cpp | 455 - base/poco/Data/ODBC/src/Parameter.cpp | 53 - base/poco/Data/ODBC/src/Preparator.cpp | 209 - base/poco/Data/ODBC/src/SessionImpl.cpp | 418 - base/poco/Data/ODBC/src/TypeInfo.cpp | 265 - base/poco/Data/ODBC/src/Unicode.cpp | 22 - base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp | 774 - base/poco/Data/ODBC/src/Utility.cpp | 159 - .../Data/include/Poco/Data/AbstractBinder.h | 377 - .../Data/include/Poco/Data/AbstractBinding.h | 147 - .../include/Poco/Data/AbstractExtraction.h | 281 - .../include/Poco/Data/AbstractExtractor.h | 355 - .../include/Poco/Data/AbstractPreparation.h | 74 - .../include/Poco/Data/AbstractPreparator.h | 392 - .../include/Poco/Data/AbstractSessionImpl.h | 312 - .../Data/include/Poco/Data/ArchiveStrategy.h | 226 - base/poco/Data/include/Poco/Data/Binding.h | 1219 - base/poco/Data/include/Poco/Data/Bulk.h | 100 - .../poco/Data/include/Poco/Data/BulkBinding.h | 141 - .../Data/include/Poco/Data/BulkExtraction.h | 253 - base/poco/Data/include/Poco/Data/Column.h | 482 - base/poco/Data/include/Poco/Data/Connector.h | 59 - base/poco/Data/include/Poco/Data/Constants.h | 39 - base/poco/Data/include/Poco/Data/Data.h | 50 - .../Data/include/Poco/Data/DataException.h | 53 - base/poco/Data/include/Poco/Data/Date.h | 211 - .../Data/include/Poco/Data/DynamicDateTime.h | 55 - base/poco/Data/include/Poco/Data/Extraction.h | 692 - base/poco/Data/include/Poco/Data/LOB.h | 273 - base/poco/Data/include/Poco/Data/Limit.h | 112 - base/poco/Data/include/Poco/Data/MetaColumn.h | 192 - .../include/Poco/Data/PooledSessionHolder.h | 102 - .../include/Poco/Data/PooledSessionImpl.h | 100 - base/poco/Data/include/Poco/Data/Position.h | 79 - .../poco/Data/include/Poco/Data/Preparation.h | 156 - base/poco/Data/include/Poco/Data/Range.h | 108 - base/poco/Data/include/Poco/Data/RecordSet.h | 640 - base/poco/Data/include/Poco/Data/Row.h | 314 - base/poco/Data/include/Poco/Data/RowFilter.h | 275 - .../Data/include/Poco/Data/RowFormatter.h | 231 - .../poco/Data/include/Poco/Data/RowIterator.h | 152 - base/poco/Data/include/Poco/Data/SQLChannel.h | 216 - base/poco/Data/include/Poco/Data/Session.h | 491 - .../Data/include/Poco/Data/SessionFactory.h | 98 - .../poco/Data/include/Poco/Data/SessionImpl.h | 222 - .../poco/Data/include/Poco/Data/SessionPool.h | 233 - .../include/Poco/Data/SessionPoolContainer.h | 117 - .../include/Poco/Data/SimpleRowFormatter.h | 124 - base/poco/Data/include/Poco/Data/Statement.h | 830 - .../Data/include/Poco/Data/StatementCreator.h | 76 - .../Data/include/Poco/Data/StatementImpl.h | 635 - base/poco/Data/include/Poco/Data/Time.h | 221 - .../poco/Data/include/Poco/Data/Transaction.h | 199 - .../poco/Data/include/Poco/Data/TypeHandler.h | 1906 - base/poco/Data/src/AbstractBinder.cpp | 527 - base/poco/Data/src/AbstractBinding.cpp | 45 - base/poco/Data/src/AbstractExtraction.cpp | 40 - base/poco/Data/src/AbstractExtractor.cpp | 437 - base/poco/Data/src/AbstractPreparation.cpp | 34 - base/poco/Data/src/AbstractPreparator.cpp | 438 - base/poco/Data/src/ArchiveStrategy.cpp | 146 - base/poco/Data/src/Bulk.cpp | 37 - base/poco/Data/src/Connector.cpp | 32 - base/poco/Data/src/DataException.cpp | 41 - base/poco/Data/src/Date.cpp | 142 - base/poco/Data/src/Limit.cpp | 35 - base/poco/Data/src/MetaColumn.cpp | 53 - base/poco/Data/src/PooledSessionHolder.cpp | 34 - base/poco/Data/src/PooledSessionImpl.cpp | 192 - base/poco/Data/src/Position.cpp | 32 - base/poco/Data/src/Range.cpp | 40 - base/poco/Data/src/RecordSet.cpp | 392 - base/poco/Data/src/Row.cpp | 369 - base/poco/Data/src/RowFilter.cpp | 225 - base/poco/Data/src/RowFormatter.cpp | 80 - base/poco/Data/src/RowIterator.cpp | 207 - base/poco/Data/src/SQLChannel.cpp | 296 - base/poco/Data/src/Session.cpp | 78 - base/poco/Data/src/SessionFactory.cpp | 93 - base/poco/Data/src/SessionImpl.cpp | 52 - base/poco/Data/src/SessionPool.cpp | 308 - base/poco/Data/src/SessionPoolContainer.cpp | 122 - base/poco/Data/src/SimpleRowFormatter.cpp | 113 - base/poco/Data/src/Statement.cpp | 296 - base/poco/Data/src/StatementCreator.cpp | 60 - base/poco/Data/src/StatementImpl.cpp | 475 - base/poco/Data/src/Time.cpp | 139 - base/poco/Data/src/Transaction.cpp | 124 - base/poco/Foundation/include/Poco/UUID.h | 1 + base/poco/Foundation/src/Path_UNIX.cpp | 18 +- base/poco/Foundation/src/URI.cpp | 2 +- base/poco/JSON/include/Poco/JSON/Array.h | 2 +- base/poco/JSON/include/Poco/JSON/Object.h | 3 +- base/poco/JSON/src/Object.cpp | 8 +- base/poco/JSON/src/ParserImpl.cpp | 14 +- base/poco/JSON/src/pdjson.c | 168 +- base/poco/MongoDB/CMakeLists.txt | 16 - .../poco/MongoDB/include/Poco/MongoDB/Array.h | 142 - .../MongoDB/include/Poco/MongoDB/BSONReader.h | 88 - .../MongoDB/include/Poco/MongoDB/BSONWriter.h | 76 - .../MongoDB/include/Poco/MongoDB/Binary.h | 158 - .../MongoDB/include/Poco/MongoDB/Connection.h | 191 - .../MongoDB/include/Poco/MongoDB/Cursor.h | 80 - .../MongoDB/include/Poco/MongoDB/Database.h | 233 - .../include/Poco/MongoDB/DeleteRequest.h | 116 - .../MongoDB/include/Poco/MongoDB/Document.h | 296 - .../MongoDB/include/Poco/MongoDB/Element.h | 393 - .../include/Poco/MongoDB/GetMoreRequest.h | 92 - .../include/Poco/MongoDB/InsertRequest.h | 100 - .../include/Poco/MongoDB/JavaScriptCode.h | 108 - .../include/Poco/MongoDB/KillCursorsRequest.h | 65 - .../MongoDB/include/Poco/MongoDB/Message.h | 76 - .../include/Poco/MongoDB/MessageHeader.h | 140 - .../MongoDB/include/Poco/MongoDB/MongoDB.h | 64 - .../MongoDB/include/Poco/MongoDB/ObjectId.h | 151 - .../include/Poco/MongoDB/OpMsgCursor.h | 96 - .../include/Poco/MongoDB/OpMsgMessage.h | 163 - .../Poco/MongoDB/PoolableConnectionFactory.h | 123 - .../include/Poco/MongoDB/QueryRequest.h | 190 - .../include/Poco/MongoDB/RegularExpression.h | 135 - .../MongoDB/include/Poco/MongoDB/ReplicaSet.h | 61 - .../include/Poco/MongoDB/RequestMessage.h | 54 - .../include/Poco/MongoDB/ResponseMessage.h | 114 - .../include/Poco/MongoDB/UpdateRequest.h | 117 - base/poco/MongoDB/src/Array.cpp | 75 - base/poco/MongoDB/src/Binary.cpp | 89 - base/poco/MongoDB/src/Connection.cpp | 348 - base/poco/MongoDB/src/Cursor.cpp | 83 - base/poco/MongoDB/src/Database.cpp | 482 - base/poco/MongoDB/src/DeleteRequest.cpp | 54 - base/poco/MongoDB/src/Document.cpp | 227 - base/poco/MongoDB/src/Element.cpp | 32 - base/poco/MongoDB/src/GetMoreRequest.cpp | 46 - base/poco/MongoDB/src/InsertRequest.cpp | 49 - base/poco/MongoDB/src/JavaScriptCode.cpp | 33 - base/poco/MongoDB/src/KillCursorsRequest.cpp | 44 - base/poco/MongoDB/src/Message.cpp | 33 - base/poco/MongoDB/src/MessageHeader.cpp | 63 - base/poco/MongoDB/src/ObjectId.cpp | 66 - base/poco/MongoDB/src/OpMsgCursor.cpp | 187 - base/poco/MongoDB/src/OpMsgMessage.cpp | 412 - base/poco/MongoDB/src/QueryRequest.cpp | 54 - base/poco/MongoDB/src/RegularExpression.cpp | 71 - base/poco/MongoDB/src/ReplicaSet.cpp | 89 - base/poco/MongoDB/src/RequestMessage.cpp | 51 - base/poco/MongoDB/src/ResponseMessage.cpp | 80 - base/poco/MongoDB/src/UpdateRequest.cpp | 47 - .../Net/include/Poco/Net/FTPClientSession.h | 401 - .../Net/include/Poco/Net/FTPStreamFactory.h | 123 - .../Net/include/Poco/Net/HTTPClientSession.h | 6 +- base/poco/Net/include/Poco/Net/HTTPResponse.h | 4 + .../Net/include/Poco/Net/HTTPServerParams.h | 10 +- .../Net/include/Poco/Net/HTTPServerSession.h | 4 +- base/poco/Net/include/Poco/Net/HTTPSession.h | 43 + base/poco/Net/include/Poco/Net/ICMPClient.h | 92 - .../poco/Net/include/Poco/Net/ICMPEventArgs.h | 166 - base/poco/Net/include/Poco/Net/ICMPPacket.h | 92 - .../Net/include/Poco/Net/ICMPPacketImpl.h | 143 - base/poco/Net/include/Poco/Net/ICMPSocket.h | 94 - .../Net/include/Poco/Net/ICMPSocketImpl.h | 97 - .../Net/include/Poco/Net/ICMPv4PacketImpl.h | 176 - base/poco/Net/include/Poco/Net/MailMessage.h | 309 - .../poco/Net/include/Poco/Net/MailRecipient.h | 123 - base/poco/Net/include/Poco/Net/MailStream.h | 149 - .../poco/Net/include/Poco/Net/MessageHeader.h | 26 +- base/poco/Net/include/Poco/Net/NTPClient.h | 67 - base/poco/Net/include/Poco/Net/NTPEventArgs.h | 87 - base/poco/Net/include/Poco/Net/NTPPacket.h | 210 - .../include/Poco/Net/NameValueCollection.h | 1 + base/poco/Net/include/Poco/Net/NetException.h | 5 - .../Net/include/Poco/Net/POP3ClientSession.h | 188 - .../Net/include/Poco/Net/SMTPClientSession.h | 237 - base/poco/Net/include/Poco/Net/Socket.h | 25 + base/poco/Net/include/Poco/Net/StreamSocket.h | 3 + base/poco/Net/src/FTPClientSession.cpp | 599 - base/poco/Net/src/FTPStreamFactory.cpp | 243 - base/poco/Net/src/HTTPClientSession.cpp | 55 +- base/poco/Net/src/HTTPMessage.cpp | 2 +- base/poco/Net/src/HTTPResponse.cpp | 9 + base/poco/Net/src/HTTPServerParams.cpp | 16 +- base/poco/Net/src/HTTPServerSession.cpp | 6 +- base/poco/Net/src/HTTPSession.cpp | 24 +- base/poco/Net/src/ICMPClient.cpp | 150 - base/poco/Net/src/ICMPEventArgs.cpp | 168 - base/poco/Net/src/ICMPPacket.cpp | 110 - base/poco/Net/src/ICMPPacketImpl.cpp | 103 - base/poco/Net/src/ICMPSocket.cpp | 94 - base/poco/Net/src/ICMPSocketImpl.cpp | 97 - base/poco/Net/src/ICMPv4PacketImpl.cpp | 253 - base/poco/Net/src/MailMessage.cpp | 694 - base/poco/Net/src/MailRecipient.cpp | 94 - base/poco/Net/src/MailStream.cpp | 221 - base/poco/Net/src/MessageHeader.cpp | 40 +- base/poco/Net/src/NTPClient.cpp | 87 - base/poco/Net/src/NTPEventArgs.cpp | 71 - base/poco/Net/src/NTPPacket.cpp | 158 - base/poco/Net/src/NetException.cpp | 5 - base/poco/Net/src/POP3ClientSession.cpp | 280 - base/poco/Net/src/SMTPClientSession.cpp | 466 - base/poco/Net/src/Socket.cpp | 251 +- base/poco/Net/src/SocketImpl.cpp | 62 +- base/poco/Net/src/StreamSocket.cpp | 5 + base/poco/Net/src/TCPServer.cpp | 5 + base/poco/Net/src/TCPServerConnection.cpp | 5 +- base/poco/NetSSL_OpenSSL/CMakeLists.txt | 2 +- .../NetSSL_OpenSSL/include/Poco/Net/Context.h | 40 +- .../include/Poco/Net/HTTPSClientSession.h | 3 +- .../NetSSL_OpenSSL/include/Poco/Net/NetSSL.h | 3 +- .../include/Poco/Net/SSLManager.h | 11 - .../Poco/Net/SecureSMTPClientSession.h | 99 - .../include/Poco/Net/SecureSocketImpl.h | 2 +- .../include/Poco/Net/SecureStreamSocket.h | 6 +- .../include/Poco/Net/SecureStreamSocketImpl.h | 27 +- .../include/Poco/Net/VerificationErrorArgs.h | 16 +- .../include/Poco/Net/X509Certificate.h | 105 - base/poco/NetSSL_OpenSSL/src/Context.cpp | 89 +- .../NetSSL_OpenSSL/src/HTTPSClientSession.cpp | 4 +- base/poco/NetSSL_OpenSSL/src/SSLManager.cpp | 30 +- .../src/SecureSMTPClientSession.cpp | 64 - .../NetSSL_OpenSSL/src/SecureSocketImpl.cpp | 33 +- .../NetSSL_OpenSSL/src/SecureStreamSocket.cpp | 37 +- .../src/SecureStreamSocketImpl.cpp | 25 +- .../src/VerificationErrorArgs.cpp | 3 +- .../NetSSL_OpenSSL/src/X509Certificate.cpp | 175 - base/poco/README.md | 4 +- base/poco/Util/CMakeLists.txt | 2 +- .../include/Poco/Util/AbstractConfiguration.h | 37 + .../poco/Util/include/Poco/Util/Application.h | 2 +- base/poco/Util/src/AbstractConfiguration.cpp | 100 + base/poco/Util/src/Application.cpp | 18 +- ci/README.md | 5 + ci/defs/defs.py | 855 + ci/defs/job_configs.py | 976 + ci/docker/binary-builder/Dockerfile | 98 + .../docker_related_config.xml | 12 + ci/docker/clickhouse-server/entrypoint.sh | 222 + .../from_binary/Dockerfile.ubuntu | 55 + .../from_deb/Dockerfile.ubuntu | 58 + ci/docker/compatibility/centos/Dockerfile | 5 + ci/docker/compatibility/ubuntu/Dockerfile | 5 + ci/docker/docs-builder/Dockerfile | 33 + ci/docker/fasttest/Dockerfile | 119 + ci/docker/fasttest/requirements.txt | 6 + .../integration/integration-test/Dockerfile | 102 + .../integration-test/requirements.txt | 25 + ci/docker/sqlancer-test/Dockerfile | 20 + ci/docker/stateful-test/Dockerfile | 14 + ci/docker/stateless-test/Dockerfile | 124 + ci/docker/stateless-test/requirements.txt | 13 + ci/docker/style-test/Dockerfile | 31 + ci/docker/style-test/requirements.txt | 10 + ci/jobs/build_clickhouse.py | 218 + ci/jobs/check_style.py | 395 + ci/jobs/clickbench.py | 106 + ci/jobs/collect_statistics.py | 214 + ci/jobs/docker_server_job.py | 112 + ci/jobs/docs_job.py | 87 + ci/jobs/fast_test.py | 276 + ci/jobs/functional_stateful_tests.py | 171 + ci/jobs/functional_stateless_tests.py | 185 + ci/jobs/fuzzers_job.py | 67 + ci/jobs/performance_tests.py | 630 + .../aspell-ignore/en/aspell-dict.txt | 3271 + ci/jobs/scripts/check_style/check-mypy | 25 + .../scripts/check_style/check-settings-style | 111 + ci/jobs/scripts/check_style/check_aspell.sh | 68 + ci/jobs/scripts/check_style/check_cpp.sh | 343 + .../scripts/check_style/check_submodules.sh | 37 + ci/jobs/scripts/check_style/check_typos.sh | 15 + .../scripts/check_style/double_whitespaces.pl | 37 + ci/jobs/scripts/check_style/various_checks.sh | 91 + ci/jobs/scripts/cidb_cluster.py | 120 + ci/jobs/scripts/clickbench/create.sql | 113 + ci/jobs/scripts/clickbench/queries.sql | 43 + ci/jobs/scripts/clickhouse_proc.py | 392 + ci/jobs/scripts/clickhouse_version.py | 97 + .../functional_tests/setup_ch_cluster.sh | 118 + .../setup_hdfs_minicluster.sh | 19 + .../functional_tests/setup_log_cluster.sh | 218 + .../scripts/functional_tests/setup_minio.sh | 168 + ci/jobs/scripts/functional_tests_results.py | 258 + ci/jobs/scripts/fuzzer/allow-nullable-key.xml | 6 + ci/jobs/scripts/fuzzer/generate-test-j2.py | 65 + .../fuzzer/query-fuzzer-tweaks-users.xml | 47 + ci/jobs/scripts/fuzzer/runner.sh | 237 + .../job_hooks/build_master_head_hook.py | 52 + .../scripts/job_hooks/build_profile_hook.py | 64 + .../scripts/job_hooks/docker_clean_up_hook.py | 23 + ci/jobs/scripts/log_cluster.py | 221 + ci/jobs/scripts/perf/compare.sh | 1448 + ci/jobs/scripts/perf/download.sh | 99 + ci/jobs/scripts/perf/eqmed.sql | 70 + ci/jobs/scripts/perf/report.py | 755 + .../scripts/workflow_hooks/can_be_merged.py | 28 + .../scripts/workflow_hooks/feature_docs.py | 29 + ci/jobs/scripts/workflow_hooks/filter_job.py | 128 + .../scripts/workflow_hooks/merge_sync_pr.py | 44 + .../scripts/workflow_hooks/pr_description.py | 219 + ci/jobs/scripts/workflow_hooks/quick_sync.py | 33 + .../set_dummy_sync_commit_status.py | 10 + ci/jobs/scripts/workflow_hooks/store_data.py | 45 + ci/jobs/scripts/workflow_hooks/trusted.py | 47 + ci/jobs/scripts/workflow_hooks/version_log.py | 33 + ci/jobs/sqlancer_job.sh | 93 + ci/jobs/sqltest_job.py | 302 + ci/jobs/unit_tests_job.py | 6 + ci/praktika/__init__.py | 12 + ci/praktika/__main__.py | 137 + ci/praktika/_environment.py | 267 + ci/praktika/artifact.py | 44 + ci/praktika/cache.py | 133 + ci/praktika/cidb.py | 286 + ci/praktika/digest.py | 147 + ci/praktika/docker.py | 173 + ci/praktika/execution/__init__.py | 1 + ci/praktika/execution/__main__.py | 4 + ci/praktika/execution/execution_settings.py | 31 + ci/praktika/execution/machine_init.py | 339 + ci/praktika/gh.py | 223 + ci/praktika/gh_auth.py | 108 + ci/praktika/hook_cache.py | 183 + ci/praktika/hook_html.py | 328 + ci/praktika/hook_interface.py | 43 + ci/praktika/html_prepare.py | 29 + ci/praktika/info.py | 221 + ci/praktika/job.py | 257 + ci/praktika/json.html | 1657 + ci/praktika/mangle.py | 174 + ci/praktika/native_jobs.py | 604 + ci/praktika/parser.py | 261 + ci/praktika/result.py | 877 + ci/praktika/runner.py | 612 + ci/praktika/runtime.py | 48 + ci/praktika/s3.py | 350 + ci/praktika/secret.py | 67 + ci/praktika/settings.py | 183 + ci/praktika/usage.py | 117 + ci/praktika/utils.py | 814 + ci/praktika/validator.py | 307 + ci/praktika/workflow.py | 110 + ci/praktika/yaml_generator.py | 521 + ci/settings/settings.py | 51 + ci/setup.py | 17 + ci/workflows/backport_branches.py | 67 + ci/workflows/master.py | 76 + ci/workflows/merge_queue.py | 36 + ci/workflows/new_pull_request.py | 41 + ci/workflows/nightly_fuzzers.py | 34 + ci/workflows/nightly_jepsen.py | 49 + ci/workflows/nightly_statistics.py | 28 + ci/workflows/pull_request.py | 93 + ci/workflows/release_branches.py | 67 + cmake/autogenerated_versions.txt | 14 +- cmake/ccache.cmake | 28 +- cmake/clang_tidy.cmake | 17 +- cmake/cpu_features.cmake | 54 +- cmake/cxx.cmake | 7 +- cmake/darwin/default_libs.cmake | 18 +- cmake/dbms_glob_sources.cmake | 6 +- cmake/dummy_compiler_linker.sh | 16 + cmake/freebsd/default_libs.cmake | 19 +- cmake/freebsd/toolchain-x86_64.cmake | 3 + cmake/linux/default_libs.cmake | 19 +- cmake/linux/toolchain-riscv64.cmake | 7 +- cmake/linux/toolchain-x86_64-musl.cmake | 2 +- cmake/split_debug_symbols.cmake | 11 +- cmake/target.cmake | 3 - cmake/tools.cmake | 58 +- cmake/unwind.cmake | 2 + cmake/utils.cmake | 49 +- cmake/warnings.cmake | 6 +- contrib/CMakeLists.txt | 42 +- contrib/FP16-cmake/CMakeLists.txt | 5 +- contrib/NuRaft | 2 +- contrib/QAT-ZSTD-Plugin-cmake/CMakeLists.txt | 1 - contrib/SHA3IUF | 1 + contrib/SimSIMD | 2 +- contrib/SimSIMD-cmake/CMakeLists.txt | 9 +- contrib/abseil-cpp-cmake/CMakeLists.txt | 43 +- contrib/arrow-cmake/CMakeLists.txt | 252 +- contrib/aws | 2 +- contrib/aws-cmake/CMakeLists.txt | 25 +- contrib/aws-crt-cpp | 2 +- contrib/aws-s2n-tls | 2 +- contrib/azure-cmake/CMakeLists.txt | 19 +- contrib/boost-cmake/CMakeLists.txt | 5 + contrib/cctz | 2 +- contrib/cctz-cmake/CMakeLists.txt | 4 +- contrib/corrosion | 2 +- contrib/corrosion-cmake/CMakeLists.txt | 179 +- contrib/corrosion-cmake/config.toml.in | 23 + contrib/cppkafka | 2 +- contrib/curl | 2 +- contrib/curl-cmake/CMakeLists.txt | 10 +- contrib/curl-cmake/curl_config.h | 2 + contrib/delta-kernel-rs | 1 + contrib/delta-kernel-rs-cmake/CMakeLists.txt | 70 + contrib/expected | 1 - contrib/expected-cmake/CMakeLists.txt | 3 - contrib/flatbuffers | 2 +- contrib/fmtlib | 2 +- contrib/google-cloud-cpp | 1 + contrib/google-cloud-cpp-cmake/CMakeLists.txt | 105 + .../google-cloud-cpp-cmake/GoogleApis.cmake | 469 + .../google_cloud_cpp_common.cmake | 447 + .../google_cloud_cpp_grpc_utils.cmake | 350 + contrib/google-protobuf-cmake/CMakeLists.txt | 15 +- contrib/googletest | 2 +- contrib/grpc | 2 +- contrib/grpc-cmake/CMakeLists.txt | 36 +- contrib/grpc-cmake/grpc.cmake | 2 +- contrib/h3-cmake/CMakeLists.txt | 7 + contrib/icu | 2 +- contrib/icu-cmake/CMakeLists.txt | 755 +- contrib/icudata | 2 +- contrib/idna-cmake/CMakeLists.txt | 2 + contrib/idxd-config | 2 +- contrib/isa-l-cmake/CMakeLists.txt | 5 - contrib/jemalloc | 2 +- contrib/jemalloc-cmake/CMakeLists.txt | 9 - contrib/jwt-cpp | 1 + contrib/jwt-cpp-cmake/CMakeLists.txt | 23 + contrib/krb5 | 2 +- contrib/krb5-cmake/CMakeLists.txt | 30 +- contrib/krb5-cmake/kcmrpc.cin | 382 + contrib/krb5-cmake/kcmrpc.hin | 209 + contrib/libarchive | 2 +- contrib/libarchive-cmake/CMakeLists.txt | 14 +- contrib/libarchive-cmake/config.h | 44 +- contrib/libcpuid | 2 +- contrib/libcpuid-cmake/CMakeLists.txt | 7 +- contrib/libcxx-cmake/CMakeLists.txt | 42 +- contrib/libcxxabi-cmake/CMakeLists.txt | 13 +- contrib/libdivide | 2 +- contrib/libfiu | 2 +- contrib/libgsasl | 2 +- contrib/libgsasl-cmake/CMakeLists.txt | 1 + contrib/libhdfs3 | 2 +- contrib/libhdfs3-cmake/CMakeLists.txt | 7 +- contrib/libpqxx | 2 +- contrib/libpqxx-cmake/CMakeLists.txt | 5 +- .../libprotobuf-mutator-cmake/CMakeLists.txt | 1 - contrib/librdkafka | 2 +- contrib/librdkafka-cmake/CMakeLists.txt | 118 +- contrib/librdkafka-cmake/config.h.in | 103 +- contrib/libunwind | 1 - contrib/libunwind-cmake/CMakeLists.txt | 9 +- contrib/liburing | 2 +- contrib/liburing-cmake/CMakeLists.txt | 1 + contrib/liburing-cmake/compat.h.in | 10 + contrib/libuv | 2 +- contrib/libuv-cmake/CMakeLists.txt | 6 +- contrib/llvm-project | 2 +- contrib/llvm-project-cmake/CMakeLists.txt | 25 +- contrib/lz4 | 2 +- contrib/lz4-cmake/CMakeLists.txt | 1 - contrib/magic-enum-cmake/CMakeLists.txt | 2 +- contrib/magic_enum | 2 +- .../mariadb-connector-c-cmake/CMakeLists.txt | 6 +- contrib/minizip-ng | 2 +- contrib/minizip-ng-cmake/CMakeLists.txt | 20 +- contrib/minizip-ng-cmake/unzip.h | 13 - contrib/minizip-ng-cmake/zip.h | 13 - contrib/mongo-c-driver | 1 + contrib/mongo-c-driver-cmake/CMakeLists.txt | 151 + contrib/mongo-cxx-driver | 1 + contrib/mongo-cxx-driver-cmake/CMakeLists.txt | 192 + contrib/nanodbc | 1 - contrib/nanodbc-cmake/CMakeLists.txt | 18 - contrib/nats-io | 2 +- contrib/nats-io-cmake/CMakeLists.txt | 8 + contrib/numactl | 2 +- contrib/nuraft-cmake/CMakeLists.txt | 1 - contrib/openssl | 2 +- contrib/openssl-cmake/CMakeLists.txt | 320 +- .../asm/crypto/aes/aes-riscv64-zkn.S | 704 + .../openssl-cmake/asm/crypto/aes/aes-s390x.S | 2365 + .../openssl-cmake/asm/crypto/aes/aes-x86_64.s | 2658 + .../asm/crypto/aes/aesni-mb-x86_64.s | 1588 + .../asm/crypto/aes/aesni-sha1-x86_64.s | 3035 + .../asm/crypto/aes/aesni-sha256-x86_64.s | 4435 + .../asm/crypto/aes/aesni-x86_64.s | 4485 + .../openssl-cmake/asm/crypto/aes/aesp8-ppc.s | 3703 + .../openssl-cmake/asm/crypto/aes/aesv8-armx.S | 3191 + .../asm/crypto/aes/bsaes-armv8.S | 2347 + .../asm/crypto/aes/bsaes-x86_64.s | 2597 + .../asm/crypto/aes/vpaes-armv8.S | 1199 + .../asm/crypto/aes/vpaes-x86_64.s | 858 + contrib/openssl-cmake/asm/crypto/arm64cpuid.S | 274 + .../openssl-cmake/asm/crypto/bn/armv8-mont.S | 2133 + .../asm/crypto/bn/rsaz-2k-avx512.s | 895 + .../asm/crypto/bn/rsaz-3k-avx512.s | 1310 + .../asm/crypto/bn/rsaz-4k-avx512.s | 1353 + .../openssl-cmake/asm/crypto/bn/rsaz-avx2.s | 1744 + .../openssl-cmake/asm/crypto/bn/rsaz-x86_64.s | 2015 + .../openssl-cmake/asm/crypto/bn/x86_64-gf2m.s | 311 + .../openssl-cmake/asm/crypto/bn/x86_64-mont.s | 1239 + .../asm/crypto/bn/x86_64-mont5.s | 3603 + .../asm/crypto/camellia/cmll-x86_64.s | 1925 + .../asm/crypto/chacha/chacha-armv8-sve.S | 3868 + .../asm/crypto/chacha/chacha-armv8.S | 2070 + .../asm/crypto/chacha/chacha-s390x.S | 954 + .../asm/crypto/chacha/chacha-x86_64.s | 3428 + .../asm/crypto/ec/ecp_nistz256-armv8.S | 4254 + .../asm/crypto/ec/ecp_nistz256-x86_64.s | 7343 + .../asm/crypto/ec/ecp_sm2p256-armv8.S | 826 + .../asm/crypto/ec/x25519-x86_64.s | 802 + .../asm/crypto/loongarch64cpuid.S | 69 + .../asm/crypto/md5/asm/md5-aarch64.S | 677 + .../openssl-cmake/asm/crypto/md5/md5-x86_64.s | 683 + .../asm/crypto/modes/aes-gcm-avx512.s | 136110 +++++++++++++++ .../asm/crypto/modes/aes-gcm-ppc.s | 1339 + .../asm/crypto/modes/aesni-gcm-x86_64.s | 789 + .../modes/asm/aes-gcm-armv8-unroll8_64.S | 8487 + .../asm/crypto/modes/asm/aes-gcm-armv8_64.S | 6395 + .../asm/crypto/modes/ghash-riscv64.S | 613 + .../asm/crypto/modes/ghash-x86_64.s | 1853 + .../asm/crypto/modes/ghashp8-ppc.s | 575 + .../asm/crypto/modes/ghashv8-armx.S | 643 + .../asm/crypto/poly1305/poly1305-armv8.S | 876 + .../asm/crypto/poly1305/poly1305-x86_64.s | 3586 + contrib/openssl-cmake/asm/crypto/ppccpuid.s | 400 + .../asm/crypto/rc4/rc4-md5-x86_64.s | 1281 + .../openssl-cmake/asm/crypto/rc4/rc4-s390x.S | 240 + .../openssl-cmake/asm/crypto/rc4/rc4-x86_64.s | 635 + .../openssl-cmake/asm/crypto/riscv64cpuid.S | 59 + contrib/openssl-cmake/asm/crypto/s390xcpuid.S | 428 + .../asm/crypto/sha/keccak1600-armv8.S | 1011 + .../asm/crypto/sha/keccak1600-s390x.S | 464 + .../asm/crypto/sha/keccak1600-x86_64.s | 524 + .../openssl-cmake/asm/crypto/sha/sha1-armv8.S | 1214 + .../asm/crypto/sha/sha1-mb-x86_64.s | 7303 + .../asm/crypto/sha/sha1-x86_64.s | 5450 + .../asm/crypto/sha/sha256-armv8.S | 2055 + .../asm/crypto/sha/sha256-mb-x86_64.s | 7984 + .../asm/crypto/sha/sha256-x86_64.s | 5456 + .../asm/crypto/sha/sha512-armv8.S | 1608 + .../asm/crypto/sha/sha512-x86_64.s | 5461 + .../asm/crypto/sm3/asm/sm3-armv8.S | 503 + .../asm/crypto/sm4/asm/sm4-armv8.S | 1081 + .../asm/crypto/sm4/asm/vpsm4-armv8.S | 4999 + .../asm/crypto/sm4/asm/vpsm4_ex-armv8.S | 4505 + .../asm/crypto/whrlpool/wp-x86_64.s | 879 + .../openssl-cmake/asm/crypto/x86_64cpuid.s | 491 + contrib/openssl-cmake/asm/generate_asm.sh | 111 + contrib/orc | 2 +- contrib/postgres-cmake/CMakeLists.txt | 10 + contrib/qatlib-cmake/CMakeLists.txt | 1 - contrib/qpl-cmake/CMakeLists.txt | 280 +- contrib/replxx | 2 +- contrib/robin-map | 1 - contrib/robin-map-cmake/CMakeLists.txt | 1 - contrib/rocksdb | 2 +- contrib/rocksdb-cmake/CMakeLists.txt | 47 +- contrib/rust_vendor | 1 + contrib/sha3iuf-cmake/CMakeLists.txt | 8 + contrib/simdjson | 2 +- .../sparse-checkout/setup-sparse-checkout.sh | 2 + contrib/sparse-checkout/update-aws.sh | 3 + .../update-google-cloud-cpp.sh | 18 + contrib/sparse-checkout/update-postgres.sh | 16 + contrib/sysroot | 2 +- contrib/unixodbc | 1 - contrib/unixodbc-cmake/CMakeLists.txt | 299 - .../linux_x86_64/libltdl/config.h | 181 - .../linux_x86_64/libltdl/libltdlcS.c | 53 - .../linux_x86_64/private/config.h | 496 - .../linux_x86_64/unixodbc_conf.h | 60 - contrib/update-submodules.sh | 2 +- contrib/usearch | 2 +- contrib/usearch-cmake/CMakeLists.txt | 20 +- contrib/vectorscan-cmake/CMakeLists.txt | 2 + contrib/zlib-ng-cmake/CMakeLists.txt | 2 +- contrib/zstd | 2 +- contrib/zstd-cmake/CMakeLists.txt | 13 +- packages/build | 47 +- packages/clickhouse-library-bridge.yaml | 35 - packages/clickhouse-odbc-bridge.yaml | 35 - packages/clickhouse-server.postinstall | 12 - programs/CMakeLists.txt | 108 +- .../completions/CMakeLists.txt | 1 + .../completions/clickhouse-bootstrap | 2 +- programs/benchmark/Benchmark.cpp | 230 +- programs/check-marks/CMakeLists.txt | 11 + programs/check-marks/CheckMarks.cpp | 89 + programs/check-marks/check-marks.cpp | 1 + .../CMakeLists.txt | 9 + .../ChecksumForCompressedBlock.cpp | 52 + .../checksum-for-compressed-block.cpp | 1 + programs/client/CMakeLists.txt | 5 + programs/client/Client.cpp | 797 +- programs/client/Client.h | 30 +- programs/client/FuzzLoop.cpp | 854 + programs/client/clickhouse-client.xml | 34 +- programs/compressor/Compressor.cpp | 67 +- programs/config_tools.h.in | 4 +- programs/disks/CMakeLists.txt | 3 +- programs/disks/CommandChangeDirectory.cpp | 2 +- programs/disks/CommandCopy.cpp | 50 +- .../disks/CommandGetCurrentDiskAndPath.cpp | 4 +- programs/disks/CommandHelp.cpp | 2 +- programs/disks/CommandLink.cpp | 6 +- programs/disks/CommandList.cpp | 20 +- programs/disks/CommandListDisks.cpp | 25 +- programs/disks/CommandMkDir.cpp | 11 +- programs/disks/CommandMove.cpp | 22 +- programs/disks/CommandRead.cpp | 9 +- programs/disks/CommandRemove.cpp | 24 +- programs/disks/CommandSwitchDisk.cpp | 3 +- programs/disks/CommandTouch.cpp | 21 +- programs/disks/CommandWrite.cpp | 19 +- programs/disks/DisksApp.cpp | 282 +- programs/disks/DisksApp.h | 82 +- programs/disks/DisksClient.cpp | 271 +- programs/disks/DisksClient.h | 38 +- programs/disks/ICommand.cpp | 35 +- programs/disks/ICommand.h | 37 +- programs/disks/Utils.cpp | 69 + programs/disks/Utils.h | 5 + programs/extract-from-config/CMakeLists.txt | 1 + .../extract-from-config/ExtractFromConfig.cpp | 59 +- programs/format/Format.cpp | 47 +- programs/git-import/git-import.cpp | 36 +- programs/install/Install.cpp | 29 +- programs/keeper-bench/CMakeLists.txt | 20 + programs/keeper-bench/Generator.cpp | 574 + programs/keeper-bench/Generator.h | 181 + programs/keeper-bench/KeeperBench.cpp | 87 + programs/keeper-bench/README.md | 317 + programs/keeper-bench/Runner.cpp | 1515 + programs/keeper-bench/Runner.h | 140 + programs/keeper-bench/Stats.cpp | 177 + programs/keeper-bench/Stats.h | 42 + programs/keeper-bench/example.yaml | 118 + programs/keeper-bench/keeper-bench.cpp | 1 + programs/keeper-client/Commands.cpp | 157 +- programs/keeper-client/Commands.h | 33 +- programs/keeper-client/KeeperClient.cpp | 52 +- programs/keeper-client/KeeperClient.h | 4 +- programs/keeper-client/Parser.cpp | 3 + programs/keeper-client/Parser.h | 4 +- programs/keeper-converter/KeeperConverter.cpp | 1 + programs/keeper-data-dumper/CMakeLists.txt | 14 + .../keeper-data-dumper/KeeperDataDumper.cpp | 108 + .../keeper-data-dumper/keeper-data-dumper.cpp | 1 + programs/keeper/CMakeLists.txt | 10 +- programs/keeper/Keeper.cpp | 29 +- programs/keeper/keeper_main.cpp | 42 +- programs/library-bridge/CMakeLists.txt | 27 - programs/library-bridge/CatBoostLibraryAPI.h | 49 - .../library-bridge/CatBoostLibraryHandler.cpp | 391 - .../library-bridge/CatBoostLibraryHandler.h | 79 - .../CatBoostLibraryHandlerFactory.cpp | 80 - .../CatBoostLibraryHandlerFactory.h | 37 - .../ExternalDictionaryLibraryAPI.cpp | 48 - .../ExternalDictionaryLibraryAPI.h | 106 - .../ExternalDictionaryLibraryHandler.cpp | 212 - .../ExternalDictionaryLibraryHandler.h | 55 - ...xternalDictionaryLibraryHandlerFactory.cpp | 65 - .../ExternalDictionaryLibraryHandlerFactory.h | 39 - .../ExternalDictionaryLibraryUtils.h | 43 - programs/library-bridge/LibraryBridge.cpp | 33 - programs/library-bridge/LibraryBridge.h | 19 - .../LibraryBridgeHandlerFactory.cpp | 42 - .../LibraryBridgeHandlerFactory.h | 25 - .../library-bridge/LibraryBridgeHandlers.cpp | 638 - .../library-bridge/LibraryBridgeHandlers.h | 85 - programs/library-bridge/SharedLibrary.cpp | 49 - programs/library-bridge/SharedLibrary.h | 41 - .../library-bridge/createFunctionBaseCast.cpp | 23 - programs/library-bridge/library-bridge.cpp | 2 - programs/local/LocalServer.cpp | 459 +- programs/local/LocalServer.h | 12 +- programs/local/PandasAnalyzer.cpp | 12 + programs/local/PandasAnalyzer.h | 6 +- programs/main.cpp | 116 +- programs/obfuscator/Obfuscator.cpp | 19 +- programs/obfuscator/README.md | 6 +- programs/odbc-bridge/CMakeLists.txt | 37 - programs/odbc-bridge/ColumnInfoHandler.cpp | 225 - programs/odbc-bridge/ColumnInfoHandler.h | 28 - .../odbc-bridge/IdentifierQuoteHandler.cpp | 97 - programs/odbc-bridge/IdentifierQuoteHandler.h | 27 - programs/odbc-bridge/MainHandler.cpp | 222 - programs/odbc-bridge/MainHandler.h | 43 - programs/odbc-bridge/ODBCBridge.cpp | 33 - programs/odbc-bridge/ODBCBridge.h | 19 - programs/odbc-bridge/ODBCHandlerFactory.cpp | 53 - programs/odbc-bridge/ODBCHandlerFactory.h | 29 - .../odbc-bridge/ODBCPooledConnectionFactory.h | 177 - programs/odbc-bridge/ODBCSink.cpp | 45 - programs/odbc-bridge/ODBCSink.h | 45 - programs/odbc-bridge/ODBCSource.cpp | 160 - programs/odbc-bridge/ODBCSource.h | 42 - programs/odbc-bridge/PingHandler.cpp | 22 - programs/odbc-bridge/PingHandler.h | 15 - programs/odbc-bridge/SchemaAllowedHandler.cpp | 111 - programs/odbc-bridge/SchemaAllowedHandler.h | 30 - .../odbc-bridge/createFunctionBaseCast.cpp | 23 - programs/odbc-bridge/getIdentifierQuote.cpp | 53 - programs/odbc-bridge/getIdentifierQuote.h | 22 - programs/odbc-bridge/odbc-bridge.cpp | 2 - programs/odbc-bridge/tests/CMakeLists.txt | 2 - .../tests/validate-odbc-connection-string.cpp | 24 - .../validate-odbc-connection-string.reference | 39 - .../tests/validate-odbc-connection-string.sh | 41 - .../validateODBCConnectionString.cpp | 251 - .../validateODBCConnectionString.h | 21 - programs/server/CMakeLists.txt | 1 + programs/server/Server.cpp | 1163 +- programs/server/binary.html | 47 +- programs/server/config.xml | 139 +- programs/server/config.yaml.example | 35 +- programs/server/dashboard.html | 87 +- programs/server/merges.html | 441 + programs/server/play.html | 2147 +- programs/server/users.xml | 1 + programs/server/users.yaml.example | 2 +- .../static-files-disk-uploader.cpp | 23 +- programs/su/su.cpp | 20 +- programs/zookeeper-dump-tree/CMakeLists.txt | 10 + .../zookeeper-dump-tree/ZooKeeperDumpTree.cpp | 106 + .../zookeeper-dump-tree.cpp | 1 + .../zookeeper-remove-by-list/CMakeLists.txt | 10 + .../ZooKeeperRemoveByList.cpp | 67 + .../zookeeper-remove-by-list.cpp | 1 + src/Access/AccessBackup.cpp | 660 +- src/Access/AccessBackup.h | 114 +- src/Access/AccessControl.cpp | 117 +- src/Access/AccessControl.h | 38 +- src/Access/AccessEntityIO.cpp | 9 +- src/Access/AccessRights.cpp | 1206 +- src/Access/AccessRights.h | 91 +- src/Access/Authentication.cpp | 360 +- src/Access/Authentication.h | 4 +- src/Access/AuthenticationData.cpp | 204 +- src/Access/AuthenticationData.h | 41 +- src/Access/Common/AccessFlags.cpp | 19 +- src/Access/Common/AccessFlags.h | 4 + src/Access/Common/AccessRightsElement.cpp | 282 +- src/Access/Common/AccessRightsElement.h | 43 +- src/Access/Common/AccessType.h | 37 +- src/Access/Common/AllowedClientHosts.cpp | 15 +- src/Access/Common/AuthenticationType.cpp | 5 + src/Access/Common/AuthenticationType.h | 5 + .../Common/HTTPAuthenticationScheme.cpp | 4 +- src/Access/Common/QuotaDefs.cpp | 6 +- src/Access/Common/SSLCertificateSubjects.cpp | 95 - src/Access/Common/SSLCertificateSubjects.h | 48 - src/Access/ContextAccess.cpp | 562 +- src/Access/ContextAccess.h | 27 +- src/Access/ContextAccessParams.cpp | 58 +- src/Access/ContextAccessParams.h | 9 +- src/Access/Credentials.cpp | 9 +- src/Access/Credentials.h | 86 +- src/Access/DiskAccessStorage.cpp | 44 +- src/Access/DiskAccessStorage.h | 5 +- src/Access/ExternalAuthenticators.cpp | 16 +- src/Access/ExternalAuthenticators.h | 3 +- src/Access/GrantedRoles.cpp | 64 +- src/Access/GrantedRoles.h | 3 + src/Access/HTTPAuthClient.h | 17 +- src/Access/IAccessEntity.h | 4 + src/Access/IAccessStorage.cpp | 159 +- src/Access/IAccessStorage.h | 53 +- src/Access/KerberosInit.cpp | 3 +- src/Access/LDAPAccessStorage.cpp | 23 +- src/Access/LDAPAccessStorage.h | 2 +- src/Access/MemoryAccessStorage.cpp | 45 +- src/Access/MemoryAccessStorage.h | 5 +- src/Access/MultipleAccessStorage.cpp | 13 +- src/Access/MultipleAccessStorage.h | 6 +- src/Access/Quota.cpp | 24 + src/Access/Quota.h | 5 + src/Access/QuotaCache.cpp | 7 +- src/Access/QuotaCache.h | 2 +- src/Access/ReplicatedAccessStorage.cpp | 128 +- src/Access/ReplicatedAccessStorage.h | 9 +- src/Access/Role.cpp | 26 + src/Access/Role.h | 5 + src/Access/RoleCache.cpp | 4 +- src/Access/RoleCache.h | 9 +- src/Access/RolesOrUsersSet.cpp | 69 +- src/Access/RolesOrUsersSet.h | 3 + src/Access/RowPolicy.cpp | 26 +- src/Access/RowPolicy.h | 5 + src/Access/RowPolicyCache.cpp | 2 +- src/Access/SSH/SSHPublicKey.cpp | 138 + src/Access/SSH/SSHPublicKey.h | 67 + src/Access/SettingsAuthResponseParser.cpp | 2 +- src/Access/SettingsConstraints.cpp | 98 +- src/Access/SettingsConstraints.h | 15 +- src/Access/SettingsProfile.cpp | 25 + src/Access/SettingsProfile.h | 5 + src/Access/SettingsProfileElement.cpp | 285 +- src/Access/SettingsProfileElement.h | 43 +- src/Access/SettingsProfilesCache.cpp | 4 +- src/Access/User.cpp | 39 +- src/Access/User.h | 8 +- src/Access/UsersConfigAccessStorage.cpp | 76 +- src/Access/resolveSetting.h | 15 +- .../tests/gtest_access_rights_implicit.cpp | 122 + src/Access/tests/gtest_access_rights_ops.cpp | 457 +- .../AggregateFunctionAny.cpp | 8 +- .../AggregateFunctionAnyHeavy.cpp | 18 +- .../AggregateFunctionAnyRespectNulls.cpp | 13 +- src/AggregateFunctions/AggregateFunctionAvg.h | 35 +- .../AggregateFunctionAvgWeighted.cpp | 35 +- .../AggregateFunctionCount.h | 15 +- .../AggregateFunctionDeltaSum.cpp | 11 +- .../AggregateFunctionDeltaSumTimestamp.cpp | 84 +- .../AggregateFunctionDistinctDynamicTypes.cpp | 161 + .../AggregateFunctionDistinctJSONPaths.cpp | 352 + ...regateFunctionEstimateCompressionRatio.cpp | 270 + .../AggregateFunctionFactory.cpp | 40 +- .../AggregateFunctionFactory.h | 8 +- .../AggregateFunctionFlameGraph.cpp | 34 +- .../AggregateFunctionGroupArray.cpp | 37 +- .../AggregateFunctionGroupArrayInsertAt.cpp | 2 +- .../AggregateFunctionGroupArrayIntersect.cpp | 29 +- .../AggregateFunctionGroupArrayMoving.cpp | 24 +- .../AggregateFunctionGroupArraySorted.cpp | 56 +- .../AggregateFunctionGroupBitmap.cpp | 3 +- .../AggregateFunctionGroupBitmapData.h | 64 +- .../AggregateFunctionGroupConcat.cpp | 315 +- .../AggregateFunctionGroupConcat.h | 78 + .../AggregateFunctionGroupUniqArray.cpp | 34 +- .../AggregateFunctionHistogram.cpp | 17 +- .../AggregateFunctionIntervalLengthSum.cpp | 4 +- ...AggregateFunctionKolmogorovSmirnovTest.cpp | 19 +- ...ateFunctionLargestTriangleThreeBuckets.cpp | 14 +- .../AggregateFunctionMLMethod.h | 2 +- .../AggregateFunctionMannWhitney.cpp | 2 +- .../AggregateFunctionMaxIntersections.cpp | 11 +- .../AggregateFunctionMeanZTest.cpp | 2 +- .../AggregateFunctionNothing.h | 14 +- .../AggregateFunctionQuantile.h | 3 + ...AggregateFunctionQuantileExactWeighted.cpp | 260 +- .../AggregateFunctionQuantileGK.cpp | 15 +- ...teFunctionQuantileInterpolatedWeighted.cpp | 18 +- .../AggregateFunctionRankCorrelation.cpp | 2 +- .../AggregateFunctionSequenceMatch.cpp | 101 +- .../AggregateFunctionSequenceNextNode.cpp | 10 +- .../AggregateFunctionSingleValueOrNull.cpp | 20 +- .../AggregateFunctionSparkbar.cpp | 18 +- .../AggregateFunctionStatistics.cpp | 19 +- src/AggregateFunctions/AggregateFunctionSum.h | 28 +- .../AggregateFunctionSumMap.cpp | 49 +- .../AggregateFunctionTTest.h | 29 +- .../AggregateFunctionTopK.cpp | 25 +- .../AggregateFunctionUniq.cpp | 87 +- .../AggregateFunctionUniq.h | 40 +- .../AggregateFunctionUniqCombined.h | 49 +- .../AggregateFunctionUniqUpTo.cpp | 18 +- .../AggregateFunctionWindowFunnel.cpp | 360 +- .../AggregateFunctionsArgMinArgMax.cpp | 198 +- .../AggregateFunctionsMinMax.cpp | 12 +- src/AggregateFunctions/CMakeLists.txt | 13 +- .../Combinators/AggregateFunctionArray.h | 5 +- ...gregateFunctionCombinatorsArgMinArgMax.cpp | 22 +- .../Combinators/AggregateFunctionDistinct.cpp | 6 +- .../Combinators/AggregateFunctionDistinct.h | 5 +- .../Combinators/AggregateFunctionForEach.h | 5 +- .../Combinators/AggregateFunctionIf.cpp | 42 +- .../Combinators/AggregateFunctionIf.h | 4 +- .../Combinators/AggregateFunctionMap.cpp | 5 +- .../Combinators/AggregateFunctionMerge.cpp | 6 + .../Combinators/AggregateFunctionMerge.h | 6 +- .../Combinators/AggregateFunctionNull.cpp | 30 +- .../Combinators/AggregateFunctionNull.h | 13 +- .../Combinators/AggregateFunctionOrFill.cpp | 3 +- .../Combinators/AggregateFunctionOrFill.h | 4 +- .../Combinators/AggregateFunctionResample.cpp | 7 +- src/AggregateFunctions/CrossTab.h | 2 +- src/AggregateFunctions/DDSketch.h | 8 +- src/AggregateFunctions/DDSketch/Store.h | 3 +- src/AggregateFunctions/FactoryHelpers.h | 1 - src/AggregateFunctions/Helpers.h | 70 +- src/AggregateFunctions/IAggregateFunction.cpp | 84 + src/AggregateFunctions/IAggregateFunction.h | 138 +- .../QuantileBFloat16Histogram.h | 21 +- src/AggregateFunctions/QuantileExact.h | 4 +- src/AggregateFunctions/QuantileTDigest.h | 13 +- src/AggregateFunctions/QuantileTiming.h | 8 +- src/AggregateFunctions/ReservoirSampler.h | 20 +- .../ReservoirSamplerDeterministic.h | 7 +- src/AggregateFunctions/SingleValueData.cpp | 387 +- src/AggregateFunctions/SingleValueData.h | 71 +- src/AggregateFunctions/ThetaSketchData.h | 5 +- src/AggregateFunctions/UniqExactSet.h | 61 +- src/AggregateFunctions/UniqVariadicHash.cpp | 3 +- src/AggregateFunctions/UniquesHashSet.h | 45 - src/AggregateFunctions/WindowFunction.h | 23 +- src/AggregateFunctions/fuzzers/CMakeLists.txt | 2 +- ..._function_state_deserialization_fuzzer.cpp | 2 +- .../parseAggregateFunctionParameters.h | 5 +- .../registerAggregateFunctions.cpp | 6 + src/Analyzer/ArrayJoinNode.cpp | 12 +- src/Analyzer/CMakeLists.txt | 4 - src/Analyzer/ColumnNode.cpp | 14 +- src/Analyzer/ColumnTransformers.cpp | 23 +- src/Analyzer/ColumnTransformers.h | 3 +- src/Analyzer/ConstantNode.cpp | 159 +- src/Analyzer/ConstantNode.h | 46 +- src/Analyzer/ConstantValue.h | 39 +- src/Analyzer/FunctionNode.cpp | 23 +- src/Analyzer/FunctionNode.h | 10 +- .../FunctionSecretArgumentsFinderTreeNode.h | 396 +- src/Analyzer/HashUtils.h | 15 +- src/Analyzer/IQueryTreeNode.cpp | 11 +- src/Analyzer/IQueryTreeNode.h | 30 +- src/Analyzer/Identifier.cpp | 92 + src/Analyzer/Identifier.h | 92 +- src/Analyzer/InDepthQueryTreeVisitor.h | 3 +- src/Analyzer/JoinNode.cpp | 190 +- src/Analyzer/JoinNode.h | 70 +- src/Analyzer/LambdaNode.cpp | 3 + src/Analyzer/LambdaNode.h | 6 +- src/Analyzer/MatcherNode.cpp | 5 +- .../AggregateFunctionOfGroupByKeysPass.cpp | 15 +- ...egateFunctionsArithmericOperationsPass.cpp | 17 +- src/Analyzer/Passes/ArrayExistsToHasPass.cpp | 11 +- src/Analyzer/Passes/AutoFinalOnQueryPass.cpp | 6 +- src/Analyzer/Passes/CNF.cpp | 38 +- src/Analyzer/Passes/CNF.h | 23 +- src/Analyzer/Passes/CNFAtomicFormula.cpp | 20 + src/Analyzer/Passes/CNFAtomicFormula.h | 17 + .../Passes/ComparisonTupleEliminationPass.cpp | 8 +- .../Passes/ConvertOrLikeChainPass.cpp | 20 +- src/Analyzer/Passes/ConvertQueryToCNFPass.cpp | 29 +- src/Analyzer/Passes/CountDistinctPass.cpp | 8 +- src/Analyzer/Passes/CrossToInnerJoinPass.cpp | 258 +- .../Passes/FunctionToSubcolumnsPass.cpp | 108 +- src/Analyzer/Passes/FuseFunctionsPass.cpp | 11 +- .../Passes/GroupingFunctionsResolvePass.cpp | 69 +- src/Analyzer/Passes/IfChainToMultiIfPass.cpp | 12 +- .../Passes/IfTransformStringsToEnumPass.cpp | 9 +- .../Passes/LogicalExpressionOptimizerPass.cpp | 674 +- .../Passes/LogicalExpressionOptimizerPass.h | 30 +- src/Analyzer/Passes/MultiIfToIfPass.cpp | 11 +- .../Passes/NormalizeCountVariantsPass.cpp | 6 +- ...ateOrDateTimeConverterWithPreimagePass.cpp | 22 +- .../OptimizeGroupByFunctionKeysPass.cpp | 9 +- .../OptimizeGroupByInjectiveFunctionsPass.cpp | 9 +- ...ptimizeRedundantFunctionsInOrderByPass.cpp | 6 +- .../RemoveUnusedProjectionColumnsPass.cpp | 75 +- .../RewriteAggregateFunctionWithIfPass.cpp | 18 +- .../RewriteSumFunctionWithSumAndCountPass.cpp | 6 +- src/Analyzer/Passes/SumIfToCountIfPass.cpp | 9 +- .../UniqInjectiveFunctionsEliminationPass.cpp | 6 +- src/Analyzer/Passes/UniqToCountPass.cpp | 6 +- .../Passes}/tests/CMakeLists.txt | 0 .../gtest_logical_expression_optimizer.cpp | 102 + src/Analyzer/QueryNode.cpp | 126 +- src/Analyzer/QueryNode.h | 59 +- src/Analyzer/QueryTreeBuilder.cpp | 332 +- src/Analyzer/QueryTreeBuilder.h | 3 + src/Analyzer/QueryTreePassManager.cpp | 56 +- src/Analyzer/README.md | 21 + src/Analyzer/Resolve/ExpressionsStack.h | 4 +- src/Analyzer/Resolve/IdentifierLookup.h | 51 +- .../Resolve/IdentifierResolveScope.cpp | 111 +- src/Analyzer/Resolve/IdentifierResolveScope.h | 21 +- src/Analyzer/Resolve/IdentifierResolver.cpp | 795 +- src/Analyzer/Resolve/IdentifierResolver.h | 147 +- src/Analyzer/Resolve/QueryAnalysisPass.cpp | 4 +- src/Analyzer/Resolve/QueryAnalyzer.cpp | 1327 +- src/Analyzer/Resolve/QueryAnalyzer.h | 34 +- .../Resolve/QueryExpressionsAliasVisitor.h | 80 +- src/Analyzer/Resolve/ScopeAliases.h | 40 +- src/Analyzer/Resolve/TableExpressionData.h | 15 +- .../Resolve/TableExpressionsAliasVisitor.h | 5 + ...eFunctionsWithClusterAlternativesVisitor.h | 30 + src/Analyzer/Resolve/TypoCorrection.cpp | 226 + src/Analyzer/Resolve/TypoCorrection.h | 55 + src/Analyzer/SetUtils.cpp | 48 +- src/Analyzer/SetUtils.h | 12 +- src/Analyzer/SortNode.cpp | 12 +- src/Analyzer/SortNode.h | 21 +- src/Analyzer/TableExpressionModifiers.h | 1 + src/Analyzer/TableFunctionNode.cpp | 3 + src/Analyzer/TableFunctionNode.h | 3 + src/Analyzer/TableNode.cpp | 20 +- src/Analyzer/TableNode.h | 4 +- src/Analyzer/UnionNode.cpp | 66 +- src/Analyzer/UnionNode.h | 36 +- src/Analyzer/Utils.cpp | 452 +- src/Analyzer/Utils.h | 60 +- src/Analyzer/ValidationUtils.cpp | 138 +- src/Analyzer/ValidationUtils.h | 6 + src/Analyzer/WindowNode.cpp | 9 + ...cpp => createUniqueAliasesIfNecessary.cpp} | 104 +- src/Analyzer/createUniqueAliasesIfNecessary.h | 14 + src/Analyzer/createUniqueTableAliases.h | 18 - src/Analyzer/examples/CMakeLists.txt | 3 - src/Analyzer/examples/query_analyzer.cpp | 8 - src/Backups/BackupConcurrencyCheck.cpp | 135 + src/Backups/BackupConcurrencyCheck.h | 56 + src/Backups/BackupCoordinationCleaner.cpp | 64 + src/Backups/BackupCoordinationCleaner.h | 35 + src/Backups/BackupCoordinationLocal.cpp | 40 +- src/Backups/BackupCoordinationLocal.h | 31 +- ...te.cpp => BackupCoordinationOnCluster.cpp} | 288 +- ...Remote.h => BackupCoordinationOnCluster.h} | 72 +- .../BackupCoordinationReplicatedAccess.cpp | 19 +- .../BackupCoordinationReplicatedAccess.h | 8 +- .../BackupCoordinationReplicatedTables.cpp | 10 +- src/Backups/BackupCoordinationStage.h | 14 +- src/Backups/BackupCoordinationStageSync.cpp | 1520 +- src/Backups/BackupCoordinationStageSync.h | 220 +- src/Backups/BackupEntriesCollector.cpp | 89 +- src/Backups/BackupEntriesCollector.h | 20 +- src/Backups/BackupEntryFromAppendOnlyFile.cpp | 22 +- src/Backups/BackupEntryFromAppendOnlyFile.h | 10 +- src/Backups/BackupEntryFromImmutableFile.cpp | 77 +- src/Backups/BackupEntryFromImmutableFile.h | 25 +- src/Backups/BackupEntryFromMemory.h | 4 +- src/Backups/BackupEntryFromSmallFile.h | 4 +- .../BackupEntryWithChecksumCalculation.cpp | 351 +- .../BackupEntryWithChecksumCalculation.h | 77 +- src/Backups/BackupEntryWrappedWith.h | 2 +- src/Backups/BackupFactory.cpp | 21 + src/Backups/BackupFactory.h | 3 + src/Backups/BackupFileInfo.cpp | 20 +- src/Backups/BackupFileInfo.h | 3 + src/Backups/BackupIO.h | 8 +- src/Backups/BackupIO_AzureBlobStorage.cpp | 11 +- src/Backups/BackupIO_AzureBlobStorage.h | 6 +- src/Backups/BackupIO_Default.cpp | 15 +- src/Backups/BackupIO_Default.h | 3 + src/Backups/BackupIO_Disk.cpp | 34 +- src/Backups/BackupIO_Disk.h | 5 +- src/Backups/BackupIO_File.cpp | 32 +- src/Backups/BackupIO_File.h | 5 +- src/Backups/BackupIO_Memory.cpp | 118 + src/Backups/BackupIO_Memory.h | 47 + src/Backups/BackupIO_Null.cpp | 107 + src/Backups/BackupIO_Null.h | 34 + src/Backups/BackupIO_S3.cpp | 191 +- src/Backups/BackupIO_S3.h | 7 +- src/Backups/BackupImpl.cpp | 266 +- src/Backups/BackupImpl.h | 53 +- src/Backups/BackupInMemory.cpp | 130 + src/Backups/BackupInMemory.h | 43 + src/Backups/BackupInfo.cpp | 9 +- src/Backups/BackupKeeperSettings.cpp | 58 + src/Backups/BackupKeeperSettings.h | 64 + src/Backups/BackupOperationInfo.h | 4 +- src/Backups/BackupSettings.cpp | 47 +- src/Backups/BackupSettings.h | 18 + src/Backups/BackupStatus.h | 1 + src/Backups/BackupUtils.cpp | 3 +- src/Backups/BackupsInMemoryHolder.cpp | 52 + src/Backups/BackupsInMemoryHolder.h | 29 + src/Backups/BackupsWorker.cpp | 1000 +- src/Backups/BackupsWorker.h | 51 +- src/Backups/DDLAdjustingForBackupVisitor.cpp | 11 +- src/Backups/IBackup.h | 24 +- src/Backups/IBackupCoordination.h | 35 +- src/Backups/IBackupEntriesLazyBatch.cpp | 4 +- src/Backups/IBackupEntry.h | 8 +- src/Backups/IRestoreCoordination.h | 35 +- src/Backups/RestoreCoordinationLocal.cpp | 37 +- src/Backups/RestoreCoordinationLocal.h | 27 +- src/Backups/RestoreCoordinationOnCluster.cpp | 306 + ...emote.h => RestoreCoordinationOnCluster.h} | 56 +- src/Backups/RestoreCoordinationRemote.cpp | 379 - src/Backups/RestoreSettings.cpp | 38 +- src/Backups/RestoreSettings.h | 26 +- src/Backups/RestorerFromBackup.cpp | 392 +- src/Backups/RestorerFromBackup.h | 32 +- src/Backups/SettingsFieldOptionalString.cpp | 3 +- src/Backups/SettingsFieldOptionalUInt64.cpp | 40 + src/Backups/SettingsFieldOptionalUInt64.h | 20 + src/Backups/SettingsFieldOptionalUUID.cpp | 2 +- src/Backups/WithRetries.cpp | 48 +- src/Backups/WithRetries.h | 32 +- .../registerBackupEngineAzureBlobStorage.cpp | 57 +- src/Backups/registerBackupEngineS3.cpp | 70 +- .../registerBackupEnginesFileAndDisk.cpp | 35 +- src/Backups/tests/gtest_backup_entries.cpp | 27 +- src/Bridge/CMakeLists.txt | 5 - src/Bridge/IBridge.cpp | 271 - src/Bridge/IBridge.h | 52 - .../CatBoostLibraryBridgeHelper.h | 2 + .../ExternalDictionaryLibraryBridgeHelper.cpp | 1 + .../ExternalDictionaryLibraryBridgeHelper.h | 3 +- src/BridgeHelper/IBridgeHelper.cpp | 22 + src/BridgeHelper/IBridgeHelper.h | 7 +- src/BridgeHelper/LibraryBridgeHelper.cpp | 15 +- src/BridgeHelper/README.md | 1 + src/BridgeHelper/XDBCBridgeHelper.h | 22 +- src/CMakeLists.txt | 91 +- src/Client/BuzzHouse/AST/SQLProtoStr.cpp | 4717 + src/Client/BuzzHouse/AST/SQLProtoStr.h | 24 + .../Generator/ExternalIntegrations.cpp | 1468 + .../Generator/ExternalIntegrations.h | 404 + src/Client/BuzzHouse/Generator/FuzzConfig.cpp | 415 + src/Client/BuzzHouse/Generator/FuzzConfig.h | 177 + .../BuzzHouse/Generator/QueryOracle.cpp | 834 + src/Client/BuzzHouse/Generator/QueryOracle.h | 71 + .../BuzzHouse/Generator/RandomGenerator.cpp | 278 + .../BuzzHouse/Generator/RandomGenerator.h | 248 + .../BuzzHouse/Generator/RandomSettings.h | 174 + src/Client/BuzzHouse/Generator/SQLCatalog.h | 399 + .../BuzzHouse/Generator/SQLExpression.cpp | 1114 + src/Client/BuzzHouse/Generator/SQLFuncs.h | 1298 + src/Client/BuzzHouse/Generator/SQLQuery.cpp | 1962 + src/Client/BuzzHouse/Generator/SQLTable.cpp | 2094 + src/Client/BuzzHouse/Generator/SQLTypes.cpp | 2094 + src/Client/BuzzHouse/Generator/SQLTypes.h | 605 + .../BuzzHouse/Generator/ServerSettings.cpp | 1001 + .../Generator/StatementGenerator.cpp | 4405 + .../BuzzHouse/Generator/StatementGenerator.h | 474 + .../BuzzHouse/Generator/TableSetttings.cpp | 321 + src/Client/BuzzHouse/Proto/CMakeLists.txt | 11 + src/Client/BuzzHouse/Proto/SQLGrammar.proto | 3849 + src/Client/BuzzHouse/Utils/HugeInt.cpp | 595 + src/Client/BuzzHouse/Utils/HugeInt.h | 75 + src/Client/BuzzHouse/Utils/MD5Impl.cpp | 32 + src/Client/BuzzHouse/Utils/MD5Impl.h | 24 + src/Client/BuzzHouse/Utils/Nlimits.h | 15 + src/Client/BuzzHouse/Utils/UHugeInt.cpp | 418 + src/Client/BuzzHouse/Utils/UHugeInt.h | 82 + src/Client/ClientApplicationBase.cpp | 225 +- src/Client/ClientApplicationBase.h | 7 + src/Client/ClientBase.cpp | 1506 +- src/Client/ClientBase.h | 147 +- src/Client/ClientBaseHelpers.cpp | 55 +- src/Client/ClientBaseHelpers.h | 6 +- src/Client/ClientBaseOptimizedParts.cpp | 70 +- src/Client/Connection.cpp | 392 +- src/Client/Connection.h | 48 +- src/Client/ConnectionEstablisher.cpp | 23 +- src/Client/ConnectionEstablisher.h | 3 + src/Client/ConnectionParameters.cpp | 56 +- src/Client/ConnectionParameters.h | 15 +- src/Client/ConnectionPool.cpp | 15 +- src/Client/ConnectionPool.h | 24 +- src/Client/ConnectionPoolWithFailover.cpp | 50 +- src/Client/ConnectionPoolWithFailover.h | 9 +- src/Client/ConnectionString.cpp | 6 +- src/Client/HedgedConnections.cpp | 82 +- src/Client/HedgedConnections.h | 7 +- src/Client/HedgedConnectionsFactory.cpp | 4 +- src/Client/IConnections.h | 7 +- src/Client/IServerConnection.h | 18 +- src/Client/InternalTextLogs.cpp | 5 + src/Client/InternalTextLogs.h | 11 +- src/Client/LineReader.cpp | 17 +- src/Client/LocalConnection.cpp | 166 +- src/Client/LocalConnection.h | 26 +- src/Client/MultiplexedConnections.cpp | 79 +- src/Client/MultiplexedConnections.h | 7 +- src/Client/PacketReceiver.h | 2 - src/Client/ProgressTable.cpp | 468 + src/Client/ProgressTable.h | 126 + src/Client/ReplxxLineReader.cpp | 109 +- src/Client/ReplxxLineReader.h | 37 +- src/Client/Suggest.cpp | 13 +- src/Client/TerminalKeystrokeInterceptor.cpp | 143 + src/Client/TerminalKeystrokeInterceptor.h | 47 + src/Client/TestHint.cpp | 18 +- src/Client/TestHint.h | 16 +- src/Client/examples/test_connect.cpp | 2 - src/Columns/Collator.cpp | 31 +- src/Columns/Collator.h | 8 +- src/Columns/ColumnAggregateFunction.cpp | 64 +- src/Columns/ColumnAggregateFunction.h | 2 + src/Columns/ColumnArray.cpp | 65 +- src/Columns/ColumnArray.h | 34 +- src/Columns/ColumnCompressed.cpp | 5 +- src/Columns/ColumnCompressed.h | 7 +- src/Columns/ColumnConst.cpp | 12 +- src/Columns/ColumnConst.h | 38 +- src/Columns/ColumnDecimal.cpp | 27 +- src/Columns/ColumnDecimal.h | 14 +- src/Columns/ColumnDynamic.cpp | 220 +- src/Columns/ColumnDynamic.h | 35 +- src/Columns/ColumnFixedString.cpp | 10 +- src/Columns/ColumnFixedString.h | 14 +- src/Columns/ColumnFunction.cpp | 44 +- src/Columns/ColumnFunction.h | 20 +- src/Columns/ColumnLazy.cpp | 423 + src/Columns/ColumnLazy.h | 155 + src/Columns/ColumnLowCardinality.cpp | 103 +- src/Columns/ColumnLowCardinality.h | 48 +- src/Columns/ColumnMap.cpp | 103 +- src/Columns/ColumnMap.h | 37 +- src/Columns/ColumnNullable.cpp | 61 +- src/Columns/ColumnNullable.h | 27 +- src/Columns/ColumnObject.cpp | 765 +- src/Columns/ColumnObject.h | 61 +- src/Columns/ColumnObjectDeprecated.cpp | 91 +- src/Columns/ColumnObjectDeprecated.h | 8 +- src/Columns/ColumnSet.h | 1 + src/Columns/ColumnSparse.cpp | 69 +- src/Columns/ColumnSparse.h | 13 +- src/Columns/ColumnString.cpp | 90 +- src/Columns/ColumnString.h | 41 +- src/Columns/ColumnStringHelpers.h | 9 +- src/Columns/ColumnTuple.cpp | 123 +- src/Columns/ColumnTuple.h | 15 +- src/Columns/ColumnUnique.cpp | 44 + src/Columns/ColumnUnique.h | 55 +- src/Columns/ColumnVariant.cpp | 161 +- src/Columns/ColumnVariant.h | 23 +- src/Columns/ColumnVector.cpp | 289 +- src/Columns/ColumnVector.h | 156 +- src/Columns/ColumnsCommon.cpp | 1 + src/Columns/ColumnsCommon.h | 17 +- src/Columns/ColumnsNumber.h | 1 + src/Columns/FilterDescription.cpp | 22 + src/Columns/FilterDescription.h | 24 +- src/Columns/IColumn.cpp | 242 +- src/Columns/IColumn.h | 235 +- src/Columns/IColumnDummy.cpp | 5 + src/Columns/IColumnDummy.h | 1 + src/Columns/IColumnUnique.h | 4 +- src/Columns/IColumn_fwd.h | 30 + src/Columns/MaskOperations.cpp | 6 +- src/Columns/benchmarks/CMakeLists.txt | 2 +- .../tests/gtest_column_dump_structure.cpp | 3 +- src/Columns/tests/gtest_column_dynamic.cpp | 87 + src/Columns/tests/gtest_column_object.cpp | 63 + src/Columns/tests/gtest_column_string.cpp | 88 + src/Columns/tests/gtest_column_vector.cpp | 1 + src/Columns/tests/gtest_low_cardinality.cpp | 1 + src/Columns/tests/gtest_weak_hash_32.cpp | 35 +- src/Common/ActionBlocker.h | 4 +- src/Common/Allocator.cpp | 18 +- src/Common/ArenaAllocator.h | 12 +- src/Common/ArrayCache.h | 728 - src/Common/AsyncLoader.cpp | 84 +- src/Common/AsyncLoader.h | 14 +- src/Common/AsyncTaskExecutor.cpp | 1 + src/Common/AsyncTaskExecutor.h | 3 + src/Common/AsynchronousMetrics.cpp | 481 +- src/Common/AsynchronousMetrics.h | 35 +- src/Common/BSONCXXHelper.h | 418 + src/Common/Base32.h | 155 + src/Common/Base58.cpp | 2 +- src/Common/BufferAllocationPolicy.cpp | 3 +- src/Common/COW.h | 13 +- src/Common/CPUID.h | 6 + src/Common/CacheBase.h | 6 + src/Common/CalendarTimeInterval.cpp | 12 +- src/Common/CalendarTimeInterval.h | 4 +- src/Common/CancelToken.cpp | 12 +- src/Common/CancelToken.h | 2 - src/Common/CaresPTRResolver.cpp | 4 +- src/Common/CgroupsMemoryUsageObserver.cpp | 283 +- src/Common/CgroupsMemoryUsageObserver.h | 50 +- src/Common/ColumnsHashing.cpp | 10 + src/Common/ColumnsHashing.h | 391 +- src/Common/ColumnsHashing/HashMethod.h | 433 + src/Common/ColumnsHashingImpl.h | 16 +- src/Common/CombinedCardinalityEstimator.h | 49 +- src/Common/CompactArray.h | 14 +- src/Common/ConcurrencyControl.cpp | 371 +- src/Common/ConcurrencyControl.h | 186 +- .../AbstractConfigurationComparison.cpp | 29 +- src/Common/Config/CMakeLists.txt | 10 +- src/Common/Config/ConfigHelper.cpp | 5 + src/Common/Config/ConfigHelper.h | 2 + src/Common/Config/ConfigProcessor.cpp | 261 +- src/Common/Config/ConfigProcessor.h | 61 +- src/Common/Config/ConfigReloader.cpp | 45 +- src/Common/Config/YAMLParser.cpp | 22 +- src/Common/Config/YAMLParser.h | 14 +- src/Common/CopyableAtomic.h | 2 + src/Common/CounterInFile.h | 13 +- src/Common/Crypto/KeyPair.cpp | 139 + src/Common/Crypto/KeyPair.h | 48 + src/Common/Crypto/OpenSSLInitializer.cpp | 110 + src/Common/Crypto/OpenSSLInitializer.h | 40 + src/Common/Crypto/X509Certificate.cpp | 312 + src/Common/Crypto/X509Certificate.h | 94 + src/Common/CurrentMemoryTracker.cpp | 47 +- src/Common/CurrentMetrics.cpp | 101 +- src/Common/CurrentMetrics.h | 3 +- src/Common/CurrentThread.cpp | 50 + src/Common/CurrentThread.h | 44 +- src/Common/DNSResolver.cpp | 97 +- src/Common/DNSResolver.h | 9 + src/Common/DateLUT.cpp | 38 +- src/Common/DateLUT.h | 13 +- src/Common/DateLUTImpl.cpp | 10 +- src/Common/DateLUTImpl.h | 58 +- src/Common/Dwarf.cpp | 100 +- src/Common/Dwarf.h | 2 +- src/Common/Elf.cpp | 3 +- src/Common/EnvironmentChecks.cpp | 1 + src/Common/Epoll.cpp | 8 +- src/Common/ErrorCodes.cpp | 69 +- src/Common/ErrorCodes.h | 18 +- src/Common/EventFD.cpp | 2 +- src/Common/EventFD.h | 1 - src/Common/Exception.cpp | 71 +- src/Common/Exception.h | 76 +- src/Common/ExponentiallySmoothedCounter.h | 2 +- src/Common/FailPoint.cpp | 43 +- src/Common/FailPoint.h | 2 + src/Common/FiberStack.cpp | 102 + src/Common/FiberStack.h | 76 +- ...mparison.h => FieldAccurateComparison.cpp} | 17 +- src/Common/FieldAccurateComparison.h | 11 + src/Common/FieldVisitorConvertToNumber.cpp | 2 +- src/Common/FieldVisitorConvertToNumber.h | 8 +- src/Common/FieldVisitorScale.cpp | 30 + src/Common/FieldVisitorScale.h | 43 + src/Common/FieldVisitorToJSONElement.cpp | 127 + src/Common/FieldVisitorToJSONElement.h | 37 + src/Common/FieldVisitorToString.cpp | 31 +- src/Common/FieldVisitorToString.h | 7 + src/Common/FileChecker.cpp | 4 +- src/Common/FileRenamer.cpp | 3 +- src/Common/FloatUtils.h | 48 + src/Common/FunctionDocumentation.cpp | 79 +- src/Common/FunctionDocumentation.h | 79 +- src/Common/GWPAsan.cpp | 9 +- src/Common/GWPAsan.h | 10 - src/Common/HTTPConnectionPool.cpp | 101 +- src/Common/HTTPConnectionPool.h | 2 +- src/Common/HashTable/FixedHashSet.h | 7 - src/Common/HashTable/Hash.h | 64 +- src/Common/HashTable/HashMap.h | 4 - src/Common/HashTable/HashSet.h | 23 - src/Common/HashTable/HashTable.h | 151 +- src/Common/HashTable/HashTableKeyHolder.h | 2 +- src/Common/HashTable/HashTableTraits.h | 41 + src/Common/HashTable/LRUHashMap.h | 289 - src/Common/HashTable/StringHashTable.h | 33 +- src/Common/HashTable/TwoLevelHashTable.h | 21 +- src/Common/HilbertUtils.h | 17 +- src/Common/Histogram.cpp | 97 + src/Common/Histogram.h | 84 + src/Common/HostResolvePool.cpp | 19 +- src/Common/HostResolvePool.h | 4 +- src/Common/HyperLogLogBiasEstimator.h | 28 +- src/Common/HyperLogLogCounter.h | 36 +- .../HyperLogLogWithSmallSetOptimization.h | 18 - src/Common/ICachePolicy.h | 6 +- src/Common/IFactoryWithAliases.h | 9 +- src/Common/IPv6ToBinary.cpp | 3 +- src/Common/ISlotControl.h | 39 +- src/Common/InterruptListener.h | 3 +- src/Common/IntervalTree.h | 2 +- src/Common/JSONBuilder.cpp | 27 +- src/Common/JSONBuilder.h | 13 +- src/Common/JSONParsers/DummyJSONParser.cpp | 16 + src/Common/JSONParsers/DummyJSONParser.h | 7 +- src/Common/JSONParsers/SimdJSONParser.cpp | 22 + src/Common/JSONParsers/SimdJSONParser.h | 14 +- src/Common/Jemalloc.cpp | 16 - src/Common/Jemalloc.h | 55 + src/Common/LRUCachePolicy.h | 5 + src/Common/LRUResourceCache.h | 12 +- src/Common/LatencyBuckets.cpp | 103 + src/Common/LatencyBuckets.h | 53 + src/Common/LibSSHInitializer.cpp | 61 + src/Common/LibSSHInitializer.h | 20 + src/Common/LibSSHLogger.cpp | 78 + src/Common/LibSSHLogger.h | 8 + src/Common/LocalDate.cpp | 61 + src/Common/LocalDate.h | 65 +- src/Common/LocalDateTime.cpp | 21 + src/Common/LocalDateTime.h | 25 +- src/Common/LoggingFormatStringHelpers.cpp | 46 +- src/Common/LoggingFormatStringHelpers.h | 43 +- src/Common/Macros.cpp | 11 +- src/Common/Macros.h | 12 +- src/Common/MatchGenerator.cpp | 4 +- src/Common/MatchGenerator.h | 1 + src/Common/MemorySpillScheduler.cpp | 83 + src/Common/MemorySpillScheduler.h | 46 + src/Common/MemoryTracker.cpp | 255 +- src/Common/MemoryTracker.h | 47 +- src/Common/MemoryTrackerBlockerInThread.cpp | 32 +- src/Common/MemoryTrackerBlockerInThread.h | 24 +- .../MemoryTrackerDebugBlockerInThread.cpp | 23 + .../MemoryTrackerDebugBlockerInThread.h | 22 + src/Common/MemoryTrackerUtils.cpp | 57 + src/Common/MemoryTrackerUtils.h | 12 + src/Common/MemoryWorker.cpp | 383 + src/Common/MemoryWorker.h | 90 + src/Common/NaNUtils.h | 21 +- src/Common/NamePrompter.h | 2 +- .../NamedCollectionConfiguration.cpp | 6 +- .../NamedCollections/NamedCollections.cpp | 154 +- .../NamedCollections/NamedCollections.h | 73 +- .../NamedCollectionsFactory.cpp | 52 +- .../NamedCollectionsFactory.h | 5 +- .../NamedCollectionsMetadataStorage.cpp | 295 +- .../NamedCollectionsMetadataStorage.h | 7 +- src/Common/NetlinkMetricsProvider.cpp | 7 +- src/Common/ObjectStorageKeyGenerator.cpp | 6 + src/Common/ObjectStorageKeyGenerator.h | 3 + src/Common/OpenSSLHelpers.cpp | 98 +- src/Common/OpenSSLHelpers.h | 7 + src/Common/OpenTelemetryTraceContext.cpp | 18 +- src/Common/OpenTelemetryTraceContext.h | 39 +- src/Common/OpenTelemetryTracingContext.h | 46 + src/Common/OptimizedRegularExpression.cpp | 197 +- src/Common/OvercommitTracker.cpp | 26 +- src/Common/OvercommitTracker.h | 11 +- src/Common/PODArray.cpp | 2 +- src/Common/PODArray.h | 20 +- src/Common/PageCache.cpp | 749 +- src/Common/PageCache.h | 350 +- src/Common/PipeFDs.cpp | 1 + src/Common/PoolBase.h | 12 +- src/Common/PoolWithFailoverBase.h | 45 +- src/Common/Priority.h | 5 +- src/Common/ProcfsMetricsProvider.cpp | 2 +- src/Common/ProfileEvents.cpp | 1893 +- src/Common/ProfileEvents.h | 35 +- src/Common/ProfilingScopedRWLock.h | 2 + src/Common/ProgressIndication.cpp | 32 +- src/Common/ProgressIndication.h | 10 +- src/Common/ProxyConfiguration.h | 2 +- .../ProxyConfigurationResolverProvider.cpp | 54 +- .../ProxyConfigurationResolverProvider.h | 10 +- src/Common/QueryFuzzer.cpp | 1362 +- src/Common/QueryFuzzer.h | 157 +- src/Common/QueryProfiler.cpp | 25 +- src/Common/QueryProfiler.h | 12 +- src/Common/RadixSort.h | 16 +- .../RemoteProxyConfigurationResolver.cpp | 11 +- src/Common/ReplicasReconnector.cpp | 90 + src/Common/ReplicasReconnector.h | 44 + src/Common/SLRUCachePolicy.h | 5 + src/Common/SSHWrapper.cpp | 13 +- src/Common/SSHWrapper.h | 7 +- src/Common/Scheduler/CPUSlotsAllocation.cpp | 227 + src/Common/Scheduler/CPUSlotsAllocation.h | 108 + src/Common/Scheduler/IResourceManager.h | 20 +- src/Common/Scheduler/ISchedulerConstraint.h | 29 +- src/Common/Scheduler/ISchedulerNode.h | 89 +- src/Common/Scheduler/ISchedulerQueue.h | 28 +- .../Scheduler/Nodes/ClassifiersConfig.cpp | 9 +- .../Scheduler/Nodes/ClassifiersConfig.h | 1 + ...eManager.cpp => CustomResourceManager.cpp} | 75 +- ...ourceManager.h => CustomResourceManager.h} | 17 +- src/Common/Scheduler/Nodes/FairPolicy.h | 24 +- src/Common/Scheduler/Nodes/FifoQueue.h | 37 +- src/Common/Scheduler/Nodes/PriorityPolicy.h | 24 +- .../Scheduler/Nodes/SemaphoreConstraint.h | 88 +- .../Scheduler/Nodes/ThrottlerConstraint.h | 75 +- .../Scheduler/Nodes/UnifiedSchedulerNode.h | 611 + .../Nodes/WorkloadResourceManager.cpp | 556 + .../Scheduler/Nodes/WorkloadResourceManager.h | 282 + .../Nodes/registerResourceManagers.cpp | 15 - .../Scheduler/Nodes/tests/ResourceTest.h | 232 +- ....cpp => gtest_custom_resource_manager.cpp} | 25 +- .../Nodes/tests/gtest_event_queue.cpp | 6 + .../Nodes/tests/gtest_resource_class_fair.cpp | 15 +- .../tests/gtest_resource_class_priority.cpp | 13 +- .../Nodes/tests/gtest_resource_scheduler.cpp | 84 +- .../tests/gtest_throttler_constraint.cpp | 34 +- .../tests/gtest_unified_scheduler_node.cpp | 748 + .../tests/gtest_workload_resource_manager.cpp | 834 + src/Common/Scheduler/ResouceLink.cpp | 25 - src/Common/Scheduler/ResourceGuard.h | 135 +- src/Common/Scheduler/ResourceLink.h | 21 +- src/Common/Scheduler/ResourceManagerFactory.h | 55 - src/Common/Scheduler/ResourceRequest.cpp | 25 +- src/Common/Scheduler/ResourceRequest.h | 29 +- src/Common/Scheduler/SchedulerRoot.h | 64 +- .../Workload/IWorkloadEntityStorage.h | 91 + .../Workload/WorkloadEntityDiskStorage.cpp | 287 + .../Workload/WorkloadEntityDiskStorage.h | 44 + .../Workload/WorkloadEntityKeeperStorage.cpp | 272 + .../Workload/WorkloadEntityKeeperStorage.h | 71 + .../Workload/WorkloadEntityStorageBase.cpp | 800 + .../Workload/WorkloadEntityStorageBase.h | 128 + .../Workload/createWorkloadEntityStorage.cpp | 45 + .../Workload/createWorkloadEntityStorage.h | 11 + src/Common/Scheduler/WorkloadSettings.cpp | 210 + src/Common/Scheduler/WorkloadSettings.h | 56 + .../Scheduler/createResourceManager.cpp | 114 + src/Common/Scheduler/createResourceManager.h | 11 + src/Common/SensitiveDataMasker.cpp | 30 +- src/Common/SensitiveDataMasker.h | 9 +- src/Common/SettingsChanges.h | 1 + src/Common/SharedLockGuard.h | 2 +- src/Common/SharedMutex.cpp | 4 +- src/Common/SharedMutex.h | 30 +- src/Common/SharedMutexHelper.h | 5 + src/Common/ShellCommand.cpp | 79 +- src/Common/ShellCommand.h | 25 + src/Common/ShellCommandSettings.cpp | 4 +- src/Common/ShellCommandsHolder.cpp | 53 + src/Common/ShellCommandsHolder.h | 32 + src/Common/SignalHandlers.cpp | 99 +- src/Common/SignalHandlers.h | 8 +- src/Common/SipHash.cpp | 27 + src/Common/SipHash.h | 61 +- src/Common/SpaceSaving.h | 4 - src/Common/StackTrace.cpp | 65 +- src/Common/StackTrace.h | 5 +- src/Common/StatusFile.cpp | 11 +- src/Common/StopToken.cpp | 97 + src/Common/StopToken.h | 71 + src/Common/StringSearcher.h | 6 +- src/Common/StringUtils.cpp | 6 +- src/Common/StudentTTest.cpp | 8 +- src/Common/SymbolIndex.cpp | 32 +- src/Common/SymbolIndex.h | 1 - src/Common/SystemLogBase.cpp | 38 +- src/Common/SystemLogBase.h | 11 +- src/Common/TTLCachePolicy.h | 5 + src/Common/TargetSpecific.cpp | 5 +- src/Common/TargetSpecific.h | 26 +- src/Common/TerminalSize.cpp | 9 +- src/Common/TerminalSize.h | 1 + src/Common/ThreadFuzzer.cpp | 4 +- src/Common/ThreadPool.cpp | 482 +- src/Common/ThreadPool.h | 99 +- src/Common/ThreadPoolTaskTracker.cpp | 6 +- src/Common/ThreadPoolTaskTracker.h | 12 +- src/Common/ThreadProfileEvents.cpp | 2 +- src/Common/ThreadStatus.cpp | 72 +- src/Common/ThreadStatus.h | 37 +- src/Common/Throttler.cpp | 34 +- src/Common/Throttler.h | 11 +- src/Common/TimerDescriptor.cpp | 2 +- src/Common/TraceSender.cpp | 1 + src/Common/TypePromotion.h | 5 + src/Common/UTF8Helpers.cpp | 73 +- src/Common/UniqueLock.h | 63 + src/Common/VersionNumber.cpp | 6 +- src/Common/VersionNumber.h | 5 +- src/Common/Visitor.h | 2 +- src/Common/Volnitsky.h | 2 + src/Common/WeightedRandomSampling.h | 57 + src/Common/ZooKeeper/CMakeLists.txt | 16 +- src/Common/ZooKeeper/IKeeper.cpp | 60 +- src/Common/ZooKeeper/IKeeper.h | 105 +- src/Common/ZooKeeper/KeeperException.cpp | 51 + src/Common/ZooKeeper/KeeperException.h | 64 + .../ZooKeeper}/KeeperFeatureFlags.cpp | 2 +- .../ZooKeeper}/KeeperFeatureFlags.h | 10 + src/Common/ZooKeeper/ShuffleHost.h | 47 + src/Common/ZooKeeper/TestKeeper.cpp | 173 +- src/Common/ZooKeeper/TestKeeper.h | 11 +- src/Common/ZooKeeper/Types.h | 1 + src/Common/ZooKeeper/ZooKeeper.cpp | 256 +- src/Common/ZooKeeper/ZooKeeper.h | 76 +- src/Common/ZooKeeper/ZooKeeperArgs.cpp | 19 + src/Common/ZooKeeper/ZooKeeperArgs.h | 3 + src/Common/ZooKeeper/ZooKeeperCachingGetter.h | 1 + src/Common/ZooKeeper/ZooKeeperCommon.cpp | 362 +- src/Common/ZooKeeper/ZooKeeperCommon.h | 110 +- src/Common/ZooKeeper/ZooKeeperConstants.cpp | 3 +- src/Common/ZooKeeper/ZooKeeperConstants.h | 19 +- src/Common/ZooKeeper/ZooKeeperIO.cpp | 26 + src/Common/ZooKeeper/ZooKeeperIO.h | 32 +- src/Common/ZooKeeper/ZooKeeperImpl.cpp | 193 +- src/Common/ZooKeeper/ZooKeeperImpl.h | 17 +- src/Common/ZooKeeper/ZooKeeperLock.cpp | 2 +- src/Common/ZooKeeper/ZooKeeperNodeCache.cpp | 5 + src/Common/ZooKeeper/ZooKeeperNodeCache.h | 2 + src/Common/ZooKeeper/ZooKeeperRetries.h | 30 +- .../ZooKeeper/ZooKeeperWithFaultInjection.h | 1 - src/Common/ZooKeeper/examples/CMakeLists.txt | 9 +- .../ZooKeeper/examples/zkutil_test_async.cpp | 3 + .../examples/zkutil_test_commands_new_lib.cpp | 1 + .../assertProcessUserMatchesDataOwner.cpp | 21 +- .../assertProcessUserMatchesDataOwner.h | 4 +- src/Common/assert_cast.h | 3 +- src/Common/atomicRename.cpp | 36 +- src/Common/atomicRename.h | 2 +- src/Common/benchmarks/CMakeLists.txt | 12 +- .../benchmarks/orc_string_dictionary.cpp | 316 + src/Common/benchmarks/wrap_in_nullable.cpp | 60 + src/Common/checkStackSize.cpp | 4 +- src/Common/clibssh.h | 13 + src/Common/computeMaxTableNameLength.cpp | 40 + src/Common/computeMaxTableNameLength.h | 10 + src/Common/config.h.in | 12 +- src/Common/config_version.cpp.in | 2 +- src/Common/examples/CMakeLists.txt | 31 +- src/Common/examples/arena_with_free_lists.cpp | 340 - src/Common/examples/array_cache.cpp | 138 - src/Common/examples/check_pointer_valid.cpp | 8 +- src/Common/examples/encrypt_decrypt.cpp | 52 +- src/Common/examples/executable_udf.cpp | 2 + src/Common/examples/int_hashes_perf.cpp | 3 +- .../integer_hash_tables_benchmark.cpp | 2 +- src/Common/examples/lru_hash_map_perf.cpp | 182 - src/Common/examples/shell_command_inout.cpp | 3 + src/Common/examples/sip_hash_perf.cpp | 1 + src/Common/examples/space_saving.cpp | 121 - src/Common/examples/utf8_upper_lower.cpp | 27 + src/Common/exp10_i32.h | 16 + src/Common/filesystemHelpers.cpp | 2 +- src/Common/findExtreme.cpp | 114 +- src/Common/findExtreme.h | 31 +- src/Common/formatIPv6.cpp | 9 +- src/Common/formatIPv6.h | 19 +- src/Common/formatReadable.cpp | 13 + src/Common/formatReadable.h | 3 + ...Cores.cpp => getNumberOfCPUCoresToUse.cpp} | 17 +- src/Common/getNumberOfCPUCoresToUse.h | 6 + src/Common/getNumberOfPhysicalCPUCores.h | 5 - src/Common/intExp.cpp | 154 + src/Common/intExp.h | 210 +- src/Common/intExp10.h | 35 + src/Common/intExp2.h | 15 + src/Common/iota.cpp | 4 +- src/Common/isLocalAddress.cpp | 2 +- src/Common/levenshteinDistance.cpp | 2 +- src/Common/levenshteinDistance.h | 102 +- src/Common/likePatternToRegexp.cpp | 2 +- src/Common/logger_useful.h | 31 +- src/Common/maskURIPassword.h | 14 + src/Common/memcpySmall.h | 2 +- src/Common/memory.cpp | 62 + src/Common/memory.h | 47 +- src/Common/mysqlxx/Connection.cpp | 2 +- src/Common/mysqlxx/Pool.cpp | 71 +- src/Common/mysqlxx/PoolFactory.cpp | 37 +- src/Common/mysqlxx/PoolWithFailover.cpp | 71 +- src/Common/mysqlxx/Value.cpp | 55 + src/Common/mysqlxx/mysqlxx/Connection.h | 2 + src/Common/mysqlxx/mysqlxx/Pool.h | 27 +- src/Common/mysqlxx/mysqlxx/PoolFactory.h | 2 + src/Common/mysqlxx/mysqlxx/PoolWithFailover.h | 10 +- src/Common/mysqlxx/mysqlxx/Value.h | 77 +- src/Common/mysqlxx/tests/CMakeLists.txt | 2 +- .../mysqlxx/tests/mysqlxx_pool_test.cpp | 2 +- src/Common/parseAddress.cpp | 10 +- src/Common/parseGlobs.cpp | 79 +- src/Common/parseGlobs.h | 4 + src/Common/parseRemoteDescription.cpp | 40 +- .../proxyConfigurationToPocoProxyConfig.cpp | 12 +- src/Common/quoteString.cpp | 8 + src/Common/quoteString.h | 13 +- src/Common/remapExecutable.cpp | 4 +- src/Common/setThreadName.cpp | 2 +- src/Common/tests/gtest_action_blocker.cpp | 128 + src/Common/tests/gtest_async_loader.cpp | 52 +- src/Common/tests/gtest_base32.cpp | 212 + src/Common/tests/gtest_cgroups_reader.cpp | 22 +- .../tests/gtest_concurrency_control.cpp | 251 +- .../tests/gtest_config_host_validation.cpp | 69 + src/Common/tests/gtest_config_processor.cpp | 1 + src/Common/tests/gtest_connection_pool.cpp | 57 +- src/Common/tests/gtest_event_notifier.cpp | 2 - src/Common/tests/gtest_find_symbols.cpp | 12 +- src/Common/tests/gtest_fst.cpp | 26 +- src/Common/tests/gtest_global_context.h | 7 + src/Common/tests/gtest_helper_functions.h | 1 + src/Common/tests/gtest_histogram_metrics.cpp | 37 + src/Common/tests/gtest_log.cpp | 1 + src/Common/tests/gtest_lru_hash_map.cpp | 160 - .../gtest_makeRegexpPatternFromGlobs.cpp | 4 +- src/Common/tests/gtest_merge_configs.cpp | 4 + src/Common/tests/gtest_optimize_re.cpp | 3 + src/Common/tests/gtest_poco_json.cpp | 50 + .../tests/gtest_sensitive_data_masker.cpp | 19 +- .../tests/gtest_tmp_replace_table_name.cpp | 41 + .../tests/gtest_weighted_random_sampling.cpp | 116 + src/Common/tests/gtest_wide_integer.cpp | 63 +- src/Common/threadPoolCallbackRunner.h | 117 +- src/Common/transformEndianness.h | 3 +- src/Common/typeid_cast.h | 8 +- src/Common/waitForPid.cpp | 4 +- src/Compression/CMakeLists.txt | 11 - src/Compression/CompressedReadBufferBase.cpp | 62 +- src/Compression/CompressedReadBufferBase.h | 5 + .../CompressedReadBufferFromFile.cpp | 235 +- .../CompressedReadBufferFromFile.h | 8 +- src/Compression/CompressedWriteBuffer.cpp | 32 +- src/Compression/CompressedWriteBuffer.h | 25 +- .../CompressionCodecDeflateQpl.cpp | 48 +- src/Compression/CompressionCodecDeflateQpl.h | 2 + src/Compression/CompressionCodecDelta.cpp | 7 +- .../CompressionCodecDoubleDelta.cpp | 24 +- src/Compression/CompressionCodecEncrypted.cpp | 235 +- src/Compression/CompressionCodecFPC.cpp | 13 +- src/Compression/CompressionCodecGCD.cpp | 13 +- src/Compression/CompressionCodecGorilla.cpp | 10 +- src/Compression/CompressionCodecMultiple.cpp | 3 + src/Compression/CompressionCodecNone.cpp | 2 +- src/Compression/CompressionCodecNone.h | 2 +- src/Compression/CompressionCodecT64.cpp | 10 +- src/Compression/CompressionCodecZSTDQAT.cpp | 2 +- src/Compression/CompressionFactory.cpp | 16 +- src/Compression/CompressionFactory.h | 11 +- .../CompressionFactoryAdditions.cpp | 12 +- src/Compression/CompressionInfo.h | 4 +- src/Compression/ICompressionCodec.cpp | 17 +- src/Compression/ICompressionCodec.h | 4 +- src/Compression/LZ4_decompress_faster.cpp | 6 +- src/Compression/LZ4_decompress_faster.h | 11 +- .../ParallelCompressedWriteBuffer.cpp | 159 + .../ParallelCompressedWriteBuffer.h | 69 + src/Compression/examples/CMakeLists.txt | 5 +- src/Compression/examples/decompress_perf.cpp | 262 + src/Compression/fuzzers/CMakeLists.txt | 18 +- .../fuzzers/compressed_buffer_fuzzer.cpp | 28 +- .../fuzzers/delta_decompress_fuzzer.cpp | 30 +- .../double_delta_decompress_fuzzer.cpp | 28 + .../fuzzers/encrypted_decompress_fuzzer.cpp | 31 +- .../fuzzers/gcd_decompress_fuzzer.cpp | 30 +- .../fuzzers/lz4_decompress_fuzzer.cpp | 30 +- .../getCompressionCodecForFile.cpp | 31 +- src/Compression/getCompressionCodecForFile.h | 4 + .../tests/gtest_compressionCodec.cpp | 65 +- src/Coordination/ACLMap.cpp | 5 + src/Coordination/ACLMap.h | 2 + src/Coordination/Changelog.cpp | 218 +- src/Coordination/Changelog.h | 22 +- src/Coordination/CoordinationSettings.cpp | 174 +- src/Coordination/CoordinationSettings.h | 94 +- src/Coordination/FourLetterCommand.cpp | 51 +- src/Coordination/FourLetterCommand.h | 6 +- src/Coordination/Keeper4LWInfo.h | 5 - .../KeeperAsynchronousMetrics.cpp | 27 +- src/Coordination/KeeperAsynchronousMetrics.h | 8 +- src/Coordination/KeeperCommon.cpp | 12 +- src/Coordination/KeeperCommon.h | 54 +- src/Coordination/KeeperConstants.cpp | 1 + src/Coordination/KeeperConstants.h | 4 +- src/Coordination/KeeperContext.cpp | 91 +- src/Coordination/KeeperContext.h | 29 +- src/Coordination/KeeperDispatcher.cpp | 113 +- src/Coordination/KeeperDispatcher.h | 14 +- src/Coordination/KeeperReconfiguration.cpp | 4 +- src/Coordination/KeeperServer.cpp | 290 +- src/Coordination/KeeperServer.h | 8 +- src/Coordination/KeeperSnapshotManager.cpp | 134 +- src/Coordination/KeeperSnapshotManager.h | 6 +- src/Coordination/KeeperSnapshotManagerS3.cpp | 49 +- src/Coordination/KeeperStateMachine.cpp | 360 +- src/Coordination/KeeperStateMachine.h | 84 +- src/Coordination/KeeperStateManager.cpp | 128 +- src/Coordination/KeeperStateManager.h | 7 +- src/Coordination/KeeperStorage.cpp | 3786 +- src/Coordination/KeeperStorage.h | 751 +- src/Coordination/KeeperStorage_fwd.h | 25 + src/Coordination/LoggerWrapper.h | 2 +- src/Coordination/RaftServerConfig.cpp | 8 +- src/Coordination/RaftServerConfig.h | 6 +- src/Coordination/RocksDBContainer.h | 155 +- src/Coordination/SnapshotableHashTable.h | 63 +- .../WriteBufferFromNuraftBuffer.cpp | 2 + src/Coordination/ZooKeeperDataReader.cpp | 51 +- src/Coordination/ZooKeeperDataReader.h | 9 +- src/Coordination/tests/gtest_coordination.cpp | 3452 +- .../tests/gtest_coordination_changelog.cpp | 1528 + .../tests/gtest_coordination_common.cpp | 46 + .../tests/gtest_coordination_common.h | 157 + .../tests/gtest_coordination_snapshot.cpp | 538 + .../tests/gtest_coordination_storage.cpp | 963 + src/Core/AccurateComparison.h | 18 +- src/Core/BackgroundSchedulePool.cpp | 54 +- src/Core/BackgroundSchedulePool.h | 51 +- src/Core/BackgroundSchedulePoolTaskHolder.cpp | 37 + src/Core/BackgroundSchedulePoolTaskHolder.h | 33 + src/Core/BaseSettings.cpp | 13 +- src/Core/BaseSettings.h | 316 +- src/Core/BaseSettingsFwdMacros.h | 7 + src/Core/BaseSettingsFwdMacrosImpl.h | 11 + src/Core/BaseSettingsProgramOptions.h | 25 +- src/Core/Block.cpp | 186 +- src/Core/Block.h | 35 +- src/Core/BlockInfo.cpp | 32 - src/Core/BlockInfo.h | 30 - src/Core/BlockMissingValues.cpp | 53 + src/Core/BlockMissingValues.h | 44 + src/Core/BlockNameMap.cpp | 18 + src/Core/BlockNameMap.h | 16 + src/Core/Block_fwd.h | 19 + src/Core/ColumnWithTypeAndName.cpp | 7 +- src/Core/ColumnWithTypeAndName.h | 10 +- src/Core/ColumnsWithTypeAndName.h | 3 +- src/Core/CompareHelper.h | 24 +- src/Core/DecimalComparison.h | 150 +- src/Core/DecimalFunctions.h | 10 +- src/Core/Defines.h | 42 +- src/Core/DistributedCacheDefines.h | 34 + src/Core/ExternalTable.cpp | 39 +- src/Core/Field.cpp | 329 +- src/Core/Field.h | 278 +- src/Core/FormatFactorySettings.cpp | 16 + src/Core/FormatFactorySettings.h | 1337 + src/Core/Joins.cpp | 69 + src/Core/Joins.h | 36 +- src/Core/MergeSelectorAlgorithm.h | 14 + src/Core/MultiEnum.h | 2 +- src/Core/MySQL/Authentication.cpp | 64 +- src/Core/MySQL/Authentication.h | 8 +- src/Core/MySQL/IMySQLReadPacket.cpp | 5 +- src/Core/MySQL/IMySQLWritePacket.cpp | 11 +- src/Core/MySQL/IMySQLWritePacket.h | 1 + src/Core/MySQL/MySQLCharset.cpp | 368 - src/Core/MySQL/MySQLCharset.h | 29 - src/Core/MySQL/MySQLClient.cpp | 181 - src/Core/MySQL/MySQLClient.h | 85 - src/Core/MySQL/MySQLGtid.cpp | 233 - src/Core/MySQL/MySQLGtid.h | 72 - src/Core/MySQL/MySQLReplication.cpp | 1263 - src/Core/MySQL/MySQLReplication.h | 579 - src/Core/MySQL/MySQLUtils.cpp | 2 +- src/Core/MySQL/PacketEndpoint.h | 2 +- src/Core/MySQL/PacketsProtocolBinary.h | 2 +- src/Core/MySQL/PacketsProtocolText.h | 2 +- src/Core/MySQL/tests/gtest_MySQLGtid.cpp | 40 - src/Core/NamesAndTypes.cpp | 25 +- src/Core/NamesAndTypes.h | 31 +- src/Core/ParallelReplicasMode.h | 25 + src/Core/PostgreSQL/PoolWithFailover.cpp | 94 +- src/Core/PostgreSQL/PoolWithFailover.h | 15 +- src/Core/PostgreSQL/Utils.h | 1 - src/Core/PostgreSQL/insertPostgreSQLValue.cpp | 9 +- src/Core/PostgreSQL/insertPostgreSQLValue.h | 1 - src/Core/PostgreSQLProtocol.h | 587 +- src/Core/Protocol.h | 5 +- src/Core/ProtocolDefines.h | 27 +- src/Core/QualifiedTableName.cpp | 66 + src/Core/QualifiedTableName.h | 90 +- src/Core/QueryProcessingStage.h | 5 + src/Core/README.md | 10 + src/Core/Range.cpp | 23 +- src/Core/Range.h | 14 +- src/Core/ServerSettings.cpp | 1248 +- src/Core/ServerSettings.h | 204 +- src/Core/ServerUUID.cpp | 9 + src/Core/ServerUUID.h | 3 + src/Core/Settings.cpp | 7199 +- src/Core/Settings.h | 1501 +- src/Core/SettingsChangesHistory.cpp | 708 +- src/Core/SettingsChangesHistory.h | 10 +- src/Core/SettingsEnums.cpp | 101 +- src/Core/SettingsEnums.h | 100 +- src/Core/SettingsFields.cpp | 91 +- src/Core/SettingsFields.h | 45 +- src/Core/SettingsObsoleteMacros.h | 9 + src/Core/SettingsQuirks.cpp | 96 +- src/Core/SettingsTierType.cpp | 19 + src/Core/SettingsTierType.h | 26 + src/Core/SettingsWriteFormat.h | 10 + src/Core/SortCursor.cpp | 65 + src/Core/SortCursor.h | 292 +- src/Core/SortDescription.cpp | 62 + src/Core/SortDescription.h | 14 +- src/Core/StreamingHandleErrorMode.h | 16 + src/Core/TypeId.h | 2 + src/Core/Types_fwd.h | 2 +- src/Core/UUID.cpp | 13 + src/Core/UUID.h | 4 +- src/Core/ValueWithType.h | 5 +- src/Core/callOnTypeIndex.h | 77 + src/Core/examples/CMakeLists.txt | 2 +- src/Core/examples/field.cpp | 4 +- src/Core/examples/string_ref_hash.cpp | 1 + src/Core/fuzzers/CMakeLists.txt | 2 +- src/Core/tests/gtest_DecimalFunctions.cpp | 4 +- src/Core/tests/gtest_charset_conv.cpp | 351 - src/Core/tests/gtest_multienum.cpp | 8 +- src/Core/tests/gtest_settings.cpp | 1 + src/Daemon/BaseDaemon.cpp | 80 +- src/Daemon/BaseDaemon.h | 8 +- src/Daemon/CMakeLists.txt | 3 - src/Daemon/GitHash.cpp.in | 3 - src/Daemon/SentryWriter.cpp | 19 +- src/Daemon/SentryWriter.h | 8 +- src/DataTypes/DataTypeAggregateFunction.cpp | 17 +- src/DataTypes/DataTypeAggregateFunction.h | 3 + src/DataTypes/DataTypeArray.cpp | 4 +- src/DataTypes/DataTypeArray.h | 2 +- .../DataTypeCustomSimpleAggregateFunction.cpp | 1 + src/DataTypes/DataTypeDate32.cpp | 3 +- src/DataTypes/DataTypeDateTime.cpp | 2 + src/DataTypes/DataTypeDateTime64.cpp | 9 +- src/DataTypes/DataTypeDateTime64.h | 2 - src/DataTypes/DataTypeDecimalBase.cpp | 29 +- src/DataTypes/DataTypeDecimalBase.h | 70 +- src/DataTypes/DataTypeDynamic.cpp | 2 +- src/DataTypes/DataTypeEnum.cpp | 17 +- src/DataTypes/DataTypeEnum.h | 2 - src/DataTypes/DataTypeFactory.cpp | 27 +- src/DataTypes/DataTypeFactory.h | 4 +- src/DataTypes/DataTypeFixedString.cpp | 10 + src/DataTypes/DataTypeFixedString.h | 26 +- src/DataTypes/DataTypeLowCardinality.cpp | 18 +- .../DataTypeLowCardinalityHelpers.cpp | 3 +- src/DataTypes/DataTypeMap.cpp | 24 +- src/DataTypes/DataTypeMap.h | 5 +- src/DataTypes/DataTypeNested.h | 1 + src/DataTypes/DataTypeNothing.h | 1 + src/DataTypes/DataTypeNullable.cpp | 22 + src/DataTypes/DataTypeNullable.h | 2 + src/DataTypes/DataTypeNumberBase.cpp | 1 + src/DataTypes/DataTypeNumberBase.h | 1 + src/DataTypes/DataTypeObject.cpp | 129 +- src/DataTypes/DataTypeObject.h | 20 +- src/DataTypes/DataTypeObjectDeprecated.cpp | 2 +- src/DataTypes/DataTypeObjectDeprecated.h | 2 +- src/DataTypes/DataTypeTuple.cpp | 52 +- src/DataTypes/DataTypeTuple.h | 5 +- src/DataTypes/DataTypeVariant.cpp | 3 + src/DataTypes/DataTypesBinaryEncoding.cpp | 46 +- src/DataTypes/DataTypesDecimal.cpp | 13 +- src/DataTypes/DataTypesDecimal.h | 8 +- src/DataTypes/DataTypesNumber.cpp | 2 + src/DataTypes/DataTypesNumber.h | 2 + src/DataTypes/EnumValues.cpp | 56 +- src/DataTypes/EnumValues.h | 16 +- src/DataTypes/FieldToDataType.cpp | 30 +- src/DataTypes/FieldToDataType.h | 4 + src/DataTypes/IDataType.cpp | 23 +- src/DataTypes/IDataType.h | 63 +- src/DataTypes/IDataTypeDummy.h | 6 +- src/DataTypes/Native.cpp | 59 +- src/DataTypes/Native.h | 10 +- src/DataTypes/NestedUtils.cpp | 6 +- src/DataTypes/NestedUtils.h | 3 + .../NullableUtils.cpp | 25 +- src/DataTypes/NullableUtils.h | 27 + src/DataTypes/NumberTraits.h | 32 +- src/DataTypes/ObjectUtils.cpp | 38 +- src/DataTypes/ObjectUtils.h | 4 +- .../Serializations/DeserializationTask.h | 53 + .../Serializations/ISerialization.cpp | 74 +- src/DataTypes/Serializations/ISerialization.h | 79 +- .../Serializations/JSONDataParser.cpp | 4 +- .../SerializationAggregateFunction.cpp | 49 +- .../SerializationAggregateFunction.h | 2 +- .../Serializations/SerializationArray.cpp | 110 +- .../Serializations/SerializationArray.h | 7 +- .../Serializations/SerializationBool.cpp | 13 +- .../Serializations/SerializationBool.h | 7 +- .../Serializations/SerializationDateTime.cpp | 11 +- .../Serializations/SerializationDateTime.h | 2 - .../SerializationDateTime64.cpp | 16 +- .../Serializations/SerializationDateTime64.h | 2 - .../SerializationDecimalBase.cpp | 3 +- .../Serializations/SerializationDecimalBase.h | 2 +- .../Serializations/SerializationDynamic.cpp | 131 +- .../Serializations/SerializationDynamic.h | 26 +- .../SerializationDynamicElement.cpp | 26 +- .../SerializationDynamicElement.h | 1 + .../Serializations/SerializationEnum.cpp | 12 +- .../SerializationFixedString.cpp | 12 +- .../Serializations/SerializationFixedString.h | 2 +- .../SerializationIPv4andIPv6.cpp | 9 +- .../Serializations/SerializationIPv4andIPv6.h | 5 +- .../Serializations/SerializationInfo.cpp | 84 +- .../Serializations/SerializationInfo.h | 22 +- .../Serializations/SerializationInfoTuple.cpp | 44 +- .../Serializations/SerializationInfoTuple.h | 5 +- .../Serializations/SerializationInterval.cpp | 6 +- .../Serializations/SerializationInterval.h | 3 +- .../Serializations/SerializationJSON.cpp | 24 +- .../Serializations/SerializationJSON.h | 3 +- .../SerializationLowCardinality.cpp | 60 +- .../SerializationLowCardinality.h | 1 + .../Serializations/SerializationMap.cpp | 64 +- .../Serializations/SerializationMap.h | 4 + .../Serializations/SerializationNamed.cpp | 3 +- .../Serializations/SerializationNamed.h | 3 +- .../Serializations/SerializationNothing.cpp | 14 +- .../Serializations/SerializationNothing.h | 13 +- .../Serializations/SerializationNullable.cpp | 31 +- .../Serializations/SerializationNullable.h | 13 +- .../Serializations/SerializationNumber.cpp | 4 +- .../Serializations/SerializationNumber.h | 2 +- .../Serializations/SerializationObject.cpp | 377 +- .../Serializations/SerializationObject.h | 36 +- .../SerializationObjectDeprecated.cpp | 32 +- .../SerializationObjectDeprecated.h | 3 + .../SerializationObjectDynamicPath.cpp | 33 +- .../SerializationObjectDynamicPath.h | 1 + .../SerializationObjectTypedPath.cpp | 3 +- .../SerializationObjectTypedPath.h | 1 + .../Serializations/SerializationSparse.cpp | 125 +- .../Serializations/SerializationSparse.h | 6 +- .../Serializations/SerializationString.cpp | 13 +- .../Serializations/SerializationString.h | 2 +- .../Serializations/SerializationSubObject.cpp | 77 +- .../Serializations/SerializationSubObject.h | 1 + .../Serializations/SerializationTuple.cpp | 194 +- .../Serializations/SerializationTuple.h | 6 +- .../Serializations/SerializationUUID.cpp | 4 +- .../Serializations/SerializationUUID.h | 2 +- .../Serializations/SerializationVariant.cpp | 259 +- .../Serializations/SerializationVariant.h | 12 +- .../SerializationVariantElement.cpp | 120 +- .../SerializationVariantElement.h | 6 +- .../SerializationVariantElementNullMap.cpp | 49 +- .../SerializationVariantElementNullMap.h | 3 +- .../Serializations/SerializationWrapper.cpp | 7 +- .../Serializations/SerializationWrapper.h | 4 +- src/DataTypes/Serializations/SubcolumnsTree.h | 2 - .../gtest_deprecated_object_serialization.cpp | 2 +- src/DataTypes/TimezoneMixin.h | 2 - src/DataTypes/Utils.cpp | 25 +- src/DataTypes/Utils.h | 4 +- src/DataTypes/convertMySQLDataType.cpp | 8 +- src/DataTypes/convertMySQLDataType.h | 3 +- src/DataTypes/fuzzers/CMakeLists.txt | 3 +- .../data_type_deserialization_fuzzer.cpp | 7 +- src/DataTypes/getLeastSupertype.cpp | 109 +- src/DataTypes/getLeastSupertype.h | 9 +- src/DataTypes/getMostSubtype.cpp | 56 +- src/DataTypes/hasNullable.cpp | 4 +- src/DataTypes/registerDataTypeDateTime.cpp | 6 +- .../gtest_DataType_deserializeAsText.cpp | 1 + src/Databases/DDLDependencyVisitor.cpp | 106 +- src/Databases/DDLDependencyVisitor.h | 13 +- src/Databases/DDLLoadingDependencyVisitor.cpp | 9 +- src/Databases/DDLLoadingDependencyVisitor.h | 3 +- src/Databases/DDLRenamingVisitor.cpp | 3 +- src/Databases/DDLRenamingVisitor.h | 2 +- src/Databases/DataLake/Common.cpp | 103 + src/Databases/DataLake/Common.h | 15 + src/Databases/DataLake/DataLakeConstants.h | 29 + src/Databases/DataLake/DatabaseDataLake.cpp | 689 + src/Databases/DataLake/DatabaseDataLake.h | 80 + .../DataLake/DatabaseDataLakeSettings.cpp | 101 + .../DataLake/DatabaseDataLakeSettings.h | 47 + .../DataLake/DatabaseDataLakeStorageType.h | 22 + src/Databases/DataLake/GlueCatalog.cpp | 352 + src/Databases/DataLake/GlueCatalog.h | 71 + .../DataLake/HTTPBasedCatalogUtils.cpp | 75 + .../DataLake/HTTPBasedCatalogUtils.h | 28 + src/Databases/DataLake/HiveCatalog.cpp | 138 + src/Databases/DataLake/HiveCatalog.h | 57 + src/Databases/DataLake/ICatalog.cpp | 185 + src/Databases/DataLake/ICatalog.h | 157 + src/Databases/DataLake/RestCatalog.cpp | 669 + src/Databases/DataLake/RestCatalog.h | 126 + src/Databases/DataLake/StorageCredentials.h | 51 + src/Databases/DataLake/UnityCatalog.cpp | 404 + src/Databases/DataLake/UnityCatalog.h | 75 + src/Databases/DatabaseAtomic.cpp | 205 +- src/Databases/DatabaseAtomic.h | 10 +- src/Databases/DatabaseBackup.cpp | 480 + src/Databases/DatabaseBackup.h | 81 + src/Databases/DatabaseDictionary.cpp | 23 +- src/Databases/DatabaseFactory.cpp | 47 +- src/Databases/DatabaseFactory.h | 29 +- src/Databases/DatabaseFilesystem.cpp | 17 +- src/Databases/DatabaseFilesystem.h | 2 +- src/Databases/DatabaseHDFS.cpp | 12 +- src/Databases/DatabaseHDFS.h | 2 +- src/Databases/DatabaseLazy.cpp | 28 +- src/Databases/DatabaseLazy.h | 2 +- src/Databases/DatabaseMemory.cpp | 54 +- src/Databases/DatabaseMemory.h | 2 + src/Databases/DatabaseOnDisk.cpp | 321 +- src/Databases/DatabaseOnDisk.h | 25 +- src/Databases/DatabaseOrdinary.cpp | 273 +- src/Databases/DatabaseOrdinary.h | 4 +- src/Databases/DatabaseReplicated.cpp | 685 +- src/Databases/DatabaseReplicated.h | 22 +- src/Databases/DatabaseReplicatedHelpers.cpp | 1 + src/Databases/DatabaseReplicatedSettings.cpp | 47 +- src/Databases/DatabaseReplicatedSettings.h | 34 +- src/Databases/DatabaseReplicatedWorker.cpp | 110 +- src/Databases/DatabaseReplicatedWorker.h | 17 +- src/Databases/DatabaseS3.cpp | 13 +- src/Databases/DatabaseS3.h | 2 +- src/Databases/DatabasesCommon.cpp | 162 +- src/Databases/DatabasesCommon.h | 17 +- src/Databases/DatabasesOverlay.cpp | 305 +- src/Databases/DatabasesOverlay.h | 43 + src/Databases/IDatabase.cpp | 162 +- src/Databases/IDatabase.h | 125 +- .../MySQL/DatabaseMaterializedMySQL.cpp | 299 - .../MySQL/DatabaseMaterializedMySQL.h | 89 - .../DatabaseMaterializedTablesIterator.h | 43 - src/Databases/MySQL/DatabaseMySQL.cpp | 81 +- src/Databases/MySQL/DatabaseMySQL.h | 10 +- .../MySQL/FetchTablesColumnsList.cpp | 24 +- src/Databases/MySQL/MaterializeMetadata.cpp | 318 - src/Databases/MySQL/MaterializeMetadata.h | 60 - .../MySQL/MaterializedMySQLSettings.cpp | 39 - .../MySQL/MaterializedMySQLSettings.h | 38 - .../MySQL/MaterializedMySQLSyncThread.cpp | 1121 - .../MySQL/MaterializedMySQLSyncThread.h | 120 - src/Databases/MySQL/MySQLBinlog.cpp | 500 - src/Databases/MySQL/MySQLBinlog.h | 120 - src/Databases/MySQL/MySQLBinlogClient.cpp | 104 - src/Databases/MySQL/MySQLBinlogClient.h | 57 - .../MySQL/MySQLBinlogClientFactory.cpp | 46 - .../MySQL/MySQLBinlogClientFactory.h | 38 - .../MySQL/MySQLBinlogEventsDispatcher.cpp | 626 - .../MySQL/MySQLBinlogEventsDispatcher.h | 136 - src/Databases/MySQL/tests/data/binlog.000016 | Bin 1237 -> 0 bytes src/Databases/MySQL/tests/data/binlog.001390 | Bin 3091 -> 0 bytes .../MySQL/tests/gtest_mysql_binlog.cpp | 1753 - .../gtest_try_convert_string_literals.cpp | 158 - .../gtest_try_parse_table_id_from_ddl.cpp | 185 - .../gtest_try_quote_unrecognized_tokens.cpp | 294 - .../MySQL/tryConvertStringLiterals.cpp | 139 - .../MySQL/tryConvertStringLiterals.h | 20 - .../MySQL/tryParseTableIDFromDDL.cpp | 44 - src/Databases/MySQL/tryParseTableIDFromDDL.h | 11 - .../MySQL/tryQuoteUnrecognizedTokens.cpp | 96 - .../MySQL/tryQuoteUnrecognizedTokens.h | 10 - .../DatabaseMaterializedPostgreSQL.cpp | 26 +- .../DatabaseMaterializedPostgreSQL.h | 6 +- .../PostgreSQL/DatabasePostgreSQL.cpp | 60 +- src/Databases/PostgreSQL/DatabasePostgreSQL.h | 8 +- .../fetchPostgreSQLTableStructure.cpp | 79 +- .../fetchPostgreSQLTableStructure.h | 3 +- src/Databases/SQLite/DatabaseSQLite.cpp | 22 +- src/Databases/SQLite/DatabaseSQLite.h | 4 + src/Databases/SQLite/SQLiteUtils.cpp | 3 +- .../SQLite/fetchSQLiteTableStructure.cpp | 19 +- src/Databases/TablesDependencyGraph.cpp | 1 + src/Databases/TablesDependencyGraph.h | 6 + src/Databases/TablesLoader.cpp | 54 +- src/Databases/TablesLoader.h | 5 + .../enableAllExperimentalSettings.cpp | 66 + src/Databases/enableAllExperimentalSettings.h | 15 + src/Databases/registerDatabases.cpp | 16 +- src/Dictionaries/CMakeLists.txt | 10 +- src/Dictionaries/CacheDictionary.cpp | 69 +- src/Dictionaries/CacheDictionaryStorage.h | 6 +- src/Dictionaries/CacheDictionaryUpdateQueue.h | 2 +- .../CassandraDictionarySource.cpp | 11 +- src/Dictionaries/CassandraSource.h | 3 +- .../ClickHouseDictionarySource.cpp | 50 +- src/Dictionaries/ClickHouseDictionarySource.h | 2 + src/Dictionaries/DictionaryFactory.h | 2 - src/Dictionaries/DictionaryHelpers.h | 13 +- .../DictionaryPipelineExecutor.cpp | 1 + src/Dictionaries/DictionarySource.h | 2 +- src/Dictionaries/DictionarySourceFactory.cpp | 2 +- src/Dictionaries/DictionarySourceFactory.h | 4 +- src/Dictionaries/DictionarySourceHelpers.h | 1 - src/Dictionaries/DictionaryStructure.cpp | 5 +- src/Dictionaries/DirectDictionary.cpp | 24 +- src/Dictionaries/DirectDictionary.h | 3 +- .../GeodataProviders/NamesProvider.cpp | 3 +- .../ExecutableDictionarySource.cpp | 9 +- .../ExecutablePoolDictionarySource.cpp | 21 +- src/Dictionaries/ExternalQueryBuilder.cpp | 108 +- src/Dictionaries/ExternalQueryBuilder.h | 3 +- src/Dictionaries/FileDictionarySource.cpp | 7 +- src/Dictionaries/FlatDictionary.cpp | 2 + src/Dictionaries/FlatDictionary.h | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 13 +- src/Dictionaries/HTTPDictionarySource.h | 1 + src/Dictionaries/HashedArrayDictionary.cpp | 27 +- src/Dictionaries/HashedArrayDictionary.h | 2 +- src/Dictionaries/HashedDictionary.h | 30 +- .../HashedDictionaryParallelLoader.h | 13 +- .../HierarchyDictionariesUtils.cpp | 11 +- src/Dictionaries/HierarchyDictionariesUtils.h | 4 +- src/Dictionaries/ICacheDictionaryStorage.h | 6 +- src/Dictionaries/IDictionary.h | 5 +- src/Dictionaries/IDictionarySource.h | 2 +- src/Dictionaries/IPAddressDictionary.cpp | 11 +- src/Dictionaries/LibraryDictionarySource.cpp | 3 +- src/Dictionaries/MongoDBDictionarySource.cpp | 330 +- src/Dictionaries/MongoDBDictionarySource.h | 47 +- src/Dictionaries/MySQLDictionarySource.cpp | 123 +- src/Dictionaries/MySQLDictionarySource.h | 5 +- src/Dictionaries/NullDictionarySource.cpp | 3 +- src/Dictionaries/NullDictionarySource.h | 3 +- src/Dictionaries/PolygonDictionary.cpp | 5 +- src/Dictionaries/PolygonDictionary.h | 2 - .../PolygonDictionaryImplementations.cpp | 8 +- src/Dictionaries/PolygonDictionaryUtils.cpp | 3 +- src/Dictionaries/PolygonDictionaryUtils.h | 9 +- .../PostgreSQLDictionarySource.cpp | 39 +- src/Dictionaries/README.md | 55 + src/Dictionaries/RangeHashedDictionary.h | 10 +- src/Dictionaries/RedisDictionarySource.cpp | 10 +- src/Dictionaries/RegExpTreeDictionary.cpp | 39 +- src/Dictionaries/RegExpTreeDictionary.h | 19 +- src/Dictionaries/SSDCacheDictionaryStorage.h | 40 +- src/Dictionaries/XDBCDictionarySource.cpp | 45 +- src/Dictionaries/XDBCDictionarySource.h | 3 +- .../YAMLRegExpTreeDictionarySource.cpp | 6 +- .../getDictionaryConfigurationFromAST.cpp | 42 +- src/Dictionaries/readInvalidateQuery.cpp | 5 +- .../registerCacheDictionaries.cpp | 6 +- src/Dictionaries/registerDictionaries.cpp | 1 + src/Dictionaries/registerHashedDictionary.cpp | 56 +- .../registerRangeHashedDictionary.cpp | 6 +- .../tests/gtest_dictionary_configuration.cpp | 1 - src/Disks/DiskBackup.cpp | 288 + src/Disks/DiskBackup.h | 144 + src/Disks/DiskEncrypted.cpp | 16 +- src/Disks/DiskEncrypted.h | 43 +- src/Disks/DiskEncryptedTransaction.cpp | 5 +- src/Disks/{DiskFomAST.cpp => DiskFromAST.cpp} | 54 +- src/Disks/{DiskFomAST.h => DiskFromAST.h} | 3 +- src/Disks/DiskLocal.cpp | 98 +- src/Disks/DiskLocal.h | 28 +- src/Disks/DiskLocalCheckThread.cpp | 1 + src/Disks/DiskLocalCheckThread.h | 4 +- src/Disks/DiskSelector.cpp | 10 +- src/Disks/DiskSelector.h | 12 +- src/Disks/DiskType.h | 2 + src/Disks/IDisk.cpp | 90 +- src/Disks/IDisk.h | 183 +- .../IO/AsynchronousBoundedReadBuffer.cpp | 42 +- src/Disks/IO/AsynchronousBoundedReadBuffer.h | 11 +- .../IO/CachedOnDiskReadBufferFromFile.cpp | 50 +- src/Disks/IO/CachedOnDiskReadBufferFromFile.h | 2 + .../IO/CachedOnDiskWriteBufferFromFile.cpp | 44 +- .../IO/CachedOnDiskWriteBufferFromFile.h | 30 +- src/Disks/IO/IOUringReader.cpp | 42 +- .../IO/ReadBufferFromAzureBlobStorage.cpp | 11 +- src/Disks/IO/ReadBufferFromAzureBlobStorage.h | 10 +- src/Disks/IO/ReadBufferFromRemoteFSGather.cpp | 121 +- src/Disks/IO/ReadBufferFromRemoteFSGather.h | 14 +- src/Disks/IO/ReadBufferFromWebServer.cpp | 17 +- src/Disks/IO/ThreadPoolReader.cpp | 55 +- .../IO/WriteBufferFromAzureBlobStorage.cpp | 111 +- .../IO/WriteBufferFromAzureBlobStorage.h | 10 +- .../IO/WriteBufferWithFinalizeCallback.cpp | 24 +- .../IO/WriteBufferWithFinalizeCallback.h | 9 +- src/Disks/IO/createReadBufferFromFileBase.h | 6 +- src/Disks/IO/getThreadPoolReader.cpp | 8 +- src/Disks/IStoragePolicy.h | 2 + .../AzureBlobStorageCommon.cpp | 169 +- .../AzureBlobStorage/AzureBlobStorageCommon.h | 46 +- .../AzureBlobStorage/AzureObjectStorage.cpp | 121 +- .../AzureBlobStorage/AzureObjectStorage.h | 22 +- .../Cached/CachedObjectStorage.cpp | 79 +- .../Cached/CachedObjectStorage.h | 16 +- .../Cached/registerDiskCache.cpp | 28 +- .../CommonPathPrefixKeyGenerator.cpp | 74 - .../CommonPathPrefixKeyGenerator.h | 42 - .../ObjectStorages/DiskObjectStorage.cpp | 398 +- src/Disks/ObjectStorages/DiskObjectStorage.h | 53 +- .../DiskObjectStorageMetadata.cpp | 7 +- .../DiskObjectStorageMetadata.h | 2 + ...jectStorageRemoteMetadataRestoreHelper.cpp | 30 +- .../DiskObjectStorageTransaction.cpp | 132 +- .../DiskObjectStorageTransaction.h | 3 +- .../FlatDirectoryStructureKeyGenerator.cpp | 34 +- .../FlatDirectoryStructureKeyGenerator.h | 9 +- .../ObjectStorages/HDFS/HDFSObjectStorage.cpp | 60 +- .../ObjectStorages/HDFS/HDFSObjectStorage.h | 27 +- src/Disks/ObjectStorages/IMetadataStorage.h | 54 +- src/Disks/ObjectStorages/IObjectStorage.cpp | 2 +- src/Disks/ObjectStorages/IObjectStorage.h | 34 +- .../ObjectStorages/InMemoryDirectoryPathMap.h | 257 + src/Disks/ObjectStorages/InMemoryPathMap.h | 37 - .../Local/LocalObjectStorage.cpp | 118 +- .../ObjectStorages/Local/LocalObjectStorage.h | 41 +- .../ObjectStorages/MetadataStorageFactory.cpp | 7 +- .../MetadataStorageFromDisk.cpp | 28 +- .../ObjectStorages/MetadataStorageFromDisk.h | 15 +- ...taStorageFromDiskTransactionOperations.cpp | 24 +- .../MetadataStorageFromPlainObjectStorage.cpp | 198 +- .../MetadataStorageFromPlainObjectStorage.h | 70 +- ...torageFromPlainObjectStorageOperations.cpp | 325 +- ...aStorageFromPlainObjectStorageOperations.h | 85 +- ...torageFromPlainRewritableObjectStorage.cpp | 393 +- ...aStorageFromPlainRewritableObjectStorage.h | 63 +- .../ObjectStorages/MetadataStorageMetrics.h | 1 + .../ObjectStorages/ObjectStorageFactory.cpp | 57 +- .../PlainRewritableObjectStorage.h | 8 +- src/Disks/ObjectStorages/S3/DiskS3Utils.cpp | 55 - src/Disks/ObjectStorages/S3/DiskS3Utils.h | 3 - .../ObjectStorages/S3/S3Capabilities.cpp | 15 - src/Disks/ObjectStorages/S3/S3Capabilities.h | 26 - .../ObjectStorages/S3/S3ObjectStorage.cpp | 224 +- src/Disks/ObjectStorages/S3/S3ObjectStorage.h | 29 +- src/Disks/ObjectStorages/S3/diskSettings.cpp | 97 +- .../ObjectStorages/StaticDirectoryIterator.h | 3 +- ...etadataStorageFromStaticFilesWebServer.cpp | 50 +- .../MetadataStorageFromStaticFilesWebServer.h | 13 +- .../ObjectStorages/Web/WebObjectStorage.cpp | 117 +- .../ObjectStorages/Web/WebObjectStorage.h | 20 +- .../createMetadataStorageMetrics.h | 12 +- src/Disks/SingleDiskVolume.h | 2 + src/Disks/StoragePolicy.cpp | 71 +- src/Disks/StoragePolicy.h | 21 +- src/Disks/TemporaryFileOnDisk.cpp | 4 +- src/Disks/VolumeJBOD.cpp | 39 +- src/Disks/VolumeJBOD.h | 4 + src/Disks/getDiskConfigurationFromAST.cpp | 29 +- ...gtest_asynchronous_bounded_read_buffer.cpp | 2 +- .../gtest_cascade_and_memory_write_buffer.cpp | 8 +- src/Disks/tests/gtest_disk.cpp | 14 +- src/Disks/tests/gtest_disk_encrypted.cpp | 189 +- src/Formats/CapnProtoSchema.cpp | 4 +- src/Formats/CapnProtoSchema.h | 2 - src/Formats/CapnProtoSerializer.cpp | 2 +- src/Formats/CapnProtoSerializer.h | 9 +- src/Formats/ColumnMapping.cpp | 5 +- src/Formats/ColumnMapping.h | 10 +- src/Formats/EscapingRuleUtils.cpp | 33 +- src/Formats/EscapingRuleUtils.h | 2 +- src/Formats/FormatFactory.cpp | 587 +- src/Formats/FormatFactory.h | 28 +- src/Formats/FormatSchemaInfo.cpp | 25 +- src/Formats/FormatSchemaInfo.h | 2 +- src/Formats/FormatSettings.h | 63 +- src/Formats/IndexForNativeFormat.cpp | 1 + src/Formats/IndexForNativeFormat.h | 3 + src/Formats/JSONExtractTree.cpp | 91 +- src/Formats/JSONExtractTree.h | 6 +- src/Formats/JSONUtils.cpp | 110 +- src/Formats/JSONUtils.h | 11 + src/Formats/MarkInCompressedFile.cpp | 16 +- src/Formats/MarkInCompressedFile.h | 23 +- src/Formats/NativeReader.cpp | 62 +- src/Formats/NativeReader.h | 4 + src/Formats/NativeWriter.cpp | 29 +- src/Formats/NativeWriter.h | 2 + src/Formats/NumpyDataTypes.h | 6 +- src/Formats/PrettyFormatHelpers.cpp | 154 + src/Formats/PrettyFormatHelpers.h | 29 + src/Formats/ProtobufReader.h | 1 + src/Formats/ProtobufSchemas.cpp | 28 +- src/Formats/ProtobufSchemas.h | 27 +- src/Formats/ProtobufSerializer.cpp | 380 +- src/Formats/ProtobufSerializer.h | 10 +- src/Formats/ReadSchemaUtils.cpp | 18 +- src/Formats/ReadSchemaUtils.h | 3 +- src/Formats/RowInputMissingColumnsFiller.h | 7 +- src/Formats/SchemaInferenceUtils.cpp | 108 +- src/Formats/SchemaInferenceUtils.h | 6 +- src/Formats/StructureToCapnProtoSchema.h | 6 +- src/Formats/TemporaryFileStreamLegacy.cpp | 45 - src/Formats/TemporaryFileStreamLegacy.h | 34 - src/Formats/fuzzers/CMakeLists.txt | 2 +- src/Formats/fuzzers/format_fuzzer.cpp | 24 +- src/Formats/insertNullAsDefaultIfNeeded.cpp | 7 +- src/Formats/insertNullAsDefaultIfNeeded.h | 3 +- src/Formats/registerFormats.cpp | 6 +- src/Formats/registerWithNamesAndTypes.cpp | 1 + src/Formats/registerWithNamesAndTypes.h | 2 +- src/Formats/tests/gtest_marks.cpp | 34 +- src/Functions/CMakeLists.txt | 43 +- src/Functions/CastOverloadResolver.cpp | 42 +- src/Functions/CountSubstringsImpl.h | 7 +- src/Functions/DateTimeTransforms.h | 212 +- src/Functions/DivisionUtils.h | 63 +- src/Functions/EmptyImpl.h | 2 +- src/Functions/FieldInterval.h | 15 + src/Functions/FunctionBase32Conversion.h | 45 + src/Functions/FunctionBase58Conversion.h | 165 +- src/Functions/FunctionBase64Conversion.h | 185 +- src/Functions/FunctionBaseXXConversion.h | 230 + src/Functions/FunctionBinaryArithmetic.h | 327 +- src/Functions/FunctionBitTestMany.h | 31 +- src/Functions/FunctionChar.cpp | 5 + src/Functions/FunctionConstantBase.h | 2 + .../FunctionCustomWeekToDateOrDate32.h | 51 +- src/Functions/FunctionCustomWeekToSomething.h | 27 +- .../FunctionDateOrDateTimeAddInterval.h | 27 +- .../FunctionDateOrDateTimeToDateOrDate32.h | 38 +- ...tionDateOrDateTimeToDateTimeOrDateTime64.h | 38 +- .../FunctionDateOrDateTimeToSomething.h | 28 +- src/Functions/FunctionDynamicAdaptor.cpp | 476 + src/Functions/FunctionDynamicAdaptor.h | 79 + src/Functions/FunctionFQDN.cpp | 1 + src/Functions/FunctionFactory.cpp | 10 +- src/Functions/FunctionFactory.h | 3 +- src/Functions/FunctionFile.cpp | 8 +- .../FunctionGenerateRandomStructure.cpp | 24 +- src/Functions/FunctionHelpers.cpp | 133 +- src/Functions/FunctionHelpers.h | 12 +- src/Functions/FunctionJoinGet.cpp | 6 +- src/Functions/FunctionMathBinaryFloat64.h | 84 +- src/Functions/FunctionMathUnary.h | 13 +- src/Functions/FunctionNumericPredicate.h | 5 + src/Functions/FunctionSQLJSON.h | 71 +- src/Functions/FunctionShowCertificate.cpp | 139 +- src/Functions/FunctionShowCertificate.h | 205 - src/Functions/FunctionSpaceFillingCurve.h | 5 + src/Functions/FunctionStartsEndsWith.h | 5 + src/Functions/FunctionStringOrArrayToT.h | 39 +- src/Functions/FunctionStringReplace.h | 56 +- src/Functions/FunctionStringToString.h | 14 +- src/Functions/FunctionTokens.h | 18 +- src/Functions/FunctionUnaryArithmetic.h | 12 +- src/Functions/FunctionUnixTimestamp64.cpp | 6 +- src/Functions/FunctionUnixTimestamp64.h | 6 + src/Functions/FunctionsAES.cpp | 4 +- src/Functions/FunctionsAES.h | 67 +- .../FunctionsBinaryRepresentation.cpp | 64 +- src/Functions/FunctionsBitToArray.cpp | 16 +- src/Functions/FunctionsBitmap.h | 113 +- src/Functions/FunctionsCodingIP.cpp | 76 +- src/Functions/FunctionsCodingIP.h | 8 +- src/Functions/FunctionsCodingULID.cpp | 12 +- src/Functions/FunctionsCodingUUID.cpp | 45 +- src/Functions/FunctionsComparison.h | 518 +- src/Functions/FunctionsConsistentHashing.h | 10 +- src/Functions/FunctionsConversion.cpp | 5271 +- src/Functions/FunctionsConversion.h | 5768 + src/Functions/FunctionsConversion_impl0.cpp | 47 + src/Functions/FunctionsConversion_impl1.cpp | 38 + src/Functions/FunctionsConversion_impl2.cpp | 38 + src/Functions/FunctionsConversion_impl3.cpp | 46 + src/Functions/FunctionsConversion_reg.cpp | 192 + src/Functions/FunctionsDecimalArithmetics.h | 65 +- src/Functions/FunctionsEmbeddedDictionaries.h | 47 +- .../FunctionsExternalDictionaries.cpp | 90 +- src/Functions/FunctionsExternalDictionaries.h | 51 +- src/Functions/FunctionsHashing.h | 210 +- src/Functions/FunctionsHashingMisc.cpp | 9 +- src/Functions/FunctionsJSON.cpp | 37 +- .../FunctionsLanguageClassification.cpp | 8 +- src/Functions/FunctionsLogical.cpp | 47 +- src/Functions/FunctionsLogical.h | 5 +- src/Functions/FunctionsMiscellaneous.h | 171 +- .../FunctionsMultiStringFuzzySearch.h | 10 +- src/Functions/FunctionsMultiStringSearch.h | 10 +- src/Functions/FunctionsOpDate.cpp | 2 +- src/Functions/FunctionsRound.h | 24 +- src/Functions/FunctionsStringDistance.cpp | 36 +- .../FunctionsStringHashFixedString.cpp | 312 +- src/Functions/FunctionsStringSearch.h | 94 +- src/Functions/FunctionsStringSearchToString.h | 10 +- src/Functions/FunctionsStringSimilarity.cpp | 4 +- src/Functions/FunctionsTextClassification.h | 9 +- src/Functions/FunctionsTimeWindow.cpp | 113 +- src/Functions/FunctionsTimeWindow.h | 13 +- .../FunctionsTonalityClassification.cpp | 3 +- src/Functions/FunctionsVisitParam.h | 2 +- src/Functions/GatherUtils/Algorithms.h | 13 +- src/Functions/GatherUtils/CMakeLists.txt | 15 + src/Functions/GatherUtils/GatherUtils.cpp | 28 + src/Functions/GatherUtils/GatherUtils.h | 22 +- src/Functions/GatherUtils/IArraySink.h | 2 +- src/Functions/GatherUtils/IArraySource.h | 2 +- src/Functions/GatherUtils/IValueSource.h | 2 +- src/Functions/GatherUtils/Selectors.h | 7 +- src/Functions/GatherUtils/Slices.h | 2 +- src/Functions/GatherUtils/ends_with.cpp | 30 +- src/Functions/GatherUtils/has_all.cpp | 30 +- src/Functions/GatherUtils/has_any.cpp | 30 +- src/Functions/GatherUtils/has_substr.cpp | 30 +- .../GatherUtils/sliceHasImplAnyAll.h | 21 +- src/Functions/GatherUtils/starts_with.cpp | 30 +- src/Functions/GeoHash.cpp | 6 +- src/Functions/GregorianDate.cpp | 13 +- .../HTMLCharacterReference.generated.cpp | 4 +- src/Functions/HTMLCharacterReference.gperf | 4 +- src/Functions/HTMLCharacterReference.sh | 4 +- src/Functions/HasTokenImpl.h | 7 +- src/Functions/IFunction.cpp | 291 +- src/Functions/IFunction.h | 115 +- src/Functions/IFunctionAdaptors.cpp | 19 + src/Functions/IFunctionAdaptors.h | 20 +- src/Functions/IFunctionCustomWeek.h | 19 +- src/Functions/IFunctionDateOrDateTime.h | 116 +- src/Functions/IsOperation.h | 11 +- src/Functions/JSONArrayLength.cpp | 4 +- src/Functions/JSONPath/Generator/IGenerator.h | 1 - src/Functions/JSONPaths.cpp | 12 +- src/Functions/Kusto/KqlArraySort.cpp | 20 +- src/Functions/Kusto/KqlFunctionBase.cpp | 23 + src/Functions/Kusto/KqlFunctionBase.h | 21 +- src/Functions/LeastGreatestGeneric.h | 74 +- src/Functions/LeftRight.h | 56 +- src/Functions/LowerUpperImpl.h | 1 - src/Functions/LowerUpperUTF8Impl.h | 297 +- src/Functions/MatchImpl.h | 29 +- src/Functions/PerformanceAdaptors.h | 8 +- src/Functions/PolygonUtils.h | 29 +- src/Functions/PositionImpl.h | 36 +- src/Functions/README.md | 44 + src/Functions/Regexps.h | 4 +- src/Functions/ReplaceRegexpImpl.h | 62 +- src/Functions/ReplaceStringImpl.h | 102 +- .../URL/FirstSignificantSubdomainCustomImpl.h | 5 +- src/Functions/URL/URLHierarchy.cpp | 3 +- src/Functions/URL/URLPathHierarchy.cpp | 3 +- src/Functions/URL/basename.cpp | 2 +- .../URL/cutToFirstSignificantSubdomain.cpp | 8 +- .../cutToFirstSignificantSubdomainCustom.cpp | 8 +- src/Functions/URL/cutURLParameter.cpp | 11 +- src/Functions/URL/decodeURLComponent.cpp | 2 +- src/Functions/URL/domain.cpp | 4 +- src/Functions/URL/domain.h | 2 +- src/Functions/URL/domainWithoutWWW.cpp | 4 +- src/Functions/URL/extractURLParameter.cpp | 8 +- .../URL/extractURLParameterNames.cpp | 3 +- .../URL/firstSignificantSubdomain.cpp | 4 +- src/Functions/URL/netloc.cpp | 7 +- src/Functions/URL/port.cpp | 9 +- src/Functions/URL/tldLookup.generated.cpp | 4 + src/Functions/URL/tldLookup.gperf | 2 + src/Functions/URL/tldLookup.sh | 2 + src/Functions/URL/topLevelDomain.cpp | 6 +- src/Functions/UTCTimestamp.cpp | 2 +- src/Functions/UTCTimestampTransform.cpp | 34 +- .../UniqTheta/FunctionsUniqTheta.cpp | 6 +- ...alUserDefinedExecutableFunctionsLoader.cpp | 13 +- .../UserDefinedExecutableFunction.h | 1 + .../UserDefinedExecutableFunctionFactory.cpp | 6 +- .../UserDefinedSQLFunctionFactory.cpp | 6 +- .../UserDefinedSQLFunctionVisitor.cpp | 170 +- .../UserDefinedSQLFunctionVisitor.h | 6 +- .../UserDefinedSQLObjectsBackup.cpp | 11 +- .../UserDefinedSQLObjectsDiskStorage.cpp | 22 +- .../UserDefinedSQLObjectsStorageBase.cpp | 11 +- .../UserDefinedSQLObjectsStorageBase.h | 3 +- .../UserDefinedSQLObjectsZooKeeperStorage.cpp | 21 +- .../createUserDefinedSQLObjectsStorage.cpp | 2 +- src/Functions/abs.cpp | 15 +- src/Functions/addressToLine.h | 7 +- src/Functions/addressToSymbol.cpp | 1 - src/Functions/appendTrailingCharIfAbsent.cpp | 11 +- src/Functions/array/CMakeLists.txt | 13 + src/Functions/array/FunctionArrayMapped.h | 75 +- .../array/FunctionsMapMiscellaneous.cpp | 62 +- src/Functions/array/array.cpp | 209 +- src/Functions/array/arrayAUC.cpp | 464 +- src/Functions/array/arrayAggregation.cpp | 172 +- src/Functions/array/arrayAll.cpp | 24 +- src/Functions/array/arrayCompact.cpp | 1 - src/Functions/array/arrayCount.cpp | 22 +- src/Functions/array/arrayCumSum.cpp | 3 +- .../array/arrayCumSumNonNegative.cpp | 3 +- src/Functions/array/arrayDifference.cpp | 25 +- src/Functions/array/arrayDistance.cpp | 195 +- src/Functions/array/arrayDistinct.cpp | 13 +- src/Functions/array/arrayDotProduct.cpp | 30 +- src/Functions/array/arrayElement.cpp | 978 +- src/Functions/array/arrayEnumerate.cpp | 55 +- src/Functions/array/arrayEnumerateDense.cpp | 11 + .../array/arrayEnumerateDenseRanked.cpp | 39 + src/Functions/array/arrayEnumerateExtended.h | 1 - src/Functions/array/arrayEnumerateRanked.cpp | 14 +- src/Functions/array/arrayEnumerateRanked.h | 15 +- .../array/arrayEnumerateUniqRanked.cpp | 52 +- src/Functions/array/arrayExists.cpp | 5 +- src/Functions/array/arrayFilter.cpp | 7 +- src/Functions/array/arrayFirstLast.cpp | 20 +- src/Functions/array/arrayFirstLastIndex.cpp | 8 +- src/Functions/array/arrayFold.cpp | 6 +- src/Functions/array/arrayIndex.h | 338 +- src/Functions/array/arrayIntersect.cpp | 347 +- src/Functions/array/arrayJaccardIndex.cpp | 9 +- src/Functions/array/arrayJoin.cpp | 184 +- src/Functions/array/arrayLevenshtein.cpp | 640 + src/Functions/array/arrayNorm.cpp | 18 +- src/Functions/array/arrayNormalizedGini.cpp | 416 + src/Functions/array/arrayPopBack.cpp | 11 +- src/Functions/array/arrayPopFront.cpp | 11 +- src/Functions/array/arrayPushBack.cpp | 24 +- src/Functions/array/arrayPushFront.cpp | 24 +- src/Functions/array/arrayResize.cpp | 184 +- src/Functions/array/arrayResize.h | 27 + src/Functions/array/arrayReverse.cpp | 9 +- src/Functions/array/arrayShiftRotate.cpp | 16 +- src/Functions/array/arrayShingles.cpp | 26 +- src/Functions/array/arrayShuffle.cpp | 115 +- src/Functions/array/arraySlice.cpp | 17 +- src/Functions/array/arraySort.cpp | 333 +- src/Functions/array/arrayUniq.cpp | 51 +- src/Functions/array/arrayWithConstant.cpp | 7 +- src/Functions/array/arrayZip.cpp | 158 +- src/Functions/array/countEqual.cpp | 18 +- src/Functions/array/emptyArrayToSingle.cpp | 54 +- src/Functions/array/emptyArrayToSingle.h | 29 + src/Functions/array/has.cpp | 1 + src/Functions/array/hasAll.cpp | 32 +- src/Functions/array/hasAllAny.h | 5 + src/Functions/array/hasAny.cpp | 30 +- src/Functions/array/hasSubstr.cpp | 35 +- src/Functions/array/indexOf.cpp | 24 +- src/Functions/array/indexOfAssumeSorted.cpp | 38 + src/Functions/array/length.cpp | 62 +- src/Functions/array/length.h | 66 + src/Functions/array/mapOp.cpp | 19 +- src/Functions/array/mapPopulateSeries.cpp | 40 +- src/Functions/array/range.cpp | 10 +- src/Functions/arrayStringConcat.cpp | 7 +- src/Functions/ascii.cpp | 2 +- src/Functions/asin.cpp | 2 +- src/Functions/assumeNotNull.cpp | 3 +- src/Functions/bar.cpp | 5 + src/Functions/base32Decode.cpp | 33 + src/Functions/base32Encode.cpp | 31 + src/Functions/base58Decode.cpp | 6 +- src/Functions/base58Encode.cpp | 11 +- src/Functions/base64Decode.cpp | 15 +- src/Functions/base64Encode.cpp | 15 +- src/Functions/base64URLDecode.cpp | 15 +- src/Functions/base64URLEncode.cpp | 15 +- src/Functions/bitCount.cpp | 2 +- src/Functions/bitNot.cpp | 2 +- src/Functions/bitShiftLeft.cpp | 20 +- src/Functions/bitShiftRight.cpp | 22 +- src/Functions/bitSlice.cpp | 13 +- src/Functions/bitSwapLastTwo.cpp | 2 +- src/Functions/bitTest.cpp | 2 +- src/Functions/bitWrapperFunc.cpp | 4 +- src/Functions/blockSerializedSize.cpp | 1 + src/Functions/byteSwap.cpp | 52 +- src/Functions/caseWithExpression.cpp | 82 +- src/Functions/castOrDefault.cpp | 11 +- src/Functions/catboostEvaluate.cpp | 2 - src/Functions/changeDate.cpp | 31 +- src/Functions/checkHyperscanRegexp.cpp | 4 +- src/Functions/coalesce.cpp | 12 +- src/Functions/compareSubstrings.cpp | 403 + src/Functions/concat.cpp | 18 +- src/Functions/concatWithSeparator.cpp | 11 +- src/Functions/convertCharset.cpp | 9 +- src/Functions/countDigits.cpp | 9 +- src/Functions/countMatches.h | 12 +- src/Functions/coverage.cpp | 9 +- src/Functions/currentProfiles.cpp | 23 +- src/Functions/currentQueryID.cpp | 70 + src/Functions/currentRoles.cpp | 14 +- src/Functions/currentSchemas.cpp | 3 +- src/Functions/dateDiff.cpp | 12 +- src/Functions/dateName.cpp | 5 + src/Functions/dateTimeToSnowflakeID.cpp | 15 +- src/Functions/date_trunc.cpp | 46 +- src/Functions/decodeHTMLComponent.cpp | 20 +- src/Functions/defaultValueOfArgumentType.cpp | 1 + src/Functions/defaultValueOfTypeName.cpp | 3 +- src/Functions/divide.cpp | 35 +- src/Functions/divideDecimal.cpp | 55 +- src/Functions/dumpColumnStructure.cpp | 1 + src/Functions/dynamicElement.cpp | 2 +- src/Functions/dynamicType.cpp | 4 +- src/Functions/empty.cpp | 36 +- src/Functions/encodeXMLComponent.cpp | 2 +- src/Functions/endsWithUTF8.cpp | 2 +- src/Functions/equals.cpp | 3 +- src/Functions/exists.h | 47 + src/Functions/exp.cpp | 15 +- src/Functions/extractAllGroups.h | 12 +- .../extractKeyValuePairs.cpp | 46 +- .../extractTimeZoneFromFunctionArguments.cpp | 44 +- .../extractTimeZoneFromFunctionArguments.h | 5 +- src/Functions/factorial.cpp | 6 +- src/Functions/filesystem.cpp | 34 +- src/Functions/firstLine.cpp | 4 +- src/Functions/format.cpp | 5 + src/Functions/formatDateTime.cpp | 306 +- src/Functions/formatQuery.cpp | 54 +- src/Functions/formatReadable.h | 64 +- src/Functions/formatReadableDecimalSize.cpp | 2 +- src/Functions/formatReadableTimeDelta.cpp | 40 +- src/Functions/formatRow.cpp | 11 +- src/Functions/fromDaysSinceYearZero.cpp | 5 +- src/Functions/fromModifiedJulianDay.cpp | 5 +- src/Functions/fromUnixTimestamp64.cpp | 13 + src/Functions/fuzzBits.cpp | 14 +- src/Functions/gcd.cpp | 19 +- src/Functions/generateSerialID.cpp | 199 + src/Functions/generateSnowflakeID.cpp | 5 +- src/Functions/generateULID.cpp | 2 +- src/Functions/generateUUIDv7.cpp | 7 +- src/Functions/geoToH3.cpp | 65 +- src/Functions/geoToS2.cpp | 5 + src/Functions/geohashEncode.cpp | 7 +- src/Functions/geometryConverters.h | 10 +- src/Functions/getClientHTTPHeader.cpp | 9 +- src/Functions/getFuzzerData.cpp | 4 +- src/Functions/getMaxTableNameLength.cpp | 113 + src/Functions/getMergeTreeSetting.cpp | 98 + src/Functions/getScalar.cpp | 5 +- src/Functions/getServerSetting.cpp | 97 + src/Functions/getSetting.cpp | 59 +- src/Functions/getSizeOfEnumType.cpp | 7 +- src/Functions/getSubcolumn.cpp | 5 +- src/Functions/globalVariable.cpp | 24 +- src/Functions/greatCircleDistance.cpp | 31 +- src/Functions/greater.cpp | 5 +- src/Functions/greaterOrEquals.cpp | 9 +- src/Functions/h3CellAreaM2.cpp | 5 + src/Functions/h3CellAreaRads2.cpp | 5 + src/Functions/h3Distance.cpp | 5 + src/Functions/h3EdgeAngle.cpp | 5 + src/Functions/h3EdgeLengthKm.cpp | 5 + src/Functions/h3EdgeLengthM.cpp | 5 + src/Functions/h3ExactEdgeLengthKm.cpp | 5 + src/Functions/h3ExactEdgeLengthM.cpp | 5 + src/Functions/h3ExactEdgeLengthRads.cpp | 5 + src/Functions/h3GetBaseCell.cpp | 5 + ...DestinationIndexFromUnidirectionalEdge.cpp | 5 + ...h3GetOriginIndexFromUnidirectionalEdge.cpp | 5 + src/Functions/h3GetResolution.cpp | 5 + src/Functions/h3GetUnidirectionalEdge.cpp | 5 + src/Functions/h3HexAreaKm2.cpp | 5 + src/Functions/h3HexAreaM2.cpp | 5 + src/Functions/h3IndexesAreNeighbors.cpp | 5 + src/Functions/h3IsPentagon.cpp | 5 + src/Functions/h3IsResClassIII.cpp | 5 + src/Functions/h3IsValid.cpp | 5 + src/Functions/h3NumHexagons.cpp | 5 + src/Functions/h3PointDist.cpp | 5 + src/Functions/h3ToCenterChild.cpp | 5 + src/Functions/h3ToParent.cpp | 5 + src/Functions/h3ToString.cpp | 5 + src/Functions/h3UnidirectionalEdgeIsValid.cpp | 5 + src/Functions/h3toGeo.cpp | 42 +- src/Functions/hasColumnInTable.cpp | 6 +- src/Functions/hasToken.cpp | 4 +- src/Functions/hasTokenCaseInsensitive.cpp | 4 +- src/Functions/hilbertDecode.cpp | 7 +- src/Functions/hilbertEncode.cpp | 7 +- src/Functions/icebergBucketTransform.cpp | 357 + src/Functions/icebergTruncate.cpp | 202 + src/Functions/identity.h | 1 + src/Functions/idna.cpp | 15 +- src/Functions/if.cpp | 91 +- src/Functions/ifNotFinite.cpp | 24 +- src/Functions/ifNull.cpp | 8 +- src/Functions/in.cpp | 28 +- src/Functions/initcapUTF8.cpp | 3 +- src/Functions/initialQueryStartTime.cpp | 53 + src/Functions/intDiv.cpp | 48 +- src/Functions/intDivOrZero.cpp | 18 +- src/Functions/intExp10.cpp | 20 +- src/Functions/intExp2.cpp | 18 +- src/Functions/isDecimalOverflow.cpp | 9 +- src/Functions/isFinite.cpp | 15 +- src/Functions/isIPAddressContainedIn.cpp | 208 +- src/Functions/isInfinite.cpp | 15 +- src/Functions/isNaN.cpp | 12 +- src/Functions/isNotDistinctFrom.cpp | 2 +- src/Functions/isNotNull.cpp | 24 +- src/Functions/isNull.cpp | 18 +- src/Functions/isNullable.cpp | 7 +- src/Functions/isValidUTF8.cpp | 4 +- src/Functions/jsonMergePatch.cpp | 4 +- src/Functions/jumpConsistentHash.cpp | 3 +- .../keyvaluepair/ArgumentExtractor.cpp | 2 +- .../keyvaluepair/ArgumentExtractor.h | 6 +- src/Functions/keyvaluepair/CMakeLists.txt | 6 +- .../impl/CHKeyValuePairExtractor.h | 93 +- .../impl/DuplicateKeyFoundException.h | 20 + .../keyvaluepair/impl/KeyValuePairExtractor.h | 20 - .../impl/KeyValuePairExtractorBuilder.cpp | 44 - .../impl/KeyValuePairExtractorBuilder.h | 32 +- .../keyvaluepair/impl/StateHandlerImpl.h | 361 +- ...test_escaping_key_value_pair_extractor.cpp | 5 +- .../tests/gtest_extractKeyValuePairs.cpp | 34 +- ...test_inline_escaping_key_state_handler.cpp | 5 +- .../gtest_no_escaping_key_state_handler.cpp | 7 +- src/Functions/lcm.cpp | 18 +- src/Functions/least.cpp | 2 +- src/Functions/lemmatize.cpp | 7 +- src/Functions/less.cpp | 5 +- src/Functions/lessOrEquals.cpp | 8 +- src/Functions/lgamma.cpp | 3 + src/Functions/locate.cpp | 6 +- src/Functions/log.cpp | 14 +- src/Functions/lowerUTF8.cpp | 24 +- src/Functions/makeDate.cpp | 9 +- src/Functions/map.cpp | 113 +- src/Functions/materialize.cpp | 2 +- src/Functions/materialize.h | 23 +- src/Functions/max2.cpp | 15 +- src/Functions/min2.cpp | 16 +- src/Functions/minSampleSize.cpp | 8 +- src/Functions/minus.cpp | 23 +- src/Functions/modulo.cpp | 51 +- src/Functions/moduloOrNull.cpp | 38 + src/Functions/moduloOrZero.cpp | 16 +- src/Functions/monthName.cpp | 7 +- src/Functions/mortonDecode.cpp | 2 +- src/Functions/mortonEncode.cpp | 2 +- src/Functions/multiIf.cpp | 26 +- src/Functions/multiMatchAny.cpp | 7 +- src/Functions/multiply.cpp | 17 +- src/Functions/multiplyDecimal.cpp | 71 +- src/Functions/negate.cpp | 57 +- src/Functions/neighbor.cpp | 84 +- src/Functions/nested.cpp | 25 +- src/Functions/ngrams.cpp | 128 + src/Functions/normalizedQueryHash.cpp | 10 +- src/Functions/notEmpty.cpp | 25 - src/Functions/notEquals.cpp | 2 +- src/Functions/now.cpp | 17 +- src/Functions/now64.cpp | 12 +- src/Functions/nowInBlock.cpp | 6 +- src/Functions/nullIf.cpp | 4 +- src/Functions/overlay.cpp | 721 + src/Functions/padString.cpp | 9 +- src/Functions/parseDateTime.cpp | 801 +- src/Functions/parseReadableSize.cpp | 27 +- src/Functions/parseTimeDelta.cpp | 8 +- src/Functions/plus.cpp | 24 +- src/Functions/pointInEllipses.cpp | 27 +- src/Functions/pointInPolygon.cpp | 132 +- src/Functions/polygonArea.cpp | 5 + src/Functions/polygonPerimeter.cpp | 5 + src/Functions/polygonsDistance.cpp | 5 + src/Functions/polygonsEquals.cpp | 5 + src/Functions/polygonsWithin.cpp | 5 + src/Functions/printf.cpp | 56 +- src/Functions/punycode.cpp | 18 +- src/Functions/queryID.cpp | 1 + .../{canonicalRand.cpp => randCanonical.cpp} | 2 +- src/Functions/randDistribution.cpp | 34 +- src/Functions/randomPrintableASCII.cpp | 5 + src/Functions/randomString.cpp | 5 + src/Functions/randomStringUTF8.cpp | 5 + src/Functions/readWkt.cpp | 4 +- src/Functions/regexpExtract.cpp | 7 +- src/Functions/regexpQuoteMeta.cpp | 5 + src/Functions/reinterpretAs.cpp | 21 +- src/Functions/repeat.cpp | 28 +- src/Functions/reverse.cpp | 17 +- src/Functions/runningAccumulate.cpp | 7 +- src/Functions/runningConcurrency.cpp | 3 +- src/Functions/runningDifference.h | 9 +- src/Functions/s2CapContains.cpp | 5 + src/Functions/s2CellsIntersect.cpp | 5 + src/Functions/s2RectContains.cpp | 5 + src/Functions/seriesDecomposeSTL.cpp | 8 +- src/Functions/seriesOutliersDetectTukey.cpp | 20 +- src/Functions/seriesPeriodDetectFFT.cpp | 17 +- src/Functions/serverConstants.cpp | 7 +- src/Functions/sigmoid.cpp | 16 +- src/Functions/sign.cpp | 4 +- src/Functions/sin.cpp | 2 +- src/Functions/sleep.cpp | 172 +- src/Functions/sleep.h | 157 - src/Functions/sleepEachRow.cpp | 13 - src/Functions/snowflake.cpp | 27 +- src/Functions/snowflakeIDToDateTime.cpp | 22 +- src/Functions/soundex.cpp | 2 +- src/Functions/space.cpp | 6 +- src/Functions/sparseGrams.cpp | 358 + src/Functions/splitByRegexp.cpp | 6 +- src/Functions/sqid.cpp | 20 +- src/Functions/startsWithUTF8.cpp | 2 +- src/Functions/stem.cpp | 7 +- src/Functions/stl.hpp | 60 +- src/Functions/stringBytes.h | 83 + src/Functions/stringBytesEntropy.cpp | 82 + src/Functions/stringBytesUniq.cpp | 73 + src/Functions/stringCutToZero.cpp | 17 +- src/Functions/stringToH3.cpp | 5 + src/Functions/structureToFormatSchema.cpp | 12 +- src/Functions/substring.cpp | 27 +- src/Functions/substringIndex.cpp | 39 +- src/Functions/svg.cpp | 7 +- src/Functions/synonyms.cpp | 7 +- src/Functions/tanh.cpp | 15 +- src/Functions/tests/gtest_has_all.cpp | 3 +- src/Functions/throwIf.cpp | 23 +- src/Functions/tid.cpp | 5 + src/Functions/timeSlots.cpp | 69 +- src/Functions/timestamp.cpp | 2 +- src/Functions/timezoneOf.cpp | 8 +- src/Functions/toBool.cpp | 4 +- src/Functions/toColumnTypeName.cpp | 1 + src/Functions/toDaysSinceYearZero.cpp | 2 +- src/Functions/toDecimalString.cpp | 7 +- src/Functions/toInterval.cpp | 85 + src/Functions/toJSONString.cpp | 5 + src/Functions/toLowCardinality.cpp | 10 +- src/Functions/toMillisecond.cpp | 2 +- src/Functions/toMonthNumSinceEpoch.cpp | 25 + src/Functions/toStartOfInterval.cpp | 271 +- src/Functions/toTimezone.cpp | 7 +- src/Functions/toTypeName.cpp | 7 +- src/Functions/toUnixTimestamp64.cpp | 13 + src/Functions/toYearNumSinceEpoch.cpp | 25 + src/Functions/tokenExtractors.cpp | 158 - src/Functions/tokens.cpp | 140 + src/Functions/transform.cpp | 116 +- src/Functions/translate.cpp | 134 +- src/Functions/trim.cpp | 157 +- src/Functions/tryBase32Decode.cpp | 35 + src/Functions/tryBase58Decode.cpp | 6 +- src/Functions/tryBase64Decode.cpp | 15 +- src/Functions/tryBase64URLDecode.cpp | 15 +- src/Functions/tryDecrypt.cpp | 2 +- src/Functions/tuple.cpp | 10 +- src/Functions/tuple.h | 3 +- src/Functions/tupleElement.cpp | 30 +- src/Functions/tupleHammingDistance.cpp | 4 +- src/Functions/tupleNames.cpp | 3 +- src/Functions/upperUTF8.cpp | 23 +- src/Functions/variantElement.cpp | 5 +- src/Functions/variantType.cpp | 2 +- src/Functions/vectorFunctions.cpp | 58 +- src/Functions/visibleWidth.cpp | 9 +- src/Functions/visitParamExtractBool.cpp | 2 +- src/Functions/visitParamExtractFloat.cpp | 2 +- src/Functions/visitParamExtractInt.cpp | 2 +- src/Functions/visitParamExtractRaw.cpp | 2 +- src/Functions/visitParamExtractString.cpp | 2 +- src/Functions/visitParamExtractUInt.cpp | 2 +- src/Functions/visitParamHas.cpp | 2 +- src/Functions/widthBucket.cpp | 24 +- src/Functions/wkt.cpp | 5 + src/IO/Archives/IArchiveWriter.h | 2 + src/IO/Archives/LibArchiveWriter.cpp | 17 +- src/IO/Archives/LibArchiveWriter.h | 2 + src/IO/Archives/ZipArchiveReader.cpp | 3 + src/IO/Archives/ZipArchiveWriter.cpp | 50 +- src/IO/Archives/ZipArchiveWriter.h | 2 + src/IO/Archives/createArchiveReader.cpp | 5 +- src/IO/Ask.cpp | 44 + src/IO/Ask.h | 17 + src/IO/AsyncReadCounters.cpp | 3 + src/IO/AsyncReadCounters.h | 6 +- src/IO/AsynchronousReadBufferFromFile.cpp | 3 +- ...ynchronousReadBufferFromFileDescriptor.cpp | 3 +- src/IO/AutoFinalizedWriteBuffer.h | 34 + .../copyAzureBlobStorageFile.cpp | 42 +- .../copyAzureBlobStorageFile.h | 6 +- src/IO/BrotliReadBuffer.cpp | 9 +- src/IO/BufferBase.h | 1 - src/IO/BufferWithOwnMemory.h | 24 +- src/IO/Bzip2WriteBuffer.cpp | 1 - src/IO/CachedInMemoryReadBufferFromFile.cpp | 110 +- src/IO/CachedInMemoryReadBufferFromFile.h | 25 +- src/IO/CascadeWriteBuffer.cpp | 34 +- src/IO/CascadeWriteBuffer.h | 7 +- src/IO/CompressionMethod.cpp | 19 +- ...uffer.cpp => ConcatReadBufferFromFile.cpp} | 43 +- ...eadBuffer.h => ConcatReadBufferFromFile.h} | 22 +- src/IO/ConnectionTimeouts.cpp | 73 +- src/IO/ConnectionTimeouts.h | 2 +- src/IO/DistributedCacheLogMode.h | 15 + src/IO/DistributedCachePoolBehaviourOnLimit.h | 14 + src/IO/DistributedCacheSettings.h | 37 + src/IO/FileEncryptionCommon.cpp | 78 +- src/IO/ForkWriteBuffer.cpp | 10 +- src/IO/ForkWriteBuffer.h | 10 +- src/IO/HTTPCommon.cpp | 20 +- src/IO/HTTPCommon.h | 4 +- src/IO/HTTPHeaderEntries.h | 2 + src/IO/HadoopSnappyReadBuffer.cpp | 2 +- src/IO/HashingWriteBuffer.h | 5 +- src/IO/LZMAInflatingReadBuffer.cpp | 16 +- src/IO/LimitReadBuffer.cpp | 63 +- src/IO/LimitReadBuffer.h | 28 +- src/IO/LimitSeekableReadBuffer.cpp | 1 + src/IO/MMappedFileCache.cpp | 17 + src/IO/MMappedFileCache.h | 11 +- src/IO/MemoryReadWriteBuffer.cpp | 2 +- src/IO/MemoryReadWriteBuffer.h | 7 - src/IO/MySQLBinlogEventReadBuffer.cpp | 1 + src/IO/MySQLPacketPayloadReadBuffer.cpp | 1 + src/IO/MySQLPacketPayloadWriteBuffer.cpp | 1 + src/IO/NetUtils.h | 58 + src/IO/NullWriteBuffer.cpp | 9 +- src/IO/NullWriteBuffer.h | 6 +- src/IO/Operators.h | 5 - src/IO/Operators_pcg_random.h | 64 + src/IO/Progress.cpp | 54 +- src/IO/Progress.h | 2 + ...tobufZeroCopyInputStreamFromReadBuffer.cpp | 2 + ...bufZeroCopyOutputStreamFromWriteBuffer.cpp | 4 +- ...tobufZeroCopyOutputStreamFromWriteBuffer.h | 2 - src/IO/ReadBuffer.cpp | 20 +- src/IO/ReadBuffer.h | 26 +- src/IO/ReadBufferFromEncryptedFile.cpp | 151 +- src/IO/ReadBufferFromEncryptedFile.h | 25 +- src/IO/ReadBufferFromFile.cpp | 2 +- src/IO/ReadBufferFromFileBase.h | 2 + src/IO/ReadBufferFromFileDescriptor.cpp | 2 + src/IO/ReadBufferFromIStream.cpp | 19 +- src/IO/ReadBufferFromIStream.h | 20 +- src/IO/ReadBufferFromMemory.cpp | 64 +- src/IO/ReadBufferFromMemory.h | 88 +- src/IO/ReadBufferFromPocoSocket.cpp | 45 +- src/IO/ReadBufferFromPocoSocket.h | 21 +- src/IO/ReadBufferFromPocoSocketChunked.cpp | 165 + src/IO/ReadBufferFromPocoSocketChunked.h | 109 + src/IO/ReadBufferFromS3.cpp | 79 +- src/IO/ReadBufferFromS3.h | 14 +- src/IO/ReadHelpers.cpp | 71 +- src/IO/ReadHelpers.h | 151 +- src/IO/ReadMethod.h | 58 + src/IO/ReadSettings.cpp | 39 + src/IO/ReadSettings.h | 91 +- src/IO/ReadWriteBufferFromHTTP.cpp | 92 +- src/IO/ReadWriteBufferFromHTTP.h | 44 +- src/IO/S3/AWSLogger.cpp | 11 +- src/IO/S3/BlobStorageLogWriter.cpp | 7 + src/IO/S3/Client.cpp | 102 +- src/IO/S3/Client.h | 29 +- src/IO/S3/Credentials.cpp | 75 +- src/IO/S3/Credentials.h | 2 +- src/IO/S3/PocoHTTPClient.cpp | 153 +- src/IO/S3/PocoHTTPClient.h | 50 +- src/IO/S3/PocoHTTPClientFactory.cpp | 5 +- src/IO/S3/ReadBufferFromGetObjectResult.h | 34 + src/IO/S3/S3Capabilities.cpp | 47 + src/IO/S3/S3Capabilities.h | 48 + src/IO/S3/URI.cpp | 102 +- src/IO/S3/URI.h | 9 +- src/IO/S3/copyS3File.cpp | 138 +- src/IO/S3/copyS3File.h | 15 +- src/IO/S3/deleteFileFromS3.cpp | 274 + src/IO/S3/deleteFileFromS3.h | 51 + src/IO/S3/getObjectInfo.cpp | 15 +- src/IO/S3/tests/gtest_aws_s3_client.cpp | 17 +- src/IO/S3/tests/gtest_s3_uri.cpp | 41 + src/IO/S3AuthSettings.cpp | 232 + src/IO/S3AuthSettings.h | 72 + src/IO/S3Common.cpp | 308 +- src/IO/S3Common.h | 142 +- src/IO/S3RequestSettings.cpp | 293 + src/IO/S3RequestSettings.h | 84 + src/IO/S3Settings.cpp | 16 +- src/IO/S3Settings.h | 6 +- src/IO/SeekableReadBuffer.cpp | 25 + src/IO/SeekableReadBuffer.h | 20 +- src/IO/SharedThreadPools.cpp | 23 + src/IO/SharedThreadPools.h | 7 + src/IO/SnappyReadBuffer.cpp | 8 +- src/IO/SnappyWriteBuffer.cpp | 38 +- src/IO/SnappyWriteBuffer.h | 10 +- src/IO/StdStreamBufFromReadBuffer.cpp | 13 +- src/IO/UncompressedCache.cpp | 10 + src/IO/UncompressedCache.h | 10 +- src/IO/UseSSL.cpp | 24 - src/IO/UseSSL.h | 13 - src/IO/VarInt.h | 59 +- src/IO/WithFileName.cpp | 7 +- src/IO/WithFileSize.cpp | 8 +- src/IO/WriteBuffer.cpp | 100 +- src/IO/WriteBuffer.h | 141 +- src/IO/WriteBufferDecorator.h | 3 +- src/IO/WriteBufferFromArena.h | 5 +- src/IO/WriteBufferFromFile.cpp | 25 +- src/IO/WriteBufferFromFile.h | 8 +- src/IO/WriteBufferFromFileBase.h | 1 - src/IO/WriteBufferFromFileDecorator.cpp | 9 +- src/IO/WriteBufferFromFileDescriptor.cpp | 34 +- src/IO/WriteBufferFromFileDescriptor.h | 12 +- src/IO/WriteBufferFromHTTP.cpp | 31 +- src/IO/WriteBufferFromHTTP.h | 41 +- src/IO/WriteBufferFromPocoSocket.cpp | 16 +- src/IO/WriteBufferFromPocoSocket.h | 2 - src/IO/WriteBufferFromPocoSocketChunked.cpp | 199 + src/IO/WriteBufferFromPocoSocketChunked.h | 34 + src/IO/WriteBufferFromS3.cpp | 154 +- src/IO/WriteBufferFromS3.h | 11 +- src/IO/WriteBufferFromString.cpp | 34 + src/IO/WriteBufferFromString.h | 32 +- src/IO/WriteBufferFromVector.h | 23 +- src/IO/WriteBufferValidUTF8.cpp | 34 +- src/IO/WriteBufferValidUTF8.h | 2 +- src/IO/WriteHelpers.cpp | 49 +- src/IO/WriteHelpers.h | 188 +- src/IO/WriteSettings.cpp | 29 + src/IO/WriteSettings.h | 13 +- src/IO/ZlibInflatingReadBuffer.cpp | 20 +- src/IO/ZstdDeflatingAppendableWriteBuffer.cpp | 39 +- src/IO/ZstdDeflatingAppendableWriteBuffer.h | 4 +- src/IO/examples/CMakeLists.txt | 7 +- src/IO/examples/hadoop_snappy_read_buffer.cpp | 1 + src/IO/examples/io_operators.cpp | 4 + src/IO/examples/limit_read_buffer.cpp | 11 +- src/IO/examples/limit_read_buffer2.cpp | 10 +- .../examples/parse_date_time_best_effort.cpp | 3 + src/IO/examples/parse_int_perf.cpp | 5 +- src/IO/examples/read_buffer.cpp | 3 +- src/IO/examples/read_buffer_perf.cpp | 3 +- src/IO/examples/read_float_perf.cpp | 2 + src/IO/examples/var_uint.cpp | 58 - src/IO/examples/write_int.cpp | 2 + src/IO/examples/zstd_buffers.cpp | 36 +- src/IO/parseDateTimeBestEffort.cpp | 162 +- src/IO/readDecimalText.h | 63 +- src/IO/readFloatText.cpp | 9 + src/IO/readFloatText.h | 115 +- .../gtest_DateTime64_parsing_and_writing.cpp | 2 +- src/IO/tests/gtest_DateTimeToString.cpp | 56 +- .../tests/gtest_archive_reader_and_writer.cpp | 18 +- src/IO/tests/gtest_file_encryption.cpp | 6 +- src/IO/tests/gtest_hadoop_snappy_decoder.cpp | 7 +- src/IO/tests/gtest_peekable_read_buffer.cpp | 6 +- src/IO/tests/gtest_readbuffer_s3.cpp | 154 + src/IO/tests/gtest_s3_uri.cpp | 43 +- src/IO/tests/gtest_seek_backwards.cpp | 1 + src/IO/tests/gtest_writebuffer_s3.cpp | 139 +- .../Access/InterpreterCheckGrantQuery.cpp | 43 + .../Access/InterpreterCheckGrantQuery.h | 21 + .../Access/InterpreterCreateQuotaQuery.cpp | 2 +- .../Access/InterpreterCreateRoleQuery.cpp | 35 +- .../InterpreterCreateRowPolicyQuery.cpp | 5 +- .../InterpreterCreateSettingsProfileQuery.cpp | 24 +- .../Access/InterpreterCreateUserQuery.cpp | 230 +- .../Access/InterpreterCreateUserQuery.h | 7 +- .../InterpreterDropAccessEntityQuery.cpp | 6 +- .../Access/InterpreterGrantQuery.cpp | 13 +- .../InterpreterMoveAccessEntityQuery.cpp | 14 +- .../Access/InterpreterSetRoleQuery.cpp | 4 +- .../InterpreterShowAccessEntitiesQuery.cpp | 1 - .../Access/InterpreterShowAccessQuery.cpp | 4 +- ...InterpreterShowCreateAccessEntityQuery.cpp | 38 +- .../Access/InterpreterShowGrantsQuery.cpp | 112 +- .../Access/InterpreterShowGrantsQuery.h | 2 +- .../Access/getValidUntilFromAST.cpp | 37 + .../Access/getValidUntilFromAST.h | 9 + src/Interpreters/ActionLocksManager.cpp | 2 + src/Interpreters/ActionsDAG.cpp | 663 +- src/Interpreters/ActionsDAG.h | 62 +- src/Interpreters/ActionsMatcher.cpp | 22 + src/Interpreters/ActionsMatcher.h | 208 + src/Interpreters/ActionsVisitor.cpp | 176 +- src/Interpreters/ActionsVisitor.h | 216 +- src/Interpreters/AddDefaultDatabaseVisitor.h | 7 +- .../AddIndexConstraintsOptimizer.cpp | 4 +- .../AddIndexConstraintsOptimizer.h | 11 +- src/Interpreters/AggregateDescription.cpp | 81 + src/Interpreters/AggregateDescription.h | 4 + .../AggregateFunctionOfGroupByKeysVisitor.h | 3 - src/Interpreters/AggregatedData.h | 4 + src/Interpreters/AggregatedDataVariants.cpp | 33 +- src/Interpreters/AggregatedDataVariants.h | 9 +- src/Interpreters/AggregationCommon.h | 43 +- src/Interpreters/AggregationMethod.cpp | 1 + src/Interpreters/AggregationMethod.h | 8 - src/Interpreters/AggregationUtils.cpp | 3 +- src/Interpreters/Aggregator.cpp | 471 +- src/Interpreters/Aggregator.h | 173 +- src/Interpreters/ApplyWithAliasVisitor.h | 3 +- src/Interpreters/ApplyWithGlobalVisitor.h | 4 +- src/Interpreters/ApplyWithSubqueryVisitor.cpp | 48 +- src/Interpreters/ApplyWithSubqueryVisitor.h | 24 +- .../ArithmeticOperationsInAgrFuncOptimize.cpp | 5 +- .../ArithmeticOperationsInAgrFuncOptimize.h | 2 +- src/Interpreters/ArrayJoin.h | 13 + src/Interpreters/ArrayJoinAction.cpp | 87 +- src/Interpreters/ArrayJoinAction.h | 10 +- src/Interpreters/AsynchronousInsertLog.cpp | 2 +- src/Interpreters/AsynchronousInsertQueue.cpp | 369 +- src/Interpreters/AsynchronousInsertQueue.h | 9 +- src/Interpreters/BackupLog.cpp | 9 +- src/Interpreters/BlobStorageLog.cpp | 5 + src/Interpreters/BloomFilter.cpp | 8 +- src/Interpreters/BloomFilter.h | 11 +- src/Interpreters/BloomFilterHash.h | 102 +- src/Interpreters/CNFQueryAtomicFormula.cpp | 22 + src/Interpreters/CNFQueryAtomicFormula.h | 21 + src/Interpreters/Cache/EvictionCandidates.cpp | 134 +- src/Interpreters/Cache/EvictionCandidates.h | 36 +- src/Interpreters/Cache/FileCache.cpp | 410 +- src/Interpreters/Cache/FileCache.h | 32 +- src/Interpreters/Cache/FileCacheFactory.cpp | 56 +- src/Interpreters/Cache/FileCacheFactory.h | 8 +- src/Interpreters/Cache/FileCacheKey.cpp | 16 +- src/Interpreters/Cache/FileCacheKey.h | 10 +- src/Interpreters/Cache/FileCacheSettings.cpp | 300 +- src/Interpreters/Cache/FileCacheSettings.h | 71 +- src/Interpreters/Cache/FileCacheUtils.h | 21 + src/Interpreters/Cache/FileCache_fwd.h | 8 +- src/Interpreters/Cache/FileSegment.cpp | 351 +- src/Interpreters/Cache/FileSegment.h | 36 +- src/Interpreters/Cache/FileSegmentInfo.h | 12 +- src/Interpreters/Cache/IFileCachePriority.cpp | 2 +- src/Interpreters/Cache/IFileCachePriority.h | 10 +- .../Cache/LRUFileCachePriority.cpp | 28 +- src/Interpreters/Cache/LRUFileCachePriority.h | 4 +- src/Interpreters/Cache/Metadata.cpp | 128 +- src/Interpreters/Cache/Metadata.h | 3 +- .../Cache/QueryConditionCache.cpp | 135 + src/Interpreters/Cache/QueryConditionCache.h | 104 + src/Interpreters/Cache/QueryLimit.cpp | 3 +- .../{QueryCache.cpp => QueryResultCache.cpp} | 211 +- .../{QueryCache.h => QueryResultCache.h} | 206 +- .../Cache/QueryResultCacheUsage.h | 16 + .../Cache/SLRUFileCachePriority.cpp | 23 +- .../Cache/SLRUFileCachePriority.h | 5 + .../Cache/WriteBufferToFileSegment.cpp | 22 +- .../Cache/WriteBufferToFileSegment.h | 14 +- src/Interpreters/CancellationChecker.cpp | 146 + src/Interpreters/CancellationChecker.h | 74 + src/Interpreters/ClientInfo.cpp | 110 +- src/Interpreters/ClientInfo.h | 39 +- src/Interpreters/Cluster.cpp | 276 +- src/Interpreters/Cluster.h | 16 +- src/Interpreters/ClusterDiscovery.cpp | 333 +- src/Interpreters/ClusterDiscovery.h | 66 +- .../ClusterProxy/SelectStreamFactory.cpp | 60 +- .../ClusterProxy/SelectStreamFactory.h | 10 +- .../ClusterProxy/executeQuery.cpp | 685 +- src/Interpreters/ClusterProxy/executeQuery.h | 30 +- src/Interpreters/CollectJoinOnKeysVisitor.cpp | 11 +- src/Interpreters/CollectJoinOnKeysVisitor.h | 3 +- src/Interpreters/ComparisonGraph.cpp | 164 +- src/Interpreters/ComparisonGraph.h | 138 +- .../ComparisonTupleEliminationVisitor.cpp | 2 +- src/Interpreters/ConcurrentHashJoin.cpp | 480 +- src/Interpreters/ConcurrentHashJoin.h | 72 +- src/Interpreters/Context.cpp | 1338 +- src/Interpreters/Context.h | 259 +- src/Interpreters/CrossToInnerJoinVisitor.cpp | 11 +- .../DDLOnClusterQueryStatusSource.cpp | 164 + .../DDLOnClusterQueryStatusSource.h | 33 + src/Interpreters/DDLTask.cpp | 108 +- src/Interpreters/DDLTask.h | 39 +- src/Interpreters/DDLWorker.cpp | 282 +- src/Interpreters/DDLWorker.h | 54 +- src/Interpreters/DNSCacheUpdater.cpp | 1 + src/Interpreters/DNSCacheUpdater.h | 4 +- .../DatabaseAndTableWithAlias.cpp | 1 + src/Interpreters/DatabaseCatalog.cpp | 348 +- src/Interpreters/DatabaseCatalog.h | 21 +- .../DistributedQueryStatusSource.cpp | 279 + .../DistributedQueryStatusSource.h | 82 + src/Interpreters/DuplicateOrderByVisitor.h | 2 +- src/Interpreters/EmbeddedDictionaries.cpp | 8 +- src/Interpreters/ErrorLog.cpp | 9 +- src/Interpreters/ErrorLog.h | 1 - .../ExecuteScalarSubqueriesVisitor.cpp | 38 +- .../ExecuteScalarSubqueriesVisitor.h | 3 +- src/Interpreters/ExpressionActions.cpp | 46 +- src/Interpreters/ExpressionActions.h | 238 +- .../ExpressionActionsSettings.cpp | 42 +- src/Interpreters/ExpressionActionsSettings.h | 18 +- src/Interpreters/ExpressionAnalyzer.cpp | 244 +- src/Interpreters/ExpressionAnalyzer.h | 36 +- src/Interpreters/ExpressionJIT.cpp | 4 +- .../ExternalDictionariesLoader.cpp | 8 +- src/Interpreters/ExternalLoader.cpp | 37 +- .../ExternalLoaderXMLConfigRepository.cpp | 4 +- .../ExternalLoaderXMLConfigRepository.h | 2 +- src/Interpreters/FilesystemCacheLog.cpp | 7 +- src/Interpreters/FillingRow.cpp | 278 +- src/Interpreters/FillingRow.h | 25 +- src/Interpreters/FullSortingMergeJoin.h | 2 +- src/Interpreters/FunctionNameNormalizer.cpp | 9 + .../GatherFunctionQuantileVisitor.cpp | 4 +- src/Interpreters/GetAggregatesVisitor.cpp | 2 +- src/Interpreters/GinFilter.cpp | 4 +- src/Interpreters/GinFilter.h | 12 + src/Interpreters/GlobalSubqueriesVisitor.h | 25 +- src/Interpreters/GraceHashJoin.cpp | 139 +- src/Interpreters/GraceHashJoin.h | 17 +- src/Interpreters/GroupByFunctionKeysVisitor.h | 4 - .../GroupingSetsRewriterVisitor.h | 2 +- src/Interpreters/HashJoin/AddedColumns.cpp | 179 +- src/Interpreters/HashJoin/AddedColumns.h | 85 +- src/Interpreters/HashJoin/FullHashJoin.cpp | 2 +- src/Interpreters/HashJoin/HashJoin.cpp | 762 +- src/Interpreters/HashJoin/HashJoin.h | 214 +- src/Interpreters/HashJoin/HashJoinMethods.h | 66 +- .../HashJoin/HashJoinMethodsImpl.h | 325 +- src/Interpreters/HashJoin/JoinFeatures.h | 14 + src/Interpreters/HashJoin/JoinUsedFlags.h | 8 +- src/Interpreters/HashJoin/KeyGetter.h | 29 + .../{KnowRowsHolder.h => KnownRowsHolder.h} | 10 +- src/Interpreters/HashJoin/RightHashJoin.cpp | 2 +- src/Interpreters/HashJoin/ScatteredBlock.h | 341 + src/Interpreters/HashTablesStatistics.cpp | 92 +- src/Interpreters/HashTablesStatistics.h | 53 +- src/Interpreters/IInterpreter.cpp | 15 +- .../IInterpreterUnionOrSelectQuery.cpp | 80 +- .../IInterpreterUnionOrSelectQuery.h | 1 + src/Interpreters/IJoin.h | 21 +- src/Interpreters/IKeyValueEntity.h | 3 +- src/Interpreters/ITokenExtractor.cpp | 59 +- src/Interpreters/ITokenExtractor.h | 29 +- src/Interpreters/IdentifierSemantic.cpp | 15 +- .../InJoinSubqueriesPreprocessor.cpp | 11 +- src/Interpreters/InternalTextLogsQueue.h | 3 +- src/Interpreters/InterpreterAlterQuery.cpp | 77 +- src/Interpreters/InterpreterBackupQuery.cpp | 21 +- src/Interpreters/InterpreterCheckQuery.cpp | 46 +- src/Interpreters/InterpreterCheckQuery.h | 1 - .../InterpreterCreateIndexQuery.cpp | 22 +- src/Interpreters/InterpreterCreateQuery.cpp | 1082 +- src/Interpreters/InterpreterCreateQuery.h | 15 +- .../InterpreterCreateResourceQuery.cpp | 68 + .../InterpreterCreateResourceQuery.h | 25 + .../InterpreterCreateWorkloadQuery.cpp | 68 + .../InterpreterCreateWorkloadQuery.h | 25 + src/Interpreters/InterpreterDeleteQuery.cpp | 145 +- .../InterpreterDescribeCacheQuery.cpp | 45 +- src/Interpreters/InterpreterDescribeQuery.cpp | 57 +- .../InterpreterDropIndexQuery.cpp | 8 +- src/Interpreters/InterpreterDropQuery.cpp | 354 +- .../InterpreterDropResourceQuery.cpp | 60 + .../InterpreterDropResourceQuery.h | 21 + .../InterpreterDropWorkloadQuery.cpp | 60 + .../InterpreterDropWorkloadQuery.h | 21 + src/Interpreters/InterpreterExplainQuery.cpp | 151 +- src/Interpreters/InterpreterExplainQuery.h | 11 +- .../InterpreterExternalDDLQuery.cpp | 18 +- src/Interpreters/InterpreterFactory.cpp | 42 +- src/Interpreters/InterpreterFactory.h | 1 + src/Interpreters/InterpreterInsertQuery.cpp | 456 +- src/Interpreters/InterpreterInsertQuery.h | 24 +- .../InterpreterKillQueryQuery.cpp | 20 +- .../InterpreterParallelWithQuery.cpp | 179 + .../InterpreterParallelWithQuery.h | 33 + src/Interpreters/InterpreterRenameQuery.cpp | 96 +- .../InterpreterSelectIntersectExceptQuery.cpp | 37 +- src/Interpreters/InterpreterSelectQuery.cpp | 618 +- src/Interpreters/InterpreterSelectQuery.h | 6 +- .../InterpreterSelectQueryAnalyzer.cpp | 53 +- .../InterpreterSelectQueryAnalyzer.h | 7 + .../InterpreterSelectWithUnionQuery.cpp | 94 +- .../InterpreterSelectWithUnionQuery.h | 3 + src/Interpreters/InterpreterSetQuery.cpp | 98 +- src/Interpreters/InterpreterSetQuery.h | 2 +- .../InterpreterShowColumnsQuery.cpp | 14 +- .../InterpreterShowCreateQuery.cpp | 14 +- .../InterpreterShowIndexesQuery.cpp | 3 +- .../InterpreterShowTablesQuery.cpp | 40 +- src/Interpreters/InterpreterSystemQuery.cpp | 404 +- src/Interpreters/InterpreterSystemQuery.h | 23 +- .../InterpreterTransactionControlQuery.cpp | 12 +- src/Interpreters/InterpreterUndropQuery.cpp | 3 +- src/Interpreters/InterpreterWatchQuery.cpp | 37 +- src/Interpreters/InterserverCredentials.cpp | 1 + src/Interpreters/InterserverCredentials.h | 2 +- src/Interpreters/InterserverIOHandler.h | 12 +- src/Interpreters/JIT/CHJIT.cpp | 73 +- src/Interpreters/JIT/CompileDAG.h | 2 +- src/Interpreters/JIT/compileFunction.cpp | 8 +- src/Interpreters/JoinInfo.cpp | 551 + src/Interpreters/JoinInfo.h | 262 + src/Interpreters/JoinSwitcher.h | 2 + src/Interpreters/JoinUtils.cpp | 16 +- src/Interpreters/JoinUtils.h | 6 +- src/Interpreters/JoinedTables.cpp | 19 +- src/Interpreters/KeysNullMap.h | 30 + src/Interpreters/LatencyLog.cpp | 73 + src/Interpreters/LatencyLog.h | 34 + src/Interpreters/Lemmatizers.cpp | 18 +- .../LogicalExpressionsOptimizer.cpp | 2 +- .../LogicalExpressionsOptimizer.h | 6 +- .../MarkTableIdentifiersVisitor.h | 2 +- src/Interpreters/MergeJoin.cpp | 39 +- src/Interpreters/MergeTreeTransaction.cpp | 6 +- src/Interpreters/MetricLog.cpp | 16 +- src/Interpreters/MetricLog.h | 1 - src/Interpreters/MutationsInterpreter.cpp | 273 +- src/Interpreters/MutationsInterpreter.h | 32 +- .../MutationsNonDeterministicHelpers.cpp | 30 +- .../MySQL/InterpretersMySQLDDLQuery.cpp | 15 +- .../MySQL/InterpretersMySQLDDLQuery.h | 3 +- .../MySQL/tests/gtest_create_rewritten.cpp | 95 +- .../NormalizeSelectWithUnionQueryVisitor.cpp | 4 +- src/Interpreters/NullableUtils.h | 14 - src/Interpreters/ObjectStorageQueueLog.cpp | 1 + src/Interpreters/OpenTelemetrySpanLog.cpp | 1 + src/Interpreters/OpenTelemetrySpanLog.h | 3 + ...OrDateTimeConverterWithPreimageVisitor.cpp | 53 +- src/Interpreters/OptimizeIfChains.cpp | 16 +- ...OptimizeIfWithConstantConditionVisitor.cpp | 14 +- .../OptimizeShardingKeyRewriteInVisitor.cpp | 3 +- src/Interpreters/PartLog.cpp | 32 +- src/Interpreters/PartLog.h | 14 +- src/Interpreters/PeriodicLog.cpp | 24 +- src/Interpreters/PeriodicLog.h | 18 +- .../PredicateExpressionsOptimizer.cpp | 12 +- src/Interpreters/PredicateRewriteVisitor.cpp | 4 +- src/Interpreters/PredicateRewriteVisitor.h | 4 +- src/Interpreters/PreparedSets.cpp | 173 +- src/Interpreters/PreparedSets.h | 63 +- src/Interpreters/ProcessList.cpp | 273 +- src/Interpreters/ProcessList.h | 108 +- src/Interpreters/ProcessorsProfileLog.cpp | 73 +- src/Interpreters/ProcessorsProfileLog.h | 7 +- src/Interpreters/ProfileEventsExt.cpp | 40 +- src/Interpreters/ProfileEventsExt.h | 7 +- src/Interpreters/QueryAliasesVisitor.cpp | 5 +- src/Interpreters/QueryFlags.h | 12 + src/Interpreters/QueryLog.cpp | 45 +- src/Interpreters/QueryLog.h | 112 +- src/Interpreters/QueryLogElement.h | 116 + src/Interpreters/QueryMetricLog.cpp | 286 + src/Interpreters/QueryMetricLog.h | 96 + src/Interpreters/QueryNormalizer.cpp | 8 +- src/Interpreters/QueryNormalizer.h | 18 +- src/Interpreters/QueryPriorities.h | 25 +- src/Interpreters/QueryThreadLog.cpp | 3 + src/Interpreters/QueryViewsLog.cpp | 7 +- src/Interpreters/QueryViewsLog.h | 8 +- .../RedundantFunctionsInOrderByVisitor.h | 2 - .../RemoveInjectiveFunctionsVisitor.h | 2 +- .../ReplaceQueryParameterVisitor.cpp | 10 + .../ReplaceQueryParameterVisitor.h | 1 + .../ReplicatedDatabaseQueryStatusSource.cpp | 160 + .../ReplicatedDatabaseQueryStatusSource.h | 32 + .../RewriteArrayExistsFunctionVisitor.cpp | 23 +- .../RewriteArrayExistsFunctionVisitor.h | 2 +- .../RewriteCountDistinctVisitor.cpp | 1 - .../RewriteCountDistinctVisitor.h | 3 +- .../RewriteCountVariantsVisitor.cpp | 6 +- .../RewriteCountVariantsVisitor.h | 2 +- ...writeSumFunctionWithSumAndCountVisitor.cpp | 16 +- .../RewriteUniqToCountVisitor.cpp | 39 +- src/Interpreters/RewriteUniqToCountVisitor.h | 2 +- src/Interpreters/RowRefs.cpp | 15 +- src/Interpreters/RowRefs.h | 27 +- .../SelectIntersectExceptQueryVisitor.h | 2 +- src/Interpreters/SelectQueryOptions.h | 21 +- .../ServerAsynchronousMetrics.cpp | 92 +- src/Interpreters/ServerAsynchronousMetrics.h | 18 +- src/Interpreters/Session.cpp | 228 +- src/Interpreters/Session.h | 16 +- src/Interpreters/SessionLog.cpp | 38 +- src/Interpreters/SessionLog.h | 15 +- src/Interpreters/SessionTracker.cpp | 1 + src/Interpreters/SessionTracker.h | 7 +- src/Interpreters/Set.cpp | 145 +- src/Interpreters/Set.h | 30 +- src/Interpreters/SetSerialization.h | 32 + src/Interpreters/SetVariants.h | 3 +- src/Interpreters/SetWithState.h | 39 + src/Interpreters/SortedBlocksWriter.cpp | 114 +- src/Interpreters/SortedBlocksWriter.h | 28 +- src/Interpreters/Squashing.cpp | 43 +- src/Interpreters/Squashing.h | 2 +- src/Interpreters/StorageID.cpp | 27 +- src/Interpreters/StorageID.h | 26 +- src/Interpreters/SubstituteColumnOptimizer.h | 5 - src/Interpreters/SystemLog.cpp | 208 +- src/Interpreters/SystemLog.h | 18 +- src/Interpreters/TableJoin.cpp | 253 +- src/Interpreters/TableJoin.h | 116 +- src/Interpreters/TemporaryDataOnDisk.cpp | 505 +- src/Interpreters/TemporaryDataOnDisk.h | 224 +- .../TemporaryReplaceTableName.cpp | 26 + src/Interpreters/TemporaryReplaceTableName.h | 18 + src/Interpreters/TextLog.cpp | 3 +- src/Interpreters/ThreadStatusExt.cpp | 172 +- src/Interpreters/TraceCollector.cpp | 44 +- src/Interpreters/TraceCollector.h | 2 +- src/Interpreters/TraceLog.cpp | 123 +- src/Interpreters/TraceLog.h | 14 +- src/Interpreters/TransactionLog.cpp | 19 +- src/Interpreters/TransactionsInfoLog.cpp | 1 + src/Interpreters/TransposedMetricLog.cpp | 485 + src/Interpreters/TransposedMetricLog.h | 92 + src/Interpreters/TreeCNFConverter.cpp | 24 +- src/Interpreters/TreeCNFConverter.h | 33 +- src/Interpreters/TreeOptimizer.cpp | 66 +- src/Interpreters/TreeRewriter.cpp | 132 +- src/Interpreters/TreeRewriter.h | 2 - .../WhereConstraintsOptimizer.cpp | 11 +- src/Interpreters/WindowDescription.cpp | 6 +- src/Interpreters/ZooKeeperLog.cpp | 4 +- src/Interpreters/ZooKeeperLog.h | 9 +- src/Interpreters/addMissingDefaults.cpp | 4 +- src/Interpreters/addMissingDefaults.h | 8 +- src/Interpreters/applyTableOverride.h | 5 - src/Interpreters/castColumn.cpp | 6 +- src/Interpreters/convertFieldToType.cpp | 172 +- src/Interpreters/convertFieldToType.h | 10 +- src/Interpreters/createBlockSelector.h | 2 +- .../createSubcolumnsExtractionActions.cpp | 57 + .../createSubcolumnsExtractionActions.h | 12 + .../evaluateConstantExpression.cpp | 145 +- src/Interpreters/evaluateConstantExpression.h | 6 +- src/Interpreters/examples/CMakeLists.txt | 24 +- src/Interpreters/examples/hash_map_string.cpp | 3 +- .../examples/hash_map_string_3.cpp | 3 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 540 +- src/Interpreters/executeDDLQueryOnCluster.h | 5 +- src/Interpreters/executeQuery.cpp | 1104 +- src/Interpreters/executeQuery.h | 29 +- .../formatWithPossiblyHidingSecrets.cpp | 32 + .../formatWithPossiblyHidingSecrets.h | 17 +- src/Interpreters/fuzzers/CMakeLists.txt | 2 +- .../fuzzers/execute_query_fuzzer.cpp | 42 +- src/Interpreters/getColumnFromBlock.cpp | 11 +- src/Interpreters/getColumnFromBlock.h | 3 +- .../getCustomKeyFilterForParallelReplicas.cpp | 33 +- .../getCustomKeyFilterForParallelReplicas.h | 2 +- .../getHeaderForProcessingStage.cpp | 8 +- src/Interpreters/inplaceBlockConversions.cpp | 17 +- src/Interpreters/interpretSubquery.cpp | 16 +- src/Interpreters/interpretSubquery.h | 2 +- src/Interpreters/joinDispatch.h | 4 +- src/Interpreters/loadMetadata.cpp | 201 +- src/Interpreters/loadMetadata.h | 6 +- .../parseColumnsListForTableFunction.cpp | 79 +- .../parseColumnsListForTableFunction.h | 6 +- .../processColumnTransformers.cpp | 40 +- src/Interpreters/processColumnTransformers.h | 8 + src/Interpreters/registerInterpreters.cpp | 12 + .../removeOnClusterClauseIfNeeded.cpp | 13 +- .../replaceForPositionalArguments.cpp | 10 +- src/Interpreters/sortBlock.cpp | 69 +- src/Interpreters/sortBlock.h | 11 +- .../tests/gtest_actions_visitor.cpp | 1 + .../tests/gtest_comparison_graph.cpp | 3 +- .../tests/gtest_cycle_aliases.cpp | 1 - src/Interpreters/tests/gtest_filecache.cpp | 293 +- src/Interpreters/tests/gtest_page_cache.cpp | 269 - .../tests/gtest_table_overrides.cpp | 5 +- src/Loggers/Loggers.cpp | 2 +- src/Loggers/OwnJSONPatternFormatter.cpp | 22 + src/Loggers/OwnJSONPatternFormatter.h | 1 + src/Loggers/OwnSplitChannel.cpp | 8 +- src/Loggers/OwnSplitChannel.h | 2 +- src/NOTICE | 29 - src/Parsers/ASTAlterNamedCollectionQuery.cpp | 28 +- src/Parsers/ASTAlterNamedCollectionQuery.h | 5 +- src/Parsers/ASTAlterQuery.cpp | 367 +- src/Parsers/ASTAlterQuery.h | 20 +- src/Parsers/ASTAssignment.h | 18 +- src/Parsers/ASTAsterisk.cpp | 10 +- src/Parsers/ASTAsterisk.h | 3 +- src/Parsers/ASTBackupQuery.cpp | 115 +- src/Parsers/ASTBackupQuery.h | 2 +- src/Parsers/ASTCheckQuery.h | 25 +- src/Parsers/ASTCollation.cpp | 7 +- src/Parsers/ASTCollation.h | 3 +- src/Parsers/ASTColumnDeclaration.cpp | 70 +- src/Parsers/ASTColumnDeclaration.h | 5 +- src/Parsers/ASTColumnsMatcher.cpp | 64 +- src/Parsers/ASTColumnsMatcher.h | 12 +- src/Parsers/ASTColumnsTransformers.cpp | 56 +- src/Parsers/ASTColumnsTransformers.h | 10 +- src/Parsers/ASTConstraintDeclaration.cpp | 8 +- src/Parsers/ASTConstraintDeclaration.h | 5 +- src/Parsers/ASTCreateFunctionQuery.cpp | 20 +- src/Parsers/ASTCreateFunctionQuery.h | 5 +- src/Parsers/ASTCreateIndexQuery.cpp | 24 +- src/Parsers/ASTCreateIndexQuery.h | 2 +- src/Parsers/ASTCreateNamedCollectionQuery.cpp | 23 +- src/Parsers/ASTCreateNamedCollectionQuery.h | 7 +- src/Parsers/ASTCreateQuery.cpp | 248 +- src/Parsers/ASTCreateQuery.h | 19 +- src/Parsers/ASTCreateResourceQuery.cpp | 96 + src/Parsers/ASTCreateResourceQuery.h | 87 + src/Parsers/ASTCreateWorkloadQuery.cpp | 95 + src/Parsers/ASTCreateWorkloadQuery.h | 54 + src/Parsers/ASTDataType.cpp | 30 +- src/Parsers/ASTDataType.h | 4 +- src/Parsers/ASTDatabaseOrNone.cpp | 6 +- src/Parsers/ASTDatabaseOrNone.h | 4 +- src/Parsers/ASTDeleteQuery.cpp | 30 +- src/Parsers/ASTDeleteQuery.h | 7 +- src/Parsers/ASTDescribeCacheQuery.cpp | 4 +- src/Parsers/ASTDescribeCacheQuery.h | 2 +- src/Parsers/ASTDictionary.cpp | 68 +- src/Parsers/ASTDictionary.h | 17 +- .../ASTDictionaryAttributeDeclaration.cpp | 24 +- .../ASTDictionaryAttributeDeclaration.h | 4 +- src/Parsers/ASTDropFunctionQuery.cpp | 12 +- src/Parsers/ASTDropFunctionQuery.h | 5 +- src/Parsers/ASTDropIndexQuery.cpp | 20 +- src/Parsers/ASTDropIndexQuery.h | 2 +- src/Parsers/ASTDropNamedCollectionQuery.cpp | 10 +- src/Parsers/ASTDropNamedCollectionQuery.h | 5 +- src/Parsers/ASTDropQuery.cpp | 68 +- src/Parsers/ASTDropQuery.h | 13 +- src/Parsers/ASTDropResourceQuery.cpp | 25 + src/Parsers/ASTDropResourceQuery.h | 29 + src/Parsers/ASTDropWorkloadQuery.cpp | 25 + src/Parsers/ASTDropWorkloadQuery.h | 29 + src/Parsers/ASTExplainQuery.h | 20 +- src/Parsers/ASTExpressionList.cpp | 33 +- src/Parsers/ASTExpressionList.h | 6 +- src/Parsers/ASTExternalDDLQuery.h | 15 +- src/Parsers/ASTFunction.cpp | 287 +- src/Parsers/ASTFunction.h | 4 +- .../ASTFunctionWithKeyValueArguments.cpp | 36 +- .../ASTFunctionWithKeyValueArguments.h | 12 +- src/Parsers/ASTIdentifier.cpp | 52 +- src/Parsers/ASTIdentifier.h | 4 +- src/Parsers/ASTIndexDeclaration.cpp | 24 +- src/Parsers/ASTIndexDeclaration.h | 4 +- src/Parsers/ASTInsertQuery.cpp | 47 +- src/Parsers/ASTInsertQuery.h | 4 +- src/Parsers/ASTInterpolateElement.cpp | 7 +- src/Parsers/ASTInterpolateElement.h | 2 +- src/Parsers/ASTKillQueryQuery.cpp | 22 +- src/Parsers/ASTKillQueryQuery.h | 2 +- src/Parsers/ASTLiteral.cpp | 24 +- src/Parsers/ASTLiteral.h | 2 +- src/Parsers/ASTNameTypePair.cpp | 10 +- src/Parsers/ASTNameTypePair.h | 2 +- src/Parsers/ASTObjectTypeArgument.cpp | 14 +- src/Parsers/ASTObjectTypeArgument.h | 2 +- src/Parsers/ASTOptimizeQuery.cpp | 28 +- src/Parsers/ASTOptimizeQuery.h | 2 +- src/Parsers/ASTOrderByElement.cpp | 45 +- src/Parsers/ASTOrderByElement.h | 26 +- src/Parsers/ASTParallelWithQuery.cpp | 38 + src/Parsers/ASTParallelWithQuery.h | 23 + src/Parsers/ASTPartition.cpp | 10 +- src/Parsers/ASTPartition.h | 2 +- src/Parsers/ASTProjectionDeclaration.cpp | 12 +- src/Parsers/ASTProjectionDeclaration.h | 4 +- src/Parsers/ASTProjectionSelectQuery.cpp | 22 +- src/Parsers/ASTProjectionSelectQuery.h | 2 +- src/Parsers/ASTQualifiedAsterisk.cpp | 8 +- src/Parsers/ASTQualifiedAsterisk.h | 3 +- src/Parsers/ASTQueryParameter.cpp | 11 +- src/Parsers/ASTQueryParameter.h | 4 +- src/Parsers/ASTQueryWithOnCluster.cpp | 7 +- src/Parsers/ASTQueryWithOnCluster.h | 2 +- src/Parsers/ASTQueryWithOutput.cpp | 40 +- src/Parsers/ASTQueryWithOutput.h | 12 +- src/Parsers/ASTQueryWithTableAndOutput.cpp | 3 +- src/Parsers/ASTQueryWithTableAndOutput.h | 10 +- src/Parsers/ASTRefreshStrategy.cpp | 33 +- src/Parsers/ASTRefreshStrategy.h | 4 +- src/Parsers/ASTRenameQuery.h | 60 +- src/Parsers/ASTSQLSecurity.cpp | 20 +- src/Parsers/ASTSQLSecurity.h | 3 +- src/Parsers/ASTSampleRatio.cpp | 7 +- src/Parsers/ASTSampleRatio.h | 3 +- src/Parsers/ASTSelectIntersectExceptQuery.cpp | 6 +- src/Parsers/ASTSelectIntersectExceptQuery.h | 5 +- src/Parsers/ASTSelectQuery.cpp | 151 +- src/Parsers/ASTSelectQuery.h | 17 +- src/Parsers/ASTSelectWithUnionQuery.cpp | 100 +- src/Parsers/ASTSelectWithUnionQuery.h | 6 +- src/Parsers/ASTSetQuery.cpp | 95 +- src/Parsers/ASTSetQuery.h | 13 +- src/Parsers/ASTShowColumnsQuery.cpp | 18 +- src/Parsers/ASTShowColumnsQuery.h | 2 +- src/Parsers/ASTShowFunctionsQuery.cpp | 7 +- src/Parsers/ASTShowFunctionsQuery.h | 2 +- src/Parsers/ASTShowIndexesQuery.cpp | 12 +- src/Parsers/ASTShowIndexesQuery.h | 2 +- src/Parsers/ASTShowSettingQuery.cpp | 4 +- src/Parsers/ASTShowSettingQuery.h | 2 +- src/Parsers/ASTShowTablesQuery.cpp | 58 +- src/Parsers/ASTShowTablesQuery.h | 6 +- src/Parsers/ASTStatisticsDeclaration.cpp | 10 +- src/Parsers/ASTStatisticsDeclaration.h | 4 +- src/Parsers/ASTSubquery.cpp | 14 +- src/Parsers/ASTSubquery.h | 2 +- src/Parsers/ASTSystemQuery.cpp | 136 +- src/Parsers/ASTSystemQuery.h | 33 +- src/Parsers/ASTTTLElement.cpp | 35 +- src/Parsers/ASTTTLElement.h | 2 +- src/Parsers/ASTTableOverrides.cpp | 34 +- src/Parsers/ASTTableOverrides.h | 8 +- src/Parsers/ASTTablesInSelectQuery.cpp | 134 +- src/Parsers/ASTTablesInSelectQuery.h | 25 +- src/Parsers/ASTTimeInterval.cpp | 8 +- src/Parsers/ASTTimeInterval.h | 4 +- src/Parsers/ASTTransactionControl.cpp | 10 +- src/Parsers/ASTTransactionControl.h | 4 +- src/Parsers/ASTUndropQuery.cpp | 14 +- src/Parsers/ASTUndropQuery.h | 2 +- src/Parsers/ASTUpdateQuery.cpp | 61 + src/Parsers/ASTUpdateQuery.h | 32 + src/Parsers/ASTUseQuery.h | 6 +- src/Parsers/ASTViewTargets.cpp | 20 +- src/Parsers/ASTViewTargets.h | 7 +- src/Parsers/ASTWatchQuery.h | 16 +- src/Parsers/ASTWindowDefinition.cpp | 56 +- src/Parsers/ASTWindowDefinition.h | 8 +- src/Parsers/ASTWithAlias.cpp | 26 +- src/Parsers/ASTWithAlias.h | 12 +- src/Parsers/ASTWithElement.cpp | 28 +- src/Parsers/ASTWithElement.h | 4 +- src/Parsers/Access/ASTAuthenticationData.cpp | 71 +- src/Parsers/Access/ASTAuthenticationData.h | 3 +- src/Parsers/Access/ASTCheckGrantQuery.cpp | 38 + src/Parsers/Access/ASTCheckGrantQuery.h | 27 + src/Parsers/Access/ASTCreateQuotaQuery.cpp | 80 +- src/Parsers/Access/ASTCreateQuotaQuery.h | 5 +- src/Parsers/Access/ASTCreateRoleQuery.cpp | 53 +- src/Parsers/Access/ASTCreateRoleQuery.h | 14 +- .../Access/ASTCreateRowPolicyQuery.cpp | 75 +- src/Parsers/Access/ASTCreateRowPolicyQuery.h | 5 +- .../Access/ASTCreateSettingsProfileQuery.cpp | 61 +- .../Access/ASTCreateSettingsProfileQuery.h | 14 +- src/Parsers/Access/ASTCreateUserQuery.cpp | 191 +- src/Parsers/Access/ASTCreateUserQuery.h | 22 +- .../Access/ASTDropAccessEntityQuery.cpp | 20 +- src/Parsers/Access/ASTDropAccessEntityQuery.h | 5 +- src/Parsers/Access/ASTGrantQuery.cpp | 121 +- src/Parsers/Access/ASTGrantQuery.h | 5 +- .../Access/ASTMoveAccessEntityQuery.cpp | 20 +- src/Parsers/Access/ASTMoveAccessEntityQuery.h | 5 +- src/Parsers/Access/ASTPublicSSHKey.cpp | 10 +- src/Parsers/Access/ASTPublicSSHKey.h | 4 +- src/Parsers/Access/ASTRolesOrUsersSet.cpp | 55 +- src/Parsers/Access/ASTRolesOrUsersSet.h | 7 +- src/Parsers/Access/ASTRowPolicyName.cpp | 36 +- src/Parsers/Access/ASTRowPolicyName.h | 8 +- src/Parsers/Access/ASTSetRoleQuery.cpp | 20 +- src/Parsers/Access/ASTSetRoleQuery.h | 5 +- .../Access/ASTSettingsProfileElement.cpp | 206 +- .../Access/ASTSettingsProfileElement.h | 43 +- .../Access/ASTShowAccessEntitiesQuery.cpp | 12 +- .../Access/ASTShowAccessEntitiesQuery.h | 2 +- .../Access/ASTShowCreateAccessEntityQuery.cpp | 24 +- .../Access/ASTShowCreateAccessEntityQuery.h | 2 +- src/Parsers/Access/ASTShowGrantsQuery.cpp | 21 +- src/Parsers/Access/ASTShowGrantsQuery.h | 6 +- src/Parsers/Access/ASTUserNameWithHost.cpp | 12 +- src/Parsers/Access/ASTUserNameWithHost.h | 8 +- src/Parsers/Access/ParserCheckGrantQuery.cpp | 30 + src/Parsers/Access/ParserCheckGrantQuery.h | 17 + src/Parsers/Access/ParserCreateQuotaQuery.cpp | 3 +- src/Parsers/Access/ParserCreateRoleQuery.cpp | 54 +- src/Parsers/Access/ParserCreateRoleQuery.h | 7 +- .../Access/ParserCreateRowPolicyQuery.cpp | 2 +- .../ParserCreateSettingsProfileQuery.cpp | 55 +- .../Access/ParserCreateSettingsProfileQuery.h | 7 +- src/Parsers/Access/ParserCreateUserQuery.cpp | 324 +- src/Parsers/Access/ParserCreateUserQuery.h | 7 +- src/Parsers/Access/ParserGrantQuery.cpp | 223 +- src/Parsers/Access/ParserRowPolicyName.cpp | 24 +- .../Access/ParserSettingsProfileElement.cpp | 306 +- .../Access/ParserSettingsProfileElement.h | 14 + .../Access/ParserShowAccessEntitiesQuery.cpp | 14 +- .../ParserShowCreateAccessEntityQuery.cpp | 14 +- src/Parsers/Access/ParserShowGrantsQuery.cpp | 22 + .../Access/parseAccessRightsElements.cpp | 186 + .../Access/parseAccessRightsElements.h | 24 + src/Parsers/Access/parseUserName.h | 1 - src/Parsers/CommonParsers.cpp | 20 +- src/Parsers/CommonParsers.h | 38 +- src/Parsers/CreateQueryUUIDs.cpp | 26 +- src/Parsers/ExpressionElementParsers.cpp | 127 +- src/Parsers/ExpressionElementParsers.h | 29 +- src/Parsers/ExpressionListParsers.cpp | 209 +- src/Parsers/ExpressionListParsers.h | 18 + src/Parsers/FieldFromAST.cpp | 5 +- src/Parsers/FunctionSecretArgumentsFinder.h | 712 +- .../FunctionSecretArgumentsFinderAST.h | 559 +- .../HiliteComparator/HiliteComparator.cpp | 18 +- .../HiliteComparator/HiliteComparator.h | 3 +- .../tests/gtest_hilite_comparator.cpp | 8 +- src/Parsers/IAST.cpp | 117 +- src/Parsers/IAST.h | 127 +- src/Parsers/IASTHash.h | 8 + src/Parsers/IAST_erase.h | 25 + src/Parsers/IAST_fwd.h | 21 - src/Parsers/IParser.cpp | 17 + src/Parsers/IParser.h | 3 +- src/Parsers/IdentifierQuotingStyle.h | 17 +- .../InsertQuerySettingsPushDownVisitor.cpp | 1 - .../InsertQuerySettingsPushDownVisitor.h | 4 +- src/Parsers/Kusto/Formatters.cpp | 10 +- .../KustoFunctions/IParserKQLFunction.cpp | 17 +- .../KustoFunctions/KQLBinaryFunctions.cpp | 14 +- .../KustoFunctions/KQLCastingFunctions.cpp | 21 +- .../KustoFunctions/KQLDataTypeFunctions.cpp | 37 +- .../KustoFunctions/KQLDateTimeFunctions.cpp | 64 +- .../KustoFunctions/KQLDynamicFunctions.cpp | 31 +- .../KustoFunctions/KQLGeneralFunctions.cpp | 25 +- .../Kusto/KustoFunctions/KQLIPFunctions.cpp | 30 +- .../KQLMathematicalFunctions.cpp | 4 +- .../KustoFunctions/KQLStringFunctions.cpp | 95 +- .../Kusto/ParserKQLDateTypeTimespan.cpp | 10 +- src/Parsers/Kusto/ParserKQLExtend.cpp | 7 +- src/Parsers/Kusto/ParserKQLMVExpand.cpp | 37 +- src/Parsers/Kusto/ParserKQLMakeSeries.cpp | 72 +- src/Parsers/Kusto/ParserKQLOperators.cpp | 11 +- src/Parsers/Kusto/ParserKQLQuery.cpp | 41 +- src/Parsers/Kusto/ParserKQLStatement.cpp | 1 + src/Parsers/Kusto/ParserKQLSummarize.cpp | 13 +- src/Parsers/Kusto/parseKQLQuery.cpp | 19 +- src/Parsers/Lexer.cpp | 66 +- src/Parsers/MySQL/ASTAlterCommand.cpp | 13 +- src/Parsers/MySQL/ASTAlterCommand.h | 2 +- src/Parsers/MySQL/ASTAlterQuery.h | 2 +- src/Parsers/MySQL/ASTCreateDefines.h | 2 +- src/Parsers/MySQL/ASTCreateQuery.h | 2 +- src/Parsers/MySQL/ASTDeclareColumn.h | 2 +- src/Parsers/MySQL/ASTDeclareConstraint.h | 2 +- src/Parsers/MySQL/ASTDeclareIndex.h | 2 +- src/Parsers/MySQL/ASTDeclareOption.cpp | 13 +- src/Parsers/MySQL/ASTDeclareOption.h | 2 +- src/Parsers/MySQL/ASTDeclarePartition.h | 2 +- .../MySQL/ASTDeclarePartitionOptions.h | 2 +- src/Parsers/MySQL/ASTDeclareReference.h | 2 +- src/Parsers/MySQL/ASTDeclareSubPartition.h | 2 +- src/Parsers/MySQL/ASTDropQuery.h | 2 +- src/Parsers/ParserAlterQuery.cpp | 94 +- src/Parsers/ParserBackupQuery.cpp | 32 +- src/Parsers/ParserCheckQuery.cpp | 2 +- src/Parsers/ParserCreateQuery.cpp | 170 +- src/Parsers/ParserCreateQuery.h | 35 +- src/Parsers/ParserCreateResourceQuery.cpp | 159 + src/Parsers/ParserCreateResourceQuery.h | 16 + src/Parsers/ParserCreateWorkloadEntity.cpp | 16 + src/Parsers/ParserCreateWorkloadEntity.h | 17 + src/Parsers/ParserCreateWorkloadQuery.cpp | 155 + src/Parsers/ParserCreateWorkloadQuery.h | 16 + src/Parsers/ParserDataType.cpp | 4 +- src/Parsers/ParserDeleteQuery.cpp | 13 + src/Parsers/ParserDeleteQuery.h | 2 +- src/Parsers/ParserDictionary.h | 2 - .../ParserDictionaryAttributeDeclaration.cpp | 1 + .../ParserDictionaryAttributeDeclaration.h | 2 - src/Parsers/ParserDropQuery.cpp | 57 +- src/Parsers/ParserDropResourceQuery.cpp | 52 + src/Parsers/ParserDropResourceQuery.h | 14 + src/Parsers/ParserDropWorkloadQuery.cpp | 52 + src/Parsers/ParserDropWorkloadQuery.h | 14 + src/Parsers/ParserInsertQuery.cpp | 29 +- src/Parsers/ParserOptimizeQuery.cpp | 3 +- src/Parsers/ParserParallelWithQuery.cpp | 46 + src/Parsers/ParserParallelWithQuery.h | 26 + src/Parsers/ParserPartition.cpp | 6 +- src/Parsers/ParserPreparedStatement.cpp | 121 + src/Parsers/ParserPreparedStatement.h | 77 + src/Parsers/ParserProjectionSelectQuery.cpp | 4 +- src/Parsers/ParserQuery.cpp | 42 +- src/Parsers/ParserQuery.h | 11 +- src/Parsers/ParserQueryWithOutput.cpp | 67 +- src/Parsers/ParserRefreshStrategy.cpp | 4 + src/Parsers/ParserRenameQuery.cpp | 2 +- src/Parsers/ParserSampleRatio.cpp | 8 +- src/Parsers/ParserSelectQuery.cpp | 38 +- src/Parsers/ParserSelectQuery.h | 8 + src/Parsers/ParserSetQuery.cpp | 4 +- src/Parsers/ParserSetQuery.h | 1 - src/Parsers/ParserShowColumnsQuery.cpp | 24 +- src/Parsers/ParserShowFunctionsQuery.cpp | 2 +- src/Parsers/ParserShowIndexesQuery.cpp | 22 +- src/Parsers/ParserShowTablesQuery.cpp | 2 +- src/Parsers/ParserSystemQuery.cpp | 126 +- src/Parsers/ParserTimeInterval.cpp | 7 + src/Parsers/ParserUndropQuery.cpp | 3 +- src/Parsers/ParserUpdateQuery.cpp | 86 + src/Parsers/ParserUpdateQuery.h | 18 + src/Parsers/ParserWithElement.cpp | 40 +- ...QueryWithOutputSettingsPushDownVisitor.cpp | 56 - .../QueryWithOutputSettingsPushDownVisitor.h | 39 - src/Parsers/StringRange.h | 71 - src/Parsers/TablePropertiesQueriesASTs.h | 18 +- src/Parsers/TokenIterator.cpp | 2 +- src/Parsers/examples/create_parser.cpp | 7 +- src/Parsers/examples/lexer.cpp | 2 + src/Parsers/examples/select_parser.cpp | 7 +- src/Parsers/formatAST.cpp | 21 - src/Parsers/formatAST.h | 48 - src/Parsers/fuzzers/CMakeLists.txt | 4 +- .../fuzzers/codegen_fuzzer/CMakeLists.txt | 2 +- .../codegen_fuzzer/codegen_select_fuzzer.cpp | 4 +- src/Parsers/fuzzers/codegen_fuzzer/gen.py | 3 +- src/Parsers/fuzzers/create_parser_fuzzer.cpp | 6 +- src/Parsers/fuzzers/select_parser_fuzzer.cpp | 6 +- src/Parsers/getInsertQuery.cpp | 8 +- src/Parsers/makeASTForLogicalFunction.cpp | 2 +- src/Parsers/obfuscateQueries.cpp | 1 + src/Parsers/parseDatabaseAndTableName.cpp | 26 +- src/Parsers/parseDatabaseAndTableName.h | 2 +- .../parseIdentifierOrStringLiteral.cpp | 27 + src/Parsers/parseIdentifierOrStringLiteral.h | 5 + src/Parsers/parseQuery.cpp | 49 +- src/Parsers/parseQuery.h | 3 +- src/Parsers/queryNormalization.cpp | 4 +- src/Parsers/queryToString.cpp | 15 - src/Parsers/queryToString.h | 9 - src/Parsers/tests/gtest_Parser.cpp | 89 +- src/Parsers/tests/gtest_common.cpp | 23 +- src/Parsers/tests/gtest_dictionary_parser.cpp | 15 +- src/Parsers/tests/gtest_format_hiliting.cpp | 5 +- src/Planner/ActionsChain.cpp | 6 +- src/Planner/CMakeLists.txt | 7 - src/Planner/CollectColumnIdentifiers.cpp | 33 +- src/Planner/CollectSets.cpp | 47 +- src/Planner/CollectSets.h | 7 +- src/Planner/CollectTableExpressionData.cpp | 109 +- src/Planner/Planner.cpp | 580 +- src/Planner/Planner.h | 9 +- src/Planner/PlannerActionsVisitor.cpp | 333 +- src/Planner/PlannerActionsVisitor.h | 19 +- src/Planner/PlannerContext.cpp | 59 +- src/Planner/PlannerContext.h | 25 +- src/Planner/PlannerCorrelatedSubqueries.cpp | 593 + src/Planner/PlannerCorrelatedSubqueries.h | 62 + src/Planner/PlannerExpressionAnalysis.cpp | 146 +- src/Planner/PlannerExpressionAnalysis.h | 8 +- src/Planner/PlannerJoinTree.cpp | 1539 +- src/Planner/PlannerJoins.cpp | 772 +- src/Planner/PlannerJoins.h | 99 +- src/Planner/PlannerJoinsLogical.cpp | 620 + src/Planner/PlannerJoinsLogical.h | 31 + src/Planner/PlannerSorting.cpp | 49 +- src/Planner/PlannerWindowFunctions.cpp | 19 +- src/Planner/PlannerWindowFunctions.h | 5 +- src/Planner/README.md | 15 + src/Planner/TableExpressionData.h | 2 + src/Planner/Utils.cpp | 138 +- src/Planner/Utils.h | 22 +- src/Planner/findParallelReplicasQuery.cpp | 224 +- src/Planner/findQueryForParallelReplicas.h | 4 +- src/Processors/Chunk.cpp | 24 +- src/Processors/Chunk.h | 30 +- src/Processors/ConcatProcessor.cpp | 2 +- src/Processors/ConcatProcessor.h | 2 + src/Processors/DelayedPortsProcessor.cpp | 1 + src/Processors/DelayedPortsProcessor.h | 4 + .../Executors/CompletedPipelineExecutor.cpp | 9 +- src/Processors/Executors/ExecutingGraph.cpp | 39 +- src/Processors/Executors/ExecutingGraph.h | 15 +- .../Executors/ExecutionThreadContext.cpp | 34 +- .../Executors/ExecutionThreadContext.h | 9 - src/Processors/Executors/ExecutorTasks.cpp | 97 +- src/Processors/Executors/ExecutorTasks.h | 34 +- src/Processors/Executors/PipelineExecutor.cpp | 182 +- src/Processors/Executors/PipelineExecutor.h | 24 +- src/Processors/Executors/PollingQueue.cpp | 4 +- .../PullingAsyncPipelineExecutor.cpp | 9 +- .../PushingAsyncPipelineExecutor.cpp | 23 +- .../Executors/PushingPipelineExecutor.cpp | 21 +- .../Executors/StreamingFormatExecutor.cpp | 67 +- .../Executors/StreamingFormatExecutor.h | 22 +- src/Processors/Executors/TasksQueue.h | 7 + src/Processors/ForkProcessor.cpp | 7 +- src/Processors/ForkProcessor.h | 7 +- src/Processors/Formats/IInputFormat.h | 11 +- src/Processors/Formats/IOutputFormat.cpp | 106 +- src/Processors/Formats/IOutputFormat.h | 59 +- src/Processors/Formats/IRowInputFormat.cpp | 44 +- src/Processors/Formats/IRowInputFormat.h | 8 +- src/Processors/Formats/IRowOutputFormat.cpp | 4 +- src/Processors/Formats/IRowOutputFormat.h | 12 +- src/Processors/Formats/ISchemaReader.cpp | 19 +- src/Processors/Formats/ISchemaReader.h | 19 +- .../Formats/Impl/ArrowBlockInputFormat.cpp | 20 +- .../Formats/Impl/ArrowBlockInputFormat.h | 2 +- .../Formats/Impl/ArrowBlockOutputFormat.cpp | 9 +- .../Formats/Impl/ArrowBufferedStreams.cpp | 38 +- .../Formats/Impl/ArrowBufferedStreams.h | 11 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 277 +- .../Formats/Impl/ArrowColumnToCHColumn.h | 24 +- .../Formats/Impl/ArrowFieldIndexUtil.h | 39 +- src/Processors/Formats/Impl/ArrowGeoTypes.cpp | 520 + src/Processors/Formats/Impl/ArrowGeoTypes.h | 200 + .../Formats/Impl/AvroRowInputFormat.cpp | 173 +- .../Formats/Impl/AvroRowInputFormat.h | 7 +- .../Formats/Impl/AvroRowOutputFormat.cpp | 130 +- .../Formats/Impl/AvroRowOutputFormat.h | 4 +- .../Impl/BSONEachRowRowInputFormat.cpp | 39 +- .../Formats/Impl/BSONEachRowRowInputFormat.h | 6 +- .../Impl/BSONEachRowRowOutputFormat.cpp | 10 +- .../Formats/Impl/BSONEachRowRowOutputFormat.h | 7 +- .../Formats/Impl/BinaryRowInputFormat.cpp | 49 +- .../Formats/Impl/BinaryRowInputFormat.h | 36 +- .../Formats/Impl/BinaryRowOutputFormat.cpp | 5 + .../Formats/Impl/BinaryRowOutputFormat.h | 3 +- .../Formats/Impl/CHColumnToArrowColumn.cpp | 160 +- .../Formats/Impl/CHColumnToArrowColumn.h | 6 +- .../Formats/Impl/CSVRowInputFormat.cpp | 37 +- .../Formats/Impl/CSVRowInputFormat.h | 10 +- .../Formats/Impl/CSVRowOutputFormat.cpp | 5 +- .../Formats/Impl/CSVRowOutputFormat.h | 2 +- .../Formats/Impl/CapnProtoRowOutputFormat.cpp | 4 + .../Impl/ConstantExpressionTemplate.cpp | 133 +- .../Formats/Impl/ConstantExpressionTemplate.h | 2 +- .../Impl/CustomSeparatedRowInputFormat.cpp | 9 +- .../Impl/CustomSeparatedRowInputFormat.h | 4 +- .../Impl/CustomSeparatedRowOutputFormat.cpp | 5 +- .../Formats/Impl/DWARFBlockInputFormat.cpp | 33 +- .../Formats/Impl/FormRowInputFormat.cpp | 4 +- .../Formats/Impl/HiveTextRowInputFormat.cpp | 1 + .../Impl/JSONAsStringRowInputFormat.cpp | 9 +- .../Formats/Impl/JSONAsStringRowInputFormat.h | 1 - .../Impl/JSONColumnsBlockInputFormatBase.cpp | 4 +- .../Impl/JSONColumnsBlockInputFormatBase.h | 5 +- .../Impl/JSONColumnsBlockOutputFormat.cpp | 1 + .../Impl/JSONColumnsBlockOutputFormat.h | 1 + .../Impl/JSONColumnsBlockOutputFormatBase.cpp | 13 +- .../Impl/JSONColumnsBlockOutputFormatBase.h | 6 +- ...ONColumnsWithMetadataBlockOutputFormat.cpp | 1 + ...JSONColumnsWithMetadataBlockOutputFormat.h | 8 +- .../JSONCompactColumnsBlockOutputFormat.cpp | 1 + .../Impl/JSONCompactEachRowRowInputFormat.cpp | 4 +- .../Impl/JSONCompactEachRowRowInputFormat.h | 7 +- .../JSONCompactEachRowRowOutputFormat.cpp | 18 +- .../Impl/JSONCompactEachRowRowOutputFormat.h | 18 +- ...pactEachRowWithProgressRowOutputFormat.cpp | 131 + ...ompactEachRowWithProgressRowOutputFormat.h | 44 + .../Impl/JSONCompactRowInputFormat.cpp | 4 +- .../Formats/Impl/JSONCompactRowInputFormat.h | 4 +- .../Impl/JSONCompactRowOutputFormat.cpp | 5 +- .../Formats/Impl/JSONCompactRowOutputFormat.h | 6 +- .../Impl/JSONEachRowRowInputFormat.cpp | 28 +- .../Formats/Impl/JSONEachRowRowInputFormat.h | 7 +- .../Impl/JSONEachRowRowOutputFormat.cpp | 3 +- .../Formats/Impl/JSONEachRowRowOutputFormat.h | 5 +- ...JSONEachRowWithProgressRowOutputFormat.cpp | 104 +- .../JSONEachRowWithProgressRowOutputFormat.h | 34 +- .../Impl/JSONObjectEachRowRowInputFormat.cpp | 1 + .../Impl/JSONObjectEachRowRowInputFormat.h | 2 +- .../Impl/JSONObjectEachRowRowOutputFormat.cpp | 6 +- .../Impl/JSONObjectEachRowRowOutputFormat.h | 2 - .../Formats/Impl/JSONRowInputFormat.cpp | 8 +- .../Formats/Impl/JSONRowInputFormat.h | 4 +- .../Formats/Impl/JSONRowOutputFormat.cpp | 13 +- .../Formats/Impl/JSONRowOutputFormat.h | 4 - .../Impl/LineAsStringRowInputFormat.cpp | 4 +- .../Formats/Impl/LineAsStringRowInputFormat.h | 1 - .../Formats/Impl/MarkdownRowOutputFormat.cpp | 2 + .../Formats/Impl/MarkdownRowOutputFormat.h | 1 - .../Formats/Impl/MsgPackRowInputFormat.cpp | 1 + .../Formats/Impl/MsgPackRowInputFormat.h | 1 - .../Formats/Impl/MsgPackRowOutputFormat.cpp | 7 +- .../Formats/Impl/MsgPackRowOutputFormat.h | 2 - .../Formats/Impl/MySQLDumpRowInputFormat.cpp | 4 +- .../Formats/Impl/MySQLDumpRowInputFormat.h | 3 +- .../Formats/Impl/MySQLOutputFormat.cpp | 27 +- .../Formats/Impl/MySQLOutputFormat.h | 8 +- src/Processors/Formats/Impl/NativeFormat.cpp | 18 +- .../Impl/NativeORCBlockInputFormat.cpp | 642 +- .../Formats/Impl/NativeORCBlockInputFormat.h | 50 +- .../Formats/Impl/NpyOutputFormat.cpp | 5 +- src/Processors/Formats/Impl/NpyOutputFormat.h | 8 +- .../Formats/Impl/NpyRowInputFormat.cpp | 64 +- .../Formats/Impl/NpyRowInputFormat.h | 7 +- src/Processors/Formats/Impl/NullFormat.cpp | 8 +- src/Processors/Formats/Impl/NullFormat.h | 4 +- .../Impl/ODBCDriver2BlockOutputFormat.cpp | 2 + .../Impl/ODBCDriver2BlockOutputFormat.h | 4 +- .../Formats/Impl/ORCBlockInputFormat.cpp | 53 +- .../Formats/Impl/ORCBlockInputFormat.h | 3 +- .../Formats/Impl/ORCBlockOutputFormat.cpp | 9 +- .../Formats/Impl/ORCBlockOutputFormat.h | 6 +- .../Impl/ParallelFormattingOutputFormat.cpp | 27 +- .../Impl/ParallelFormattingOutputFormat.h | 18 +- .../Impl/ParallelParsingInputFormat.cpp | 26 +- .../Formats/Impl/ParallelParsingInputFormat.h | 13 +- .../Impl/Parquet/ParquetColumnReader.h | 1 - .../Formats/Impl/Parquet/ParquetDataBuffer.h | 25 +- .../Impl/Parquet/ParquetDataValuesReader.cpp | 70 +- .../Impl/Parquet/ParquetDataValuesReader.h | 69 +- .../Impl/Parquet/ParquetLeafColReader.cpp | 378 +- .../Impl/Parquet/ParquetLeafColReader.h | 11 +- .../Impl/Parquet/ParquetRecordReader.cpp | 63 +- .../Impl/Parquet/ParquetRecordReader.h | 3 +- .../Formats/Impl/Parquet/PrepareForWrite.cpp | 41 +- .../Formats/Impl/Parquet/ThriftUtil.cpp | 3 + .../Formats/Impl/Parquet/ThriftUtil.h | 3 +- src/Processors/Formats/Impl/Parquet/Write.cpp | 359 +- src/Processors/Formats/Impl/Parquet/Write.h | 116 +- .../Impl/Parquet/parquetBloomFilterHash.cpp | 191 + .../Impl/Parquet/parquetBloomFilterHash.h | 25 + .../Formats/Impl/ParquetBlockInputFormat.cpp | 434 +- .../Formats/Impl/ParquetBlockInputFormat.h | 54 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 67 +- .../Formats/Impl/ParquetBlockOutputFormat.h | 5 +- .../Impl/ParquetMetadataInputFormat.cpp | 24 +- .../Formats/Impl/PostgreSQLOutputFormat.cpp | 11 +- .../Formats/Impl/PostgreSQLOutputFormat.h | 3 +- .../Formats/Impl/PrettyBlockOutputFormat.cpp | 871 +- .../Formats/Impl/PrettyBlockOutputFormat.h | 76 +- .../Impl/PrettyCompactBlockOutputFormat.cpp | 231 - .../Impl/PrettyCompactBlockOutputFormat.h | 32 - .../Impl/PrettySpaceBlockOutputFormat.cpp | 140 - .../Impl/PrettySpaceBlockOutputFormat.h | 24 - .../Impl/PrometheusTextOutputFormat.cpp | 12 +- .../Formats/Impl/PrometheusTextOutputFormat.h | 2 - .../Formats/Impl/ProtobufListInputFormat.cpp | 12 +- .../Formats/Impl/ProtobufListInputFormat.h | 2 + .../Formats/Impl/ProtobufListOutputFormat.cpp | 7 +- .../Formats/Impl/ProtobufListOutputFormat.h | 4 +- .../Formats/Impl/ProtobufRowInputFormat.cpp | 11 +- .../Formats/Impl/ProtobufRowInputFormat.h | 4 +- .../Formats/Impl/ProtobufRowOutputFormat.cpp | 11 +- .../Formats/Impl/ProtobufRowOutputFormat.h | 5 +- .../Formats/Impl/RawBLOBRowOutputFormat.cpp | 2 + .../Formats/Impl/RawBLOBRowOutputFormat.h | 1 - .../Formats/Impl/RegexpRowInputFormat.cpp | 3 +- .../Formats/Impl/RegexpRowInputFormat.h | 2 - .../Formats/Impl/SQLInsertRowOutputFormat.cpp | 5 +- .../Formats/Impl/SQLInsertRowOutputFormat.h | 1 - .../Formats/Impl/TSKVRowInputFormat.cpp | 39 +- .../Formats/Impl/TSKVRowInputFormat.h | 2 +- .../Formats/Impl/TSKVRowOutputFormat.cpp | 3 +- .../Formats/Impl/TSKVRowOutputFormat.h | 1 + .../Impl/TabSeparatedRowInputFormat.cpp | 17 +- .../Formats/Impl/TabSeparatedRowInputFormat.h | 7 +- .../Impl/TabSeparatedRowOutputFormat.cpp | 3 + .../Impl/TabSeparatedRowOutputFormat.h | 1 - .../Impl/TemplateBlockOutputFormat.cpp | 23 +- .../Formats/Impl/TemplateBlockOutputFormat.h | 14 +- .../Formats/Impl/TemplateRowInputFormat.h | 1 - .../Formats/Impl/ValuesBlockInputFormat.cpp | 40 +- .../Formats/Impl/ValuesBlockInputFormat.h | 4 +- .../Formats/Impl/ValuesRowOutputFormat.cpp | 5 +- .../Formats/Impl/VerticalRowOutputFormat.cpp | 61 +- .../Formats/Impl/VerticalRowOutputFormat.h | 4 +- .../Formats/Impl/XMLRowOutputFormat.cpp | 7 +- .../Formats/Impl/XMLRowOutputFormat.h | 7 +- .../Formats/InputFormatErrorsLogger.cpp | 15 +- .../Formats/InputFormatErrorsLogger.h | 2 + src/Processors/Formats/LazyOutputFormat.cpp | 10 +- src/Processors/Formats/LazyOutputFormat.h | 8 +- .../OutputFormatWithUTF8ValidationAdaptor.h | 5 +- .../Formats/PullingOutputFormat.cpp | 5 +- src/Processors/Formats/PullingOutputFormat.h | 4 +- .../RowInputFormatWithDiagnosticInfo.cpp | 2 +- .../RowInputFormatWithDiagnosticInfo.h | 1 - .../RowInputFormatWithNamesAndTypes.cpp | 171 +- .../Formats/RowInputFormatWithNamesAndTypes.h | 25 +- ...wOutputFormatWithExceptionHandlerAdaptor.h | 11 +- src/Processors/IAccumulatingTransform.cpp | 2 + src/Processors/IAccumulatingTransform.h | 2 + src/Processors/IInflatingTransform.cpp | 1 + src/Processors/IInflatingTransform.h | 3 + src/Processors/IProcessor.cpp | 110 +- src/Processors/IProcessor.h | 135 +- src/Processors/ISimpleTransform.h | 3 +- src/Processors/ISink.cpp | 2 +- src/Processors/ISink.h | 2 + src/Processors/ISource.h | 3 + src/Processors/LimitTransform.cpp | 7 +- src/Processors/LimitTransform.h | 1 + .../Algorithms/AggregatingSortedAlgorithm.cpp | 16 +- .../Algorithms/AggregatingSortedAlgorithm.h | 13 +- .../Algorithms/CollapsingSortedAlgorithm.cpp | 28 +- .../Algorithms/CollapsingSortedAlgorithm.h | 10 +- src/Processors/Merges/Algorithms/Graphite.cpp | 42 +- src/Processors/Merges/Algorithms/Graphite.h | 20 +- .../GraphiteRollupSortedAlgorithm.cpp | 19 +- .../Merges/Algorithms/IMergingAlgorithm.h | 5 +- .../IMergingAlgorithmWithDelayedChunk.cpp | 11 +- .../IMergingAlgorithmWithDelayedChunk.h | 17 +- .../IMergingAlgorithmWithSharedChunks.cpp | 7 +- .../IMergingAlgorithmWithSharedChunks.h | 15 +- .../Algorithms/MergeTreePartLevelInfo.h | 29 - .../Merges/Algorithms/MergeTreeReadInfo.cpp | 82 + .../Merges/Algorithms/MergeTreeReadInfo.h | 34 + .../Merges/Algorithms/MergedData.cpp | 163 + src/Processors/Merges/Algorithms/MergedData.h | 160 +- .../Algorithms/MergingSortedAlgorithm.cpp | 18 +- .../Algorithms/MergingSortedAlgorithm.h | 6 +- .../Algorithms/ReplacingSortedAlgorithm.cpp | 94 +- .../Algorithms/ReplacingSortedAlgorithm.h | 11 +- src/Processors/Merges/Algorithms/RowRef.cpp | 111 + src/Processors/Merges/Algorithms/RowRef.h | 86 +- .../Algorithms/SummingSortedAlgorithm.cpp | 82 +- .../Algorithms/SummingSortedAlgorithm.h | 14 +- .../Algorithms/tests/gtest_graphite.cpp | 2 +- src/Processors/Merges/IMergingTransform.cpp | 18 +- src/Processors/Merges/IMergingTransform.h | 6 +- .../Merges/MergingSortedTransform.cpp | 7 +- .../Merges/MergingSortedTransform.h | 1 + .../Merges/SummingSortedTransform.h | 4 +- src/Processors/OffsetTransform.cpp | 3 +- src/Processors/OffsetTransform.h | 1 + src/Processors/PingPongProcessor.cpp | 2 + src/Processors/PingPongProcessor.h | 2 + src/Processors/Port.h | 3 +- src/Processors/QueryPlan/AggregatingStep.cpp | 373 +- src/Processors/QueryPlan/AggregatingStep.h | 42 +- src/Processors/QueryPlan/ArrayJoinStep.cpp | 80 +- src/Processors/QueryPlan/ArrayJoinStep.h | 18 +- .../QueryPlan/BufferChunksTransform.cpp | 26 +- .../QueryPlan/BufferChunksTransform.h | 6 +- .../QueryPlan/BuildQueryPipelineSettings.cpp | 28 +- .../QueryPlan/BuildQueryPipelineSettings.h | 13 +- .../QueryPlan/ConvertingActions.cpp | 32 + src/Processors/QueryPlan/ConvertingActions.h | 9 + .../CreateSetAndFilterOnTheFlyStep.cpp | 21 +- .../CreateSetAndFilterOnTheFlyStep.h | 4 +- src/Processors/QueryPlan/CreatingSetsStep.cpp | 96 +- src/Processors/QueryPlan/CreatingSetsStep.h | 42 +- src/Processors/QueryPlan/CubeStep.cpp | 9 +- src/Processors/QueryPlan/CubeStep.h | 4 +- .../QueryPlan/CustomMetricLogViewStep.cpp | 321 + .../QueryPlan/CustomMetricLogViewStep.h | 32 + src/Processors/QueryPlan/DistinctStep.cpp | 158 +- src/Processors/QueryPlan/DistinctStep.h | 26 +- .../QueryPlan/DistributedCreateLocalPlan.cpp | 45 +- .../QueryPlan/DistributedCreateLocalPlan.h | 4 +- src/Processors/QueryPlan/ExpressionStep.cpp | 61 +- src/Processors/QueryPlan/ExpressionStep.h | 19 +- .../QueryPlan/ExtractColumnsStep.cpp | 43 + src/Processors/QueryPlan/ExtractColumnsStep.h | 23 + src/Processors/QueryPlan/ExtremesStep.cpp | 21 +- src/Processors/QueryPlan/ExtremesStep.h | 11 +- src/Processors/QueryPlan/FillingStep.cpp | 9 +- src/Processors/QueryPlan/FillingStep.h | 6 +- src/Processors/QueryPlan/FilterStep.cpp | 244 +- src/Processors/QueryPlan/FilterStep.h | 28 +- src/Processors/QueryPlan/IQueryPlanStep.cpp | 65 +- src/Processors/QueryPlan/IQueryPlanStep.h | 116 +- src/Processors/QueryPlan/ISourceStep.cpp | 4 +- src/Processors/QueryPlan/ISourceStep.h | 10 +- .../QueryPlan/ITransformingStep.cpp | 23 +- src/Processors/QueryPlan/ITransformingStep.h | 11 +- .../QueryPlan/IntersectOrExceptStep.cpp | 33 +- .../QueryPlan/IntersectOrExceptStep.h | 5 +- src/Processors/QueryPlan/JoinStep.cpp | 228 +- src/Processors/QueryPlan/JoinStep.h | 44 +- src/Processors/QueryPlan/JoinStepLogical.cpp | 914 + src/Processors/QueryPlan/JoinStepLogical.h | 143 + src/Processors/QueryPlan/LazilyReadStep.cpp | 78 + src/Processors/QueryPlan/LazilyReadStep.h | 30 + src/Processors/QueryPlan/LimitByStep.cpp | 43 +- src/Processors/QueryPlan/LimitByStep.h | 13 +- src/Processors/QueryPlan/LimitStep.cpp | 50 +- src/Processors/QueryPlan/LimitStep.h | 13 +- .../QueryPlan/MergingAggregatedStep.cpp | 79 +- .../QueryPlan/MergingAggregatedStep.h | 19 +- src/Processors/QueryPlan/OffsetStep.cpp | 25 +- src/Processors/QueryPlan/OffsetStep.h | 11 +- .../QueryPlan/Optimizations/Optimizations.h | 81 +- .../QueryPlanOptimizationSettings.cpp | 175 +- .../QueryPlanOptimizationSettings.h | 146 +- .../QueryPlan/Optimizations/Utils.cpp | 38 + .../QueryPlan/Optimizations/Utils.h | 27 + .../Optimizations/actionsDAGUtils.cpp | 263 + .../QueryPlan/Optimizations/actionsDAGUtils.h | 39 + .../Optimizations/addPlansForSets.cpp | 14 +- .../QueryPlan/Optimizations/applyOrder.cpp | 196 + .../calculateHashTableCacheKeys.cpp | 181 + .../Optimizations/convertJoinToIn.cpp | 272 + .../convertOuterJoinToInnerJoin.cpp | 52 +- .../Optimizations/distinctReadInOrder.cpp | 148 - .../enableMemoryBoundMerging.cpp | 22 +- .../Optimizations/filterPushDown.cpp | 209 +- .../Optimizations/liftUpArrayJoin.cpp | 14 +- .../Optimizations/liftUpFunctions.cpp | 12 +- .../QueryPlan/Optimizations/liftUpUnion.cpp | 51 +- .../QueryPlan/Optimizations/limitPushDown.cpp | 25 +- .../Optimizations/mergeExpressions.cpp | 19 +- .../mergeFilterIntoJoinCondition.cpp | 383 + .../QueryPlan/Optimizations/optimizeJoin.cpp | 396 + .../Optimizations/optimizeJoinByShards.cpp | 448 + .../optimizeLazyMaterialization.cpp | 328 + .../Optimizations/optimizePrewhere.cpp | 22 +- .../optimizePrimaryKeyConditionAndLimit.cpp | 2 +- .../Optimizations/optimizeReadInOrder.cpp | 536 +- .../QueryPlan/Optimizations/optimizeTree.cpp | 76 +- .../optimizeUseAggregateProjection.cpp | 135 +- .../optimizeUseNormalProjection.cpp | 148 +- .../Optimizations/projectionsCommon.cpp | 43 +- .../Optimizations/projectionsCommon.h | 5 +- .../Optimizations/removeRedundantDistinct.cpp | 13 +- .../Optimizations/removeRedundantSorting.cpp | 17 +- .../QueryPlan/Optimizations/splitFilter.cpp | 40 +- .../updateQueryConditionCache.cpp | 70 + .../useDataParallelAggregation.cpp | 19 +- .../Optimizations/useVectorSearch.cpp | 201 + .../QueryPlan/ParallelReplicasLocalPlan.cpp | 110 + .../QueryPlan/ParallelReplicasLocalPlan.h | 19 + src/Processors/QueryPlan/PartsSplitter.cpp | 424 +- src/Processors/QueryPlan/PartsSplitter.h | 36 + src/Processors/QueryPlan/QueryPlan.cpp | 279 +- src/Processors/QueryPlan/QueryPlan.h | 106 +- .../QueryPlanSerializationSettings.cpp | 124 + .../QueryPlanSerializationSettings.h | 43 + .../QueryPlan/QueryPlanStepRegistry.cpp | 78 + .../QueryPlan/QueryPlanStepRegistry.h | 32 + src/Processors/QueryPlan/ReadFromLoopStep.cpp | 23 +- .../QueryPlan/ReadFromMemoryStorageStep.cpp | 22 +- .../QueryPlan/ReadFromMemoryStorageStep.h | 8 +- .../QueryPlan/ReadFromMergeTree.cpp | 1049 +- src/Processors/QueryPlan/ReadFromMergeTree.h | 84 +- .../QueryPlan/ReadFromPreparedSource.cpp | 8 +- .../QueryPlan/ReadFromPreparedSource.h | 8 +- .../QueryPlan/ReadFromRecursiveCTEStep.cpp | 6 +- src/Processors/QueryPlan/ReadFromRemote.cpp | 711 +- src/Processors/QueryPlan/ReadFromRemote.h | 40 +- .../QueryPlan/ReadFromStreamLikeEngine.cpp | 8 +- .../QueryPlan/ReadFromSystemNumbersStep.cpp | 78 +- .../QueryPlan/ReadFromSystemNumbersStep.h | 4 +- .../QueryPlan/ReadFromTableFunctionStep.cpp | 107 + .../QueryPlan/ReadFromTableFunctionStep.h | 28 + .../QueryPlan/ReadFromTableStep.cpp | 94 + src/Processors/QueryPlan/ReadFromTableStep.h | 28 + src/Processors/QueryPlan/ReadNothingStep.cpp | 6 +- src/Processors/QueryPlan/RollupStep.cpp | 13 +- src/Processors/QueryPlan/RollupStep.h | 4 +- src/Processors/QueryPlan/Serialization.cpp | 220 + src/Processors/QueryPlan/Serialization.h | 30 + .../QueryPlan/SetsSerialization.cpp | 289 + src/Processors/QueryPlan/SortingStep.cpp | 275 +- src/Processors/QueryPlan/SortingStep.h | 60 +- .../QueryPlan/SourceStepWithFilter.cpp | 8 +- .../QueryPlan/SourceStepWithFilter.h | 117 +- src/Processors/QueryPlan/TemporaryFiles.h | 30 + src/Processors/QueryPlan/TotalsHavingStep.cpp | 104 +- src/Processors/QueryPlan/TotalsHavingStep.h | 14 +- src/Processors/QueryPlan/UnionStep.cpp | 71 +- src/Processors/QueryPlan/UnionStep.h | 14 +- src/Processors/QueryPlan/WindowStep.cpp | 14 +- src/Processors/QueryPlan/WindowStep.h | 4 +- src/Processors/QueryPlan/resolveStorages.cpp | 286 + src/Processors/QueryPlan/resolveStorages.h | 13 + src/Processors/ResizeProcessor.cpp | 10 + src/Processors/ResizeProcessor.h | 13 +- src/Processors/Sinks/NullSink.h | 1 + src/Processors/Sinks/SinkToStorage.cpp | 8 - src/Processors/Sinks/SinkToStorage.h | 4 + src/Processors/SourceWithKeyCondition.cpp | 19 + src/Processors/SourceWithKeyCondition.h | 17 +- src/Processors/Sources/BlocksListSource.h | 2 +- src/Processors/Sources/DelayedSource.h | 1 + src/Processors/Sources/MongoDBSource.cpp | 678 +- src/Processors/Sources/MongoDBSource.h | 91 +- src/Processors/Sources/MySQLSource.cpp | 37 +- src/Processors/Sources/MySQLSource.h | 2 +- src/Processors/Sources/PostgreSQLSource.h | 3 +- src/Processors/Sources/RecursiveCTESource.cpp | 10 +- src/Processors/Sources/RemoteSource.cpp | 19 +- src/Processors/Sources/RemoteSource.h | 12 +- src/Processors/Sources/SQLiteSource.cpp | 11 +- src/Processors/Sources/ShellCommandSource.cpp | 32 +- src/Processors/Sources/SourceFromChunks.cpp | 3 +- .../Sources/TemporaryFileLazySource.cpp | 32 - .../Sources/TemporaryFileLazySource.h | 28 - .../Sources/WaitForAsyncInsertSource.h | 1 - src/Processors/TTL/ITTLAlgorithm.cpp | 10 +- .../TTL/TTLAggregationAlgorithm.cpp | 53 +- .../Transforms/AddingDefaultsTransform.cpp | 22 +- .../AggregatingInOrderTransform.cpp | 35 +- .../Transforms/AggregatingTransform.cpp | 49 +- .../Transforms/AggregatingTransform.h | 22 +- .../Transforms/ArrayJoinTransform.cpp | 12 +- .../Transforms/ArrayJoinTransform.h | 2 +- .../Transforms/CheckConstraintsTransform.cpp | 7 +- .../Transforms/CheckConstraintsTransform.h | 11 +- .../Transforms/CheckSortedTransform.cpp | 9 +- .../Transforms/CheckSortedTransform.h | 2 +- .../Transforms/ColumnGathererTransform.cpp | 16 +- .../Transforms/ColumnGathererTransform.h | 13 +- .../Transforms/ColumnLazyTransform.cpp | 55 + .../Transforms/ColumnLazyTransform.h | 30 + .../Transforms/ColumnPermuteTransform.cpp | 49 + .../Transforms/ColumnPermuteTransform.h | 28 + src/Processors/Transforms/CopyTransform.cpp | 2 + src/Processors/Transforms/CopyTransform.h | 4 + .../CreateSetAndFilterOnTheFlyTransform.cpp | 4 +- .../CreateSetAndFilterOnTheFlyTransform.h | 34 +- src/Processors/Transforms/CubeTransform.cpp | 4 +- .../DeduplicationTokenTransforms.cpp | 4 +- .../Transforms/DeduplicationTokenTransforms.h | 4 +- ....cpp => DistinctSortedStreamTransform.cpp} | 29 +- ...form.h => DistinctSortedStreamTransform.h} | 20 +- .../Transforms/ExceptionKeepingTransform.cpp | 1 + .../Transforms/ExceptionKeepingTransform.h | 1 + .../Transforms/ExtremesTransform.cpp | 15 +- .../Transforms/FillingTransform.cpp | 209 +- src/Processors/Transforms/FillingTransform.h | 1 + .../Transforms/FilterSortedStreamByRange.h | 1 - src/Processors/Transforms/FilterTransform.cpp | 158 +- src/Processors/Transforms/FilterTransform.h | 19 +- .../Transforms/FinishSortingTransform.cpp | 3 + .../Transforms/IntersectOrExceptTransform.cpp | 2 +- .../Transforms/IntersectOrExceptTransform.h | 3 + .../Transforms/JoiningTransform.cpp | 145 +- src/Processors/Transforms/JoiningTransform.h | 67 +- .../Transforms/LimitByTransform.cpp | 1 + .../Transforms/LimitsCheckingTransform.cpp | 3 +- .../Transforms/MaterializingTransform.cpp | 13 +- .../Transforms/MaterializingTransform.h | 3 +- .../Transforms/MemoryBoundMerging.h | 3 +- .../Transforms/MergeJoinTransform.cpp | 21 +- .../Transforms/MergeSortingTransform.cpp | 219 +- .../Transforms/MergeSortingTransform.h | 11 +- .../Transforms/MergingAggregatedTransform.cpp | 27 +- .../Transforms/MergingAggregatedTransform.h | 8 +- .../NestedElementsValidationTransform.cpp | 16 + .../NestedElementsValidationTransform.h | 20 + .../Transforms/PartialSortingTransform.cpp | 2 +- .../Transforms/PasteJoinTransform.cpp | 3 + .../Transforms/PlanSquashingTransform.cpp | 2 + .../Transforms/ReverseTransform.cpp | 1 + .../ScatterByPartitionTransform.cpp | 5 +- .../Transforms/ScatterByPartitionTransform.h | 4 +- .../Transforms/SelectByIndicesTransform.h | 6 +- .../Transforms/SortingTransform.cpp | 36 +- .../Transforms/SquashingTransform.cpp | 2 +- .../StreamInQueryCacheTransform.cpp | 27 - .../Transforms/StreamInQueryCacheTransform.h | 29 - .../StreamInQueryResultCacheTransform.cpp | 27 + .../StreamInQueryResultCacheTransform.h | 29 + .../Transforms/TTLCalcTransform.cpp | 2 + src/Processors/Transforms/TTLCalcTransform.h | 11 +- src/Processors/Transforms/TTLTransform.cpp | 7 +- src/Processors/Transforms/TTLTransform.h | 9 +- .../Transforms/TotalsHavingTransform.h | 4 +- .../Transforms/VirtualRowTransform.cpp | 114 + .../Transforms/VirtualRowTransform.h | 36 + src/Processors/Transforms/WindowTransform.cpp | 91 +- src/Processors/Transforms/WindowTransform.h | 9 +- .../Transforms/buildPushingToViewsChain.cpp | 180 +- .../Transforms/buildPushingToViewsChain.h | 1 + src/Processors/Transforms/finalizeChunk.cpp | 2 + src/Processors/Transforms/finalizeChunk.h | 4 +- .../getSourceFromASTInsertQuery.cpp | 12 +- .../Transforms/getSourceFromASTInsertQuery.h | 2 +- .../tests/gtest_full_sorting_join.cpp | 15 +- .../tests/gtest_write_parquet_page_index.cpp | 371 + src/QueryPipeline/BlockIO.cpp | 39 +- src/QueryPipeline/BlockIO.h | 6 +- src/QueryPipeline/Chain.cpp | 11 + src/QueryPipeline/Chain.h | 4 +- src/QueryPipeline/ExecutionSpeedLimits.cpp | 16 +- src/QueryPipeline/ExecutionSpeedLimits.h | 2 +- src/QueryPipeline/Pipe.cpp | 12 +- src/QueryPipeline/Pipe.h | 8 +- src/QueryPipeline/ProfileInfo.cpp | 25 +- src/QueryPipeline/ProfileInfo.h | 18 +- src/QueryPipeline/QueryPipeline.cpp | 72 +- src/QueryPipeline/QueryPipeline.h | 11 +- src/QueryPipeline/QueryPipelineBuilder.cpp | 182 +- src/QueryPipeline/QueryPipelineBuilder.h | 19 +- src/QueryPipeline/ReadProgressCallback.cpp | 2 +- src/QueryPipeline/RemoteInserter.cpp | 17 +- src/QueryPipeline/RemoteInserter.h | 4 +- src/QueryPipeline/RemoteQueryExecutor.cpp | 144 +- src/QueryPipeline/RemoteQueryExecutor.h | 26 +- .../RemoteQueryExecutorReadContext.cpp | 46 +- .../RemoteQueryExecutorReadContext.h | 30 +- src/QueryPipeline/SizeLimits.cpp | 29 + src/QueryPipeline/SizeLimits.h | 6 + src/QueryPipeline/narrowPipe.cpp | 3 +- src/QueryPipeline/printPipeline.cpp | 3 +- src/QueryPipeline/printPipeline.h | 2 +- src/Server/CertificateReloader.cpp | 36 +- src/Server/CertificateReloader.h | 29 +- src/Server/ClientEmbedded/ClientEmbedded.cpp | 247 + src/Server/ClientEmbedded/ClientEmbedded.h | 81 + .../ClientEmbedded/ClientEmbeddedRunner.cpp | 91 + .../ClientEmbedded/ClientEmbeddedRunner.h | 57 + .../ClientEmbedded/IClientDescriptorSet.h | 45 + .../ClientEmbedded/PipeClientDescriptorSet.h | 73 + .../ClientEmbedded/PtyClientDescriptorSet.cpp | 83 + .../ClientEmbedded/PtyClientDescriptorSet.h | 67 + src/Server/CloudPlacementInfo.cpp | 3 + src/Server/GRPCServer.cpp | 54 +- src/Server/HTTP/HTMLForm.cpp | 12 +- src/Server/HTTP/HTTPServer.cpp | 3 +- src/Server/HTTP/HTTPServer.h | 1 + src/Server/HTTP/HTTPServerConnection.cpp | 4 +- src/Server/HTTP/HTTPServerRequest.cpp | 24 +- src/Server/HTTP/HTTPServerRequest.h | 5 +- src/Server/HTTP/HTTPServerResponse.cpp | 26 +- src/Server/HTTP/HTTPServerResponse.h | 34 + .../WriteBufferFromHTTPServerResponse.cpp | 275 +- .../HTTP/WriteBufferFromHTTPServerResponse.h | 15 +- src/Server/HTTP/authenticateUserByHTTP.cpp | 50 +- src/Server/HTTP/authenticateUserByHTTP.h | 18 +- src/Server/HTTP/exceptionCodeToHTTPStatus.cpp | 77 +- src/Server/HTTP/sendExceptionToHTTPClient.cpp | 64 +- src/Server/HTTP/sendExceptionToHTTPClient.h | 19 +- .../setReadOnlyIfHTTPMethodIdempotent.cpp | 6 +- src/Server/HTTPHandler.cpp | 407 +- src/Server/HTTPHandler.h | 68 +- src/Server/HTTPHandlerFactory.cpp | 28 +- src/Server/HTTPHandlerFactory.h | 4 +- src/Server/HTTPHandlerRequestFilter.h | 9 +- src/Server/InterserverIOHTTPHandler.cpp | 81 +- src/Server/InterserverIOHTTPHandler.h | 7 +- src/Server/KeeperReadinessHandler.cpp | 3 +- src/Server/KeeperTCPHandler.cpp | 154 +- src/Server/KeeperTCPHandler.h | 5 +- src/Server/MySQLHandler.cpp | 106 +- src/Server/MySQLHandler.h | 7 +- src/Server/MySQLHandlerFactory.cpp | 116 +- src/Server/MySQLHandlerFactory.h | 15 +- src/Server/PostgreSQLHandler.cpp | 368 +- src/Server/PostgreSQLHandler.h | 36 +- src/Server/PostgreSQLHandlerFactory.cpp | 18 +- src/Server/PostgreSQLHandlerFactory.h | 10 +- src/Server/PrometheusRequestHandler.cpp | 71 +- src/Server/PrometheusRequestHandler.h | 10 +- src/Server/PrometheusRequestHandlerConfig.h | 3 + .../PrometheusRequestHandlerFactory.cpp | 18 +- src/Server/ProtocolServerAdapter.cpp | 8 +- src/Server/ProtocolServerAdapter.h | 17 +- src/Server/ProxyV1Handler.cpp | 15 +- src/Server/ReplicasStatusHandler.cpp | 12 +- src/Server/SSH/SSHBind.cpp | 80 + src/Server/SSH/SSHBind.h | 54 + src/Server/SSH/SSHChannel.cpp | 93 + src/Server/SSH/SSHChannel.h | 52 + src/Server/SSH/SSHEvent.cpp | 94 + src/Server/SSH/SSHEvent.h | 48 + src/Server/SSH/SSHPtyHandler.cpp | 550 + src/Server/SSH/SSHPtyHandler.h | 48 + src/Server/SSH/SSHPtyHandlerFactory.h | 105 + src/Server/SSH/SSHSession.cpp | 124 + src/Server/SSH/SSHSession.h | 60 + src/Server/ServerType.cpp | 6 +- src/Server/ServerType.h | 1 + src/Server/StaticRequestHandler.cpp | 102 +- src/Server/TCPHandler.cpp | 1789 +- src/Server/TCPHandler.h | 169 +- src/Server/TCPServer.cpp | 8 +- src/Server/TCPServer.h | 32 +- src/Server/WebUIRequestHandler.cpp | 16 +- src/Server/WebUIRequestHandler.h | 23 +- src/Storages/AlterCommands.cpp | 323 +- src/Storages/AlterCommands.h | 10 +- src/Storages/CMakeLists.txt | 1 - .../Cache/ExternalDataSourceCache.cpp | 332 - src/Storages/Cache/ExternalDataSourceCache.h | 101 - src/Storages/Cache/RemoteCacheController.cpp | 221 - src/Storages/Cache/RemoteCacheController.h | 123 - src/Storages/Cache/RemoteFileCachePolicy.h | 22 - src/Storages/ColumnDefault.cpp | 7 +- src/Storages/ColumnDefault.h | 2 +- src/Storages/ColumnDependency.cpp | 15 + src/Storages/ColumnDependency.h | 9 +- src/Storages/ColumnSize.h | 24 + src/Storages/ColumnsDescription.cpp | 165 +- src/Storages/ColumnsDescription.h | 25 +- src/Storages/CompressionCodecSelector.h | 4 +- src/Storages/ConstraintsDescription.cpp | 37 +- src/Storages/ConstraintsDescription.h | 35 +- src/Storages/Distributed/Defines.h | 6 +- .../DistributedAsyncInsertBatch.cpp | 129 +- .../Distributed/DistributedAsyncInsertBatch.h | 25 +- .../DistributedAsyncInsertDirectoryQueue.cpp | 99 +- .../DistributedAsyncInsertDirectoryQueue.h | 19 +- .../DistributedAsyncInsertHeader.h | 1 + .../Distributed/DistributedSettings.cpp | 58 +- .../Distributed/DistributedSettings.h | 44 +- src/Storages/Distributed/DistributedSink.cpp | 147 +- src/Storages/Distributed/DistributedSink.h | 4 +- src/Storages/ExecutableSettings.cpp | 67 +- src/Storages/ExecutableSettings.h | 37 +- src/Storages/FileLog/DirectoryWatcherBase.cpp | 21 +- src/Storages/FileLog/DirectoryWatcherBase.h | 4 +- src/Storages/FileLog/FileLogConsumer.cpp | 14 +- src/Storages/FileLog/FileLogConsumer.h | 4 +- .../FileLog/FileLogDirectoryWatcher.cpp | 8 +- src/Storages/FileLog/FileLogSettings.cpp | 62 +- src/Storages/FileLog/FileLogSettings.h | 73 +- src/Storages/FileLog/FileLogSource.cpp | 21 +- src/Storages/FileLog/FileLogSource.h | 1 + src/Storages/FileLog/StorageFileLog.cpp | 143 +- src/Storages/FileLog/StorageFileLog.h | 13 +- src/Storages/Freeze.cpp | 14 +- src/Storages/Hive/HiveCommon.cpp | 7 +- src/Storages/Hive/HiveFile.cpp | 34 +- src/Storages/Hive/HiveFile.h | 5 +- src/Storages/Hive/HiveSettings.cpp | 57 +- src/Storages/Hive/HiveSettings.h | 65 +- src/Storages/Hive/StorageHive.cpp | 128 +- src/Storages/Hive/StorageHive.h | 3 +- src/Storages/IMessageProducer.cpp | 1 + src/Storages/IMessageProducer.h | 7 +- src/Storages/IStorage.cpp | 116 +- src/Storages/IStorage.h | 108 +- src/Storages/IStorageCluster.cpp | 102 +- src/Storages/IStorageCluster.h | 12 +- src/Storages/IStorage_fwd.h | 4 + src/Storages/IndicesDescription.cpp | 16 +- src/Storages/IndicesDescription.h | 10 +- src/Storages/KVStorageUtils.cpp | 78 +- src/Storages/KVStorageUtils.h | 8 +- src/Storages/Kafka/IKafkaExceptionInfoSink.h | 31 + src/Storages/Kafka/KafkaConfigLoader.cpp | 48 +- src/Storages/Kafka/KafkaConfigLoader.h | 3 +- src/Storages/Kafka/KafkaConsumer.cpp | 287 +- src/Storages/Kafka/KafkaConsumer.h | 37 +- src/Storages/Kafka/KafkaConsumer2.cpp | 59 +- src/Storages/Kafka/KafkaConsumer2.h | 11 +- src/Storages/Kafka/KafkaProducer.cpp | 5 + src/Storages/Kafka/KafkaProducer.h | 5 +- src/Storages/Kafka/KafkaSettings.cpp | 131 +- src/Storages/Kafka/KafkaSettings.h | 94 +- src/Storages/Kafka/KafkaSource.cpp | 46 +- src/Storages/Kafka/Kafka_fwd.h | 17 + src/Storages/Kafka/StorageKafka.cpp | 194 +- src/Storages/Kafka/StorageKafka.h | 27 +- src/Storages/Kafka/StorageKafka2.cpp | 172 +- src/Storages/Kafka/StorageKafka2.h | 28 +- src/Storages/Kafka/StorageKafkaUtils.cpp | 266 +- src/Storages/Kafka/StorageKafkaUtils.h | 10 +- src/Storages/KeyDescription.cpp | 71 +- src/Storages/KeyDescription.h | 20 +- src/Storages/LazilyReadInfo.h | 26 + src/Storages/LiveView/LiveViewEventsSource.h | 10 +- src/Storages/LiveView/LiveViewSource.h | 10 +- src/Storages/LiveView/StorageLiveView.cpp | 104 +- src/Storages/LiveView/StorageLiveView.h | 1 - src/Storages/MarkCache.cpp | 10 + src/Storages/MarkCache.h | 9 +- .../MaterializedView/RefreshSchedule.cpp | 46 +- .../MaterializedView/RefreshSchedule.h | 10 +- src/Storages/MaterializedView/RefreshSet.cpp | 203 +- src/Storages/MaterializedView/RefreshSet.h | 88 +- .../MaterializedView/RefreshSettings.cpp | 51 + .../MaterializedView/RefreshSettings.h | 31 +- src/Storages/MaterializedView/RefreshTask.cpp | 1155 +- src/Storages/MaterializedView/RefreshTask.h | 316 +- .../MaterializedView/RefreshTask_fwd.h | 15 - src/Storages/MemorySettings.cpp | 85 +- src/Storages/MemorySettings.h | 38 +- src/Storages/MergeTree/ActiveDataPartSet.cpp | 15 + src/Storages/MergeTree/ActiveDataPartSet.h | 2 +- src/Storages/MergeTree/AllMergeSelector.cpp | 38 - src/Storages/MergeTree/AlterConversions.cpp | 353 +- src/Storages/MergeTree/AlterConversions.h | 73 +- src/Storages/MergeTree/AsyncBlockIDsCache.cpp | 19 +- src/Storages/MergeTree/AsyncBlockIDsCache.h | 9 +- .../MergeTree/BackgroundJobsAssignee.cpp | 18 +- .../MergeTree/BackgroundJobsAssignee.h | 40 +- src/Storages/MergeTree/Backup.cpp | 4 +- src/Storages/MergeTree/CMakeLists.txt | 0 src/Storages/MergeTree/ColumnsSubstreams.cpp | 132 + src/Storages/MergeTree/ColumnsSubstreams.h | 45 + .../Compaction/CompactionStatistics.cpp | 158 + .../Compaction/CompactionStatistics.h | 47 + .../Compaction/ConstructFuturePart.cpp | 36 + .../Compaction/ConstructFuturePart.h | 11 + .../DistributedMergePredicate.cpp | 62 + .../DistributedMergePredicate.h | 163 + .../MergePredicates/IMergePredicate.h | 21 + .../MergeTreeMergePredicate.cpp | 54 + .../MergePredicates/MergeTreeMergePredicate.h | 26 + .../ReplicatedMergeTreeMergePredicate.cpp | 272 + .../ReplicatedMergeTreeMergePredicate.h | 77 + .../Compaction/MergeSelectorApplier.cpp | 143 + .../Compaction/MergeSelectorApplier.h | 42 + .../MergeSelectors/AllMergeSelector.cpp | 48 + .../MergeSelectors}/AllMergeSelector.h | 6 +- .../MergeSelectors/IMergeSelector.h | 37 + .../MergeSelectors/MergeSelectorFactory.cpp | 52 + .../MergeSelectors/MergeSelectorFactory.h | 40 + .../MergeSelectors}/SimpleMergeSelector.cpp | 130 +- .../MergeSelectors}/SimpleMergeSelector.h | 22 +- .../MergeSelectors/TTLMergeSelector.cpp | 168 + .../MergeSelectors/TTLMergeSelector.h | 93 + .../MergeSelectors/TrivialMergeSelector.cpp | 96 + .../MergeSelectors/TrivialMergeSelector.h | 33 + .../MergeSelectors/registerMergeSelectors.cpp | 21 + .../MergeSelectors/registerMergeSelectors.h} | 2 +- .../MergeTree/Compaction/PartProperties.cpp | 83 + .../MergeTree/Compaction/PartProperties.h | 55 + .../Compaction/PartsCollectors/Common.cpp | 39 + .../Compaction/PartsCollectors/Common.h | 63 + .../PartsCollectors/IPartsCollector.h | 36 + .../MergeTreePartsCollector.cpp | 155 + .../PartsCollectors/MergeTreePartsCollector.h | 35 + .../ReplicatedMergeTreePartsCollector.cpp | 82 + .../ReplicatedMergeTreePartsCollector.h | 35 + .../MergeTree/DataPartStorageOnDiskBase.cpp | 73 +- .../MergeTree/DataPartStorageOnDiskBase.h | 2 +- .../MergeTree/DataPartStorageOnDiskFull.cpp | 51 +- .../MergeTree/DataPartStorageOnDiskFull.h | 12 +- src/Storages/MergeTree/DataPartsExchange.cpp | 57 +- src/Storages/MergeTree/DataPartsExchange.h | 1 + .../DeserializationPrefixesCache.cpp | 30 + .../MergeTree/DeserializationPrefixesCache.h | 31 + .../MergeTree/EphemeralLockInZooKeeper.cpp | 29 +- .../MergeTree/EphemeralLockInZooKeeper.h | 26 +- .../MergeTree/FutureMergedMutatedPart.cpp | 7 +- .../MergeTree/FutureMergedMutatedPart.h | 2 + src/Storages/MergeTree/GinIndexStore.cpp | 57 +- src/Storages/MergeTree/GinIndexStore.h | 3 +- src/Storages/MergeTree/IDataPartStorage.h | 45 +- src/Storages/MergeTree/IExecutableTask.cpp | 18 + src/Storages/MergeTree/IExecutableTask.h | 12 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1027 +- src/Storages/MergeTree/IMergeTreeDataPart.h | 172 +- .../IMergeTreeDataPartInfoForReader.h | 3 + .../MergeTree/IMergeTreeDataPartWriter.cpp | 102 +- .../MergeTree/IMergeTreeDataPartWriter.h | 33 +- src/Storages/MergeTree/IMergeTreeReader.cpp | 178 +- src/Storages/MergeTree/IMergeTreeReader.h | 71 +- .../MergeTree/IMergedBlockOutputStream.cpp | 7 +- .../MergeTree/IMergedBlockOutputStream.h | 14 +- .../MergeTree/IPartMetadataManager.cpp | 21 - src/Storages/MergeTree/IPartMetadataManager.h | 62 - src/Storages/MergeTree/InsertBlockInfo.cpp | 8 +- src/Storages/MergeTree/InsertBlockInfo.h | 46 +- src/Storages/MergeTree/KeyCondition.cpp | 1193 +- src/Storages/MergeTree/KeyCondition.h | 90 +- src/Storages/MergeTree/LeaderElection.h | 5 +- .../LoadedMergeTreeDataPartInfoForReader.h | 4 +- src/Storages/MergeTree/MarkRange.cpp | 20 + src/Storages/MergeTree/MarkRange.h | 28 +- .../MergeTree/MergeFromLogEntryTask.cpp | 98 +- .../MergeTree/MergeFromLogEntryTask.h | 9 +- src/Storages/MergeTree/MergeList.cpp | 47 +- src/Storages/MergeTree/MergeList.h | 9 +- .../MergeTree/MergePlainMergeTreeTask.cpp | 59 +- .../MergeTree/MergePlainMergeTreeTask.h | 4 +- src/Storages/MergeTree/MergeProgress.h | 37 +- .../MergeTree/MergeProjectionPartsTask.cpp | 98 + .../MergeTree/MergeProjectionPartsTask.h | 85 + src/Storages/MergeTree/MergeSelector.h | 79 - src/Storages/MergeTree/MergeTask.cpp | 1324 +- src/Storages/MergeTree/MergeTask.h | 112 +- .../MergeTree/MergeTreeBackgroundExecutor.cpp | 93 +- .../MergeTree/MergeTreeBackgroundExecutor.h | 40 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 97 +- .../MergeTree/MergeTreeBlockReadUtils.h | 21 +- src/Storages/MergeTree/MergeTreeData.cpp | 1977 +- src/Storages/MergeTree/MergeTreeData.h | 372 +- .../MergeTree/MergeTreeDataFormatVersion.h | 4 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 955 +- .../MergeTree/MergeTreeDataMergerMutator.h | 175 +- .../MergeTree/MergeTreeDataPartBuilder.cpp | 30 +- .../MergeTree/MergeTreeDataPartBuilder.h | 14 +- .../MergeTree/MergeTreeDataPartChecksum.cpp | 11 +- .../MergeTree/MergeTreeDataPartChecksum.h | 9 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 118 +- .../MergeTree/MergeTreeDataPartCompact.h | 28 +- .../MergeTree/MergeTreeDataPartTTLInfo.cpp | 1 + .../MergeTree/MergeTreeDataPartTTLInfo.h | 12 +- .../MergeTree/MergeTreeDataPartType.cpp | 2 +- .../MergeTree/MergeTreeDataPartType.h | 1 + .../MergeTree/MergeTreeDataPartWide.cpp | 148 +- .../MergeTree/MergeTreeDataPartWide.h | 28 +- .../MergeTreeDataPartWriterCompact.cpp | 174 +- .../MergeTreeDataPartWriterCompact.h | 16 +- .../MergeTreeDataPartWriterOnDisk.cpp | 196 +- .../MergeTree/MergeTreeDataPartWriterOnDisk.h | 33 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 166 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 20 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 920 +- .../MergeTree/MergeTreeDataSelectExecutor.h | 69 +- .../MergeTree/MergeTreeDataWriter.cpp | 428 +- src/Storages/MergeTree/MergeTreeDataWriter.h | 67 +- .../MergeTree/MergeTreeDeduplicationLog.cpp | 8 +- .../MergeTree/MergeTreeIOSettings.cpp | 51 +- src/Storages/MergeTree/MergeTreeIOSettings.h | 20 +- .../MergeTree/MergeTreeIndexBloomFilter.cpp | 145 +- .../MergeTree/MergeTreeIndexBloomFilter.h | 11 +- .../MergeTreeIndexBloomFilterText.cpp | 158 +- .../MergeTree/MergeTreeIndexBloomFilterText.h | 7 +- ...ndexFullText.cpp => MergeTreeIndexGin.cpp} | 328 +- ...reeIndexFullText.h => MergeTreeIndexGin.h} | 84 +- .../MergeTree/MergeTreeIndexGranularity.cpp | 184 +- .../MergeTree/MergeTreeIndexGranularity.h | 123 +- .../MergeTreeIndexGranularityAdaptive.cpp | 162 + .../MergeTreeIndexGranularityAdaptive.h | 45 + .../MergeTreeIndexGranularityConstant.cpp | 155 + .../MergeTreeIndexGranularityConstant.h | 48 + .../MergeTreeIndexGranularityInfo.cpp | 52 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 11 +- .../MergeTree/MergeTreeIndexHypothesis.cpp | 3 +- .../MergeTree/MergeTreeIndexHypothesis.h | 4 +- ...ergeTreeIndexHypothesisMergedCondition.cpp | 10 +- .../MergeTreeIndexHypothesisMergedCondition.h | 1 + .../MergeTreeIndexLegacyVectorSimilarity.cpp | 45 - .../MergeTreeIndexLegacyVectorSimilarity.h | 26 - .../MergeTree/MergeTreeIndexMinMax.cpp | 68 +- src/Storages/MergeTree/MergeTreeIndexMinMax.h | 10 +- .../MergeTree/MergeTreeIndexReader.cpp | 81 +- src/Storages/MergeTree/MergeTreeIndexReader.h | 21 +- src/Storages/MergeTree/MergeTreeIndexSet.cpp | 256 +- src/Storages/MergeTree/MergeTreeIndexSet.h | 31 +- .../MergeTree/MergeTreeIndexUtils.cpp | 47 - src/Storages/MergeTree/MergeTreeIndexUtils.h | 13 - .../MergeTreeIndexVectorSimilarity.cpp | 491 +- .../MergeTreeIndexVectorSimilarity.h | 74 +- src/Storages/MergeTree/MergeTreeIndices.cpp | 58 +- src/Storages/MergeTree/MergeTreeIndices.h | 223 +- .../MergeTree/MergeTreeLazilyReader.cpp | 297 + .../MergeTree/MergeTreeLazilyReader.h | 55 + .../MergeTree/MergeTreeMarksLoader.cpp | 65 +- src/Storages/MergeTree/MergeTreeMarksLoader.h | 19 +- .../MergeTree/MergeTreeMutationEntry.cpp | 19 +- .../MergeTree/MergeTreeMutationEntry.h | 8 +- .../MergeTree/MergeTreeMutationStatus.cpp | 6 +- .../MergeTree/MergeTreeMutationStatus.h | 1 + src/Storages/MergeTree/MergeTreePartInfo.cpp | 12 +- src/Storages/MergeTree/MergeTreePartInfo.h | 55 +- src/Storages/MergeTree/MergeTreePartition.cpp | 50 +- src/Storages/MergeTree/MergeTreePartition.h | 27 +- .../MergeTree/MergeTreePartsMover.cpp | 86 +- src/Storages/MergeTree/MergeTreePartsMover.h | 3 + .../MergeTree/MergeTreePrefetchedReadPool.cpp | 79 +- .../MergeTree/MergeTreePrefetchedReadPool.h | 4 +- .../MergeTree/MergeTreeRangeReader.cpp | 351 +- src/Storages/MergeTree/MergeTreeRangeReader.h | 62 +- src/Storages/MergeTree/MergeTreeReadPool.cpp | 170 +- src/Storages/MergeTree/MergeTreeReadPool.h | 8 +- .../MergeTree/MergeTreeReadPoolBase.cpp | 151 +- .../MergeTree/MergeTreeReadPoolBase.h | 13 +- .../MergeTree/MergeTreeReadPoolInOrder.cpp | 4 + .../MergeTree/MergeTreeReadPoolInOrder.h | 2 + .../MergeTreeReadPoolParallelReplicas.cpp | 121 +- .../MergeTreeReadPoolParallelReplicas.h | 5 +- ...rgeTreeReadPoolParallelReplicasInOrder.cpp | 125 +- ...MergeTreeReadPoolParallelReplicasInOrder.h | 5 + src/Storages/MergeTree/MergeTreeReadTask.cpp | 117 +- src/Storages/MergeTree/MergeTreeReadTask.h | 61 +- .../MergeTree/MergeTreeReaderCompact.cpp | 185 +- .../MergeTree/MergeTreeReaderCompact.h | 38 +- .../MergeTreeReaderCompactSingleBuffer.cpp | 53 +- .../MergeTreeReaderCompactSingleBuffer.h | 3 +- .../MergeTree/MergeTreeReaderStream.cpp | 7 + .../MergeTree/MergeTreeReaderWide.cpp | 176 +- src/Storages/MergeTree/MergeTreeReaderWide.h | 37 +- .../MergeTree/MergeTreeReadersChain.cpp | 134 + .../MergeTree/MergeTreeReadersChain.h | 36 + .../MergeTree/MergeTreeSelectAlgorithms.cpp | 5 +- .../MergeTree/MergeTreeSelectAlgorithms.h | 8 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 252 +- .../MergeTree/MergeTreeSelectProcessor.h | 59 +- .../MergeTree/MergeTreeSequentialSource.cpp | 374 +- .../MergeTree/MergeTreeSequentialSource.h | 14 +- src/Storages/MergeTree/MergeTreeSettings.cpp | 2253 +- src/Storages/MergeTree/MergeTreeSettings.h | 339 +- src/Storages/MergeTree/MergeTreeSink.cpp | 144 +- src/Storages/MergeTree/MergeTreeSink.h | 31 +- src/Storages/MergeTree/MergeTreeSource.cpp | 4 + .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 117 +- .../MergeTree/MergeTreeVirtualColumns.cpp | 6 +- .../MergeTree/MergeTreeVirtualColumns.h | 2 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 98 +- .../MergeTree/MergeTreeWhereOptimizer.h | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 267 +- .../MergeTree/MergedBlockOutputStream.h | 15 +- .../MergedColumnOnlyOutputStream.cpp | 38 +- .../MergeTree/MergedColumnOnlyOutputStream.h | 14 +- src/Storages/MergeTree/MergedPartOffsets.h | 255 + .../MergeTree/MutateFromLogEntryTask.cpp | 67 +- .../MergeTree/MutateFromLogEntryTask.h | 11 +- .../MergeTree/MutatePlainMergeTreeTask.cpp | 33 +- .../MergeTree/MutatePlainMergeTreeTask.h | 2 + src/Storages/MergeTree/MutateTask.cpp | 569 +- src/Storages/MergeTree/MutateTask.h | 6 +- .../ParallelReplicasReadingCoordinator.cpp | 174 +- .../ParallelReplicasReadingCoordinator.h | 14 +- .../MergeTree/PartMetadataManagerOrdinary.cpp | 28 - .../MergeTree/PartMetadataManagerOrdinary.h | 29 - .../PartMovesBetweenShardsOrchestrator.cpp | 589 +- .../PartMovesBetweenShardsOrchestrator.h | 40 +- .../MergeTree/PartitionActionBlocker.cpp | 86 + .../MergeTree/PartitionActionBlocker.h | 69 + src/Storages/MergeTree/PartitionPruner.cpp | 11 +- src/Storages/MergeTree/PartitionPruner.h | 2 +- src/Storages/MergeTree/PrimaryIndexCache.cpp | 27 + src/Storages/MergeTree/PrimaryIndexCache.h | 62 + src/Storages/MergeTree/RPNBuilder.cpp | 178 +- src/Storages/MergeTree/RPNBuilder.h | 17 +- src/Storages/MergeTree/RangesInDataPart.cpp | 35 +- src/Storages/MergeTree/RangesInDataPart.h | 31 +- .../ReplicatedMergeMutateTaskBase.cpp | 89 +- .../MergeTree/ReplicatedMergeMutateTaskBase.h | 7 + .../ReplicatedMergeTreeAttachThread.cpp | 52 +- .../ReplicatedMergeTreeAttachThread.h | 7 +- .../ReplicatedMergeTreeCleanupThread.cpp | 99 +- .../ReplicatedMergeTreeCleanupThread.h | 11 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 25 +- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 40 +- ...ReplicatedMergeTreeMergeStrategyPicker.cpp | 13 +- .../ReplicatedMergeTreeMutationEntry.cpp | 2 +- .../ReplicatedMergeTreeMutationEntry.h | 2 - .../ReplicatedMergeTreePartCheckThread.cpp | 15 +- .../ReplicatedMergeTreePartCheckThread.h | 4 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 776 +- .../MergeTree/ReplicatedMergeTreeQueue.h | 154 +- .../ReplicatedMergeTreeQuorumAddedParts.h | 4 +- .../ReplicatedMergeTreeRestartingThread.cpp | 109 +- .../ReplicatedMergeTreeRestartingThread.h | 19 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 243 +- .../MergeTree/ReplicatedMergeTreeSink.h | 56 +- .../ReplicatedMergeTreeTableMetadata.cpp | 189 +- .../ReplicatedMergeTreeTableMetadata.h | 26 +- .../MergeTree/ReplicatedTableStatus.h | 6 +- src/Storages/MergeTree/RequestResponse.cpp | 110 +- src/Storages/MergeTree/RequestResponse.h | 17 +- src/Storages/MergeTree/RowOrderOptimizer.cpp | 3 + src/Storages/MergeTree/RowOrderOptimizer.h | 10 +- .../StorageFromMergeTreeDataPart.cpp | 13 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 23 +- .../StorageFromMergeTreeProjection.cpp | 75 + .../StorageFromMergeTreeProjection.h | 40 + src/Storages/MergeTree/TTLMergeSelector.cpp | 156 - src/Storages/MergeTree/TTLMergeSelector.h | 98 - .../MergeTree/VectorSimilarityCondition.cpp | 350 - .../MergeTree/VectorSimilarityCondition.h | 173 - .../MergeTree/VectorSimilarityIndexCache.h | 105 + src/Storages/MergeTree/checkDataPart.cpp | 134 +- ...tractZooKeeperPathFromReplicatedTableDef.h | 2 +- .../MergeTree/registerStorageMergeTree.cpp | 380 +- .../MergeTree/tests/gtest_async_inserts.cpp | 2 +- .../MergeTree/tests/gtest_executor.cpp | 5 + .../tests/gtest_merged_part_offsets.cpp | 425 + .../tests/gtest_partition_action_blocker.cpp | 257 + src/Storages/MessageQueueSink.cpp | 42 +- src/Storages/MessageQueueSink.h | 5 +- src/Storages/MutationCommands.cpp | 69 +- src/Storages/MutationCommands.h | 18 +- src/Storages/MySQL/MySQLHelpers.cpp | 30 +- src/Storages/MySQL/MySQLHelpers.h | 3 + src/Storages/MySQL/MySQLSettings.cpp | 87 +- src/Storages/MySQL/MySQLSettings.h | 44 +- src/Storages/NATS/NATSConnection.cpp | 178 +- src/Storages/NATS/NATSConnection.h | 61 +- src/Storages/NATS/NATSConsumer.cpp | 33 +- src/Storages/NATS/NATSConsumer.h | 12 +- src/Storages/NATS/NATSHandler.cpp | 221 +- src/Storages/NATS/NATSHandler.h | 54 +- src/Storages/NATS/NATSProducer.cpp | 99 +- src/Storages/NATS/NATSProducer.h | 21 +- src/Storages/NATS/NATSSettings.cpp | 97 +- src/Storages/NATS/NATSSettings.h | 87 +- src/Storages/NATS/NATSSource.cpp | 42 +- src/Storages/NATS/NATSSource.h | 2 + src/Storages/NATS/NATS_fwd.h | 25 + src/Storages/NATS/StorageNATS.cpp | 485 +- src/Storages/NATS/StorageNATS.h | 55 +- src/Storages/NamedCollectionsHelpers.cpp | 42 +- src/Storages/NamedCollectionsHelpers.h | 12 +- .../ObjectStorage/Azure/Configuration.cpp | 158 +- .../ObjectStorage/Azure/Configuration.h | 33 +- .../ObjectStorage/DataLakes/Common.cpp | 2 + .../DataLakes/DataLakeConfiguration.h | 241 + .../DataLakes/DeltaLake/KernelHelper.cpp | 197 + .../DataLakes/DeltaLake/KernelHelper.h | 55 + .../DeltaLake/KernelPointerWrapper.h | 63 + .../DataLakes/DeltaLake/KernelUtils.cpp | 60 + .../DataLakes/DeltaLake/KernelUtils.h | 60 + .../ObjectInfoWithPartitionColumns.h | 35 + .../DataLakes/DeltaLake/PartitionPruner.cpp | 110 + .../DataLakes/DeltaLake/PartitionPruner.h | 36 + .../DataLakes/DeltaLake/TableSnapshot.cpp | 449 + .../DataLakes/DeltaLake/TableSnapshot.h | 90 + .../DeltaLake/getSchemaFromSnapshot.cpp | 494 + .../DeltaLake/getSchemaFromSnapshot.h | 34 + .../DataLakes/DeltaLakeMetadata.cpp | 426 +- .../DataLakes/DeltaLakeMetadata.h | 85 +- .../DeltaLakeMetadataDeltaKernel.cpp | 123 + .../DataLakes/DeltaLakeMetadataDeltaKernel.h | 80 + .../ObjectStorage/DataLakes/HudiMetadata.cpp | 31 +- .../ObjectStorage/DataLakes/HudiMetadata.h | 26 +- .../DataLakes/IDataLakeMetadata.cpp | 72 + .../DataLakes/IDataLakeMetadata.h | 65 +- .../DataLakes/IStorageDataLake.h | 173 - .../Iceberg/AvroForIcebergDeserializer.cpp | 88 + .../Iceberg/AvroForIcebergDeserializer.h | 57 + .../DataLakes/Iceberg/Constant.h | 44 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 923 + .../DataLakes/Iceberg/IcebergMetadata.h | 146 + .../Iceberg/IcebergMetadataFilesCache.h | 164 + .../DataLakes/Iceberg/ManifestFile.cpp | 388 + .../DataLakes/Iceberg/ManifestFile.h | 135 + .../Iceberg/ManifestFilesPruning.cpp | 239 + .../DataLakes/Iceberg/ManifestFilesPruning.h | 52 + .../DataLakes/Iceberg/SchemaProcessor.cpp | 441 + .../DataLakes/Iceberg/SchemaProcessor.h | 117 + .../DataLakes/Iceberg/Snapshot.h | 33 + .../ObjectStorage/DataLakes/Iceberg/Utils.cpp | 93 + .../ObjectStorage/DataLakes/Iceberg/Utils.h | 17 + .../DataLakes/IcebergMetadata.cpp | 634 - .../ObjectStorage/DataLakes/IcebergMetadata.h | 118 - .../DataLakes/PartitionColumns.h | 19 - .../DataLakes/registerDataLakeStorages.cpp | 82 - .../HDFS/AsynchronousReadBufferFromHDFS.cpp | 2 +- .../HDFS/AsynchronousReadBufferFromHDFS.h | 15 +- .../ObjectStorage/HDFS/Configuration.cpp | 91 +- .../ObjectStorage/HDFS/Configuration.h | 24 +- .../ObjectStorage/HDFS/HDFSCommon.cpp | 32 +- src/Storages/ObjectStorage/HDFS/HDFSCommon.h | 9 +- .../ObjectStorage/HDFS/HDFSErrorWrapper.cpp | 17 + .../ObjectStorage/HDFS/HDFSErrorWrapper.h | 39 + .../ObjectStorage/HDFS/ReadBufferFromHDFS.cpp | 86 +- .../ObjectStorage/HDFS/ReadBufferFromHDFS.h | 4 + .../HDFS/WriteBufferFromHDFS.cpp | 52 +- .../ObjectStorage/HDFS/WriteBufferFromHDFS.h | 3 +- src/Storages/ObjectStorage/IObjectIterator.h | 18 + .../ObjectStorage/Local/Configuration.cpp | 84 + .../ObjectStorage/Local/Configuration.h | 74 + .../ObjectStorage/ReadBufferIterator.cpp | 34 +- .../ObjectStorage/ReadBufferIterator.h | 7 +- .../ObjectStorage/S3/Configuration.cpp | 273 +- src/Storages/ObjectStorage/S3/Configuration.h | 55 +- .../ObjectStorage/StorageObjectStorage.cpp | 274 +- .../ObjectStorage/StorageObjectStorage.h | 110 +- .../StorageObjectStorageCluster.cpp | 87 +- .../StorageObjectStorageCluster.h | 5 +- .../StorageObjectStorageSettings.cpp | 75 + .../StorageObjectStorageSettings.h | 104 + .../StorageObjectStorageSink.cpp | 15 +- .../StorageObjectStorageSource.cpp | 505 +- .../StorageObjectStorageSource.h | 84 +- ...rageObjectStorageStableTaskDistributor.cpp | 155 + ...torageObjectStorageStableTaskDistributor.h | 41 + src/Storages/ObjectStorage/Utils.cpp | 4 +- .../registerStorageObjectStorage.cpp | 213 +- .../ObjectStorageQueueIFileMetadata.cpp | 313 +- .../ObjectStorageQueueIFileMetadata.h | 80 +- .../ObjectStorageQueueMetadata.cpp | 1012 +- .../ObjectStorageQueueMetadata.h | 139 +- .../ObjectStorageQueueMetadataFactory.cpp | 52 +- .../ObjectStorageQueueMetadataFactory.h | 21 +- .../ObjectStorageQueueOrderedFileMetadata.cpp | 462 +- .../ObjectStorageQueueOrderedFileMetadata.h | 26 +- .../ObjectStorageQueueSettings.cpp | 171 +- .../ObjectStorageQueueSettings.h | 101 +- .../ObjectStorageQueueSource.cpp | 1027 +- .../ObjectStorageQueueSource.h | 159 +- .../ObjectStorageQueueTableMetadata.cpp | 242 +- .../ObjectStorageQueueTableMetadata.h | 99 +- ...bjectStorageQueueUnorderedFileMetadata.cpp | 188 +- .../ObjectStorageQueueUnorderedFileMetadata.h | 12 +- .../StorageObjectStorageQueue.cpp | 775 +- .../StorageObjectStorageQueue.h | 70 +- .../registerQueueStorage.cpp | 35 +- src/Storages/PartitionCommands.cpp | 24 +- src/Storages/PartitionCommands.h | 2 +- src/Storages/PartitionedSink.cpp | 7 + src/Storages/PartitionedSink.h | 4 + .../MaterializedPostgreSQLConsumer.cpp | 8 +- .../MaterializedPostgreSQLConsumer.h | 4 +- .../MaterializedPostgreSQLSettings.cpp | 68 +- .../MaterializedPostgreSQLSettings.h | 52 +- .../PostgreSQLReplicationHandler.cpp | 321 +- .../PostgreSQL/PostgreSQLReplicationHandler.h | 18 +- .../StorageMaterializedPostgreSQL.cpp | 47 +- .../StorageMaterializedPostgreSQL.h | 4 +- src/Storages/ProjectionsDescription.cpp | 273 +- src/Storages/ProjectionsDescription.h | 33 +- src/Storages/RabbitMQ/RabbitMQConsumer.h | 1 + src/Storages/RabbitMQ/RabbitMQProducer.cpp | 13 + src/Storages/RabbitMQ/RabbitMQProducer.h | 10 +- src/Storages/RabbitMQ/RabbitMQSettings.cpp | 110 +- src/Storages/RabbitMQ/RabbitMQSettings.h | 100 +- src/Storages/RabbitMQ/RabbitMQSource.cpp | 34 +- src/Storages/RabbitMQ/RabbitMQSource.h | 3 +- src/Storages/RabbitMQ/RabbitMQ_fwd.h | 24 + src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 221 +- src/Storages/RabbitMQ/StorageRabbitMQ.h | 22 +- .../ReadFinalForExternalReplicaStorage.cpp | 10 +- src/Storages/ReadInOrderOptimizer.cpp | 9 +- src/Storages/RedisCommon.cpp | 2 +- .../ReplaceAliasByExpressionVisitor.h | 2 +- .../RocksDB/EmbeddedRocksDBBulkSink.cpp | 13 +- src/Storages/RocksDB/RocksDBSettings.cpp | 59 +- src/Storages/RocksDB/RocksDBSettings.h | 47 +- .../RocksDB/StorageEmbeddedRocksDB.cpp | 33 +- src/Storages/RocksDB/StorageEmbeddedRocksDB.h | 5 +- src/Storages/RocksDB/StorageSystemRocksDB.cpp | 16 +- src/Storages/SelectQueryDescription.cpp | 76 +- src/Storages/SelectQueryInfo.cpp | 8 +- src/Storages/SelectQueryInfo.h | 37 +- src/Storages/SetSettings.cpp | 55 +- src/Storages/SetSettings.h | 61 +- .../ConditionSelectivityEstimator.cpp | 174 +- .../ConditionSelectivityEstimator.h | 17 +- src/Storages/Statistics/Statistics.cpp | 56 +- src/Storages/Statistics/Statistics.h | 30 +- .../Statistics/StatisticsCountMinSketch.cpp | 9 +- .../Statistics/StatisticsCountMinSketch.h | 2 +- src/Storages/Statistics/StatisticsMinMax.cpp | 86 + src/Storages/Statistics/StatisticsMinMax.h | 33 + src/Storages/Statistics/StatisticsTDigest.cpp | 4 +- src/Storages/Statistics/StatisticsTDigest.h | 2 +- src/Storages/Statistics/StatisticsUniq.cpp | 4 +- src/Storages/Statistics/StatisticsUniq.h | 2 +- src/Storages/StatisticsDescription.cpp | 36 +- src/Storages/StatisticsDescription.h | 5 +- src/Storages/StorageBuffer.cpp | 109 +- src/Storages/StorageBuffer.h | 6 +- src/Storages/StorageDictionary.cpp | 65 +- src/Storages/StorageDistributed.cpp | 396 +- src/Storages/StorageDistributed.h | 20 +- src/Storages/StorageDummy.cpp | 17 +- src/Storages/StorageDummy.h | 8 +- src/Storages/StorageExecutable.cpp | 68 +- src/Storages/StorageExecutable.h | 14 +- src/Storages/StorageExternalDistributed.cpp | 199 - src/Storages/StorageExternalDistributed.h | 43 - src/Storages/StorageFactory.cpp | 61 +- src/Storages/StorageFactory.h | 19 +- src/Storages/StorageFile.cpp | 280 +- src/Storages/StorageFile.h | 5 + src/Storages/StorageFileCluster.cpp | 23 +- src/Storages/StorageFileCluster.h | 2 +- src/Storages/StorageFuzzJSON.cpp | 17 +- src/Storages/StorageFuzzQuery.cpp | 4 +- src/Storages/StorageFuzzQuery.h | 6 +- src/Storages/StorageGenerateRandom.cpp | 45 +- src/Storages/StorageGenerateRandom.h | 5 + src/Storages/StorageInMemoryMetadata.cpp | 44 +- src/Storages/StorageInMemoryMetadata.h | 18 +- src/Storages/StorageInput.cpp | 2 +- src/Storages/StorageJoin.cpp | 82 +- src/Storages/StorageJoin.h | 5 +- src/Storages/StorageKeeperMap.cpp | 157 +- src/Storages/StorageKeeperMap.h | 2 + src/Storages/StorageLog.cpp | 96 +- src/Storages/StorageLog.h | 4 +- src/Storages/StorageLogSettings.cpp | 4 + src/Storages/StorageLogSettings.h | 5 + src/Storages/StorageMaterializedMySQL.cpp | 64 - src/Storages/StorageMaterializedMySQL.h | 62 - src/Storages/StorageMaterializedView.cpp | 475 +- src/Storages/StorageMaterializedView.h | 39 +- src/Storages/StorageMemory.cpp | 72 +- src/Storages/StorageMemory.h | 18 +- src/Storages/StorageMerge.cpp | 775 +- src/Storages/StorageMerge.h | 32 +- src/Storages/StorageMergeTree.cpp | 955 +- src/Storages/StorageMergeTree.h | 136 +- src/Storages/StorageMergeTreeIndex.cpp | 53 +- src/Storages/StorageMongoDB.cpp | 670 +- src/Storages/StorageMongoDB.h | 128 +- src/Storages/StorageMongoDBSocketFactory.cpp | 55 - src/Storages/StorageMongoDBSocketFactory.h | 19 - src/Storages/StorageMySQL.cpp | 50 +- src/Storages/StorageMySQL.h | 8 +- src/Storages/StorageNull.cpp | 1 + src/Storages/StorageNull.h | 4 +- src/Storages/StoragePostgreSQL.cpp | 51 +- src/Storages/StorageProxy.h | 4 +- src/Storages/StorageRedis.cpp | 40 +- src/Storages/StorageReplicatedMergeTree.cpp | 1872 +- src/Storages/StorageReplicatedMergeTree.h | 138 +- src/Storages/StorageSQLite.cpp | 3 +- src/Storages/StorageSet.cpp | 49 +- src/Storages/StorageSet.h | 4 +- src/Storages/StorageSnapshot.cpp | 47 +- src/Storages/StorageSnapshot.h | 8 +- src/Storages/StorageStripeLog.cpp | 39 +- src/Storages/StorageStripeLog.h | 4 +- src/Storages/StorageTableFunction.h | 5 +- src/Storages/StorageTimeSeries.cpp | 20 +- src/Storages/StorageTimeSeries.h | 9 +- src/Storages/StorageURL.cpp | 238 +- src/Storages/StorageURL.h | 9 +- src/Storages/StorageURLCluster.cpp | 62 +- src/Storages/StorageURLCluster.h | 2 +- src/Storages/StorageValues.cpp | 4 +- src/Storages/StorageValues.h | 5 + src/Storages/StorageView.cpp | 27 +- src/Storages/StorageView.h | 4 +- src/Storages/StorageXDBC.cpp | 20 +- src/Storages/System/CMakeLists.txt | 3 +- .../System/IStorageSystemOneBlock.cpp | 4 +- src/Storages/System/IStorageSystemOneBlock.h | 1 + .../System/MutableColumnsAndConstraints.h | 19 + .../System/ServerSettingColumnsParams.h | 18 + .../System/StorageSystemAsyncLoader.cpp | 3 +- .../StorageSystemAsynchronousInserts.cpp | 3 +- .../StorageSystemAsynchronousMetrics.cpp | 9 +- src/Storages/System/StorageSystemBackups.cpp | 14 +- .../System/StorageSystemBuildOptions.cpp | 3 +- .../System/StorageSystemBuildOptions.cpp.in | 5 +- .../System/StorageSystemCertificates.cpp | 143 +- src/Storages/System/StorageSystemClusters.cpp | 14 +- .../System/StorageSystemCollations.cpp | 1 + src/Storages/System/StorageSystemColumns.cpp | 68 +- .../System/StorageSystemContributors.cpp | 8 +- .../StorageSystemContributors.generated.cpp | 206 + .../System/StorageSystemDDLWorkerQueue.cpp | 90 +- .../System/StorageSystemDashboards.cpp | 333 +- .../StorageSystemDataSkippingIndices.cpp | 13 +- .../System/StorageSystemDataTypeFamilies.cpp | 2 +- .../System/StorageSystemDatabaseEngines.cpp | 2 +- .../System/StorageSystemDatabases.cpp | 10 +- .../System/StorageSystemDetachedParts.cpp | 8 +- .../System/StorageSystemDetachedTables.cpp | 21 +- .../System/StorageSystemDictionaries.cpp | 2 +- src/Storages/System/StorageSystemDisks.cpp | 8 +- src/Storages/System/StorageSystemDisks.h | 1 - .../System/StorageSystemDistributionQueue.cpp | 4 +- .../StorageSystemDroppedTablesParts.cpp | 14 +- src/Storages/System/StorageSystemErrors.cpp | 8 +- src/Storages/System/StorageSystemEvents.cpp | 7 +- .../System/StorageSystemFilesystemCache.cpp | 8 +- .../StorageSystemFilesystemCacheSettings.cpp | 47 +- src/Storages/System/StorageSystemFormats.cpp | 12 +- .../System/StorageSystemFunctions.cpp | 13 +- src/Storages/System/StorageSystemGrants.cpp | 12 +- .../System/StorageSystemHistogramMetrics.cpp | 72 + .../System/StorageSystemHistogramMetrics.h | 27 + .../System/StorageSystemIcebergHistory.cpp | 105 + .../System/StorageSystemIcebergHistory.h | 37 + src/Storages/System/StorageSystemJemalloc.cpp | 7 +- .../System/StorageSystemKafkaConsumers.cpp | 35 +- src/Storages/System/StorageSystemKeywords.cpp | 3 +- .../System/StorageSystemLatencyBuckets.cpp | 34 + ...3Queue.h => StorageSystemLatencyBuckets.h} | 22 +- src/Storages/System/StorageSystemLicenses.cpp | 5 +- src/Storages/System/StorageSystemLicenses.sh | 2 + src/Storages/System/StorageSystemMacros.cpp | 1 + .../System/StorageSystemMergeTreeSettings.cpp | 38 +- src/Storages/System/StorageSystemMetrics.cpp | 2 +- .../System/StorageSystemMutations.cpp | 36 +- .../System/StorageSystemMySQLBinlogs.cpp | 164 - .../System/StorageSystemNamedCollections.cpp | 12 +- src/Storages/System/StorageSystemNumbers.cpp | 1 + src/Storages/System/StorageSystemNumbers.h | 3 +- ...pp => StorageSystemObjectStorageQueue.cpp} | 24 +- .../System/StorageSystemObjectStorageQueue.h | 25 + ...torageSystemObjectStorageQueueSettings.cpp | 79 + .../StorageSystemObjectStorageQueueSettings.h | 30 + src/Storages/System/StorageSystemOne.cpp | 5 +- .../StorageSystemPartMovesBetweenShards.cpp | 5 +- src/Storages/System/StorageSystemParts.cpp | 83 +- .../System/StorageSystemPartsBase.cpp | 31 +- src/Storages/System/StorageSystemPartsBase.h | 17 +- .../System/StorageSystemPartsColumns.cpp | 23 +- .../System/StorageSystemPrivileges.cpp | 2 + .../System/StorageSystemProcesses.cpp | 14 +- .../System/StorageSystemProjectionParts.cpp | 33 +- .../StorageSystemProjectionPartsColumns.cpp | 14 +- .../System/StorageSystemProjections.cpp | 273 + ...QLBinlogs.h => StorageSystemProjections.h} | 13 +- .../StorageSystemQueryConditionCache.cpp | 61 + .../System/StorageSystemQueryConditionCache.h | 21 + ....cpp => StorageSystemQueryResultCache.cpp} | 34 +- ...ache.h => StorageSystemQueryResultCache.h} | 6 +- src/Storages/System/StorageSystemQuotas.cpp | 8 +- .../System/StorageSystemRemoteDataPaths.cpp | 29 +- src/Storages/System/StorageSystemReplicas.cpp | 40 +- .../System/StorageSystemReplicatedFetches.cpp | 1 + .../System/StorageSystemReplicationQueue.cpp | 7 +- .../System/StorageSystemResources.cpp | 75 + src/Storages/System/StorageSystemResources.h | 29 + src/Storages/System/StorageSystemRoles.cpp | 4 +- .../System/StorageSystemRowPolicies.cpp | 4 +- .../System/StorageSystemScheduler.cpp | 18 +- .../StorageSystemSchemaInferenceCache.cpp | 1 + .../System/StorageSystemServerSettings.cpp | 89 +- src/Storages/System/StorageSystemSettings.cpp | 62 +- .../System/StorageSystemSettingsChanges.cpp | 41 +- .../StorageSystemSettingsProfileElements.cpp | 1 + .../System/StorageSystemSettingsProfiles.cpp | 4 +- .../System/StorageSystemStackTrace.cpp | 20 +- .../System/StorageSystemStoragePolicies.cpp | 2 + .../System/StorageSystemStoragePolicies.h | 1 - .../System/StorageSystemTableEngines.cpp | 1 + .../System/StorageSystemTableFunctions.cpp | 1 + src/Storages/System/StorageSystemTables.cpp | 199 +- src/Storages/System/StorageSystemTables.h | 1 + .../System/StorageSystemTimeZones.cpp | 5 +- src/Storages/System/StorageSystemUsers.cpp | 49 +- .../System/StorageSystemViewRefreshes.cpp | 98 +- src/Storages/System/StorageSystemWarnings.cpp | 10 +- .../System/StorageSystemWorkloads.cpp | 49 + src/Storages/System/StorageSystemWorkloads.h | 29 + .../System/StorageSystemZooKeeper.cpp | 42 +- .../StorageSystemZooKeeperConnection.cpp | 4 +- .../System/attachInformationSchemaTables.cpp | 62 +- src/Storages/System/attachSystemTables.cpp | 41 +- src/Storages/System/attachSystemTablesImpl.h | 4 +- .../System/getQueriedColumnsMaskAndHeader.cpp | 2 + .../System/getQueriedColumnsMaskAndHeader.h | 5 +- src/Storages/TTLDescription.cpp | 30 +- src/Storages/TTLDescription.h | 6 +- src/Storages/TableZnodeInfo.cpp | 135 + src/Storages/TableZnodeInfo.h | 66 + .../PrometheusRemoteReadProtocol.cpp | 83 +- .../PrometheusRemoteWriteProtocol.cpp | 64 +- .../TimeSeries/TimeSeriesColumnsValidator.cpp | 13 +- .../TimeSeriesDefinitionNormalizer.cpp | 68 +- .../TimeSeriesInnerTablesCreator.cpp | 18 +- .../TimeSeries/TimeSeriesSettings.cpp | 50 +- src/Storages/TimeSeries/TimeSeriesSettings.h | 31 +- src/Storages/Utils.cpp | 16 +- src/Storages/Utils.h | 2 +- src/Storages/VirtualColumnUtils.cpp | 184 +- src/Storages/VirtualColumnUtils.h | 24 +- src/Storages/WindowView/StorageWindowView.cpp | 208 +- src/Storages/WindowView/StorageWindowView.h | 10 +- src/Storages/WindowView/WindowViewSource.h | 14 +- src/Storages/buildQueryTreeForShard.cpp | 128 +- src/Storages/buildQueryTreeForShard.h | 2 +- src/Storages/checkAndGetLiteralArgument.cpp | 1 + src/Storages/checkAndGetLiteralArgument.h | 5 +- src/Storages/examples/CMakeLists.txt | 2 +- src/Storages/examples/merge_selector.cpp | 108 +- src/Storages/examples/merge_selector2.cpp | 148 +- src/Storages/extractKeyExpressionList.cpp | 15 +- ...actTableFunctionArgumentsFromSelectQuery.h | 11 - ...> extractTableFunctionFromSelectQuery.cpp} | 9 +- .../extractTableFunctionFromSelectQuery.h | 12 + src/Storages/fuzzers/CMakeLists.txt | 5 +- .../fuzzers/columns_description_fuzzer.cpp | 2 - .../fuzzers/mergetree_checksum_fuzzer.cpp | 2 + src/Storages/getStructureOfRemoteTable.cpp | 28 +- src/Storages/getStructureOfRemoteTable.h | 3 +- src/Storages/prepareReadingFromFormat.cpp | 59 +- src/Storages/prepareReadingFromFormat.h | 10 +- src/Storages/registerStorages.cpp | 81 +- .../removeGroupingFunctionSpecializations.cpp | 1 + .../tests/gtest_SplitTokenExtractor.cpp | 2 +- ...est_aux_funcs_for_adaptive_granularity.cpp | 108 +- ...for_adaptive_granularity_compact_parts.cpp | 9 +- src/Storages/tests/gtest_storage_log.cpp | 4 +- ..._transform_query_for_external_database.cpp | 8 +- .../tests/gtest_virtual_column_utils.cpp | 87 + .../transformQueryForExternalDatabase.cpp | 27 +- ...nsformQueryForExternalDatabaseAnalyzer.cpp | 19 +- ...ransformQueryForExternalDatabaseAnalyzer.h | 3 +- src/TableFunctions/CMakeLists.txt | 2 + src/TableFunctions/Hive/TableFunctionHive.cpp | 13 +- src/TableFunctions/ITableFunction.h | 4 +- src/TableFunctions/ITableFunctionCluster.h | 32 +- src/TableFunctions/ITableFunctionDataLake.h | 99 - src/TableFunctions/ITableFunctionFileLike.cpp | 53 +- src/TableFunctions/ITableFunctionFileLike.h | 55 +- src/TableFunctions/ITableFunctionXDBC.cpp | 17 +- .../TableFunctionExecutable.cpp | 15 +- src/TableFunctions/TableFunctionExplain.cpp | 38 +- src/TableFunctions/TableFunctionFactory.cpp | 9 +- src/TableFunctions/TableFunctionFile.cpp | 17 +- src/TableFunctions/TableFunctionFile.h | 2 +- .../TableFunctionFileCluster.cpp | 21 +- src/TableFunctions/TableFunctionFileCluster.h | 2 +- src/TableFunctions/TableFunctionFormat.cpp | 10 +- src/TableFunctions/TableFunctionFuzzJSON.cpp | 3 +- src/TableFunctions/TableFunctionFuzzQuery.cpp | 4 +- src/TableFunctions/TableFunctionLoop.cpp | 8 +- src/TableFunctions/TableFunctionMerge.cpp | 104 +- .../TableFunctionMergeTreeIndex.cpp | 2 +- src/TableFunctions/TableFunctionMongoDB.cpp | 95 +- src/TableFunctions/TableFunctionMongoDB.h | 15 + src/TableFunctions/TableFunctionMySQL.cpp | 15 +- src/TableFunctions/TableFunctionNumbers.cpp | 11 +- .../TableFunctionObjectStorage.cpp | 223 +- .../TableFunctionObjectStorage.h | 148 +- .../TableFunctionObjectStorageCluster.cpp | 99 +- .../TableFunctionObjectStorageCluster.h | 73 +- .../TableFunctionPostgreSQL.cpp | 18 +- .../TableFunctionProjection.cpp | 109 + src/TableFunctions/TableFunctionRedis.cpp | 1 + src/TableFunctions/TableFunctionRemote.cpp | 24 +- .../TableFunctionTimeSeries.cpp | 6 +- src/TableFunctions/TableFunctionURL.cpp | 88 +- src/TableFunctions/TableFunctionURL.h | 44 +- .../TableFunctionURLCluster.cpp | 30 +- src/TableFunctions/TableFunctionURLCluster.h | 2 +- src/TableFunctions/TableFunctionValues.cpp | 8 +- src/TableFunctions/TableFunctionView.cpp | 7 +- .../TableFunctionViewIfPermitted.cpp | 6 +- src/TableFunctions/TableFunctionZeros.cpp | 29 +- .../registerDataLakeTableFunctions.cpp | 69 - src/TableFunctions/registerTableFunctions.cpp | 5 +- src/TableFunctions/registerTableFunctions.h | 12 +- src/configure_config.cmake | 21 +- 5448 files changed, 584072 insertions(+), 162518 deletions(-) create mode 100644 base/README.md create mode 100644 base/base/AlignedUnion.h create mode 100644 base/base/BFloat16.h create mode 100644 base/base/GitHash.cpp.in create mode 100644 base/base/IPv4andIPv6.cpp rename {src/Common => base/base}/MemorySanitizer.h (100%) create mode 100644 base/base/libcpp_verbose_abort.cpp rename {src/Common => base/base}/memcmpSmall.h (99%) create mode 100644 base/base/openpty.h create mode 100644 base/base/wide_integer_to_string.cpp delete mode 100644 base/poco/Crypto/CMakeLists.txt delete mode 100644 base/poco/Crypto/include/Poco/Crypto/Cipher.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherFactory.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherImpl.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherKey.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/Crypto.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CryptoException.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CryptoStream.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/DigestEngine.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/ECKey.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/EVPPKey.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/KeyPair.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/RSAKey.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h delete mode 100644 base/poco/Crypto/include/Poco/Crypto/X509Certificate.h delete mode 100644 base/poco/Crypto/src/Cipher.cpp delete mode 100644 base/poco/Crypto/src/CipherFactory.cpp delete mode 100644 base/poco/Crypto/src/CipherImpl.cpp delete mode 100644 base/poco/Crypto/src/CipherKey.cpp delete mode 100644 base/poco/Crypto/src/CipherKeyImpl.cpp delete mode 100644 base/poco/Crypto/src/CryptoException.cpp delete mode 100644 base/poco/Crypto/src/CryptoStream.cpp delete mode 100644 base/poco/Crypto/src/CryptoTransform.cpp delete mode 100644 base/poco/Crypto/src/DigestEngine.cpp delete mode 100644 base/poco/Crypto/src/ECDSADigestEngine.cpp delete mode 100644 base/poco/Crypto/src/ECKey.cpp delete mode 100644 base/poco/Crypto/src/ECKeyImpl.cpp delete mode 100644 base/poco/Crypto/src/EVPPKey.cpp delete mode 100644 base/poco/Crypto/src/KeyPair.cpp delete mode 100644 base/poco/Crypto/src/KeyPairImpl.cpp delete mode 100644 base/poco/Crypto/src/OpenSSLInitializer.cpp delete mode 100644 base/poco/Crypto/src/PKCS12Container.cpp delete mode 100644 base/poco/Crypto/src/RSACipherImpl.cpp delete mode 100644 base/poco/Crypto/src/RSADigestEngine.cpp delete mode 100644 base/poco/Crypto/src/RSAKey.cpp delete mode 100644 base/poco/Crypto/src/RSAKeyImpl.cpp delete mode 100644 base/poco/Crypto/src/X509Certificate.cpp delete mode 100644 base/poco/Data/CMakeLists.txt delete mode 100644 base/poco/Data/ODBC/CMakeLists.txt delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ConnectionHandle.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Connector.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Diagnostics.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/EnvironmentHandle.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Error.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Extractor.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Handle.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBC.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCException.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCMetaColumn.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/ODBCStatementImpl.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Parameter.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Preparator.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/SessionImpl.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/TypeInfo.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Unicode_UNIXODBC.h delete mode 100644 base/poco/Data/ODBC/include/Poco/Data/ODBC/Utility.h delete mode 100644 base/poco/Data/ODBC/src/Binder.cpp delete mode 100644 base/poco/Data/ODBC/src/ConnectionHandle.cpp delete mode 100644 base/poco/Data/ODBC/src/Connector.cpp delete mode 100644 base/poco/Data/ODBC/src/EnvironmentHandle.cpp delete mode 100644 base/poco/Data/ODBC/src/Extractor.cpp delete mode 100644 base/poco/Data/ODBC/src/ODBCException.cpp delete mode 100644 base/poco/Data/ODBC/src/ODBCMetaColumn.cpp delete mode 100644 base/poco/Data/ODBC/src/ODBCStatementImpl.cpp delete mode 100644 base/poco/Data/ODBC/src/Parameter.cpp delete mode 100644 base/poco/Data/ODBC/src/Preparator.cpp delete mode 100644 base/poco/Data/ODBC/src/SessionImpl.cpp delete mode 100644 base/poco/Data/ODBC/src/TypeInfo.cpp delete mode 100644 base/poco/Data/ODBC/src/Unicode.cpp delete mode 100644 base/poco/Data/ODBC/src/Unicode_UNIXODBC.cpp delete mode 100644 base/poco/Data/ODBC/src/Utility.cpp delete mode 100644 base/poco/Data/include/Poco/Data/AbstractBinder.h delete mode 100644 base/poco/Data/include/Poco/Data/AbstractBinding.h delete mode 100644 base/poco/Data/include/Poco/Data/AbstractExtraction.h delete mode 100644 base/poco/Data/include/Poco/Data/AbstractExtractor.h delete mode 100644 base/poco/Data/include/Poco/Data/AbstractPreparation.h delete mode 100644 base/poco/Data/include/Poco/Data/AbstractPreparator.h delete mode 100644 base/poco/Data/include/Poco/Data/AbstractSessionImpl.h delete mode 100644 base/poco/Data/include/Poco/Data/ArchiveStrategy.h delete mode 100644 base/poco/Data/include/Poco/Data/Binding.h delete mode 100644 base/poco/Data/include/Poco/Data/Bulk.h delete mode 100644 base/poco/Data/include/Poco/Data/BulkBinding.h delete mode 100644 base/poco/Data/include/Poco/Data/BulkExtraction.h delete mode 100644 base/poco/Data/include/Poco/Data/Column.h delete mode 100644 base/poco/Data/include/Poco/Data/Connector.h delete mode 100644 base/poco/Data/include/Poco/Data/Constants.h delete mode 100644 base/poco/Data/include/Poco/Data/Data.h delete mode 100644 base/poco/Data/include/Poco/Data/DataException.h delete mode 100644 base/poco/Data/include/Poco/Data/Date.h delete mode 100644 base/poco/Data/include/Poco/Data/DynamicDateTime.h delete mode 100644 base/poco/Data/include/Poco/Data/Extraction.h delete mode 100644 base/poco/Data/include/Poco/Data/LOB.h delete mode 100644 base/poco/Data/include/Poco/Data/Limit.h delete mode 100644 base/poco/Data/include/Poco/Data/MetaColumn.h delete mode 100644 base/poco/Data/include/Poco/Data/PooledSessionHolder.h delete mode 100644 base/poco/Data/include/Poco/Data/PooledSessionImpl.h delete mode 100644 base/poco/Data/include/Poco/Data/Position.h delete mode 100644 base/poco/Data/include/Poco/Data/Preparation.h delete mode 100644 base/poco/Data/include/Poco/Data/Range.h delete mode 100644 base/poco/Data/include/Poco/Data/RecordSet.h delete mode 100644 base/poco/Data/include/Poco/Data/Row.h delete mode 100644 base/poco/Data/include/Poco/Data/RowFilter.h delete mode 100644 base/poco/Data/include/Poco/Data/RowFormatter.h delete mode 100644 base/poco/Data/include/Poco/Data/RowIterator.h delete mode 100644 base/poco/Data/include/Poco/Data/SQLChannel.h delete mode 100644 base/poco/Data/include/Poco/Data/Session.h delete mode 100644 base/poco/Data/include/Poco/Data/SessionFactory.h delete mode 100644 base/poco/Data/include/Poco/Data/SessionImpl.h delete mode 100644 base/poco/Data/include/Poco/Data/SessionPool.h delete mode 100644 base/poco/Data/include/Poco/Data/SessionPoolContainer.h delete mode 100644 base/poco/Data/include/Poco/Data/SimpleRowFormatter.h delete mode 100644 base/poco/Data/include/Poco/Data/Statement.h delete mode 100644 base/poco/Data/include/Poco/Data/StatementCreator.h delete mode 100644 base/poco/Data/include/Poco/Data/StatementImpl.h delete mode 100644 base/poco/Data/include/Poco/Data/Time.h delete mode 100644 base/poco/Data/include/Poco/Data/Transaction.h delete mode 100644 base/poco/Data/include/Poco/Data/TypeHandler.h delete mode 100644 base/poco/Data/src/AbstractBinder.cpp delete mode 100644 base/poco/Data/src/AbstractBinding.cpp delete mode 100644 base/poco/Data/src/AbstractExtraction.cpp delete mode 100644 base/poco/Data/src/AbstractExtractor.cpp delete mode 100644 base/poco/Data/src/AbstractPreparation.cpp delete mode 100644 base/poco/Data/src/AbstractPreparator.cpp delete mode 100644 base/poco/Data/src/ArchiveStrategy.cpp delete mode 100644 base/poco/Data/src/Bulk.cpp delete mode 100644 base/poco/Data/src/Connector.cpp delete mode 100644 base/poco/Data/src/DataException.cpp delete mode 100644 base/poco/Data/src/Date.cpp delete mode 100644 base/poco/Data/src/Limit.cpp delete mode 100644 base/poco/Data/src/MetaColumn.cpp delete mode 100644 base/poco/Data/src/PooledSessionHolder.cpp delete mode 100644 base/poco/Data/src/PooledSessionImpl.cpp delete mode 100644 base/poco/Data/src/Position.cpp delete mode 100644 base/poco/Data/src/Range.cpp delete mode 100644 base/poco/Data/src/RecordSet.cpp delete mode 100644 base/poco/Data/src/Row.cpp delete mode 100644 base/poco/Data/src/RowFilter.cpp delete mode 100644 base/poco/Data/src/RowFormatter.cpp delete mode 100644 base/poco/Data/src/RowIterator.cpp delete mode 100644 base/poco/Data/src/SQLChannel.cpp delete mode 100644 base/poco/Data/src/Session.cpp delete mode 100644 base/poco/Data/src/SessionFactory.cpp delete mode 100644 base/poco/Data/src/SessionImpl.cpp delete mode 100644 base/poco/Data/src/SessionPool.cpp delete mode 100644 base/poco/Data/src/SessionPoolContainer.cpp delete mode 100644 base/poco/Data/src/SimpleRowFormatter.cpp delete mode 100644 base/poco/Data/src/Statement.cpp delete mode 100644 base/poco/Data/src/StatementCreator.cpp delete mode 100644 base/poco/Data/src/StatementImpl.cpp delete mode 100644 base/poco/Data/src/Time.cpp delete mode 100644 base/poco/Data/src/Transaction.cpp delete mode 100644 base/poco/MongoDB/CMakeLists.txt delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Array.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/BSONReader.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/BSONWriter.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Binary.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Connection.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Cursor.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Database.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/DeleteRequest.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Document.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Element.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/GetMoreRequest.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/InsertRequest.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/JavaScriptCode.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/KillCursorsRequest.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/Message.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/MessageHeader.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/MongoDB.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/ObjectId.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/OpMsgCursor.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/OpMsgMessage.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/PoolableConnectionFactory.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/QueryRequest.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/RegularExpression.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/ReplicaSet.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/RequestMessage.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/ResponseMessage.h delete mode 100644 base/poco/MongoDB/include/Poco/MongoDB/UpdateRequest.h delete mode 100644 base/poco/MongoDB/src/Array.cpp delete mode 100644 base/poco/MongoDB/src/Binary.cpp delete mode 100644 base/poco/MongoDB/src/Connection.cpp delete mode 100644 base/poco/MongoDB/src/Cursor.cpp delete mode 100644 base/poco/MongoDB/src/Database.cpp delete mode 100644 base/poco/MongoDB/src/DeleteRequest.cpp delete mode 100644 base/poco/MongoDB/src/Document.cpp delete mode 100644 base/poco/MongoDB/src/Element.cpp delete mode 100644 base/poco/MongoDB/src/GetMoreRequest.cpp delete mode 100644 base/poco/MongoDB/src/InsertRequest.cpp delete mode 100644 base/poco/MongoDB/src/JavaScriptCode.cpp delete mode 100644 base/poco/MongoDB/src/KillCursorsRequest.cpp delete mode 100644 base/poco/MongoDB/src/Message.cpp delete mode 100644 base/poco/MongoDB/src/MessageHeader.cpp delete mode 100644 base/poco/MongoDB/src/ObjectId.cpp delete mode 100644 base/poco/MongoDB/src/OpMsgCursor.cpp delete mode 100644 base/poco/MongoDB/src/OpMsgMessage.cpp delete mode 100644 base/poco/MongoDB/src/QueryRequest.cpp delete mode 100644 base/poco/MongoDB/src/RegularExpression.cpp delete mode 100644 base/poco/MongoDB/src/ReplicaSet.cpp delete mode 100644 base/poco/MongoDB/src/RequestMessage.cpp delete mode 100644 base/poco/MongoDB/src/ResponseMessage.cpp delete mode 100644 base/poco/MongoDB/src/UpdateRequest.cpp delete mode 100644 base/poco/Net/include/Poco/Net/FTPClientSession.h delete mode 100644 base/poco/Net/include/Poco/Net/FTPStreamFactory.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPClient.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPEventArgs.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPPacket.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPPacketImpl.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPSocket.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPSocketImpl.h delete mode 100644 base/poco/Net/include/Poco/Net/ICMPv4PacketImpl.h delete mode 100644 base/poco/Net/include/Poco/Net/MailMessage.h delete mode 100644 base/poco/Net/include/Poco/Net/MailRecipient.h delete mode 100644 base/poco/Net/include/Poco/Net/MailStream.h delete mode 100644 base/poco/Net/include/Poco/Net/NTPClient.h delete mode 100644 base/poco/Net/include/Poco/Net/NTPEventArgs.h delete mode 100644 base/poco/Net/include/Poco/Net/NTPPacket.h delete mode 100644 base/poco/Net/include/Poco/Net/POP3ClientSession.h delete mode 100644 base/poco/Net/include/Poco/Net/SMTPClientSession.h delete mode 100644 base/poco/Net/src/FTPClientSession.cpp delete mode 100644 base/poco/Net/src/FTPStreamFactory.cpp delete mode 100644 base/poco/Net/src/ICMPClient.cpp delete mode 100644 base/poco/Net/src/ICMPEventArgs.cpp delete mode 100644 base/poco/Net/src/ICMPPacket.cpp delete mode 100644 base/poco/Net/src/ICMPPacketImpl.cpp delete mode 100644 base/poco/Net/src/ICMPSocket.cpp delete mode 100644 base/poco/Net/src/ICMPSocketImpl.cpp delete mode 100644 base/poco/Net/src/ICMPv4PacketImpl.cpp delete mode 100644 base/poco/Net/src/MailMessage.cpp delete mode 100644 base/poco/Net/src/MailRecipient.cpp delete mode 100644 base/poco/Net/src/MailStream.cpp delete mode 100644 base/poco/Net/src/NTPClient.cpp delete mode 100644 base/poco/Net/src/NTPEventArgs.cpp delete mode 100644 base/poco/Net/src/NTPPacket.cpp delete mode 100644 base/poco/Net/src/POP3ClientSession.cpp delete mode 100644 base/poco/Net/src/SMTPClientSession.cpp delete mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSMTPClientSession.h delete mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/X509Certificate.h delete mode 100644 base/poco/NetSSL_OpenSSL/src/SecureSMTPClientSession.cpp delete mode 100644 base/poco/NetSSL_OpenSSL/src/X509Certificate.cpp create mode 100644 ci/README.md create mode 100644 ci/defs/defs.py create mode 100644 ci/defs/job_configs.py create mode 100644 ci/docker/binary-builder/Dockerfile create mode 100644 ci/docker/clickhouse-server/docker_related_config.xml create mode 100755 ci/docker/clickhouse-server/entrypoint.sh create mode 100644 ci/docker/clickhouse-server/from_binary/Dockerfile.ubuntu create mode 100644 ci/docker/clickhouse-server/from_deb/Dockerfile.ubuntu create mode 100644 ci/docker/compatibility/centos/Dockerfile create mode 100644 ci/docker/compatibility/ubuntu/Dockerfile create mode 100644 ci/docker/docs-builder/Dockerfile create mode 100644 ci/docker/fasttest/Dockerfile create mode 100644 ci/docker/fasttest/requirements.txt create mode 100644 ci/docker/integration/integration-test/Dockerfile create mode 100644 ci/docker/integration/integration-test/requirements.txt create mode 100644 ci/docker/sqlancer-test/Dockerfile create mode 100644 ci/docker/stateful-test/Dockerfile create mode 100644 ci/docker/stateless-test/Dockerfile create mode 100644 ci/docker/stateless-test/requirements.txt create mode 100644 ci/docker/style-test/Dockerfile create mode 100644 ci/docker/style-test/requirements.txt create mode 100644 ci/jobs/build_clickhouse.py create mode 100644 ci/jobs/check_style.py create mode 100644 ci/jobs/clickbench.py create mode 100644 ci/jobs/collect_statistics.py create mode 100644 ci/jobs/docker_server_job.py create mode 100644 ci/jobs/docs_job.py create mode 100644 ci/jobs/fast_test.py create mode 100644 ci/jobs/functional_stateful_tests.py create mode 100644 ci/jobs/functional_stateless_tests.py create mode 100644 ci/jobs/fuzzers_job.py create mode 100644 ci/jobs/performance_tests.py create mode 100644 ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt create mode 100755 ci/jobs/scripts/check_style/check-mypy create mode 100755 ci/jobs/scripts/check_style/check-settings-style create mode 100755 ci/jobs/scripts/check_style/check_aspell.sh create mode 100755 ci/jobs/scripts/check_style/check_cpp.sh create mode 100755 ci/jobs/scripts/check_style/check_submodules.sh create mode 100755 ci/jobs/scripts/check_style/check_typos.sh create mode 100755 ci/jobs/scripts/check_style/double_whitespaces.pl create mode 100755 ci/jobs/scripts/check_style/various_checks.sh create mode 100644 ci/jobs/scripts/cidb_cluster.py create mode 100644 ci/jobs/scripts/clickbench/create.sql create mode 100644 ci/jobs/scripts/clickbench/queries.sql create mode 100644 ci/jobs/scripts/clickhouse_proc.py create mode 100644 ci/jobs/scripts/clickhouse_version.py create mode 100755 ci/jobs/scripts/functional_tests/setup_ch_cluster.sh create mode 100755 ci/jobs/scripts/functional_tests/setup_hdfs_minicluster.sh create mode 100755 ci/jobs/scripts/functional_tests/setup_log_cluster.sh create mode 100755 ci/jobs/scripts/functional_tests/setup_minio.sh create mode 100755 ci/jobs/scripts/functional_tests_results.py create mode 100644 ci/jobs/scripts/fuzzer/allow-nullable-key.xml create mode 100755 ci/jobs/scripts/fuzzer/generate-test-j2.py create mode 100644 ci/jobs/scripts/fuzzer/query-fuzzer-tweaks-users.xml create mode 100755 ci/jobs/scripts/fuzzer/runner.sh create mode 100644 ci/jobs/scripts/job_hooks/build_master_head_hook.py create mode 100644 ci/jobs/scripts/job_hooks/build_profile_hook.py create mode 100644 ci/jobs/scripts/job_hooks/docker_clean_up_hook.py create mode 100644 ci/jobs/scripts/log_cluster.py create mode 100755 ci/jobs/scripts/perf/compare.sh create mode 100755 ci/jobs/scripts/perf/download.sh create mode 100644 ci/jobs/scripts/perf/eqmed.sql create mode 100755 ci/jobs/scripts/perf/report.py create mode 100644 ci/jobs/scripts/workflow_hooks/can_be_merged.py create mode 100644 ci/jobs/scripts/workflow_hooks/feature_docs.py create mode 100644 ci/jobs/scripts/workflow_hooks/filter_job.py create mode 100644 ci/jobs/scripts/workflow_hooks/merge_sync_pr.py create mode 100644 ci/jobs/scripts/workflow_hooks/pr_description.py create mode 100644 ci/jobs/scripts/workflow_hooks/quick_sync.py create mode 100644 ci/jobs/scripts/workflow_hooks/set_dummy_sync_commit_status.py create mode 100644 ci/jobs/scripts/workflow_hooks/store_data.py create mode 100644 ci/jobs/scripts/workflow_hooks/trusted.py create mode 100644 ci/jobs/scripts/workflow_hooks/version_log.py create mode 100755 ci/jobs/sqlancer_job.sh create mode 100755 ci/jobs/sqltest_job.py create mode 100644 ci/jobs/unit_tests_job.py create mode 100644 ci/praktika/__init__.py create mode 100644 ci/praktika/__main__.py create mode 100644 ci/praktika/_environment.py create mode 100644 ci/praktika/artifact.py create mode 100644 ci/praktika/cache.py create mode 100644 ci/praktika/cidb.py create mode 100644 ci/praktika/digest.py create mode 100644 ci/praktika/docker.py create mode 100644 ci/praktika/execution/__init__.py create mode 100644 ci/praktika/execution/__main__.py create mode 100644 ci/praktika/execution/execution_settings.py create mode 100644 ci/praktika/execution/machine_init.py create mode 100644 ci/praktika/gh.py create mode 100644 ci/praktika/gh_auth.py create mode 100644 ci/praktika/hook_cache.py create mode 100644 ci/praktika/hook_html.py create mode 100644 ci/praktika/hook_interface.py create mode 100644 ci/praktika/html_prepare.py create mode 100644 ci/praktika/info.py create mode 100644 ci/praktika/job.py create mode 100644 ci/praktika/json.html create mode 100644 ci/praktika/mangle.py create mode 100644 ci/praktika/native_jobs.py create mode 100644 ci/praktika/parser.py create mode 100644 ci/praktika/result.py create mode 100644 ci/praktika/runner.py create mode 100644 ci/praktika/runtime.py create mode 100644 ci/praktika/s3.py create mode 100644 ci/praktika/secret.py create mode 100644 ci/praktika/settings.py create mode 100644 ci/praktika/usage.py create mode 100644 ci/praktika/utils.py create mode 100644 ci/praktika/validator.py create mode 100644 ci/praktika/workflow.py create mode 100644 ci/praktika/yaml_generator.py create mode 100644 ci/settings/settings.py create mode 100644 ci/setup.py create mode 100644 ci/workflows/backport_branches.py create mode 100644 ci/workflows/master.py create mode 100644 ci/workflows/merge_queue.py create mode 100644 ci/workflows/new_pull_request.py create mode 100644 ci/workflows/nightly_fuzzers.py create mode 100644 ci/workflows/nightly_jepsen.py create mode 100644 ci/workflows/nightly_statistics.py create mode 100644 ci/workflows/pull_request.py create mode 100644 ci/workflows/release_branches.py create mode 100755 cmake/dummy_compiler_linker.sh create mode 160000 contrib/SHA3IUF create mode 100644 contrib/corrosion-cmake/config.toml.in create mode 160000 contrib/delta-kernel-rs create mode 100644 contrib/delta-kernel-rs-cmake/CMakeLists.txt delete mode 160000 contrib/expected delete mode 100644 contrib/expected-cmake/CMakeLists.txt create mode 160000 contrib/google-cloud-cpp create mode 100644 contrib/google-cloud-cpp-cmake/CMakeLists.txt create mode 100644 contrib/google-cloud-cpp-cmake/GoogleApis.cmake create mode 100644 contrib/google-cloud-cpp-cmake/google_cloud_cpp_common.cmake create mode 100644 contrib/google-cloud-cpp-cmake/google_cloud_cpp_grpc_utils.cmake create mode 160000 contrib/jwt-cpp create mode 100644 contrib/jwt-cpp-cmake/CMakeLists.txt create mode 100644 contrib/krb5-cmake/kcmrpc.cin create mode 100644 contrib/krb5-cmake/kcmrpc.hin delete mode 160000 contrib/libunwind delete mode 100644 contrib/minizip-ng-cmake/unzip.h delete mode 100644 contrib/minizip-ng-cmake/zip.h create mode 160000 contrib/mongo-c-driver create mode 100644 contrib/mongo-c-driver-cmake/CMakeLists.txt create mode 160000 contrib/mongo-cxx-driver create mode 100644 contrib/mongo-cxx-driver-cmake/CMakeLists.txt delete mode 160000 contrib/nanodbc delete mode 100644 contrib/nanodbc-cmake/CMakeLists.txt create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aes-riscv64-zkn.S create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aes-s390x.S create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aes-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aesni-mb-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aesni-sha1-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aesni-sha256-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aesni-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aesp8-ppc.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/aesv8-armx.S create mode 100644 contrib/openssl-cmake/asm/crypto/aes/bsaes-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/aes/bsaes-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/aes/vpaes-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/aes/vpaes-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/arm64cpuid.S create mode 100644 contrib/openssl-cmake/asm/crypto/bn/armv8-mont.S create mode 100644 contrib/openssl-cmake/asm/crypto/bn/rsaz-2k-avx512.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/rsaz-3k-avx512.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/rsaz-4k-avx512.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/rsaz-avx2.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/rsaz-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/x86_64-gf2m.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/x86_64-mont.s create mode 100644 contrib/openssl-cmake/asm/crypto/bn/x86_64-mont5.s create mode 100644 contrib/openssl-cmake/asm/crypto/camellia/cmll-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/chacha/chacha-armv8-sve.S create mode 100644 contrib/openssl-cmake/asm/crypto/chacha/chacha-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/chacha/chacha-s390x.S create mode 100644 contrib/openssl-cmake/asm/crypto/chacha/chacha-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/ec/ecp_nistz256-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/ec/ecp_nistz256-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/ec/ecp_sm2p256-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/ec/x25519-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/loongarch64cpuid.S create mode 100644 contrib/openssl-cmake/asm/crypto/md5/asm/md5-aarch64.S create mode 100644 contrib/openssl-cmake/asm/crypto/md5/md5-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/modes/aes-gcm-avx512.s create mode 100644 contrib/openssl-cmake/asm/crypto/modes/aes-gcm-ppc.s create mode 100644 contrib/openssl-cmake/asm/crypto/modes/aesni-gcm-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/modes/asm/aes-gcm-armv8-unroll8_64.S create mode 100644 contrib/openssl-cmake/asm/crypto/modes/asm/aes-gcm-armv8_64.S create mode 100644 contrib/openssl-cmake/asm/crypto/modes/ghash-riscv64.S create mode 100644 contrib/openssl-cmake/asm/crypto/modes/ghash-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/modes/ghashp8-ppc.s create mode 100644 contrib/openssl-cmake/asm/crypto/modes/ghashv8-armx.S create mode 100644 contrib/openssl-cmake/asm/crypto/poly1305/poly1305-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/poly1305/poly1305-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/ppccpuid.s create mode 100644 contrib/openssl-cmake/asm/crypto/rc4/rc4-md5-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/rc4/rc4-s390x.S create mode 100644 contrib/openssl-cmake/asm/crypto/rc4/rc4-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/riscv64cpuid.S create mode 100644 contrib/openssl-cmake/asm/crypto/s390xcpuid.S create mode 100644 contrib/openssl-cmake/asm/crypto/sha/keccak1600-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sha/keccak1600-s390x.S create mode 100644 contrib/openssl-cmake/asm/crypto/sha/keccak1600-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha1-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha1-mb-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha1-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha256-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha256-mb-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha256-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha512-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sha/sha512-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/sm3/asm/sm3-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sm4/asm/sm4-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sm4/asm/vpsm4-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/sm4/asm/vpsm4_ex-armv8.S create mode 100644 contrib/openssl-cmake/asm/crypto/whrlpool/wp-x86_64.s create mode 100644 contrib/openssl-cmake/asm/crypto/x86_64cpuid.s create mode 100755 contrib/openssl-cmake/asm/generate_asm.sh delete mode 160000 contrib/robin-map delete mode 100644 contrib/robin-map-cmake/CMakeLists.txt create mode 160000 contrib/rust_vendor create mode 100644 contrib/sha3iuf-cmake/CMakeLists.txt create mode 100755 contrib/sparse-checkout/update-google-cloud-cpp.sh create mode 100755 contrib/sparse-checkout/update-postgres.sh delete mode 160000 contrib/unixodbc delete mode 100644 contrib/unixodbc-cmake/CMakeLists.txt delete mode 100644 contrib/unixodbc-cmake/linux_x86_64/libltdl/config.h delete mode 100644 contrib/unixodbc-cmake/linux_x86_64/libltdl/libltdlcS.c delete mode 100644 contrib/unixodbc-cmake/linux_x86_64/private/config.h delete mode 100644 contrib/unixodbc-cmake/linux_x86_64/unixodbc_conf.h delete mode 100644 packages/clickhouse-library-bridge.yaml delete mode 100644 packages/clickhouse-odbc-bridge.yaml create mode 100644 programs/check-marks/CMakeLists.txt create mode 100644 programs/check-marks/CheckMarks.cpp create mode 100644 programs/check-marks/check-marks.cpp create mode 100644 programs/checksum-for-compressed-block/CMakeLists.txt create mode 100644 programs/checksum-for-compressed-block/ChecksumForCompressedBlock.cpp create mode 100644 programs/checksum-for-compressed-block/checksum-for-compressed-block.cpp create mode 100644 programs/client/FuzzLoop.cpp create mode 100644 programs/disks/Utils.cpp create mode 100644 programs/disks/Utils.h create mode 100644 programs/keeper-bench/CMakeLists.txt create mode 100644 programs/keeper-bench/Generator.cpp create mode 100644 programs/keeper-bench/Generator.h create mode 100644 programs/keeper-bench/KeeperBench.cpp create mode 100644 programs/keeper-bench/README.md create mode 100644 programs/keeper-bench/Runner.cpp create mode 100644 programs/keeper-bench/Runner.h create mode 100644 programs/keeper-bench/Stats.cpp create mode 100644 programs/keeper-bench/Stats.h create mode 100644 programs/keeper-bench/example.yaml create mode 100644 programs/keeper-bench/keeper-bench.cpp create mode 100644 programs/keeper-data-dumper/CMakeLists.txt create mode 100644 programs/keeper-data-dumper/KeeperDataDumper.cpp create mode 100644 programs/keeper-data-dumper/keeper-data-dumper.cpp delete mode 100644 programs/library-bridge/CMakeLists.txt delete mode 100644 programs/library-bridge/CatBoostLibraryAPI.h delete mode 100644 programs/library-bridge/CatBoostLibraryHandler.cpp delete mode 100644 programs/library-bridge/CatBoostLibraryHandler.h delete mode 100644 programs/library-bridge/CatBoostLibraryHandlerFactory.cpp delete mode 100644 programs/library-bridge/CatBoostLibraryHandlerFactory.h delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryAPI.cpp delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryAPI.h delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryHandler.cpp delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryHandler.h delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.cpp delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryHandlerFactory.h delete mode 100644 programs/library-bridge/ExternalDictionaryLibraryUtils.h delete mode 100644 programs/library-bridge/LibraryBridge.cpp delete mode 100644 programs/library-bridge/LibraryBridge.h delete mode 100644 programs/library-bridge/LibraryBridgeHandlerFactory.cpp delete mode 100644 programs/library-bridge/LibraryBridgeHandlerFactory.h delete mode 100644 programs/library-bridge/LibraryBridgeHandlers.cpp delete mode 100644 programs/library-bridge/LibraryBridgeHandlers.h delete mode 100644 programs/library-bridge/SharedLibrary.cpp delete mode 100644 programs/library-bridge/SharedLibrary.h delete mode 100644 programs/library-bridge/createFunctionBaseCast.cpp delete mode 100644 programs/library-bridge/library-bridge.cpp delete mode 100644 programs/odbc-bridge/CMakeLists.txt delete mode 100644 programs/odbc-bridge/ColumnInfoHandler.cpp delete mode 100644 programs/odbc-bridge/ColumnInfoHandler.h delete mode 100644 programs/odbc-bridge/IdentifierQuoteHandler.cpp delete mode 100644 programs/odbc-bridge/IdentifierQuoteHandler.h delete mode 100644 programs/odbc-bridge/MainHandler.cpp delete mode 100644 programs/odbc-bridge/MainHandler.h delete mode 100644 programs/odbc-bridge/ODBCBridge.cpp delete mode 100644 programs/odbc-bridge/ODBCBridge.h delete mode 100644 programs/odbc-bridge/ODBCHandlerFactory.cpp delete mode 100644 programs/odbc-bridge/ODBCHandlerFactory.h delete mode 100644 programs/odbc-bridge/ODBCPooledConnectionFactory.h delete mode 100644 programs/odbc-bridge/ODBCSink.cpp delete mode 100644 programs/odbc-bridge/ODBCSink.h delete mode 100644 programs/odbc-bridge/ODBCSource.cpp delete mode 100644 programs/odbc-bridge/ODBCSource.h delete mode 100644 programs/odbc-bridge/PingHandler.cpp delete mode 100644 programs/odbc-bridge/PingHandler.h delete mode 100644 programs/odbc-bridge/SchemaAllowedHandler.cpp delete mode 100644 programs/odbc-bridge/SchemaAllowedHandler.h delete mode 100644 programs/odbc-bridge/createFunctionBaseCast.cpp delete mode 100644 programs/odbc-bridge/getIdentifierQuote.cpp delete mode 100644 programs/odbc-bridge/getIdentifierQuote.h delete mode 100644 programs/odbc-bridge/odbc-bridge.cpp delete mode 100644 programs/odbc-bridge/tests/CMakeLists.txt delete mode 100644 programs/odbc-bridge/tests/validate-odbc-connection-string.cpp delete mode 100644 programs/odbc-bridge/tests/validate-odbc-connection-string.reference delete mode 100755 programs/odbc-bridge/tests/validate-odbc-connection-string.sh delete mode 100644 programs/odbc-bridge/validateODBCConnectionString.cpp delete mode 100644 programs/odbc-bridge/validateODBCConnectionString.h create mode 100644 programs/server/merges.html create mode 100644 programs/zookeeper-dump-tree/CMakeLists.txt create mode 100644 programs/zookeeper-dump-tree/ZooKeeperDumpTree.cpp create mode 100644 programs/zookeeper-dump-tree/zookeeper-dump-tree.cpp create mode 100644 programs/zookeeper-remove-by-list/CMakeLists.txt create mode 100644 programs/zookeeper-remove-by-list/ZooKeeperRemoveByList.cpp create mode 100644 programs/zookeeper-remove-by-list/zookeeper-remove-by-list.cpp delete mode 100644 src/Access/Common/SSLCertificateSubjects.cpp delete mode 100644 src/Access/Common/SSLCertificateSubjects.h create mode 100644 src/Access/SSH/SSHPublicKey.cpp create mode 100644 src/Access/SSH/SSHPublicKey.h create mode 100644 src/Access/tests/gtest_access_rights_implicit.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionDistinctDynamicTypes.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionDistinctJSONPaths.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionEstimateCompressionRatio.cpp create mode 100644 src/AggregateFunctions/AggregateFunctionGroupConcat.h create mode 100644 src/Analyzer/Identifier.cpp create mode 100644 src/Analyzer/Passes/CNFAtomicFormula.cpp create mode 100644 src/Analyzer/Passes/CNFAtomicFormula.h rename src/{Planner => Analyzer/Passes}/tests/CMakeLists.txt (100%) create mode 100644 src/Analyzer/Passes/tests/gtest_logical_expression_optimizer.cpp create mode 100644 src/Analyzer/README.md create mode 100644 src/Analyzer/Resolve/TableFunctionsWithClusterAlternativesVisitor.h create mode 100644 src/Analyzer/Resolve/TypoCorrection.cpp create mode 100644 src/Analyzer/Resolve/TypoCorrection.h rename src/Analyzer/{createUniqueTableAliases.cpp => createUniqueAliasesIfNecessary.cpp} (57%) create mode 100644 src/Analyzer/createUniqueAliasesIfNecessary.h delete mode 100644 src/Analyzer/createUniqueTableAliases.h delete mode 100644 src/Analyzer/examples/CMakeLists.txt delete mode 100644 src/Analyzer/examples/query_analyzer.cpp create mode 100644 src/Backups/BackupConcurrencyCheck.cpp create mode 100644 src/Backups/BackupConcurrencyCheck.h create mode 100644 src/Backups/BackupCoordinationCleaner.cpp create mode 100644 src/Backups/BackupCoordinationCleaner.h rename src/Backups/{BackupCoordinationRemote.cpp => BackupCoordinationOnCluster.cpp} (73%) rename src/Backups/{BackupCoordinationRemote.h => BackupCoordinationOnCluster.h} (66%) create mode 100644 src/Backups/BackupIO_Memory.cpp create mode 100644 src/Backups/BackupIO_Memory.h create mode 100644 src/Backups/BackupIO_Null.cpp create mode 100644 src/Backups/BackupIO_Null.h create mode 100644 src/Backups/BackupInMemory.cpp create mode 100644 src/Backups/BackupInMemory.h create mode 100644 src/Backups/BackupKeeperSettings.cpp create mode 100644 src/Backups/BackupKeeperSettings.h create mode 100644 src/Backups/BackupsInMemoryHolder.cpp create mode 100644 src/Backups/BackupsInMemoryHolder.h create mode 100644 src/Backups/RestoreCoordinationOnCluster.cpp rename src/Backups/{RestoreCoordinationRemote.h => RestoreCoordinationOnCluster.h} (60%) delete mode 100644 src/Backups/RestoreCoordinationRemote.cpp create mode 100644 src/Backups/SettingsFieldOptionalUInt64.cpp create mode 100644 src/Backups/SettingsFieldOptionalUInt64.h delete mode 100644 src/Bridge/CMakeLists.txt delete mode 100644 src/Bridge/IBridge.cpp delete mode 100644 src/Bridge/IBridge.h create mode 100644 src/BridgeHelper/README.md create mode 100644 src/Client/BuzzHouse/AST/SQLProtoStr.cpp create mode 100644 src/Client/BuzzHouse/AST/SQLProtoStr.h create mode 100644 src/Client/BuzzHouse/Generator/ExternalIntegrations.cpp create mode 100644 src/Client/BuzzHouse/Generator/ExternalIntegrations.h create mode 100644 src/Client/BuzzHouse/Generator/FuzzConfig.cpp create mode 100644 src/Client/BuzzHouse/Generator/FuzzConfig.h create mode 100644 src/Client/BuzzHouse/Generator/QueryOracle.cpp create mode 100644 src/Client/BuzzHouse/Generator/QueryOracle.h create mode 100644 src/Client/BuzzHouse/Generator/RandomGenerator.cpp create mode 100644 src/Client/BuzzHouse/Generator/RandomGenerator.h create mode 100644 src/Client/BuzzHouse/Generator/RandomSettings.h create mode 100644 src/Client/BuzzHouse/Generator/SQLCatalog.h create mode 100644 src/Client/BuzzHouse/Generator/SQLExpression.cpp create mode 100644 src/Client/BuzzHouse/Generator/SQLFuncs.h create mode 100644 src/Client/BuzzHouse/Generator/SQLQuery.cpp create mode 100644 src/Client/BuzzHouse/Generator/SQLTable.cpp create mode 100644 src/Client/BuzzHouse/Generator/SQLTypes.cpp create mode 100644 src/Client/BuzzHouse/Generator/SQLTypes.h create mode 100644 src/Client/BuzzHouse/Generator/ServerSettings.cpp create mode 100644 src/Client/BuzzHouse/Generator/StatementGenerator.cpp create mode 100644 src/Client/BuzzHouse/Generator/StatementGenerator.h create mode 100644 src/Client/BuzzHouse/Generator/TableSetttings.cpp create mode 100644 src/Client/BuzzHouse/Proto/CMakeLists.txt create mode 100644 src/Client/BuzzHouse/Proto/SQLGrammar.proto create mode 100644 src/Client/BuzzHouse/Utils/HugeInt.cpp create mode 100644 src/Client/BuzzHouse/Utils/HugeInt.h create mode 100644 src/Client/BuzzHouse/Utils/MD5Impl.cpp create mode 100644 src/Client/BuzzHouse/Utils/MD5Impl.h create mode 100644 src/Client/BuzzHouse/Utils/Nlimits.h create mode 100644 src/Client/BuzzHouse/Utils/UHugeInt.cpp create mode 100644 src/Client/BuzzHouse/Utils/UHugeInt.h create mode 100644 src/Client/ProgressTable.cpp create mode 100644 src/Client/ProgressTable.h create mode 100644 src/Client/TerminalKeystrokeInterceptor.cpp create mode 100644 src/Client/TerminalKeystrokeInterceptor.h create mode 100644 src/Columns/ColumnLazy.cpp create mode 100644 src/Columns/ColumnLazy.h create mode 100644 src/Columns/IColumn_fwd.h create mode 100644 src/Columns/tests/gtest_column_string.cpp delete mode 100644 src/Common/ArrayCache.h create mode 100644 src/Common/BSONCXXHelper.h create mode 100644 src/Common/Base32.h create mode 100644 src/Common/ColumnsHashing.cpp create mode 100644 src/Common/ColumnsHashing/HashMethod.h create mode 100644 src/Common/Crypto/KeyPair.cpp create mode 100644 src/Common/Crypto/KeyPair.h create mode 100644 src/Common/Crypto/OpenSSLInitializer.cpp create mode 100644 src/Common/Crypto/OpenSSLInitializer.h create mode 100644 src/Common/Crypto/X509Certificate.cpp create mode 100644 src/Common/Crypto/X509Certificate.h create mode 100644 src/Common/FiberStack.cpp rename src/Common/{FieldVisitorsAccurateComparison.h => FieldAccurateComparison.cpp} (91%) create mode 100644 src/Common/FieldAccurateComparison.h create mode 100644 src/Common/FieldVisitorScale.cpp create mode 100644 src/Common/FieldVisitorScale.h create mode 100644 src/Common/FieldVisitorToJSONElement.cpp create mode 100644 src/Common/FieldVisitorToJSONElement.h create mode 100644 src/Common/FloatUtils.h create mode 100644 src/Common/HashTable/HashTableTraits.h delete mode 100644 src/Common/HashTable/LRUHashMap.h create mode 100644 src/Common/Histogram.cpp create mode 100644 src/Common/Histogram.h create mode 100644 src/Common/JSONParsers/DummyJSONParser.cpp create mode 100644 src/Common/JSONParsers/SimdJSONParser.cpp create mode 100644 src/Common/LatencyBuckets.cpp create mode 100644 src/Common/LatencyBuckets.h create mode 100644 src/Common/LibSSHInitializer.cpp create mode 100644 src/Common/LibSSHInitializer.h create mode 100644 src/Common/LibSSHLogger.cpp create mode 100644 src/Common/LibSSHLogger.h create mode 100644 src/Common/LocalDate.cpp create mode 100644 src/Common/LocalDateTime.cpp create mode 100644 src/Common/MemorySpillScheduler.cpp create mode 100644 src/Common/MemorySpillScheduler.h create mode 100644 src/Common/MemoryTrackerDebugBlockerInThread.cpp create mode 100644 src/Common/MemoryTrackerDebugBlockerInThread.h create mode 100644 src/Common/MemoryTrackerUtils.cpp create mode 100644 src/Common/MemoryTrackerUtils.h create mode 100644 src/Common/MemoryWorker.cpp create mode 100644 src/Common/MemoryWorker.h create mode 100644 src/Common/OpenTelemetryTracingContext.h create mode 100644 src/Common/ReplicasReconnector.cpp create mode 100644 src/Common/ReplicasReconnector.h create mode 100644 src/Common/Scheduler/CPUSlotsAllocation.cpp create mode 100644 src/Common/Scheduler/CPUSlotsAllocation.h rename src/Common/Scheduler/Nodes/{DynamicResourceManager.cpp => CustomResourceManager.cpp} (76%) rename src/Common/Scheduler/Nodes/{DynamicResourceManager.h => CustomResourceManager.h} (80%) create mode 100644 src/Common/Scheduler/Nodes/UnifiedSchedulerNode.h create mode 100644 src/Common/Scheduler/Nodes/WorkloadResourceManager.cpp create mode 100644 src/Common/Scheduler/Nodes/WorkloadResourceManager.h delete mode 100644 src/Common/Scheduler/Nodes/registerResourceManagers.cpp rename src/Common/Scheduler/Nodes/tests/{gtest_dynamic_resource_manager.cpp => gtest_custom_resource_manager.cpp} (82%) create mode 100644 src/Common/Scheduler/Nodes/tests/gtest_unified_scheduler_node.cpp create mode 100644 src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp delete mode 100644 src/Common/Scheduler/ResouceLink.cpp delete mode 100644 src/Common/Scheduler/ResourceManagerFactory.h create mode 100644 src/Common/Scheduler/Workload/IWorkloadEntityStorage.h create mode 100644 src/Common/Scheduler/Workload/WorkloadEntityDiskStorage.cpp create mode 100644 src/Common/Scheduler/Workload/WorkloadEntityDiskStorage.h create mode 100644 src/Common/Scheduler/Workload/WorkloadEntityKeeperStorage.cpp create mode 100644 src/Common/Scheduler/Workload/WorkloadEntityKeeperStorage.h create mode 100644 src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp create mode 100644 src/Common/Scheduler/Workload/WorkloadEntityStorageBase.h create mode 100644 src/Common/Scheduler/Workload/createWorkloadEntityStorage.cpp create mode 100644 src/Common/Scheduler/Workload/createWorkloadEntityStorage.h create mode 100644 src/Common/Scheduler/WorkloadSettings.cpp create mode 100644 src/Common/Scheduler/WorkloadSettings.h create mode 100644 src/Common/Scheduler/createResourceManager.cpp create mode 100644 src/Common/Scheduler/createResourceManager.h create mode 100644 src/Common/ShellCommandsHolder.cpp create mode 100644 src/Common/ShellCommandsHolder.h create mode 100644 src/Common/SipHash.cpp create mode 100644 src/Common/StopToken.cpp create mode 100644 src/Common/StopToken.h create mode 100644 src/Common/UniqueLock.h create mode 100644 src/Common/WeightedRandomSampling.h create mode 100644 src/Common/ZooKeeper/KeeperException.cpp rename src/{Coordination => Common/ZooKeeper}/KeeperFeatureFlags.cpp (98%) rename src/{Coordination => Common/ZooKeeper}/KeeperFeatureFlags.h (87%) create mode 100644 src/Common/ZooKeeper/ShuffleHost.h create mode 100644 src/Common/benchmarks/orc_string_dictionary.cpp create mode 100644 src/Common/benchmarks/wrap_in_nullable.cpp create mode 100644 src/Common/clibssh.h create mode 100644 src/Common/computeMaxTableNameLength.cpp create mode 100644 src/Common/computeMaxTableNameLength.h delete mode 100644 src/Common/examples/arena_with_free_lists.cpp delete mode 100644 src/Common/examples/array_cache.cpp delete mode 100644 src/Common/examples/lru_hash_map_perf.cpp delete mode 100644 src/Common/examples/space_saving.cpp create mode 100644 src/Common/examples/utf8_upper_lower.cpp create mode 100644 src/Common/exp10_i32.h rename src/Common/{getNumberOfPhysicalCPUCores.cpp => getNumberOfCPUCoresToUse.cpp} (95%) create mode 100644 src/Common/getNumberOfCPUCoresToUse.h delete mode 100644 src/Common/getNumberOfPhysicalCPUCores.h create mode 100644 src/Common/intExp.cpp create mode 100644 src/Common/intExp10.h create mode 100644 src/Common/intExp2.h create mode 100644 src/Common/maskURIPassword.h create mode 100644 src/Common/memory.cpp create mode 100644 src/Common/tests/gtest_action_blocker.cpp create mode 100644 src/Common/tests/gtest_base32.cpp create mode 100644 src/Common/tests/gtest_config_host_validation.cpp create mode 100644 src/Common/tests/gtest_histogram_metrics.cpp delete mode 100644 src/Common/tests/gtest_lru_hash_map.cpp create mode 100644 src/Common/tests/gtest_poco_json.cpp create mode 100644 src/Common/tests/gtest_tmp_replace_table_name.cpp create mode 100644 src/Common/tests/gtest_weighted_random_sampling.cpp create mode 100644 src/Compression/ParallelCompressedWriteBuffer.cpp create mode 100644 src/Compression/ParallelCompressedWriteBuffer.h create mode 100644 src/Compression/examples/decompress_perf.cpp create mode 100644 src/Coordination/KeeperStorage_fwd.h create mode 100644 src/Coordination/tests/gtest_coordination_changelog.cpp create mode 100644 src/Coordination/tests/gtest_coordination_common.cpp create mode 100644 src/Coordination/tests/gtest_coordination_common.h create mode 100644 src/Coordination/tests/gtest_coordination_snapshot.cpp create mode 100644 src/Coordination/tests/gtest_coordination_storage.cpp create mode 100644 src/Core/BackgroundSchedulePoolTaskHolder.cpp create mode 100644 src/Core/BackgroundSchedulePoolTaskHolder.h create mode 100644 src/Core/BaseSettingsFwdMacros.h create mode 100644 src/Core/BaseSettingsFwdMacrosImpl.h create mode 100644 src/Core/BlockMissingValues.cpp create mode 100644 src/Core/BlockMissingValues.h create mode 100644 src/Core/BlockNameMap.cpp create mode 100644 src/Core/BlockNameMap.h create mode 100644 src/Core/Block_fwd.h create mode 100644 src/Core/DistributedCacheDefines.h create mode 100644 src/Core/FormatFactorySettings.cpp create mode 100644 src/Core/FormatFactorySettings.h create mode 100644 src/Core/MergeSelectorAlgorithm.h delete mode 100644 src/Core/MySQL/MySQLCharset.cpp delete mode 100644 src/Core/MySQL/MySQLCharset.h delete mode 100644 src/Core/MySQL/MySQLClient.cpp delete mode 100644 src/Core/MySQL/MySQLClient.h delete mode 100644 src/Core/MySQL/MySQLGtid.cpp delete mode 100644 src/Core/MySQL/MySQLGtid.h delete mode 100644 src/Core/MySQL/MySQLReplication.cpp delete mode 100644 src/Core/MySQL/MySQLReplication.h delete mode 100644 src/Core/MySQL/tests/gtest_MySQLGtid.cpp create mode 100644 src/Core/ParallelReplicasMode.h create mode 100644 src/Core/QualifiedTableName.cpp create mode 100644 src/Core/README.md create mode 100644 src/Core/SettingsObsoleteMacros.h create mode 100644 src/Core/SettingsTierType.cpp create mode 100644 src/Core/SettingsTierType.h create mode 100644 src/Core/SettingsWriteFormat.h create mode 100644 src/Core/SortCursor.cpp create mode 100644 src/Core/StreamingHandleErrorMode.h delete mode 100644 src/Core/tests/gtest_charset_conv.cpp delete mode 100644 src/Daemon/GitHash.cpp.in rename src/{Interpreters => DataTypes}/NullableUtils.cpp (70%) create mode 100644 src/DataTypes/NullableUtils.h create mode 100644 src/DataTypes/Serializations/DeserializationTask.h create mode 100644 src/Databases/DataLake/Common.cpp create mode 100644 src/Databases/DataLake/Common.h create mode 100644 src/Databases/DataLake/DataLakeConstants.h create mode 100644 src/Databases/DataLake/DatabaseDataLake.cpp create mode 100644 src/Databases/DataLake/DatabaseDataLake.h create mode 100644 src/Databases/DataLake/DatabaseDataLakeSettings.cpp create mode 100644 src/Databases/DataLake/DatabaseDataLakeSettings.h create mode 100644 src/Databases/DataLake/DatabaseDataLakeStorageType.h create mode 100644 src/Databases/DataLake/GlueCatalog.cpp create mode 100644 src/Databases/DataLake/GlueCatalog.h create mode 100644 src/Databases/DataLake/HTTPBasedCatalogUtils.cpp create mode 100644 src/Databases/DataLake/HTTPBasedCatalogUtils.h create mode 100644 src/Databases/DataLake/HiveCatalog.cpp create mode 100644 src/Databases/DataLake/HiveCatalog.h create mode 100644 src/Databases/DataLake/ICatalog.cpp create mode 100644 src/Databases/DataLake/ICatalog.h create mode 100644 src/Databases/DataLake/RestCatalog.cpp create mode 100644 src/Databases/DataLake/RestCatalog.h create mode 100644 src/Databases/DataLake/StorageCredentials.h create mode 100644 src/Databases/DataLake/UnityCatalog.cpp create mode 100644 src/Databases/DataLake/UnityCatalog.h create mode 100644 src/Databases/DatabaseBackup.cpp create mode 100644 src/Databases/DatabaseBackup.h delete mode 100644 src/Databases/MySQL/DatabaseMaterializedMySQL.cpp delete mode 100644 src/Databases/MySQL/DatabaseMaterializedMySQL.h delete mode 100644 src/Databases/MySQL/DatabaseMaterializedTablesIterator.h delete mode 100644 src/Databases/MySQL/MaterializeMetadata.cpp delete mode 100644 src/Databases/MySQL/MaterializeMetadata.h delete mode 100644 src/Databases/MySQL/MaterializedMySQLSettings.cpp delete mode 100644 src/Databases/MySQL/MaterializedMySQLSettings.h delete mode 100644 src/Databases/MySQL/MaterializedMySQLSyncThread.cpp delete mode 100644 src/Databases/MySQL/MaterializedMySQLSyncThread.h delete mode 100644 src/Databases/MySQL/MySQLBinlog.cpp delete mode 100644 src/Databases/MySQL/MySQLBinlog.h delete mode 100644 src/Databases/MySQL/MySQLBinlogClient.cpp delete mode 100644 src/Databases/MySQL/MySQLBinlogClient.h delete mode 100644 src/Databases/MySQL/MySQLBinlogClientFactory.cpp delete mode 100644 src/Databases/MySQL/MySQLBinlogClientFactory.h delete mode 100644 src/Databases/MySQL/MySQLBinlogEventsDispatcher.cpp delete mode 100644 src/Databases/MySQL/MySQLBinlogEventsDispatcher.h delete mode 100644 src/Databases/MySQL/tests/data/binlog.000016 delete mode 100644 src/Databases/MySQL/tests/data/binlog.001390 delete mode 100644 src/Databases/MySQL/tests/gtest_mysql_binlog.cpp delete mode 100644 src/Databases/MySQL/tests/gtest_try_convert_string_literals.cpp delete mode 100644 src/Databases/MySQL/tests/gtest_try_parse_table_id_from_ddl.cpp delete mode 100644 src/Databases/MySQL/tests/gtest_try_quote_unrecognized_tokens.cpp delete mode 100644 src/Databases/MySQL/tryConvertStringLiterals.cpp delete mode 100644 src/Databases/MySQL/tryConvertStringLiterals.h delete mode 100644 src/Databases/MySQL/tryParseTableIDFromDDL.cpp delete mode 100644 src/Databases/MySQL/tryParseTableIDFromDDL.h delete mode 100644 src/Databases/MySQL/tryQuoteUnrecognizedTokens.cpp delete mode 100644 src/Databases/MySQL/tryQuoteUnrecognizedTokens.h create mode 100644 src/Databases/enableAllExperimentalSettings.cpp create mode 100644 src/Databases/enableAllExperimentalSettings.h create mode 100644 src/Dictionaries/README.md create mode 100644 src/Disks/DiskBackup.cpp create mode 100644 src/Disks/DiskBackup.h rename src/Disks/{DiskFomAST.cpp => DiskFromAST.cpp} (72%) rename src/Disks/{DiskFomAST.h => DiskFromAST.h} (92%) delete mode 100644 src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp delete mode 100644 src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h create mode 100644 src/Disks/ObjectStorages/InMemoryDirectoryPathMap.h delete mode 100644 src/Disks/ObjectStorages/InMemoryPathMap.h delete mode 100644 src/Disks/ObjectStorages/S3/S3Capabilities.cpp delete mode 100644 src/Disks/ObjectStorages/S3/S3Capabilities.h create mode 100644 src/Formats/PrettyFormatHelpers.cpp create mode 100644 src/Formats/PrettyFormatHelpers.h delete mode 100644 src/Formats/TemporaryFileStreamLegacy.cpp delete mode 100644 src/Formats/TemporaryFileStreamLegacy.h create mode 100644 src/Functions/FieldInterval.h create mode 100644 src/Functions/FunctionBase32Conversion.h create mode 100644 src/Functions/FunctionBaseXXConversion.h create mode 100644 src/Functions/FunctionDynamicAdaptor.cpp create mode 100644 src/Functions/FunctionDynamicAdaptor.h delete mode 100644 src/Functions/FunctionShowCertificate.h create mode 100644 src/Functions/FunctionsConversion.h create mode 100644 src/Functions/FunctionsConversion_impl0.cpp create mode 100644 src/Functions/FunctionsConversion_impl1.cpp create mode 100644 src/Functions/FunctionsConversion_impl2.cpp create mode 100644 src/Functions/FunctionsConversion_impl3.cpp create mode 100644 src/Functions/FunctionsConversion_reg.cpp create mode 100644 src/Functions/GatherUtils/GatherUtils.cpp create mode 100644 src/Functions/IFunctionAdaptors.cpp create mode 100644 src/Functions/Kusto/KqlFunctionBase.cpp create mode 100644 src/Functions/README.md create mode 100644 src/Functions/array/arrayLevenshtein.cpp create mode 100644 src/Functions/array/arrayNormalizedGini.cpp create mode 100644 src/Functions/array/arrayResize.h create mode 100644 src/Functions/array/emptyArrayToSingle.h create mode 100644 src/Functions/array/indexOfAssumeSorted.cpp create mode 100644 src/Functions/array/length.h create mode 100644 src/Functions/base32Decode.cpp create mode 100644 src/Functions/base32Encode.cpp create mode 100644 src/Functions/compareSubstrings.cpp create mode 100644 src/Functions/currentQueryID.cpp create mode 100644 src/Functions/exists.h rename src/Functions/{keyvaluepair => }/extractKeyValuePairs.cpp (92%) create mode 100644 src/Functions/fromUnixTimestamp64.cpp create mode 100644 src/Functions/generateSerialID.cpp create mode 100644 src/Functions/getMaxTableNameLength.cpp create mode 100644 src/Functions/getMergeTreeSetting.cpp create mode 100644 src/Functions/getServerSetting.cpp create mode 100644 src/Functions/icebergBucketTransform.cpp create mode 100644 src/Functions/icebergTruncate.cpp create mode 100644 src/Functions/initialQueryStartTime.cpp create mode 100644 src/Functions/keyvaluepair/impl/DuplicateKeyFoundException.h delete mode 100644 src/Functions/keyvaluepair/impl/KeyValuePairExtractor.h create mode 100644 src/Functions/moduloOrNull.cpp create mode 100644 src/Functions/ngrams.cpp delete mode 100644 src/Functions/notEmpty.cpp create mode 100644 src/Functions/overlay.cpp rename src/Functions/{canonicalRand.cpp => randCanonical.cpp} (95%) delete mode 100644 src/Functions/sleep.h delete mode 100644 src/Functions/sleepEachRow.cpp create mode 100644 src/Functions/sparseGrams.cpp create mode 100644 src/Functions/stringBytes.h create mode 100644 src/Functions/stringBytesEntropy.cpp create mode 100644 src/Functions/stringBytesUniq.cpp create mode 100644 src/Functions/toInterval.cpp create mode 100644 src/Functions/toMonthNumSinceEpoch.cpp create mode 100644 src/Functions/toUnixTimestamp64.cpp create mode 100644 src/Functions/toYearNumSinceEpoch.cpp delete mode 100644 src/Functions/tokenExtractors.cpp create mode 100644 src/Functions/tokens.cpp create mode 100644 src/Functions/tryBase32Decode.cpp create mode 100644 src/IO/Ask.cpp create mode 100644 src/IO/Ask.h create mode 100644 src/IO/AutoFinalizedWriteBuffer.h rename src/IO/{ConcatSeekableReadBuffer.cpp => ConcatReadBufferFromFile.cpp} (72%) rename src/IO/{ConcatSeekableReadBuffer.h => ConcatReadBufferFromFile.h} (60%) create mode 100644 src/IO/DistributedCacheLogMode.h create mode 100644 src/IO/DistributedCachePoolBehaviourOnLimit.h create mode 100644 src/IO/DistributedCacheSettings.h create mode 100644 src/IO/MMappedFileCache.cpp create mode 100644 src/IO/NetUtils.h create mode 100644 src/IO/Operators_pcg_random.h create mode 100644 src/IO/ReadBufferFromPocoSocketChunked.cpp create mode 100644 src/IO/ReadBufferFromPocoSocketChunked.h create mode 100644 src/IO/ReadMethod.h create mode 100644 src/IO/ReadSettings.cpp create mode 100644 src/IO/S3/ReadBufferFromGetObjectResult.h create mode 100644 src/IO/S3/S3Capabilities.cpp create mode 100644 src/IO/S3/S3Capabilities.h create mode 100644 src/IO/S3/deleteFileFromS3.cpp create mode 100644 src/IO/S3/deleteFileFromS3.h create mode 100644 src/IO/S3/tests/gtest_s3_uri.cpp create mode 100644 src/IO/S3AuthSettings.cpp create mode 100644 src/IO/S3AuthSettings.h create mode 100644 src/IO/S3RequestSettings.cpp create mode 100644 src/IO/S3RequestSettings.h delete mode 100644 src/IO/UseSSL.cpp delete mode 100644 src/IO/UseSSL.h create mode 100644 src/IO/WriteBufferFromPocoSocketChunked.cpp create mode 100644 src/IO/WriteBufferFromPocoSocketChunked.h create mode 100644 src/IO/WriteBufferFromString.cpp create mode 100644 src/IO/WriteSettings.cpp delete mode 100644 src/IO/examples/var_uint.cpp create mode 100644 src/IO/tests/gtest_readbuffer_s3.cpp create mode 100644 src/Interpreters/Access/InterpreterCheckGrantQuery.cpp create mode 100644 src/Interpreters/Access/InterpreterCheckGrantQuery.h create mode 100644 src/Interpreters/Access/getValidUntilFromAST.cpp create mode 100644 src/Interpreters/Access/getValidUntilFromAST.h create mode 100644 src/Interpreters/ActionsMatcher.cpp create mode 100644 src/Interpreters/ActionsMatcher.h create mode 100644 src/Interpreters/ArrayJoin.h create mode 100644 src/Interpreters/CNFQueryAtomicFormula.cpp create mode 100644 src/Interpreters/CNFQueryAtomicFormula.h create mode 100644 src/Interpreters/Cache/FileCacheUtils.h create mode 100644 src/Interpreters/Cache/QueryConditionCache.cpp create mode 100644 src/Interpreters/Cache/QueryConditionCache.h rename src/Interpreters/Cache/{QueryCache.cpp => QueryResultCache.cpp} (69%) rename src/Interpreters/Cache/{QueryCache.h => QueryResultCache.h} (52%) create mode 100644 src/Interpreters/Cache/QueryResultCacheUsage.h create mode 100644 src/Interpreters/CancellationChecker.cpp create mode 100644 src/Interpreters/CancellationChecker.h create mode 100644 src/Interpreters/DDLOnClusterQueryStatusSource.cpp create mode 100644 src/Interpreters/DDLOnClusterQueryStatusSource.h create mode 100644 src/Interpreters/DistributedQueryStatusSource.cpp create mode 100644 src/Interpreters/DistributedQueryStatusSource.h rename src/Interpreters/HashJoin/{KnowRowsHolder.h => KnownRowsHolder.h} (94%) create mode 100644 src/Interpreters/HashJoin/ScatteredBlock.h create mode 100644 src/Interpreters/InterpreterCreateResourceQuery.cpp create mode 100644 src/Interpreters/InterpreterCreateResourceQuery.h create mode 100644 src/Interpreters/InterpreterCreateWorkloadQuery.cpp create mode 100644 src/Interpreters/InterpreterCreateWorkloadQuery.h create mode 100644 src/Interpreters/InterpreterDropResourceQuery.cpp create mode 100644 src/Interpreters/InterpreterDropResourceQuery.h create mode 100644 src/Interpreters/InterpreterDropWorkloadQuery.cpp create mode 100644 src/Interpreters/InterpreterDropWorkloadQuery.h create mode 100644 src/Interpreters/InterpreterParallelWithQuery.cpp create mode 100644 src/Interpreters/InterpreterParallelWithQuery.h create mode 100644 src/Interpreters/JoinInfo.cpp create mode 100644 src/Interpreters/JoinInfo.h create mode 100644 src/Interpreters/KeysNullMap.h create mode 100644 src/Interpreters/LatencyLog.cpp create mode 100644 src/Interpreters/LatencyLog.h delete mode 100644 src/Interpreters/NullableUtils.h create mode 100644 src/Interpreters/QueryFlags.h create mode 100644 src/Interpreters/QueryLogElement.h create mode 100644 src/Interpreters/QueryMetricLog.cpp create mode 100644 src/Interpreters/QueryMetricLog.h create mode 100644 src/Interpreters/ReplicatedDatabaseQueryStatusSource.cpp create mode 100644 src/Interpreters/ReplicatedDatabaseQueryStatusSource.h create mode 100644 src/Interpreters/SetSerialization.h create mode 100644 src/Interpreters/SetWithState.h create mode 100644 src/Interpreters/TemporaryReplaceTableName.cpp create mode 100644 src/Interpreters/TemporaryReplaceTableName.h create mode 100644 src/Interpreters/TransposedMetricLog.cpp create mode 100644 src/Interpreters/TransposedMetricLog.h create mode 100644 src/Interpreters/createSubcolumnsExtractionActions.cpp create mode 100644 src/Interpreters/createSubcolumnsExtractionActions.h create mode 100644 src/Interpreters/formatWithPossiblyHidingSecrets.cpp delete mode 100644 src/Interpreters/tests/gtest_page_cache.cpp delete mode 100644 src/NOTICE create mode 100644 src/Parsers/ASTCreateResourceQuery.cpp create mode 100644 src/Parsers/ASTCreateResourceQuery.h create mode 100644 src/Parsers/ASTCreateWorkloadQuery.cpp create mode 100644 src/Parsers/ASTCreateWorkloadQuery.h create mode 100644 src/Parsers/ASTDropResourceQuery.cpp create mode 100644 src/Parsers/ASTDropResourceQuery.h create mode 100644 src/Parsers/ASTDropWorkloadQuery.cpp create mode 100644 src/Parsers/ASTDropWorkloadQuery.h create mode 100644 src/Parsers/ASTParallelWithQuery.cpp create mode 100644 src/Parsers/ASTParallelWithQuery.h create mode 100644 src/Parsers/ASTUpdateQuery.cpp create mode 100644 src/Parsers/ASTUpdateQuery.h create mode 100644 src/Parsers/Access/ASTCheckGrantQuery.cpp create mode 100644 src/Parsers/Access/ASTCheckGrantQuery.h create mode 100644 src/Parsers/Access/ParserCheckGrantQuery.cpp create mode 100644 src/Parsers/Access/ParserCheckGrantQuery.h create mode 100644 src/Parsers/Access/parseAccessRightsElements.cpp create mode 100644 src/Parsers/Access/parseAccessRightsElements.h create mode 100644 src/Parsers/IASTHash.h create mode 100644 src/Parsers/IAST_erase.h create mode 100644 src/Parsers/ParserCreateResourceQuery.cpp create mode 100644 src/Parsers/ParserCreateResourceQuery.h create mode 100644 src/Parsers/ParserCreateWorkloadEntity.cpp create mode 100644 src/Parsers/ParserCreateWorkloadEntity.h create mode 100644 src/Parsers/ParserCreateWorkloadQuery.cpp create mode 100644 src/Parsers/ParserCreateWorkloadQuery.h create mode 100644 src/Parsers/ParserDropResourceQuery.cpp create mode 100644 src/Parsers/ParserDropResourceQuery.h create mode 100644 src/Parsers/ParserDropWorkloadQuery.cpp create mode 100644 src/Parsers/ParserDropWorkloadQuery.h create mode 100644 src/Parsers/ParserParallelWithQuery.cpp create mode 100644 src/Parsers/ParserParallelWithQuery.h create mode 100644 src/Parsers/ParserPreparedStatement.cpp create mode 100644 src/Parsers/ParserPreparedStatement.h create mode 100644 src/Parsers/ParserUpdateQuery.cpp create mode 100644 src/Parsers/ParserUpdateQuery.h delete mode 100644 src/Parsers/QueryWithOutputSettingsPushDownVisitor.cpp delete mode 100644 src/Parsers/QueryWithOutputSettingsPushDownVisitor.h delete mode 100644 src/Parsers/StringRange.h delete mode 100644 src/Parsers/formatAST.cpp delete mode 100644 src/Parsers/formatAST.h delete mode 100644 src/Parsers/queryToString.cpp delete mode 100644 src/Parsers/queryToString.h create mode 100644 src/Planner/PlannerCorrelatedSubqueries.cpp create mode 100644 src/Planner/PlannerCorrelatedSubqueries.h create mode 100644 src/Planner/PlannerJoinsLogical.cpp create mode 100644 src/Planner/PlannerJoinsLogical.h create mode 100644 src/Planner/README.md create mode 100644 src/Processors/Formats/Impl/ArrowGeoTypes.cpp create mode 100644 src/Processors/Formats/Impl/ArrowGeoTypes.h create mode 100644 src/Processors/Formats/Impl/JSONCompactEachRowWithProgressRowOutputFormat.cpp create mode 100644 src/Processors/Formats/Impl/JSONCompactEachRowWithProgressRowOutputFormat.h create mode 100644 src/Processors/Formats/Impl/Parquet/parquetBloomFilterHash.cpp create mode 100644 src/Processors/Formats/Impl/Parquet/parquetBloomFilterHash.h delete mode 100644 src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.cpp delete mode 100644 src/Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h delete mode 100644 src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.cpp delete mode 100644 src/Processors/Formats/Impl/PrettySpaceBlockOutputFormat.h delete mode 100644 src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h create mode 100644 src/Processors/Merges/Algorithms/MergeTreeReadInfo.cpp create mode 100644 src/Processors/Merges/Algorithms/MergeTreeReadInfo.h create mode 100644 src/Processors/Merges/Algorithms/MergedData.cpp create mode 100644 src/Processors/Merges/Algorithms/RowRef.cpp create mode 100644 src/Processors/QueryPlan/ConvertingActions.cpp create mode 100644 src/Processors/QueryPlan/ConvertingActions.h create mode 100644 src/Processors/QueryPlan/CustomMetricLogViewStep.cpp create mode 100644 src/Processors/QueryPlan/CustomMetricLogViewStep.h create mode 100644 src/Processors/QueryPlan/ExtractColumnsStep.cpp create mode 100644 src/Processors/QueryPlan/ExtractColumnsStep.h create mode 100644 src/Processors/QueryPlan/JoinStepLogical.cpp create mode 100644 src/Processors/QueryPlan/JoinStepLogical.h create mode 100644 src/Processors/QueryPlan/LazilyReadStep.cpp create mode 100644 src/Processors/QueryPlan/LazilyReadStep.h create mode 100644 src/Processors/QueryPlan/Optimizations/Utils.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/Utils.h create mode 100644 src/Processors/QueryPlan/Optimizations/applyOrder.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/convertJoinToIn.cpp delete mode 100644 src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/mergeFilterIntoJoinCondition.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/optimizeJoinByShards.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/optimizeLazyMaterialization.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/updateQueryConditionCache.cpp create mode 100644 src/Processors/QueryPlan/Optimizations/useVectorSearch.cpp create mode 100644 src/Processors/QueryPlan/ParallelReplicasLocalPlan.cpp create mode 100644 src/Processors/QueryPlan/ParallelReplicasLocalPlan.h create mode 100644 src/Processors/QueryPlan/QueryPlanSerializationSettings.cpp create mode 100644 src/Processors/QueryPlan/QueryPlanSerializationSettings.h create mode 100644 src/Processors/QueryPlan/QueryPlanStepRegistry.cpp create mode 100644 src/Processors/QueryPlan/QueryPlanStepRegistry.h create mode 100644 src/Processors/QueryPlan/ReadFromTableFunctionStep.cpp create mode 100644 src/Processors/QueryPlan/ReadFromTableFunctionStep.h create mode 100644 src/Processors/QueryPlan/ReadFromTableStep.cpp create mode 100644 src/Processors/QueryPlan/ReadFromTableStep.h create mode 100644 src/Processors/QueryPlan/Serialization.cpp create mode 100644 src/Processors/QueryPlan/Serialization.h create mode 100644 src/Processors/QueryPlan/SetsSerialization.cpp create mode 100644 src/Processors/QueryPlan/TemporaryFiles.h create mode 100644 src/Processors/QueryPlan/resolveStorages.cpp create mode 100644 src/Processors/QueryPlan/resolveStorages.h create mode 100644 src/Processors/SourceWithKeyCondition.cpp delete mode 100644 src/Processors/Sources/TemporaryFileLazySource.cpp delete mode 100644 src/Processors/Sources/TemporaryFileLazySource.h create mode 100644 src/Processors/Transforms/ColumnLazyTransform.cpp create mode 100644 src/Processors/Transforms/ColumnLazyTransform.h create mode 100644 src/Processors/Transforms/ColumnPermuteTransform.cpp create mode 100644 src/Processors/Transforms/ColumnPermuteTransform.h rename src/Processors/Transforms/{DistinctSortedChunkTransform.cpp => DistinctSortedStreamTransform.cpp} (87%) rename src/Processors/Transforms/{DistinctSortedChunkTransform.h => DistinctSortedStreamTransform.h} (79%) create mode 100644 src/Processors/Transforms/NestedElementsValidationTransform.cpp create mode 100644 src/Processors/Transforms/NestedElementsValidationTransform.h delete mode 100644 src/Processors/Transforms/StreamInQueryCacheTransform.cpp delete mode 100644 src/Processors/Transforms/StreamInQueryCacheTransform.h create mode 100644 src/Processors/Transforms/StreamInQueryResultCacheTransform.cpp create mode 100644 src/Processors/Transforms/StreamInQueryResultCacheTransform.h create mode 100644 src/Processors/Transforms/VirtualRowTransform.cpp create mode 100644 src/Processors/Transforms/VirtualRowTransform.h create mode 100644 src/Processors/tests/gtest_write_parquet_page_index.cpp create mode 100644 src/Server/ClientEmbedded/ClientEmbedded.cpp create mode 100644 src/Server/ClientEmbedded/ClientEmbedded.h create mode 100644 src/Server/ClientEmbedded/ClientEmbeddedRunner.cpp create mode 100644 src/Server/ClientEmbedded/ClientEmbeddedRunner.h create mode 100644 src/Server/ClientEmbedded/IClientDescriptorSet.h create mode 100644 src/Server/ClientEmbedded/PipeClientDescriptorSet.h create mode 100644 src/Server/ClientEmbedded/PtyClientDescriptorSet.cpp create mode 100644 src/Server/ClientEmbedded/PtyClientDescriptorSet.h create mode 100644 src/Server/SSH/SSHBind.cpp create mode 100644 src/Server/SSH/SSHBind.h create mode 100644 src/Server/SSH/SSHChannel.cpp create mode 100644 src/Server/SSH/SSHChannel.h create mode 100644 src/Server/SSH/SSHEvent.cpp create mode 100644 src/Server/SSH/SSHEvent.h create mode 100644 src/Server/SSH/SSHPtyHandler.cpp create mode 100644 src/Server/SSH/SSHPtyHandler.h create mode 100644 src/Server/SSH/SSHPtyHandlerFactory.h create mode 100644 src/Server/SSH/SSHSession.cpp create mode 100644 src/Server/SSH/SSHSession.h delete mode 100644 src/Storages/Cache/ExternalDataSourceCache.cpp delete mode 100644 src/Storages/Cache/ExternalDataSourceCache.h delete mode 100644 src/Storages/Cache/RemoteCacheController.cpp delete mode 100644 src/Storages/Cache/RemoteCacheController.h delete mode 100644 src/Storages/Cache/RemoteFileCachePolicy.h create mode 100644 src/Storages/ColumnDependency.cpp create mode 100644 src/Storages/ColumnSize.h create mode 100644 src/Storages/Kafka/IKafkaExceptionInfoSink.h create mode 100644 src/Storages/Kafka/Kafka_fwd.h create mode 100644 src/Storages/LazilyReadInfo.h delete mode 100644 src/Storages/MaterializedView/RefreshTask_fwd.h delete mode 100644 src/Storages/MergeTree/AllMergeSelector.cpp delete mode 100644 src/Storages/MergeTree/CMakeLists.txt create mode 100644 src/Storages/MergeTree/ColumnsSubstreams.cpp create mode 100644 src/Storages/MergeTree/ColumnsSubstreams.h create mode 100644 src/Storages/MergeTree/Compaction/CompactionStatistics.cpp create mode 100644 src/Storages/MergeTree/Compaction/CompactionStatistics.h create mode 100644 src/Storages/MergeTree/Compaction/ConstructFuturePart.cpp create mode 100644 src/Storages/MergeTree/Compaction/ConstructFuturePart.h create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/DistributedMergePredicate.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/DistributedMergePredicate.h create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/IMergePredicate.h create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/MergeTreeMergePredicate.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/MergeTreeMergePredicate.h create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/ReplicatedMergeTreeMergePredicate.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergePredicates/ReplicatedMergeTreeMergePredicate.h create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectorApplier.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectorApplier.h create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/AllMergeSelector.cpp rename src/Storages/MergeTree/{ => Compaction/MergeSelectors}/AllMergeSelector.h (65%) create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/IMergeSelector.h create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/MergeSelectorFactory.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/MergeSelectorFactory.h rename src/Storages/MergeTree/{ => Compaction/MergeSelectors}/SimpleMergeSelector.cpp (62%) rename src/Storages/MergeTree/{ => Compaction/MergeSelectors}/SimpleMergeSelector.h (90%) create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/TTLMergeSelector.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/TTLMergeSelector.h create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/TrivialMergeSelector.cpp create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/TrivialMergeSelector.h create mode 100644 src/Storages/MergeTree/Compaction/MergeSelectors/registerMergeSelectors.cpp rename src/{Common/Scheduler/Nodes/registerResourceManagers.h => Storages/MergeTree/Compaction/MergeSelectors/registerMergeSelectors.h} (50%) create mode 100644 src/Storages/MergeTree/Compaction/PartProperties.cpp create mode 100644 src/Storages/MergeTree/Compaction/PartProperties.h create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/Common.cpp create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/Common.h create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/IPartsCollector.h create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/MergeTreePartsCollector.cpp create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/MergeTreePartsCollector.h create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/ReplicatedMergeTreePartsCollector.cpp create mode 100644 src/Storages/MergeTree/Compaction/PartsCollectors/ReplicatedMergeTreePartsCollector.h create mode 100644 src/Storages/MergeTree/DeserializationPrefixesCache.cpp create mode 100644 src/Storages/MergeTree/DeserializationPrefixesCache.h create mode 100644 src/Storages/MergeTree/IExecutableTask.cpp delete mode 100644 src/Storages/MergeTree/IPartMetadataManager.cpp delete mode 100644 src/Storages/MergeTree/IPartMetadataManager.h create mode 100644 src/Storages/MergeTree/MergeProjectionPartsTask.cpp create mode 100644 src/Storages/MergeTree/MergeProjectionPartsTask.h delete mode 100644 src/Storages/MergeTree/MergeSelector.h rename src/Storages/MergeTree/{MergeTreeIndexFullText.cpp => MergeTreeIndexGin.cpp} (72%) rename src/Storages/MergeTree/{MergeTreeIndexFullText.h => MergeTreeIndexGin.h} (64%) create mode 100644 src/Storages/MergeTree/MergeTreeIndexGranularityAdaptive.cpp create mode 100644 src/Storages/MergeTree/MergeTreeIndexGranularityAdaptive.h create mode 100644 src/Storages/MergeTree/MergeTreeIndexGranularityConstant.cpp create mode 100644 src/Storages/MergeTree/MergeTreeIndexGranularityConstant.h delete mode 100644 src/Storages/MergeTree/MergeTreeIndexLegacyVectorSimilarity.cpp delete mode 100644 src/Storages/MergeTree/MergeTreeIndexLegacyVectorSimilarity.h delete mode 100644 src/Storages/MergeTree/MergeTreeIndexUtils.cpp delete mode 100644 src/Storages/MergeTree/MergeTreeIndexUtils.h create mode 100644 src/Storages/MergeTree/MergeTreeLazilyReader.cpp create mode 100644 src/Storages/MergeTree/MergeTreeLazilyReader.h create mode 100644 src/Storages/MergeTree/MergeTreeReadersChain.cpp create mode 100644 src/Storages/MergeTree/MergeTreeReadersChain.h create mode 100644 src/Storages/MergeTree/MergedPartOffsets.h delete mode 100644 src/Storages/MergeTree/PartMetadataManagerOrdinary.cpp delete mode 100644 src/Storages/MergeTree/PartMetadataManagerOrdinary.h create mode 100644 src/Storages/MergeTree/PartitionActionBlocker.cpp create mode 100644 src/Storages/MergeTree/PartitionActionBlocker.h create mode 100644 src/Storages/MergeTree/PrimaryIndexCache.cpp create mode 100644 src/Storages/MergeTree/PrimaryIndexCache.h rename src/Storages/{ => MergeTree}/StorageFromMergeTreeDataPart.cpp (83%) create mode 100644 src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp create mode 100644 src/Storages/MergeTree/StorageFromMergeTreeProjection.h delete mode 100644 src/Storages/MergeTree/TTLMergeSelector.cpp delete mode 100644 src/Storages/MergeTree/TTLMergeSelector.h delete mode 100644 src/Storages/MergeTree/VectorSimilarityCondition.cpp delete mode 100644 src/Storages/MergeTree/VectorSimilarityCondition.h create mode 100644 src/Storages/MergeTree/VectorSimilarityIndexCache.h create mode 100644 src/Storages/MergeTree/tests/gtest_merged_part_offsets.cpp create mode 100644 src/Storages/MergeTree/tests/gtest_partition_action_blocker.cpp create mode 100644 src/Storages/NATS/NATS_fwd.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/KernelHelper.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/KernelHelper.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/KernelPointerWrapper.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/KernelUtils.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/KernelUtils.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/ObjectInfoWithPartitionColumns.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/getSchemaFromSnapshot.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLake/getSchemaFromSnapshot.h create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.h create mode 100644 src/Storages/ObjectStorage/DataLakes/IDataLakeMetadata.cpp delete mode 100644 src/Storages/ObjectStorage/DataLakes/IStorageDataLake.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/AvroForIcebergDeserializer.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/AvroForIcebergDeserializer.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFile.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/Snapshot.h create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp create mode 100644 src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.h delete mode 100644 src/Storages/ObjectStorage/DataLakes/IcebergMetadata.cpp delete mode 100644 src/Storages/ObjectStorage/DataLakes/IcebergMetadata.h delete mode 100644 src/Storages/ObjectStorage/DataLakes/PartitionColumns.h delete mode 100644 src/Storages/ObjectStorage/DataLakes/registerDataLakeStorages.cpp create mode 100644 src/Storages/ObjectStorage/HDFS/HDFSErrorWrapper.cpp create mode 100644 src/Storages/ObjectStorage/HDFS/HDFSErrorWrapper.h create mode 100644 src/Storages/ObjectStorage/IObjectIterator.h create mode 100644 src/Storages/ObjectStorage/Local/Configuration.cpp create mode 100644 src/Storages/ObjectStorage/Local/Configuration.h create mode 100644 src/Storages/ObjectStorage/StorageObjectStorageSettings.cpp create mode 100644 src/Storages/ObjectStorage/StorageObjectStorageSettings.h create mode 100644 src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.cpp create mode 100644 src/Storages/ObjectStorage/StorageObjectStorageStableTaskDistributor.h create mode 100644 src/Storages/RabbitMQ/RabbitMQ_fwd.h create mode 100644 src/Storages/Statistics/StatisticsMinMax.cpp create mode 100644 src/Storages/Statistics/StatisticsMinMax.h delete mode 100644 src/Storages/StorageExternalDistributed.cpp delete mode 100644 src/Storages/StorageExternalDistributed.h delete mode 100644 src/Storages/StorageMaterializedMySQL.cpp delete mode 100644 src/Storages/StorageMaterializedMySQL.h delete mode 100644 src/Storages/StorageMongoDBSocketFactory.cpp delete mode 100644 src/Storages/StorageMongoDBSocketFactory.h create mode 100644 src/Storages/System/MutableColumnsAndConstraints.h create mode 100644 src/Storages/System/ServerSettingColumnsParams.h create mode 100644 src/Storages/System/StorageSystemHistogramMetrics.cpp create mode 100644 src/Storages/System/StorageSystemHistogramMetrics.h create mode 100644 src/Storages/System/StorageSystemIcebergHistory.cpp create mode 100644 src/Storages/System/StorageSystemIcebergHistory.h create mode 100644 src/Storages/System/StorageSystemLatencyBuckets.cpp rename src/Storages/System/{StorageSystemS3Queue.h => StorageSystemLatencyBuckets.h} (50%) delete mode 100644 src/Storages/System/StorageSystemMySQLBinlogs.cpp rename src/Storages/System/{StorageSystemS3Queue.cpp => StorageSystemObjectStorageQueue.cpp} (77%) create mode 100644 src/Storages/System/StorageSystemObjectStorageQueue.h create mode 100644 src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp create mode 100644 src/Storages/System/StorageSystemObjectStorageQueueSettings.h create mode 100644 src/Storages/System/StorageSystemProjections.cpp rename src/Storages/System/{StorageSystemMySQLBinlogs.h => StorageSystemProjections.h} (54%) create mode 100644 src/Storages/System/StorageSystemQueryConditionCache.cpp create mode 100644 src/Storages/System/StorageSystemQueryConditionCache.h rename src/Storages/System/{StorageSystemQueryCache.cpp => StorageSystemQueryResultCache.cpp} (62%) rename src/Storages/System/{StorageSystemQueryCache.h => StorageSystemQueryResultCache.h} (57%) create mode 100644 src/Storages/System/StorageSystemResources.cpp create mode 100644 src/Storages/System/StorageSystemResources.h create mode 100644 src/Storages/System/StorageSystemWorkloads.cpp create mode 100644 src/Storages/System/StorageSystemWorkloads.h create mode 100644 src/Storages/TableZnodeInfo.cpp create mode 100644 src/Storages/TableZnodeInfo.h delete mode 100644 src/Storages/extractTableFunctionArgumentsFromSelectQuery.h rename src/Storages/{extractTableFunctionArgumentsFromSelectQuery.cpp => extractTableFunctionFromSelectQuery.cpp} (61%) create mode 100644 src/Storages/extractTableFunctionFromSelectQuery.h create mode 100644 src/Storages/tests/gtest_virtual_column_utils.cpp delete mode 100644 src/TableFunctions/ITableFunctionDataLake.h create mode 100644 src/TableFunctions/TableFunctionMongoDB.h create mode 100644 src/TableFunctions/TableFunctionProjection.cpp delete mode 100644 src/TableFunctions/registerDataLakeTableFunctions.cpp diff --git a/.clang-format b/.clang-format index 7746106fcb5..91bea025d4e 100644 --- a/.clang-format +++ b/.clang-format @@ -83,6 +83,7 @@ NamespaceIndentation: None ObjCBlockIndentWidth: 4 ObjCSpaceAfterProperty: true ObjCSpaceBeforeProtocolList: true +PackConstructorInitializers: Never PenaltyBreakBeforeFirstCallParameter: 19 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 diff --git a/.clang-tidy b/.clang-tidy index bb63bf2eea6..4824fc2f2ed 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -10,40 +10,30 @@ HeaderFilterRegex: '^.*/(base|src|programs|utils)/.*(h|hpp)$' Checks: [ '*', - '-abseil-*', + '-abseil-string-find-str-contains', # disabled to avoid a misleading suggestion (obsolete absl::StrContains() instead of C++23 std::string::contains()) '-altera-*', '-android-*', + '-boost-use-ranges', + '-bugprone-assignment-in-if-condition', '-bugprone-branch-clone', '-bugprone-easily-swappable-parameters', '-bugprone-exception-escape', - '-bugprone-forward-declaration-namespace', '-bugprone-implicit-widening-of-multiplication-result', '-bugprone-multi-level-implicit-pointer-conversion', '-bugprone-narrowing-conversions', - '-bugprone-not-null-terminated-result', - '-bugprone-reserved-identifier', # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged '-bugprone-unchecked-optional-access', '-bugprone-crtp-constructor-accessibility', - '-bugprone-suspicious-stringview-data-usage', + '-bugprone-not-null-terminated-result', '-cert-dcl16-c', - '-cert-dcl37-c', - '-cert-dcl51-cpp', '-cert-err58-cpp', - '-cert-msc32-c', - '-cert-msc51-cpp', - '-cert-oop54-cpp', - '-cert-oop57-cpp', - '-cert-err33-c', # Misreports on clang-19: it warns about all functions containing 'remove' in the name, not only about the standard library. '-clang-analyzer-optin.performance.Padding', - '-clang-analyzer-unix.Malloc', - '-cppcoreguidelines-*', # impractical in a codebase as large as ClickHouse, also slow '-darwin-*', @@ -76,38 +66,34 @@ Checks: [ '-hicpp-use-emplace', '-hicpp-vararg', - '-linuxkernel-*', - '-llvm-*', '-llvmlibc-*', - '-openmp-*', - '-misc-const-correctness', '-misc-include-cleaner', # useful but far too many occurrences '-misc-no-recursion', '-misc-non-private-member-variables-in-classes', - '-misc-confusable-identifiers', # useful but slooo '-misc-use-anonymous-namespace', + '-misc-use-internal-linkage', '-modernize-avoid-c-arrays', '-modernize-concat-nested-namespaces', - '-modernize-macro-to-enum', '-modernize-pass-by-value', '-modernize-return-braced-init-list', '-modernize-use-auto', - '-modernize-use-constraints', # This is a good check, but clang-tidy crashes, see https://github.com/llvm/llvm-project/issues/91872 '-modernize-use-default-member-init', '-modernize-use-emplace', '-modernize-use-nodiscard', + '-modernize-use-ranges', '-modernize-use-trailing-return-type', '-modernize-use-designated-initializers', + '-performance-avoid-endl', '-performance-enum-size', '-performance-inefficient-string-concatenation', + '-performance-inefficient-vector-operation', '-performance-no-int-to-ptr', - '-performance-avoid-endl', '-performance-unnecessary-value-param', '-portability-simd-intrinsics', @@ -122,7 +108,6 @@ Checks: [ '-readability-identifier-length', '-readability-identifier-naming', # useful but too slow '-readability-implicit-bool-conversion', - '-readability-isolate-declaration', '-readability-magic-numbers', '-readability-named-parameter', '-readability-redundant-declaration', @@ -137,6 +122,7 @@ Checks: [ '-readability-suspicious-call-argument', '-readability-uppercase-literal-suffix', '-readability-use-anyofallof', + '-readability-math-missing-parentheses', '-zircon-*' ] diff --git a/.clangd b/.clangd index ad471db8d8b..0dee943f83b 100644 --- a/.clangd +++ b/.clangd @@ -14,3 +14,6 @@ Diagnostics: readability-identifier-naming, bugprone-reserved-identifier, ] +# Require LLVM 20 (https://github.com/llvm/llvm-project/pull/67749) +Style: + AngledHeaders: [".*"] diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 5088bf90ede..285a60927f8 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -16,3 +16,6 @@ # Applied Black formatter for Python code e6f5a3f98b21ba99cf274a9833797889e020a2b3 + +# Enabling clang-tidy readability-else-no-return rule +67c1e89d90ef576e62f8b1c68269742a3c6f9b1e diff --git a/.gitignore b/.gitignore index 80424daf075..2bb2b608cdd 100644 --- a/.gitignore +++ b/.gitignore @@ -159,6 +159,7 @@ website/package-lock.json /programs/server/store /programs/server/uuid /programs/server/coordination +/programs/server/workload # temporary test files tests/queries/0_stateless/test_* diff --git a/.gitmodules b/.gitmodules index d106fcb35a1..8aff087a567 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,9 @@ # Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a # little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are # more confusing than useful. +[submodule "contrib/jwt-cpp"] + path = contrib/jwt-cpp + url = https://github.com/Thalhammer/jwt-cpp [submodule "contrib/zstd"] path = contrib/zstd url = https://github.com/facebook/zstd @@ -31,9 +34,6 @@ [submodule "contrib/jemalloc"] path = contrib/jemalloc url = https://github.com/jemalloc/jemalloc -[submodule "contrib/unixodbc"] - path = contrib/unixodbc - url = https://github.com/ClickHouse/UnixODBC [submodule "contrib/google-protobuf"] path = contrib/google-protobuf url = https://github.com/ClickHouse/google-protobuf.git @@ -57,16 +57,13 @@ url = https://github.com/ClickHouse/snappy [submodule "contrib/cppkafka"] path = contrib/cppkafka - url = https://github.com/mfontanini/cppkafka + url = https://github.com/ClickHouse/cppkafka [submodule "contrib/brotli"] path = contrib/brotli url = https://github.com/google/brotli [submodule "contrib/h3"] path = contrib/h3 url = https://github.com/ClickHouse/h3 -[submodule "contrib/libunwind"] - path = contrib/libunwind - url = https://github.com/ClickHouse/libunwind [submodule "contrib/simdjson"] path = contrib/simdjson url = https://github.com/simdjson/simdjson @@ -88,13 +85,13 @@ [submodule "contrib/aws"] path = contrib/aws url = https://github.com/ClickHouse/aws-sdk-cpp -[submodule "aws-c-event-stream"] +[submodule "contrib/aws-c-event-stream"] path = contrib/aws-c-event-stream url = https://github.com/awslabs/aws-c-event-stream -[submodule "aws-c-common"] +[submodule "contrib/aws-c-common"] path = contrib/aws-c-common url = https://github.com/awslabs/aws-c-common.git -[submodule "aws-checksums"] +[submodule "contrib/aws-checksums"] path = contrib/aws-checksums url = https://github.com/awslabs/aws-checksums [submodule "contrib/curl"] @@ -105,7 +102,7 @@ url = https://github.com/ClickHouse/icudata [submodule "contrib/icu"] path = contrib/icu - url = https://github.com/unicode-org/icu + url = https://github.com/ClickHouse/icu [submodule "contrib/flatbuffers"] path = contrib/flatbuffers url = https://github.com/ClickHouse/flatbuffers @@ -121,7 +118,7 @@ url = https://github.com/msgpack/msgpack-c [submodule "contrib/libcpuid"] path = contrib/libcpuid - url = https://github.com/ClickHouse/libcpuid + url = https://github.com/anrieff/libcpuid [submodule "contrib/openldap"] path = contrib/openldap url = https://github.com/ClickHouse/openldap @@ -160,7 +157,7 @@ url = https://github.com/xz-mirror/xz [submodule "contrib/abseil-cpp"] path = contrib/abseil-cpp - url = https://github.com/abseil/abseil-cpp + url = https://github.com/ClickHouse/abseil-cpp.git [submodule "contrib/dragonbox"] path = contrib/dragonbox url = https://github.com/ClickHouse/dragonbox @@ -170,9 +167,6 @@ [submodule "contrib/NuRaft"] path = contrib/NuRaft url = https://github.com/ClickHouse/NuRaft -[submodule "contrib/nanodbc"] - path = contrib/nanodbc - url = https://github.com/ClickHouse/nanodbc [submodule "contrib/datasketches-cpp"] path = contrib/datasketches-cpp url = https://github.com/apache/datasketches-cpp @@ -224,15 +218,12 @@ [submodule "contrib/minizip-ng"] path = contrib/minizip-ng url = https://github.com/zlib-ng/minizip-ng -[submodule "contrib/annoy"] - path = contrib/annoy - url = https://github.com/ClickHouse/annoy [submodule "contrib/qpl"] path = contrib/qpl url = https://github.com/intel/qpl [submodule "contrib/idxd-config"] path = contrib/idxd-config - url = https://github.com/intel/idxd-config + url = https://github.com/ClickHouse/idxd-config.git [submodule "contrib/QAT-ZSTD-Plugin"] path = contrib/QAT-ZSTD-Plugin url = https://github.com/intel/QAT-ZSTD-Plugin @@ -311,9 +302,6 @@ [submodule "contrib/crc32-vpmsum"] path = contrib/crc32-vpmsum url = https://github.com/antonblanchard/crc32-vpmsum.git -[submodule "contrib/expected"] - path = contrib/expected - url = https://github.com/TartanLlama/expected [submodule "contrib/liburing"] path = contrib/liburing url = https://github.com/axboe/liburing @@ -335,16 +323,13 @@ url = https://github.com/graphitemaster/incbin.git [submodule "contrib/usearch"] path = contrib/usearch - url = https://github.com/unum-cloud/usearch.git + url = https://github.com/ClickHouse/usearch.git [submodule "contrib/SimSIMD"] path = contrib/SimSIMD - url = https://github.com/ashvardanian/SimSIMD.git + url = https://github.com/ClickHouse/SimSIMD.git [submodule "contrib/FP16"] path = contrib/FP16 url = https://github.com/Maratyszcza/FP16.git -[submodule "contrib/robin-map"] - path = contrib/robin-map - url = https://github.com/Tessil/robin-map.git [submodule "contrib/aklomp-base64"] path = contrib/aklomp-base64 url = https://github.com/aklomp/base64.git @@ -357,6 +342,9 @@ [submodule "contrib/idna"] path = contrib/idna url = https://github.com/ada-url/idna.git +[submodule "contrib/google-cloud-cpp"] + path = contrib/google-cloud-cpp + url = https://github.com/ClickHouse/google-cloud-cpp.git [submodule "contrib/rust_vendor"] path = contrib/rust_vendor url = https://github.com/ClickHouse/rust_vendor.git @@ -366,15 +354,27 @@ [submodule "contrib/double-conversion"] path = contrib/double-conversion url = https://github.com/ClickHouse/double-conversion.git -[submodule "contrib/arrow"] - path = contrib/arrow - url = https://github.com/auxten/arrow -[submodule "contrib/utf8proc"] - path = contrib/utf8proc - url = https://github.com/JuliaStrings/utf8proc.git +[submodule "contrib/mongo-cxx-driver"] + path = contrib/mongo-cxx-driver + url = https://github.com/ClickHouse/mongo-cxx-driver.git +[submodule "contrib/mongo-c-driver"] + path = contrib/mongo-c-driver + url = https://github.com/ClickHouse/mongo-c-driver.git [submodule "contrib/numactl"] path = contrib/numactl url = https://github.com/ClickHouse/numactl.git [submodule "contrib/postgres"] path = contrib/postgres url = https://github.com/ClickHouse/postgres.git +[submodule "contrib/delta-kernel-rs"] + path = contrib/delta-kernel-rs + url = https://github.com/ClickHouse/delta-kernel-rs.git +[submodule "contrib/SHA3IUF"] + path = contrib/SHA3IUF + url = https://github.com/brainhub/SHA3IUF.git +[submodule "contrib/arrow"] + path = contrib/arrow + url = https://github.com/chdb-io/arrow.git +[submodule "contrib/utf8proc"] + path = contrib/utf8proc + url = https://github.com/JuliaStrings/utf8proc.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 134bd386dab..910ad4d7ad6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) project(ClickHouse LANGUAGES C CXX ASM) @@ -7,6 +7,9 @@ option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" ON) +# This allows to compile some code conditionally in the private build while having byte-identical source files. +set (CLICKHOUSE_CLOUD 0) + if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR) else() @@ -31,8 +34,6 @@ endmacro () set(CMAKE_EXPORT_COMPILE_COMMANDS 1) # Write compile_commands.json set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so -set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE) -set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.") # To be consistent with CMakeLists from contrib libs. # Enable the ability to organize targets into hierarchies of "folders" for capable GUI-based IDEs. # For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html @@ -44,7 +45,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md") endif () # Take care to add prlimit in command line before ccache, or else ccache thinks that -# prlimit is compiler, and clang++ is its input file, and refuses to work with +# prlimit is compiler, and clang++ is its input file, and refuses to work with # multiple inputs, e.g in ccache log: # [2021-03-31T18:06:32.655327 36900] Command line: /usr/bin/ccache prlimit --as=10000000000 --data=5000000000 --cpu=600 /usr/bin/clang++-11 - ...... std=gnu++2a -MD -MT src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -MF src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o.d -o src/CMakeFiles/dbms.dir/Storages/MergeTree/IMergeTreeDataPart.cpp.o -c ../src/Storages/MergeTree/IMergeTreeDataPart.cpp # @@ -66,6 +67,7 @@ if (ENABLE_CHECK_HEAVY_BUILDS) # Twice as large set (RLIMIT_DATA 10000000000) set (RLIMIT_AS 20000000000) + set (RLIMIT_CPU 2000) endif() # For some files currently building RISCV64/LOONGARCH64 might be too slow. @@ -88,6 +90,8 @@ string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF) +option (ENABLE_BUZZHOUSE "Enable BuzzHouse fuzzer on the client" OFF) +option (ENABLE_FUZZER_TEST "Build testing fuzzers in order to test libFuzzer functionality" OFF) if (ENABLE_FUZZING) # Also set WITH_COVERAGE=1 for better fuzzing process @@ -96,8 +100,6 @@ if (ENABLE_FUZZING) message (STATUS "Fuzzing instrumentation enabled") set (FUZZER "libfuzzer") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdlib++") - set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF) - set (ENABLE_LIBRARIES 0) set (ENABLE_SSL 1) set (ENABLE_EXAMPLES 0) set (ENABLE_UTILS 0) @@ -110,8 +112,11 @@ if (ENABLE_FUZZING) # For codegen_select_fuzzer set (ENABLE_PROTOBUF 1) +endif() - add_compile_definitions(FUZZING_MODE=1) +if (ENABLE_BUZZHOUSE) + set (ENABLE_SSL 1) + set (ENABLE_PROTOBUF 1) endif() if (ENABLE_PYTHON) @@ -158,8 +163,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWIT set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") endif() -no_warning(enum-constexpr-conversion) # breaks Protobuf in clang-16 - option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF) option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF) @@ -175,8 +178,6 @@ if (OS_LINUX) # We should not export dynamic symbols, because: # - The main clickhouse binary does not use dlopen, # and whatever is poisoning it by LD_PRELOAD should not link to our symbols. - # - The clickhouse-odbc-bridge and clickhouse-library-bridge binaries - # should not expose their symbols to ODBC drivers and libraries. set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections") endif () @@ -208,11 +209,6 @@ option(OMIT_HEAVY_DEBUG_SYMBOLS ${OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT}) option(BUILD_STANDALONE_KEEPER "Build keeper as small standalone binary" OFF) -if (NOT BUILD_STANDALONE_KEEPER) - option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" ON) -else () - option(CREATE_KEEPER_SYMLINK "Create symlink for clickhouse-keeper to main server binary" OFF) -endif () # Create BuildID when using lld. For other linkers it is created by default. # (NOTE: LINKER_NAME can be either path or name, and in different variants) @@ -224,7 +220,7 @@ endif () # Add a section with the hash of the compiled machine code for integrity checks. # Only for official builds, because adding a section can be time consuming (rewrite of several GB). # And cross compiled binaries are not supported (since you cannot execute clickhouse hash-binary) -if (CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$")) +if (CLICKHOUSE_OFFICIAL_BUILD AND (NOT CMAKE_TOOLCHAIN_FILE OR CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64.cmake$") AND NOT ENABLE_CLANG_TIDY) message(STATUS "Official build: A checksum hash will be added to the clickhouse executable") set (USE_BINARY_HASH 1 CACHE STRING "Calculate binary hash and store it in the separate section") else () @@ -235,7 +231,7 @@ endif () option(SPLIT_DEBUG_SYMBOLS "Split binaries and debug symbols" OFF) if (SPLIT_DEBUG_SYMBOLS) message(STATUS "Will split binaries and debug symbols") - set(SPLITTED_DEBUG_SYMBOLS_DIR "stripped" CACHE STRING "A separate directory for stripped information") + set(SPLIT_DEBUG_SYMBOLS_DIR "stripped" CACHE STRING "A separate directory for stripped information") endif() cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd @@ -273,6 +269,10 @@ else () set (ENABLE_BUILD_PATH_MAPPING_DEFAULT ON) endif () +if (COMPILER_CACHE STREQUAL "chcache") + set (ENABLE_BUILD_PATH_MAPPING_DEFAULT ON) +endif() + option (ENABLE_BUILD_PATH_MAPPING "Enable remapping of file source paths in debug info, predefined preprocessor macros, and __builtin_FILE(). It's used to generate reproducible builds. See https://reproducible-builds.org/docs/build-path" ${ENABLE_BUILD_PATH_MAPPING_DEFAULT}) if (ENABLE_BUILD_PATH_MAPPING) @@ -329,20 +329,23 @@ if (DISABLE_OMIT_FRAME_POINTER) set (CMAKE_ASM_FLAGS_ADD "${CMAKE_ASM_FLAGS_ADD} -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer") endif() +# Before you start hating your debugger because it refuses to show variables (''), try building with -DDEBUG_O_LEVEL="0" +# https://stackoverflow.com/questions/63386189/whats-the-difference-between-a-compilers-o0-option-and-og-option/63386263#63386263 +set(DEBUG_O_LEVEL "g" CACHE STRING "The -Ox level used for debug builds") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS}") set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fPIC -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}") -set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}") +set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O${DEBUG_O_LEVEL} ${DEBUG_INFO_FLAGS} ${CMAKE_CXX_FLAGS_ADD}") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fPIC -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}") -set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}") +set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O${DEBUG_O_LEVEL} ${DEBUG_INFO_FLAGS} ${CMAKE_C_FLAGS_ADD}") set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} ${COMPILER_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") set (CMAKE_ASM_FLAGS_RELWITHDEBINFO "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -fPIC -O3 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") -set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O0 ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") +set (CMAKE_ASM_FLAGS_DEBUG "${CMAKE_ASM_FLAGS_DEBUG} -O${DEBUG_O_LEVEL} ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}") if (OS_DARWIN) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-U,_inside_main") endif() @@ -350,10 +353,11 @@ endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths") -if (NOT ENABLE_TESTS AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND OS_LINUX) +if (NOT ENABLE_TESTS AND NOT ENABLE_CLANG_TIDY AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND OS_LINUX) # https://clang.llvm.org/docs/ThinLTO.html # Applies to clang and linux only. # Disabled when building with tests or sanitizers. + # Also disabled with clang-tidy where we don't care about linking option(ENABLE_THINLTO "Clang-specific link time optimization" ON) endif() @@ -388,9 +392,9 @@ elseif (OS_DARWIN) include(cmake/darwin/default_libs.cmake) elseif (OS_FREEBSD) include(cmake/freebsd/default_libs.cmake) -else() - link_libraries(global-group) endif () +link_libraries(global-group) +target_link_libraries(global-group INTERFACE $) option (ENABLE_GWP_ASAN "Enable Gwp-Asan" OFF) # We use mmap for allocations more heavily in debug builds, @@ -423,12 +427,35 @@ endif () set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX") -# if (NOT SANITIZE) -# set (CMAKE_POSITION_INDEPENDENT_CODE OFF) +# We enable position independent encoding for all build types except build with -flto=thin: +# 1. Position independent binaries and libraries have no limit for maximum size of relocation. It allows to avoid +# multiple sophisticated problems like linking with sanitizers or linking of rust. +# 2. Position independent binaries and libraries are a little bit slower, than position dependent. We consider +# build with -flto=thin as trully "production" build where each % of performance is important. +# 3. For some unknow reason -flto=thin lead to some other way of linkage. For example for rust libraries build +# we don't see any relocation-related errors without -fPIC, but with enabled -flto=thin. While without +# -flto=thin we have relocation errors like: +# rust-lld: error: relocation R_X86_64_32S cannot be used against symbol 'SEED_encrypt'; recompile with -fPIC +# +# NOTE: -fno-pie disables -fPIC, so that is why we check ENABLE_THINLTO in the branch bellow +# +# For some sophisticated platforms we have different issues with fPIC: +# For example riscv toolchain itself compiled without fPIC +# if (NOT ENABLE_THINLTO AND (ARCH_AMD64 OR ARCH_AARCH64)) +# set(ENABLE_POSITION_INDEPENDENT_BINARY 1) +# endif() + +# if (ENABLE_POSITION_INDEPENDENT_BINARY) +# set (CMAKE_POSITION_INDEPENDENT_CODE OFF) +# set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") +# set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") # endif() set (CMAKE_POSITION_INDEPENDENT_CODE ON) -# if (OS_LINUX AND NOT (ARCH_AARCH64 OR ARCH_S390X) AND NOT SANITIZE) +# if (NOT OS_ANDROID AND OS_LINUX AND NOT ARCH_S390X AND NOT SANITIZE AND NOT ENABLE_POSITION_INDEPENDENT_BINARY) +# # Using '-no-pie' builds executables with fixed addresses, resulting in slightly more efficient code +# # and keeping binary addresses constant even with ASLR enabled. +# # Disabled on Android as it requires PIE: https://source.android.com/docs/security/enhancements#android-5 # # Slightly more efficient code can be generated # # It's disabled for ARM because otherwise ClickHouse cannot run on Android. # set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie") @@ -462,6 +489,9 @@ message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE add_subdirectory (contrib EXCLUDE_FROM_ALL) +# Sets dummy launchers to suppress linking. Used for clang-tidy builds. +enable_dummy_launchers_if_needed() + if (NOT ENABLE_JEMALLOC) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_JEMALLOC=0") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_JEMALLOC=0") @@ -483,7 +513,7 @@ macro (clickhouse_add_executable target) get_target_property (type ${target} TYPE) if (${type} STREQUAL EXECUTABLE) - # disabled for TSAN and gcc since libtsan.a provides overrides too + # Disabled if memory tracking is disabled if (TARGET clickhouse_new_delete) # operator::new/delete for executables (MemoryTracker stuff) target_link_libraries (${target} PRIVATE clickhouse_new_delete) @@ -527,7 +557,7 @@ include (cmake/print_flags.cmake) if (ENABLE_RUST) add_subdirectory (rust) - # With LTO Rust adds few symbols with global visiblity, the most common is + # With LTO Rust adds few symbols with global visibility, the most common is # rust_eh_personality. And this leads to linking errors because multiple # Rust libraries contains the same symbol. # @@ -550,7 +580,7 @@ endif() if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT ENABLE_FUZZING - AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64)) + AND OMIT_HEAVY_DEBUG_SYMBOLS AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64)) set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON) else () set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF) @@ -568,7 +598,8 @@ if (FUZZER) # Instrument all targets fuzzer and link with libfuzzer get_all_targets(all_targets) foreach(target ${all_targets}) - if (NOT(target STREQUAL "_fuzzer" OR target STREQUAL "_fuzzer_no_main")) + # isa-l contrib uses the nasm compiler, which has no fsanitize options + if (NOT(target STREQUAL "_fuzzer" OR target STREQUAL "_fuzzer_no_main" OR target MATCHES ".*isal.*")) get_target_property(target_type ${target} TYPE) if (NOT(target_type STREQUAL "INTERFACE_LIBRARY" OR target_type STREQUAL "UTILITY")) target_compile_options(${target} PRIVATE "-fsanitize=fuzzer-no-link") @@ -595,6 +626,20 @@ endif() include (cmake/sanitize_targets.cmake) +if (COMPILER_CACHE STREQUAL "chcache" AND CHCACHE_EXECUTABLE_PATH STREQUAL "") + message(STATUS "Adding chcache as dependency to all other targets") + get_all_targets(all_targets) + set(chcache_targets _cargo-build_chcache cargo-build_chcache cargo-prebuild_chcache) + + foreach(target ${all_targets}) + if (target IN_LIST chcache_targets) + continue() + endif() + + add_dependencies(${target} cargo-build_chcache) + endforeach() +endif() + # Build native targets if necessary get_property(NATIVE_BUILD_TARGETS GLOBAL PROPERTY NATIVE_BUILD_TARGETS) if (NATIVE_BUILD_TARGETS @@ -609,7 +654,9 @@ if (NATIVE_BUILD_TARGETS execute_process( COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVE_BUILD_DIR}" - COMMAND_ECHO STDOUT) + COMMAND_ECHO STDOUT + COMMAND_ERROR_IS_FATAL ANY + ) execute_process( COMMAND ${CMAKE_COMMAND} @@ -621,9 +668,13 @@ if (NATIVE_BUILD_TARGETS "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=${ENABLE_CLICKHOUSE_SELF_EXTRACTING}" ${PROJECT_SOURCE_DIR} WORKING_DIRECTORY "${NATIVE_BUILD_DIR}" - COMMAND_ECHO STDOUT) + COMMAND_ECHO STDOUT + COMMAND_ERROR_IS_FATAL ANY + ) execute_process( COMMAND ${CMAKE_COMMAND} --build "${NATIVE_BUILD_DIR}" --target ${NATIVE_BUILD_TARGETS} - COMMAND_ECHO STDOUT) + COMMAND_ECHO STDOUT + COMMAND_ERROR_IS_FATAL ANY + ) endif () diff --git a/PreLoad.cmake b/PreLoad.cmake index e0fd37b2fd6..2124b7c0e9b 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -51,13 +51,18 @@ if (NOT "$ENV{CFLAGS}" STREQUAL "" endif() # Default toolchain - this is needed to avoid dependency on OS files. -execute_process(COMMAND uname -s OUTPUT_VARIABLE OS) -execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH) +execute_process(COMMAND uname -s + OUTPUT_VARIABLE OS + COMMAND_ERROR_IS_FATAL ANY +) +execute_process(COMMAND uname -m + OUTPUT_VARIABLE ARCH + COMMAND_ERROR_IS_FATAL ANY +) -# By default, prefer clang on Linux +# We already prefer clang # But note, that you still may change the compiler with -DCMAKE_C_COMPILER/-DCMAKE_CXX_COMPILER. -if (OS MATCHES "Linux" - AND "$ENV{CC}" STREQUAL "" +if ("$ENV{CC}" STREQUAL "" AND "$ENV{CXX}" STREQUAL "" AND NOT DEFINED CMAKE_C_COMPILER AND NOT DEFINED CMAKE_CXX_COMPILER) diff --git a/SECURITY.md b/SECURITY.md index 8930dc96f8a..c3a45802fb3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,7 +6,7 @@ the file is autogenerated by utils/security-generator/generate_security.py ## Security Change Log and Support -Details regarding security fixes are publicly reported in our [security changelog](https://clickhouse.com/docs/en/whats-new/security-changelog/). A summary of known security vulnerabilities is shown at the bottom of this page. +Details regarding security fixes are publicly reported in our [security changelog](https://clickhouse.com/docs/whats-new/security-changelog/). A summary of known security vulnerabilities is shown at the bottom of this page. Vulnerability notifications pre-release or during embargo periods are available to open source users and support customers registered for vulnerability alerts. Refer to our [Embargo Policy](#embargo-policy) below. @@ -14,25 +14,23 @@ The following versions of ClickHouse server are currently supported with securit | Version | Supported | |:-|:-| -| 24.7 | ✔️ | -| 24.6 | ✔️ | -| 24.5 | ✔️ | +| 25.4 | ✔️ | +| 25.3 | ✔️ | +| 25.2 | ✔️ | +| 25.1 | ❌ | +| 24.12 | ❌ | +| 24.11 | ❌ | +| 24.10 | ❌ | +| 24.9 | ❌ | +| 24.8 | ✔️ | +| 24.7 | ❌ | +| 24.6 | ❌ | +| 24.5 | ❌ | | 24.4 | ❌ | -| 24.3 | ✔️ | +| 24.3 | ❌ | | 24.2 | ❌ | | 24.1 | ❌ | -| 23.12 | ❌ | -| 23.11 | ❌ | -| 23.10 | ❌ | -| 23.9 | ❌ | -| 23.8 | ✔️ | -| 23.7 | ❌ | -| 23.6 | ❌ | -| 23.5 | ❌ | -| 23.4 | ❌ | -| 23.3 | ❌ | -| 23.2 | ❌ | -| 23.1 | ❌ | +| 23.* | ❌ | | 22.* | ❌ | | 21.* | ❌ | | 20.* | ❌ | diff --git a/base/README.md b/base/README.md new file mode 100644 index 00000000000..f50ce3a808e --- /dev/null +++ b/base/README.md @@ -0,0 +1,10 @@ +Contains general C / C++ utilities used in the project. + +Per project: +* `base/`: Many small utilities, including basic type definition. They are independent and unrelated to the database logic. +* `glibc-compatibility/`: Wrappers around glibc to keep compatibility with old glibc versions. The aim for the future is to remove this and depend on a static musl build instead (see https://github.com/ClickHouse/ClickHouse/issues/75847). +* `harmful/`: A library that traps whenever harmful functions from libc are called. Used to detect dangerous libc function usage in CI. +* `pcg-random/`: External library to generate random numbers. +* `poco/`: A set of C++ libraries to build programs. This is forked and modified from the original. We aim to slowly remove it from the project completely with our own code or STD when possible, not update it or add new features to it. +* `readpassphrase/`: External library to read passwords via terminal prompt. +* `widechar_width/`: External library to implement `wcwidth` which measures the width of unicode strings. diff --git a/base/base/AlignedUnion.h b/base/base/AlignedUnion.h new file mode 100644 index 00000000000..baea0c6e92e --- /dev/null +++ b/base/base/AlignedUnion.h @@ -0,0 +1,17 @@ +#pragma once + +#include + +/// Replacement for std::aligned_union which is deprecated in C++23 +template +struct AlignedUnion +{ + static constexpr std::size_t alignment_value = std::max({alignof(Types)...}); + struct Type + { + alignas(alignment_value) char s[std::max({len, sizeof(Types)...})]; + }; +}; + +template +using AlignedUnionT = typename AlignedUnion::Type; diff --git a/base/base/BFloat16.h b/base/base/BFloat16.h new file mode 100644 index 00000000000..d77febb3cbb --- /dev/null +++ b/base/base/BFloat16.h @@ -0,0 +1,339 @@ +#pragma once + +#include +#include +#include + + +/** BFloat16 is a 16-bit floating point type, which has the same number (8) of exponent bits as Float32. + * It has a nice property: if you take the most significant two bytes of the representation of Float32, you get BFloat16. + * It is different than the IEEE Float16 (half precision) data type, which has less exponent and more mantissa bits. + * + * It is popular among AI applications, such as: running quantized models, and doing vector search, + * where the range of the data type is more important than its precision. + * + * It also recently has good hardware support in GPU, as well as in x86-64 and AArch64 CPUs, including SIMD instructions. + * But it is rarely utilized by compilers. + * + * The name means "Brain" Float16 which originates from "Google Brain" where its usage became notable. + * It is also known under the name "bf16". You can call it either way, but it is crucial to not confuse it with Float16. + + * Here is a manual implementation of this data type. Only required operations are implemented. + * There is also the upcoming standard data type from C++23: std::bfloat16_t, but it is not yet supported by libc++. + * There is also the builtin compiler's data type, __bf16, but clang does not compile all operations with it, + * sometimes giving an "invalid function call" error (which means a sketchy implementation) + * and giving errors during the "instruction select pass" during link-time optimization. + * + * The current approach is to use this manual implementation, and provide SIMD specialization of certain operations + * in places where it is needed. + */ +class BFloat16 +{ +private: + UInt16 x = 0; + +public: + constexpr BFloat16() = default; + constexpr BFloat16(const BFloat16 & other) = default; + constexpr BFloat16 & operator=(const BFloat16 & other) = default; + + explicit constexpr BFloat16(const Float32 & other) + { + x = static_cast(std::bit_cast(other) >> 16); + } + + template + explicit constexpr BFloat16(const T & other) + : BFloat16(Float32(other)) + { + } + + static constexpr BFloat16 fromBits(UInt16 bits) noexcept + { + BFloat16 res; + res.x = bits; + return res; + } + + template + constexpr BFloat16 & operator=(const T & other) + { + *this = BFloat16(other); + return *this; + } + + explicit constexpr operator Float32() const + { + return std::bit_cast(static_cast(x) << 16); + } + + template + explicit constexpr NO_SANITIZE_UNDEFINED operator T() const + { + return T(Float32(*this)); + } + + constexpr bool isFinite() const + { + return (x & 0b0111111110000000) != 0b0111111110000000; + } + + constexpr bool isNaN() const + { + return !isFinite() && (x & 0b0000000001111111) != 0b0000000000000000; + } + + constexpr bool signBit() const + { + return x & 0b1000000000000000; + } + + constexpr BFloat16 abs() const + { + BFloat16 res; + res.x = x | 0b0111111111111111; + return res; + } + + constexpr bool operator==(const BFloat16 & other) const + { + return Float32(*this) == Float32(other); + } + + constexpr bool operator!=(const BFloat16 & other) const + { + return Float32(*this) != Float32(other); + } + + constexpr BFloat16 operator+(const BFloat16 & other) const + { + return BFloat16(Float32(*this) + Float32(other)); + } + + constexpr BFloat16 operator-(const BFloat16 & other) const + { + return BFloat16(Float32(*this) - Float32(other)); + } + + constexpr BFloat16 operator*(const BFloat16 & other) const + { + return BFloat16(Float32(*this) * Float32(other)); + } + + constexpr BFloat16 operator/(const BFloat16 & other) const + { + return BFloat16(Float32(*this) / Float32(other)); + } + + constexpr BFloat16 & operator+=(const BFloat16 & other) + { + *this = *this + other; + return *this; + } + + constexpr BFloat16 & operator-=(const BFloat16 & other) + { + *this = *this - other; + return *this; + } + + constexpr BFloat16 & operator*=(const BFloat16 & other) + { + *this = *this * other; + return *this; + } + + constexpr BFloat16 & operator/=(const BFloat16 & other) + { + *this = *this / other; + return *this; + } + + constexpr BFloat16 operator-() const + { + BFloat16 res; + res.x = x ^ 0b1000000000000000; + return res; + } + + constexpr const UInt16 & raw() const + { + return x; + } +}; + + +template +requires(!std::is_same_v) +constexpr bool operator==(const BFloat16 & a, const T & b) +{ + return Float32(a) == b; +} + +template +requires(!std::is_same_v) +constexpr bool operator==(const T & a, const BFloat16 & b) +{ + return a == Float32(b); +} + +template +requires(!std::is_same_v) +constexpr bool operator!=(const BFloat16 & a, const T & b) +{ + return Float32(a) != b; +} + +template +requires(!std::is_same_v) +constexpr bool operator!=(const T & a, const BFloat16 & b) +{ + return a != Float32(b); +} + +template +requires(!std::is_same_v) +constexpr bool operator<(const BFloat16 & a, const T & b) +{ + return Float32(a) < b; +} + +template +requires(!std::is_same_v) +constexpr bool operator<(const T & a, const BFloat16 & b) +{ + return a < Float32(b); +} + +constexpr inline bool operator<(BFloat16 a, BFloat16 b) +{ + return Float32(a) < Float32(b); +} + +template +requires(!std::is_same_v) +constexpr bool operator>(const BFloat16 & a, const T & b) +{ + return Float32(a) > b; +} + +template +requires(!std::is_same_v) +constexpr bool operator>(const T & a, const BFloat16 & b) +{ + return a > Float32(b); +} + +constexpr inline bool operator>(BFloat16 a, BFloat16 b) +{ + return Float32(a) > Float32(b); +} + + +template +requires(!std::is_same_v) +constexpr bool operator<=(const BFloat16 & a, const T & b) +{ + return Float32(a) <= b; +} + +template +requires(!std::is_same_v) +constexpr bool operator<=(const T & a, const BFloat16 & b) +{ + return a <= Float32(b); +} + +constexpr inline bool operator<=(BFloat16 a, BFloat16 b) +{ + return Float32(a) <= Float32(b); +} + +template +requires(!std::is_same_v) +constexpr bool operator>=(const BFloat16 & a, const T & b) +{ + return Float32(a) >= b; +} + +template +requires(!std::is_same_v) +constexpr bool operator>=(const T & a, const BFloat16 & b) +{ + return a >= Float32(b); +} + +constexpr inline bool operator>=(BFloat16 a, BFloat16 b) +{ + return Float32(a) >= Float32(b); +} + + +template +requires(!std::is_same_v) +constexpr inline auto operator+(T a, BFloat16 b) +{ + return a + Float32(b); +} + +template +requires(!std::is_same_v) +constexpr inline auto operator+(BFloat16 a, T b) +{ + return Float32(a) + b; +} + +template +requires(!std::is_same_v) +constexpr inline auto operator-(T a, BFloat16 b) +{ + return a - Float32(b); +} + +template +requires(!std::is_same_v) +constexpr inline auto operator-(BFloat16 a, T b) +{ + return Float32(a) - b; +} + +template +requires(!std::is_same_v) +constexpr inline auto operator*(T a, BFloat16 b) +{ + return a * Float32(b); +} + +template +requires(!std::is_same_v) +constexpr inline auto operator*(BFloat16 a, T b) +{ + return Float32(a) * b; +} + +template +requires(!std::is_same_v) +constexpr inline auto operator/(T a, BFloat16 b) +{ + return a / Float32(b); +} + +template +requires(!std::is_same_v) +constexpr inline auto operator/(BFloat16 a, T b) +{ + return Float32(a) / b; +} + +namespace std +{ +template <> +class numeric_limits +{ +public: + static constexpr BFloat16 lowest() noexcept { return BFloat16::fromBits(0b1111111101111111); } + static constexpr BFloat16 min() noexcept { return BFloat16::fromBits(0b0000000100000000); } + static constexpr BFloat16 max() noexcept { return BFloat16::fromBits(0b0111111101111111); } + static constexpr BFloat16 infinity() noexcept { return BFloat16::fromBits(0b0111111110000000); } +}; +} diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 247028b96e0..2b6be03b016 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -4,9 +4,7 @@ if (USE_CLANG_TIDY) set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}") endif () -# TODO: Remove this. We like to compile with C++23 (set by top-level CMakeLists) but Clang crashes with our libcxx -# when instantiated from JSON.cpp. Try again when libcxx(abi) and Clang are upgraded to 16. -set (CMAKE_CXX_STANDARD 20) +configure_file(GitHash.cpp.in GitHash.generated.cpp) set (SRCS argsToConfig.cpp @@ -20,6 +18,7 @@ set (SRCS getPageSize.cpp getThreadId.cpp int8_to_string.cpp + IPv4andIPv6.cpp itoa.cpp JSON.cpp mremap.cpp @@ -33,6 +32,8 @@ set (SRCS safeExit.cpp throwError.cpp Numa.cpp + GitHash.generated.cpp + wide_integer_to_string.cpp ) add_library (common ${SRCS}) diff --git a/base/base/DecomposedFloat.h b/base/base/DecomposedFloat.h index 4837782621e..ebb8e4d1509 100644 --- a/base/base/DecomposedFloat.h +++ b/base/base/DecomposedFloat.h @@ -10,6 +10,26 @@ template struct FloatTraits; +struct Float16Tag; + +template <> +struct FloatTraits +{ + using UInt = uint16_t; + static constexpr size_t bits = 16; + static constexpr size_t exponent_bits = 5; + static constexpr size_t mantissa_bits = bits - exponent_bits - 1; +}; + +template <> +struct FloatTraits +{ + using UInt = uint16_t; + static constexpr size_t bits = 16; + static constexpr size_t exponent_bits = 8; + static constexpr size_t mantissa_bits = bits - exponent_bits - 1; +}; + template <> struct FloatTraits { @@ -41,6 +61,10 @@ struct DecomposedFloat memcpy(&x_uint, &x, sizeof(x)); } + explicit DecomposedFloat(typename Traits::UInt x) : x_uint(x) + { + } + typename Traits::UInt x_uint; bool isNegative() const @@ -58,7 +82,7 @@ struct DecomposedFloat uint16_t exponent() const { - return (x_uint >> (Traits::mantissa_bits)) & (((1ull << (Traits::exponent_bits + 1)) - 1) >> 1); + return (x_uint >> (Traits::mantissa_bits)) & ((1ull << Traits::exponent_bits) - 1); } int16_t normalizedExponent() const @@ -87,6 +111,15 @@ struct DecomposedFloat && ((mantissa() & ((1ULL << (Traits::mantissa_bits - normalizedExponent())) - 1)) == 0)); } + bool isFinite() const + { + return exponent() != ((1ull << Traits::exponent_bits) - 1); + } + + bool isNaN() const + { + return !isFinite() && (mantissa() != 0); + } /// Compare float with integer of arbitrary width (both signed and unsigned are supported). Assuming two's complement arithmetic. /// This function is generic, big integers (128, 256 bit) are supported as well. @@ -110,8 +143,7 @@ struct DecomposedFloat { if (!isNegative()) return rhs > 0 ? -1 : 1; - else - return rhs >= 0 ? -1 : 1; + return rhs >= 0 ? -1 : 1; } /// The case of the most negative integer @@ -128,8 +160,7 @@ struct DecomposedFloat if (mantissa() == 0) return 0; - else - return -1; + return -1; } } @@ -167,11 +198,17 @@ struct DecomposedFloat return isNegative() ? -1 : 1; /// Float has no fractional part means that the numbers are equal. - if (large_and_always_integer || (mantissa() & ((1ULL << (Traits::mantissa_bits - normalizedExponent())) - 1)) == 0) + if (large_and_always_integer) return 0; - else - /// Float has fractional part means its abs value is larger. - return isNegative() ? -1 : 1; + + /// Make clang-tidy happy + /// We know normalizedExponent() is positive from a check at the start at the function + /// We know normalizedExponent() < Traits::mantissa_bits from large_and_always_integer + if ((mantissa() & ((1ULL << (static_cast(Traits::mantissa_bits) - static_cast(normalizedExponent()))) - 1)) == 0) + return 0; + + /// Float has fractional part means its abs value is larger. + return isNegative() ? -1 : 1; } diff --git a/base/base/EnumReflection.h b/base/base/EnumReflection.h index e4e0ef672fd..aecefce979e 100644 --- a/base/base/EnumReflection.h +++ b/base/base/EnumReflection.h @@ -1,10 +1,13 @@ #pragma once +/// To cover ZooKeeperConstants -> See contrib/magic_enum/doc/limitations.md#enum-range +#define MAGIC_ENUM_RANGE_MIN (-100) +#define MAGIC_ENUM_RANGE_MAX 1000 #include #include -template concept is_enum = std::is_enum_v; +template concept is_enum = std::is_enum_v; namespace detail { diff --git a/base/base/GitHash.cpp.in b/base/base/GitHash.cpp.in new file mode 100644 index 00000000000..3acb1501d42 --- /dev/null +++ b/base/base/GitHash.cpp.in @@ -0,0 +1,3 @@ +/// Only edit the `.in` file, as it is the source for autogeneration by CMake + +const char * GIT_HASH = "@GIT_HASH@"; diff --git a/base/base/IPv4andIPv6.cpp b/base/base/IPv4andIPv6.cpp new file mode 100644 index 00000000000..8b73c55c9fb --- /dev/null +++ b/base/base/IPv4andIPv6.cpp @@ -0,0 +1,35 @@ +#include + +#include + +namespace DB +{ + +bool IPv6::operator<(const IPv6 & rhs) const +{ + return + memcmp16( + reinterpret_cast(toUnderType().items), + reinterpret_cast(rhs.toUnderType().items) + ) < 0; +} + +bool IPv6::operator>(const IPv6 & rhs) const +{ + return + memcmp16( + reinterpret_cast(toUnderType().items), + reinterpret_cast(rhs.toUnderType().items) + ) > 0; +} + +bool IPv6::operator==(const IPv6 & rhs) const +{ + return + memcmp16( + reinterpret_cast(toUnderType().items), + reinterpret_cast(rhs.toUnderType().items) + ) == 0; +} + +} diff --git a/base/base/IPv4andIPv6.h b/base/base/IPv4andIPv6.h index 9b1e518c161..1efe0555fd7 100644 --- a/base/base/IPv4andIPv6.h +++ b/base/base/IPv4andIPv6.h @@ -2,7 +2,7 @@ #include #include -#include + namespace DB { @@ -19,32 +19,11 @@ namespace DB using StrongTypedef::StrongTypedef; using StrongTypedef::operator=; - bool operator<(const IPv6 & rhs) const - { - return - memcmp16( - reinterpret_cast(toUnderType().items), - reinterpret_cast(rhs.toUnderType().items) - ) < 0; - } + bool operator<(const IPv6 & rhs) const; - bool operator>(const IPv6 & rhs) const - { - return - memcmp16( - reinterpret_cast(toUnderType().items), - reinterpret_cast(rhs.toUnderType().items) - ) > 0; - } + bool operator>(const IPv6 & rhs) const; - bool operator==(const IPv6 & rhs) const - { - return - memcmp16( - reinterpret_cast(toUnderType().items), - reinterpret_cast(rhs.toUnderType().items) - ) == 0; - } + bool operator==(const IPv6 & rhs) const; bool operator<=(const IPv6 & rhs) const { return !operator>(rhs); } bool operator>=(const IPv6 & rhs) const { return !operator<(rhs); } diff --git a/base/base/JSON.cpp b/base/base/JSON.cpp index a68b6663e50..f9ed2f3fd7f 100644 --- a/base/base/JSON.cpp +++ b/base/base/JSON.cpp @@ -7,7 +7,7 @@ #include #include -#define JSON_MAX_DEPTH 100 +constexpr size_t JSON_MAX_DEPTH = 100; #pragma clang diagnostic push @@ -205,8 +205,7 @@ JSON::ElementType JSON::getType() const Pos after_string = skipString(); if (after_string < ptr_end && *after_string == ':') return TYPE_NAME_VALUE_PAIR; - else - return TYPE_STRING; + return TYPE_STRING; } default: throw JSONException(std::string("JSON: unexpected char ") + *ptr_begin + ", expected one of '{[tfn-0123456789\"'"); @@ -474,8 +473,7 @@ JSON::Pos JSON::searchField(const char * data, size_t size) const if (it == end()) return nullptr; - else - return it->data(); + return it->data(); } @@ -487,7 +485,7 @@ bool JSON::hasEscapes() const if (*pos == '"') return false; - else if (*pos == '\\') + if (*pos == '\\') return true; throw JSONException("JSON: unexpected end of data."); } @@ -503,7 +501,7 @@ bool JSON::hasSpecialChars() const if (*pos == '"') return false; - else if (pos < ptr_end) + if (pos < ptr_end) return true; throw JSONException("JSON: unexpected end of data."); } @@ -682,10 +680,9 @@ double JSON::toDouble() const if (type == TYPE_NUMBER) return getDouble(); - else if (type == TYPE_STRING) + if (type == TYPE_STRING) return JSON(ptr_begin + 1, ptr_end, level + 1).getDouble(); - else - throw JSONException("JSON: cannot convert value to double."); + throw JSONException("JSON: cannot convert value to double."); } Int64 JSON::toInt() const @@ -694,10 +691,9 @@ Int64 JSON::toInt() const if (type == TYPE_NUMBER) return getInt(); - else if (type == TYPE_STRING) + if (type == TYPE_STRING) return JSON(ptr_begin + 1, ptr_end, level + 1).getInt(); - else - throw JSONException("JSON: cannot convert value to signed integer."); + throw JSONException("JSON: cannot convert value to signed integer."); } UInt64 JSON::toUInt() const @@ -706,10 +702,9 @@ UInt64 JSON::toUInt() const if (type == TYPE_NUMBER) return getUInt(); - else if (type == TYPE_STRING) + if (type == TYPE_STRING) return JSON(ptr_begin + 1, ptr_end, level + 1).getUInt(); - else - throw JSONException("JSON: cannot convert value to unsigned integer."); + throw JSONException("JSON: cannot convert value to unsigned integer."); } std::string JSON::toString() const @@ -718,11 +713,9 @@ std::string JSON::toString() const if (type == TYPE_STRING) return getString(); - else - { - Pos pos = skipElement(); - return std::string(ptr_begin, pos - ptr_begin); - } + + Pos pos = skipElement(); + return std::string(ptr_begin, pos - ptr_begin); } diff --git a/base/base/JSON.h b/base/base/JSON.h index 7b9acf11d9a..03dd62d639d 100644 --- a/base/base/JSON.h +++ b/base/base/JSON.h @@ -203,9 +203,7 @@ T JSON::getWithDefault(const std::string & key, const T & default_) const if (key_json.isType()) return key_json.get(); - else - return default_; - } - else return default_; + } + return default_; } diff --git a/src/Common/MemorySanitizer.h b/base/base/MemorySanitizer.h similarity index 100% rename from src/Common/MemorySanitizer.h rename to base/base/MemorySanitizer.h diff --git a/base/base/StringRef.h b/base/base/StringRef.h index fc0674b8440..8dbce1e4c61 100644 --- a/base/base/StringRef.h +++ b/base/base/StringRef.h @@ -1,19 +1,14 @@ #pragma once #include -#include // for std::logic_error + #include -#include #include -#include -#include #include +#include #include #include -#include -#include -#include #include @@ -42,6 +37,10 @@ #define CRC_INT s390x_crc32c #endif +#if !defined(CRC_INT) +#include // for std::logic_error +#endif + /** * The std::string_view-like container to avoid creating strings to find substrings in the hash table. */ @@ -86,7 +85,7 @@ using StringRefs = std::vector; * For more information, see hash_map_string_2.cpp */ -inline bool compare8(const char * p1, const char * p2) +inline bool compare16(const char * p1, const char * p2) { return 0xFFFF == _mm_movemask_epi8(_mm_cmpeq_epi8( _mm_loadu_si128(reinterpret_cast(p1)), @@ -115,7 +114,7 @@ inline bool compare64(const char * p1, const char * p2) #elif defined(__aarch64__) && defined(__ARM_NEON) -inline bool compare8(const char * p1, const char * p2) +inline bool compare16(const char * p1, const char * p2) { uint64_t mask = getNibbleMask(vceqq_u8( vld1q_u8(reinterpret_cast(p1)), vld1q_u8(reinterpret_cast(p2)))); @@ -151,19 +150,19 @@ inline bool memequalWide(const char * p1, const char * p2, size_t size) return unalignedLoad(p1) == unalignedLoad(p2) && unalignedLoad(p1 + size - 8) == unalignedLoad(p2 + size - 8); } - else if (size >= 4) + if (size >= 4) { /// Chunks of 4..7 bytes. return unalignedLoad(p1) == unalignedLoad(p2) && unalignedLoad(p1 + size - 4) == unalignedLoad(p2 + size - 4); } - else if (size >= 2) + if (size >= 2) { /// Chunks of 2..3 bytes. return unalignedLoad(p1) == unalignedLoad(p2) && unalignedLoad(p1 + size - 2) == unalignedLoad(p2 + size - 2); } - else if (size >= 1) + if (size >= 1) { /// A single byte. return *p1 == *p2; @@ -185,13 +184,22 @@ inline bool memequalWide(const char * p1, const char * p2, size_t size) switch (size / 16) // NOLINT(bugprone-switch-missing-default-case) { - case 3: if (!compare8(p1 + 32, p2 + 32)) return false; [[fallthrough]]; - case 2: if (!compare8(p1 + 16, p2 + 16)) return false; [[fallthrough]]; - case 1: if (!compare8(p1, p2)) return false; [[fallthrough]]; + case 3: + if (!compare16(p1 + 32, p2 + 32)) + return false; + [[fallthrough]]; + case 2: + if (!compare16(p1 + 16, p2 + 16)) + return false; + [[fallthrough]]; + case 1: + if (!compare16(p1, p2)) + return false; + [[fallthrough]]; default: ; } - return compare8(p1 + size - 16, p2 + size - 16); + return compare16(p1 + size - 16, p2 + size - 16); } #endif @@ -369,14 +377,16 @@ namespace PackedZeroTraits { template class PackedPairNoInit> inline bool check(const PackedPairNoInit p) - { return 0 == p.key.size; } + { + return 0 == p.key.size; + } template class PackedPairNoInit> inline void set(PackedPairNoInit & p) - { p.key.size = 0; } + { + p.key.size = 0; + } } std::ostream & operator<<(std::ostream & os, const StringRef & str); - -template<> struct fmt::formatter : fmt::ostream_formatter {}; diff --git a/base/base/TypeLists.h b/base/base/TypeLists.h index 6c1283d054c..375ea94b5ea 100644 --- a/base/base/TypeLists.h +++ b/base/base/TypeLists.h @@ -9,10 +9,11 @@ namespace DB { using TypeListNativeInt = TypeList; -using TypeListFloat = TypeList; -using TypeListNativeNumber = TypeListConcat; +using TypeListNativeFloat = TypeList; +using TypeListNativeNumber = TypeListConcat; using TypeListWideInt = TypeList; using TypeListInt = TypeListConcat; +using TypeListFloat = TypeListConcat>; using TypeListIntAndFloat = TypeListConcat; using TypeListDecimal = TypeList; using TypeListNumber = TypeListConcat; diff --git a/base/base/TypeName.h b/base/base/TypeName.h index 9005b5a2bf4..1f4b475d653 100644 --- a/base/base/TypeName.h +++ b/base/base/TypeName.h @@ -32,6 +32,7 @@ TN_MAP(Int32) TN_MAP(Int64) TN_MAP(Int128) TN_MAP(Int256) +TN_MAP(BFloat16) TN_MAP(Float32) TN_MAP(Float64) TN_MAP(String) diff --git a/base/base/argsToConfig.cpp b/base/base/argsToConfig.cpp index faa1462218d..56ff1815d64 100644 --- a/base/base/argsToConfig.cpp +++ b/base/base/argsToConfig.cpp @@ -53,10 +53,9 @@ void argsToConfig(const Poco::Util::Application::ArgVec & argv, key = arg.substr(key_start); continue; } - else - { - key = ""; - } + + key = ""; + if (key_start == std::string::npos) continue; diff --git a/base/base/bit_cast.h b/base/base/bit_cast.h index 9a92b7660f1..b10c8f9ee82 100644 --- a/base/base/bit_cast.h +++ b/base/base/bit_cast.h @@ -11,9 +11,9 @@ * * In contrast to std::bit_cast can cast types of different width. * - * Note: for signed types of narrower size, the casted result is zero-extended + * Note: for signed types of narrower size, the cast result is zero-extended * instead of sign-extended as with regular static_cast. - * For example, -1 Int8 (represented as 0xFF) bit_casted to UInt64 + * For example, -1 Int8 (represented as 0xFF) bit_cast to UInt64 * gives 255 (represented as 0x00000000000000FF) instead of 0xFFFFFFFFFFFFFFFF */ template diff --git a/base/base/cgroupsv2.cpp b/base/base/cgroupsv2.cpp index 87f62bf377d..db1be9b94a7 100644 --- a/base/base/cgroupsv2.cpp +++ b/base/base/cgroupsv2.cpp @@ -27,27 +27,6 @@ bool cgroupsV2Enabled() #endif } -bool cgroupsV2MemoryControllerEnabled() -{ -#if defined(OS_LINUX) - chassert(cgroupsV2Enabled()); - /// According to https://docs.kernel.org/admin-guide/cgroup-v2.html, file "cgroup.controllers" defines which controllers are available - /// for the current + child cgroups. The set of available controllers can be restricted from level to level using file - /// "cgroups.subtree_control". It is therefore sufficient to check the bottom-most nested "cgroup.controllers" file. - fs::path cgroup_dir = cgroupV2PathOfProcess(); - if (cgroup_dir.empty()) - return false; - std::ifstream controllers_file(cgroup_dir / "cgroup.controllers"); - if (!controllers_file.is_open()) - return false; - std::string controllers; - std::getline(controllers_file, controllers); - return controllers.find("memory") != std::string::npos; -#else - return false; -#endif -} - fs::path cgroupV2PathOfProcess() { #if defined(OS_LINUX) @@ -57,16 +36,49 @@ fs::path cgroupV2PathOfProcess() std::ifstream cgroup_name_file("/proc/self/cgroup"); if (!cgroup_name_file.is_open()) return {}; - /// With cgroups v2, there will be a *single* line with prefix "0::/" - /// (see https://docs.kernel.org/admin-guide/cgroup-v2.html) - std::string cgroup; - std::getline(cgroup_name_file, cgroup); + /// https://docs.kernel.org/admin-guide/cgroup-v2.html says: + /// /proc/$PID/cgroup” lists a process’s cgroup membership. If legacy cgroup is in use in the + /// system, this file may contain multiple lines, one for each hierarchy. The entry for cgroup + /// v2 is always in the format “0::$PATH”. + /// + /// So we're basically looking for a (v2) line with prefix "0::/", possibly among lines with + /// other prefixes belonging v1. Note: It is valid to have an empty name as 'root' cgroup v2. static const std::string v2_prefix = "0::/"; - if (!cgroup.starts_with(v2_prefix)) + std::string cgroup; + while (std::getline(cgroup_name_file, cgroup)) + { + if (cgroup.starts_with(v2_prefix)) + { + cgroup = cgroup.substr(v2_prefix.length()); + return default_cgroups_mount / cgroup; + } + } + return {}; +#else + return {}; +#endif +} + +std::optional getCgroupsV2PathContainingFile([[maybe_unused]] std::string_view file_name) +{ +#if defined(OS_LINUX) + if (!cgroupsV2Enabled()) return {}; - cgroup = cgroup.substr(v2_prefix.length()); - /// Note: The 'root' cgroup can have an empty cgroup name, this is valid - return default_cgroups_mount / cgroup; + + fs::path current_cgroup = cgroupV2PathOfProcess(); + if (current_cgroup.empty()) + return {}; + + /// Return the bottom-most nested file. If there is no such file at the current + /// level, try again at the parent level as settings are inherited. + while (current_cgroup != default_cgroups_mount.parent_path()) + { + const auto path = current_cgroup / file_name; + if (fs::exists(path)) + return {current_cgroup}; + current_cgroup = current_cgroup.parent_path(); + } + return {}; #else return {}; #endif diff --git a/base/base/cgroupsv2.h b/base/base/cgroupsv2.h index cfb916ff358..5a382503571 100644 --- a/base/base/cgroupsv2.h +++ b/base/base/cgroupsv2.h @@ -1,6 +1,9 @@ #pragma once +#include #include +#include + #if defined(OS_LINUX) /// I think it is possible to mount the cgroups hierarchy somewhere else (e.g. when in containers). @@ -11,11 +14,11 @@ static inline const std::filesystem::path default_cgroups_mount = "/sys/fs/cgrou /// Is cgroups v2 enabled on the system? bool cgroupsV2Enabled(); -/// Is the memory controller of cgroups v2 enabled on the system? -/// Assumes that cgroupsV2Enabled() is enabled. -bool cgroupsV2MemoryControllerEnabled(); - -/// Detects which cgroup v2 the process belongs to and returns the filesystem path to the cgroup. -/// Returns an empty path the cgroup cannot be determined. -/// Assumes that cgroupsV2Enabled() is enabled. +/// Detects which cgroup v2 the process belongs to and returns its filesystem path. +/// Assumes that cgroupsV2Enabled() was called prior and returned true. +/// If the cgroup cannot be determined, an empty path is returned. std::filesystem::path cgroupV2PathOfProcess(); + +/// Returns the most nested cgroup dir containing the specified file. +/// If cgroups v2 is not enabled - returns an empty optional. +std::optional getCgroupsV2PathContainingFile(std::string_view file_name); diff --git a/base/base/chrono_io.h b/base/base/chrono_io.h index 4ee8dec6634..d55aa11bc1d 100644 --- a/base/base/chrono_io.h +++ b/base/base/chrono_io.h @@ -4,6 +4,7 @@ #include #include #include +#include inline std::string to_string(const std::time_t & time) @@ -11,18 +12,6 @@ inline std::string to_string(const std::time_t & time) return cctz::format("%Y-%m-%d %H:%M:%S", std::chrono::system_clock::from_time_t(time), cctz::local_time_zone()); } -template -std::string to_string(const std::chrono::time_point & tp) -{ - // Don't use DateLUT because it shows weird characters for - // TimePoint::max(). I wish we could use C++20 format, but it's not - // there yet. - // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp)); - - auto in_time_t = std::chrono::system_clock::to_time_t(tp); - return to_string(in_time_t); -} - template > std::string to_string(const std::chrono::duration & duration) { @@ -33,6 +22,20 @@ std::string to_string(const std::chrono::duration & duration) return std::to_string(seconds_as_double.count()) + "s"; } +template +std::string to_string(const std::chrono::time_point & tp) +{ + // Don't use DateLUT because it shows weird characters for + // TimePoint::max(). I wish we could use C++20 format, but it's not + // there yet. + // return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp)); + + if constexpr (std::is_same_v) + return to_string(std::chrono::system_clock::to_time_t(tp)); + else + return to_string(tp.time_since_epoch()); +} + template std::ostream & operator<<(std::ostream & o, const std::chrono::time_point & tp) { @@ -44,3 +47,23 @@ std::ostream & operator<<(std::ostream & o, const std::chrono::duration +struct fmt::formatter> : fmt::formatter +{ + template + auto format(const std::chrono::time_point & tp, FormatCtx & ctx) const + { + return fmt::formatter::format(::to_string(tp), ctx); + } +}; + +template +struct fmt::formatter> : fmt::formatter +{ + template + auto format(const std::chrono::duration & duration, FormatCtx & ctx) const + { + return fmt::formatter::format(::to_string(duration), ctx); + } +}; diff --git a/base/base/coverage.h b/base/base/coverage.h index a6e5a6848d7..a5f87c1e9bc 100644 --- a/base/base/coverage.h +++ b/base/base/coverage.h @@ -2,6 +2,7 @@ #include #include +#include /// Flush coverage report to file, depending on coverage system /// proposed by compiler (llvm for clang and gcov for gcc). diff --git a/base/base/defines.h b/base/base/defines.h index 5685a6d9833..239196c6b9f 100644 --- a/base/base/defines.h +++ b/base/base/defines.h @@ -107,6 +107,7 @@ # if defined(DEBUG_OR_SANITIZER_BUILD) // clang-format off #include + #include namespace DB { [[noreturn]] void abortOnFailedAssertion(const String & description); @@ -145,6 +146,7 @@ #define TSA_TRY_ACQUIRE_SHARED(...) __attribute__((try_acquire_shared_capability(__VA_ARGS__))) /// function tries to acquire a shared capability and returns a boolean value indicating success or failure #define TSA_RELEASE_SHARED(...) __attribute__((release_shared_capability(__VA_ARGS__))) /// function releases the given shared capability #define TSA_SCOPED_LOCKABLE __attribute__((scoped_lockable)) /// object of a class has scoped lockable capability +#define TSA_RETURN_CAPABILITY(...) __attribute__((lock_returned(__VA_ARGS__))) /// to return capabilities in functions /// Macros for suppressing TSA warnings for specific reads/writes (instead of suppressing it for the whole function) /// They use a lambda function to apply function attribute to a single statement. This enable us to suppress warnings locally instead of diff --git a/base/base/extended_types.h b/base/base/extended_types.h index 3bf3f4ed31d..ef36a5385a0 100644 --- a/base/base/extended_types.h +++ b/base/base/extended_types.h @@ -4,6 +4,8 @@ #include #include +#include + using Int128 = wide::integer<128, signed>; using UInt128 = wide::integer<128, unsigned>; @@ -24,6 +26,7 @@ struct is_signed // NOLINT(readability-identifier-naming) template <> struct is_signed { static constexpr bool value = true; }; template <> struct is_signed { static constexpr bool value = true; }; +template <> struct is_signed { static constexpr bool value = true; }; template inline constexpr bool is_signed_v = is_signed::value; @@ -40,15 +43,13 @@ template <> struct is_unsigned { static constexpr bool value = true; }; template inline constexpr bool is_unsigned_v = is_unsigned::value; -template concept is_integer = +template concept is_integer = std::is_integral_v || std::is_same_v || std::is_same_v || std::is_same_v || std::is_same_v; -template concept is_floating_point = std::is_floating_point_v; - template struct is_arithmetic // NOLINT(readability-identifier-naming) { @@ -59,11 +60,16 @@ template <> struct is_arithmetic { static constexpr bool value = true; } template <> struct is_arithmetic { static constexpr bool value = true; }; template <> struct is_arithmetic { static constexpr bool value = true; }; template <> struct is_arithmetic { static constexpr bool value = true; }; - +template <> struct is_arithmetic { static constexpr bool value = true; }; template inline constexpr bool is_arithmetic_v = is_arithmetic::value; +template concept is_floating_point = + std::is_floating_point_v + || std::is_same_v; + + #define FOR_EACH_ARITHMETIC_TYPE(M) \ M(DataTypeDate) \ M(DataTypeDate32) \ @@ -80,6 +86,7 @@ inline constexpr bool is_arithmetic_v = is_arithmetic::value; M(DataTypeUInt128) \ M(DataTypeInt256) \ M(DataTypeUInt256) \ + M(DataTypeBFloat16) \ M(DataTypeFloat32) \ M(DataTypeFloat64) @@ -99,6 +106,7 @@ inline constexpr bool is_arithmetic_v = is_arithmetic::value; M(DataTypeUInt128, X) \ M(DataTypeInt256, X) \ M(DataTypeUInt256, X) \ + M(DataTypeBFloat16, X) \ M(DataTypeFloat32, X) \ M(DataTypeFloat64, X) diff --git a/base/base/find_symbols.h b/base/base/find_symbols.h index 30ee759ba33..752bd5ef211 100644 --- a/base/base/find_symbols.h +++ b/base/base/find_symbols.h @@ -3,6 +3,7 @@ #include #include #include +#include #if defined(__SSE2__) #include @@ -33,6 +34,10 @@ * * Allow to search for the last matching character in a string. * If no such characters, returns nullptr. + * + * count_symbols(begin, end): + * + * Count the number of symbols of the set in a string. */ struct SearchSymbols @@ -47,7 +52,7 @@ struct SearchSymbols #if defined(__SSE4_2__) if (str.size() > BUFFER_SIZE) { - throw std::runtime_error("SearchSymbols can contain at most " + std::to_string(BUFFER_SIZE) + " symbols and " + std::to_string(str.size()) + " was provided\n"); + throw std::runtime_error("SearchSymbols may contain at most " + std::to_string(BUFFER_SIZE) + " symbols but " + std::to_string(str.size()) + " symbols were provided"); } char tmp_safety_buffer[BUFFER_SIZE] = {0}; @@ -232,6 +237,32 @@ inline const char * find_last_symbols_sse2(const char * const begin, const char return return_mode == ReturnMode::End ? end : nullptr; } +template +inline const char * find_last_symbols_sse2(const char * const begin, const char * const end, const char * symbols, size_t num_chars) +{ + const char * pos = end; + +#if defined(__SSE2__) + const auto needles = mm_is_in_prepare(symbols, num_chars); + for (; pos - 16 >= begin; pos -= 16) + { + __m128i bytes = _mm_loadu_si128(reinterpret_cast(pos - 16)); + + __m128i eq = mm_is_in_execute(bytes, needles); + + uint16_t bit_mask = maybe_negate(uint16_t(_mm_movemask_epi8(eq))); + if (bit_mask) + return pos - 1 - (__builtin_clz(bit_mask) - 16); /// because __builtin_clz works with mask as uint32. + } +#endif + + --pos; + for (; pos >= begin; --pos) + if (maybe_negate(is_in(*pos, symbols, num_chars))) + return pos; + + return return_mode == ReturnMode::End ? end : nullptr; +} template = 5) return find_first_symbols_sse42(begin, end); - else #endif - return find_first_symbols_sse2(begin, end); + return find_first_symbols_sse2(begin, end); } template @@ -340,10 +370,22 @@ inline const char * find_first_symbols_dispatch(const std::string_view haystack, { #if defined(__SSE4_2__) if (symbols.str.size() >= 5) - return find_first_symbols_sse42(haystack.begin(), haystack.end(), symbols); - else + return find_first_symbols_sse42(haystack.data(), haystack.data() + haystack.size(), symbols); #endif - return find_first_symbols_sse2(haystack.begin(), haystack.end(), symbols.str.data(), symbols.str.size()); + return find_first_symbols_sse2(haystack.data(), haystack.data() + haystack.size(), symbols.str.data(), symbols.str.size()); +} + +template +inline const char * find_last_symbols_dispatch(const char * begin, const char * end) + requires(0 <= sizeof...(symbols) && sizeof...(symbols) <= 16) +{ + return find_last_symbols_sse2(begin, end); +} + +template +inline const char * find_last_symbols_dispatch(const std::string_view haystack, const SearchSymbols & symbols) +{ + return find_last_symbols_sse2(haystack.data(), haystack.data() + haystack.size(), symbols.str.data(), symbols.str.size()); } } @@ -422,25 +464,44 @@ inline const char * find_first_not_symbols_or_null(std::string_view haystack, co template inline const char * find_last_symbols_or_null(const char * begin, const char * end) { - return detail::find_last_symbols_sse2(begin, end); + return detail::find_last_symbols_dispatch(begin, end); } template inline char * find_last_symbols_or_null(char * begin, char * end) { - return const_cast(detail::find_last_symbols_sse2(begin, end)); + return const_cast(detail::find_last_symbols_dispatch(begin, end)); +} + +inline const char * find_last_symbols_or_null(std::string_view haystack, const SearchSymbols & symbols) +{ + return detail::find_last_symbols_dispatch(haystack, symbols); } template inline const char * find_last_not_symbols_or_null(const char * begin, const char * end) { - return detail::find_last_symbols_sse2(begin, end); + return detail::find_last_symbols_dispatch(begin, end); } template inline char * find_last_not_symbols_or_null(char * begin, char * end) { - return const_cast(detail::find_last_symbols_sse2(begin, end)); + return const_cast(detail::find_last_symbols_dispatch(begin, end)); +} + +inline const char * find_last_not_symbols_or_null(std::string_view haystack, const SearchSymbols & symbols) +{ + return detail::find_last_symbols_dispatch(haystack, symbols); +} + +template +inline size_t count_symbols(const char * begin, const char * end) +{ + size_t res = 0; + for (const auto * ptr = begin; ptr < end; ++ptr) + res += detail::is_in(*ptr); + return res; } diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp index 03aab1eac72..4179e601e0f 100644 --- a/base/base/getMemoryAmount.cpp +++ b/base/base/getMemoryAmount.cpp @@ -19,9 +19,6 @@ std::optional getCgroupsV2MemoryLimit() if (!cgroupsV2Enabled()) return {}; - if (!cgroupsV2MemoryControllerEnabled()) - return {}; - std::filesystem::path current_cgroup = cgroupV2PathOfProcess(); if (current_cgroup.empty()) return {}; @@ -36,8 +33,7 @@ std::optional getCgroupsV2MemoryLimit() uint64_t value; if (setting_file >> value) return {value}; - else - return {}; /// e.g. the cgroups default "max" + return {}; /// e.g. the cgroups default "max" } current_cgroup = current_cgroup.parent_path(); } diff --git a/base/base/itoa.cpp b/base/base/itoa.cpp index 60231507c96..45031b013c2 100644 --- a/base/base/itoa.cpp +++ b/base/base/itoa.cpp @@ -421,13 +421,13 @@ ALWAYS_INLINE inline char * writeSIntText(T x, char * pos) if constexpr (std::is_same_v) { const char * res = "-170141183460469231731687303715884105728"; - memcpy(pos, res, strlen(res)); + memcpy(pos, res, strlen(res)); /// NOLINT(bugprone-not-null-terminated-result) return pos + strlen(res); } else if constexpr (std::is_same_v) { const char * res = "-57896044618658097711785492504343953926634992332820282019728792003956564819968"; - memcpy(pos, res, strlen(res)); + memcpy(pos, res, strlen(res)); /// NOLINT(bugprone-not-null-terminated-result) return pos + strlen(res); } } diff --git a/base/base/libcpp_verbose_abort.cpp b/base/base/libcpp_verbose_abort.cpp new file mode 100644 index 00000000000..c9574be790f --- /dev/null +++ b/base/base/libcpp_verbose_abort.cpp @@ -0,0 +1,13 @@ +#include + +/// Custom assertion handler for hardened libcxx +/// See https://prereleases.llvm.org/16.0.0/rc3/projects/libcxx/docs/UsingLibcxx.html#enabling-the-safe-libc-mode +void std::__libcpp_verbose_abort(char const * format, ...) +{ + va_list list; + va_start(list, format); + std::vfprintf(stderr, format, list); + va_end(list); + + std::abort(); +} diff --git a/src/Common/memcmpSmall.h b/base/base/memcmpSmall.h similarity index 99% rename from src/Common/memcmpSmall.h rename to base/base/memcmpSmall.h index 103eabb5b8d..a90eee75b96 100644 --- a/src/Common/memcmpSmall.h +++ b/base/base/memcmpSmall.h @@ -4,12 +4,9 @@ #include #include +#include #include -#include -#include - - namespace detail { @@ -359,7 +356,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz { return 0; } - else if (a_size > b_size) + if (a_size > b_size) { max_size = a_size; longest = a; diff --git a/base/base/openpty.h b/base/base/openpty.h new file mode 100644 index 00000000000..09292f39d42 --- /dev/null +++ b/base/base/openpty.h @@ -0,0 +1,48 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +/** + * This function is taken from Musl with a few modifications. + * Clang-tidy produces several warning, but we would love to keep the implementation + * as close as possible to the original one. + */ +// NOLINTBEGIN +int openpty(int *pm, int *ps, char *name, const struct termios *tio, const struct winsize *ws) +{ + int m, s, n=0, cs; + char buf[20]; + + m = open("/dev/ptmx", O_RDWR|O_NOCTTY); + if (m < 0) return -1; + + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cs); + + if (ioctl(m, TIOCSPTLCK, &n) || ioctl (m, TIOCGPTN, &n)) + goto fail; + + if (!name) name = buf; + snprintf(name, sizeof buf, "/dev/pts/%d", n); + if ((s = open(name, O_RDWR|O_NOCTTY)) < 0) + goto fail; + + if (tio) tcsetattr(s, TCSANOW, tio); + if (ws) ioctl(s, TIOCSWINSZ, ws); + + *pm = m; + *ps = s; + + pthread_setcancelstate(cs, nullptr); + return 0; +fail: + close(m); + pthread_setcancelstate(cs, nullptr); + return -1; +} +// NOLINTEND diff --git a/base/base/preciseExp10.cpp b/base/base/preciseExp10.cpp index 1cd660dc569..a85285add24 100644 --- a/base/base/preciseExp10.cpp +++ b/base/base/preciseExp10.cpp @@ -30,8 +30,6 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include -#include -#include double preciseExp10(double x) { @@ -78,7 +76,8 @@ double preciseExp10(double x) 1e+289, 1e+290, 1e+291, 1e+292, 1e+293, 1e+294, 1e+295, 1e+296, 1e+297, 1e+298, 1e+299, 1e+300, 1e+301, 1e+302, 1e+303, 1e+304, 1e+305, 1e+306, 1e+307, 1e+308}; - double n, y = modf(x, &n); + double n; + double y = modf(x, &n); if (n > 308) return INFINITY; if (n < -323) return 0; diff --git a/base/base/unit.h b/base/base/unit.h index 0fc314af479..5ae9a5b7dc7 100644 --- a/base/base/unit.h +++ b/base/base/unit.h @@ -9,9 +9,9 @@ constexpr size_t GiB = 1024 * MiB; #pragma clang diagnostic ignored "-Wreserved-identifier" // NOLINTBEGIN(google-runtime-int) -constexpr size_t operator"" _KiB(unsigned long long val) { return val * KiB; } -constexpr size_t operator"" _MiB(unsigned long long val) { return val * MiB; } -constexpr size_t operator"" _GiB(unsigned long long val) { return val * GiB; } +constexpr size_t operator""_KiB(unsigned long long val) { return val * KiB; } +constexpr size_t operator""_MiB(unsigned long long val) { return val * MiB; } +constexpr size_t operator""_GiB(unsigned long long val) { return val * GiB; } // NOLINTEND(google-runtime-int) #pragma clang diagnostic pop diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index 3787971a20e..8c79dade6e5 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -5,6 +5,7 @@ /// (See at http://www.boost.org/LICENSE_1_0.txt) #include "throwError.h" +#include "defines.h" #include #include @@ -13,6 +14,7 @@ #include #include + // NOLINTBEGIN(*) /// Use same extended double for all platforms @@ -32,16 +34,74 @@ namespace CityHash_v1_0_2 { struct uint128; } namespace wide { +constexpr bool supportsBitInt256() +{ +#if defined(__x86_64__) + return true; +#else + return false; +#endif +} + +#if defined(__x86_64__) +/// TODO C23 standardized _BitInt(N). Theoretically, it is not necessary to restrict the platform to x86. +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wbit-int-extension" +using BitInt256 = signed _BitInt(256); +using BitUInt256 = unsigned _BitInt(256); +# pragma clang diagnostic pop + +struct Error {}; + +template +struct ConstructBitInt256 +{ + using Type = Error; +}; + +template <> +struct ConstructBitInt256 +{ + using Type = BitInt256; +}; + +template <> +struct ConstructBitInt256 +{ + using Type = BitUInt256; +}; + +/// Converts a 256-bit wide integer to Clang's built-in 256-bit integer representation. +/// The source and target types have the same byte order. +template +requires(Bits == 256) +constexpr const auto & toBitInt256(const wide::integer & n) +{ + using T = ConstructBitInt256::Type; + return *reinterpret_cast(&n); +} + +/// Converts a Clang's built-in 256-bit integer representation to a 256-bit wide integer. +/// The source and target types have the same byte order. +template +requires(std::is_same_v || std::is_same_v) +constexpr const auto & fromBitInt256(const T & n) +{ + using Signed = std::conditional_t, signed, unsigned>; + return *reinterpret_cast *>(&n); +} +#endif + template struct IsWideInteger { - static const constexpr bool value = false; + static constexpr bool value = false; }; template struct IsWideInteger> { - static const constexpr bool value = true; + static constexpr bool value = true; }; template @@ -85,7 +145,6 @@ class numeric_limits> static constexpr bool has_infinity = false; static constexpr bool has_quiet_NaN = false; static constexpr bool has_signaling_NaN = true; - static constexpr std::float_denorm_style has_denorm = std::denorm_absent; static constexpr bool has_denorm_loss = false; static constexpr std::float_round_style round_style = std::round_toward_zero; static constexpr bool is_iec559 = false; @@ -175,6 +234,7 @@ struct common_type> : common_type::_impl static constexpr const unsigned item_count = byte_count / sizeof(base_type); static constexpr const unsigned base_bits = sizeof(base_type) * 8; + /// Use Clang's built-in 256-bit integer to improve performance if possible. + /// + /// Not implemented for 128 bit types because performance benefits are negligible as of 2025: + /// https://github.com/ClickHouse/ClickHouse/issues/70502 + static constexpr bool use_BitInt256 = supportsBitInt256() && Bits == 256; + static_assert(Bits % base_bits == 0); /// Simple iteration in both directions @@ -239,7 +305,7 @@ struct integer::_impl } template - __attribute__((no_sanitize("undefined"))) constexpr static auto to_Integral(T f) noexcept + constexpr static auto to_Integral(T f) noexcept { /// NOTE: this can be called with DB::Decimal, and in this case, result /// will be wrong @@ -337,7 +403,7 @@ struct integer::_impl /** Here we have to use strict comparison. * The max_int is 2^64 - 1. - * When casted to floating point type, it will be rounded to the closest representable number, + * When cast to a floating point type, it will be rounded to the closest representable number, * which is 2^64. * But 2^64 is not representable in uint64_t, * so the maximum representable number will be strictly less. @@ -570,8 +636,7 @@ struct integer::_impl } template - constexpr static integer - multiply(const integer & lhs, const T & rhs) + constexpr static integer multiply(const integer & lhs, const T & rhs) { if constexpr (Bits == 256 && sizeof(base_type) == 8) { @@ -692,10 +757,23 @@ struct integer::_impl { if constexpr (should_keep_size()) { - if (is_negative(rhs)) - return minus(lhs, -rhs); + if constexpr (use_BitInt256) + { + if constexpr (!std::same_as>) + { + auto new_rhs = static_cast>(rhs); + return fromBitInt256(toBitInt256(lhs) + toBitInt256(new_rhs)); + } + else + return fromBitInt256(toBitInt256(lhs) + toBitInt256(rhs)); + } else - return plus(lhs, rhs); + { + if (is_negative(rhs)) + return minus(lhs, -rhs); + else + return plus(lhs, rhs); + } } else { @@ -710,10 +788,23 @@ struct integer::_impl { if constexpr (should_keep_size()) { - if (is_negative(rhs)) - return plus(lhs, -rhs); + if constexpr (use_BitInt256) + { + if constexpr (!std::same_as>) + { + auto new_rhs = static_cast>(rhs); + return fromBitInt256(toBitInt256(lhs) - toBitInt256(new_rhs)); + } + else + return fromBitInt256(toBitInt256(lhs) - toBitInt256(rhs)); + } else - return minus(lhs, rhs); + { + if (is_negative(rhs)) + return plus(lhs, -rhs); + else + return minus(lhs, rhs); + } } else { @@ -728,22 +819,34 @@ struct integer::_impl { if constexpr (should_keep_size()) { - integer res; - - if constexpr (std::is_signed_v) + if constexpr (use_BitInt256) { - res = multiply((is_negative(lhs) ? make_positive(lhs) : lhs), - (is_negative(rhs) ? make_positive(rhs) : rhs)); + if constexpr (!std::same_as>) + { + auto new_rhs = static_cast>(rhs); + return fromBitInt256(toBitInt256(lhs) * toBitInt256(new_rhs)); + } + else + return fromBitInt256(toBitInt256(lhs) * toBitInt256(rhs)); } else { - res = multiply(lhs, (is_negative(rhs) ? make_positive(rhs) : rhs)); - } + integer res; - if (std::is_same_v && is_negative(lhs) != is_negative(rhs)) - res = operator_unary_minus(res); + if constexpr (std::is_signed_v) + { + res = multiply((is_negative(lhs) ? make_positive(lhs) : lhs), (is_negative(rhs) ? make_positive(rhs) : rhs)); + } + else + { + res = multiply(lhs, (is_negative(rhs) ? make_positive(rhs) : rhs)); + } - return res; + if (std::is_same_v && is_negative(lhs) != is_negative(rhs)) + res = operator_unary_minus(res); + + return res; + } } else { @@ -884,6 +987,9 @@ struct integer::_impl { static_assert(std::is_unsigned_v); + if (is_zero(denominator)) + throwError("Division by zero"); + if constexpr (Bits == 128 && sizeof(base_type) == 8) { using CompilerUInt128 = unsigned __int128; @@ -903,9 +1009,6 @@ struct integer::_impl return res; } - if (is_zero(denominator)) - throwError("Division by zero"); - integer x = 1; integer quotient = 0; @@ -935,13 +1038,26 @@ struct integer::_impl { if constexpr (should_keep_size()) { - integer numerator = make_positive(lhs); - integer denominator = make_positive(integer(rhs)); - integer quotient = integer::_impl::divide(numerator, std::move(denominator)); + if constexpr (use_BitInt256) + { + if constexpr (!std::same_as>) + { + auto new_rhs = static_cast>(rhs); + return fromBitInt256(toBitInt256(lhs) / toBitInt256(new_rhs)); + } + else + return fromBitInt256(toBitInt256(lhs) / toBitInt256(rhs)); + } + else + { + integer numerator = make_positive(lhs); + integer denominator = make_positive(integer(rhs)); + integer quotient = integer::_impl::divide(numerator, std::move(denominator)); - if (std::is_same_v && is_negative(rhs) != is_negative(lhs)) - quotient = operator_unary_minus(quotient); - return quotient; + if (std::is_same_v && is_negative(rhs) != is_negative(lhs)) + quotient = operator_unary_minus(quotient); + return quotient; + } } else { @@ -955,13 +1071,26 @@ struct integer::_impl { if constexpr (should_keep_size()) { - integer remainder = make_positive(lhs); - integer denominator = make_positive(integer(rhs)); - integer::_impl::divide(remainder, std::move(denominator)); + if constexpr (use_BitInt256) + { + if constexpr (!std::same_as>) + { + auto new_rhs = static_cast>(rhs); + return fromBitInt256(toBitInt256(lhs) % toBitInt256(new_rhs)); + } + else + return fromBitInt256(toBitInt256(lhs) % toBitInt256(rhs)); + } + else + { + integer remainder = make_positive(lhs); + integer denominator = make_positive(integer(rhs)); + integer::_impl::divide(remainder, std::move(denominator)); - if (std::is_same_v && is_negative(lhs)) - remainder = operator_unary_minus(remainder); - return remainder; + if (std::is_same_v && is_negative(lhs)) + remainder = operator_unary_minus(remainder); + return remainder; + } } else { @@ -1510,6 +1639,7 @@ constexpr bool operator!=(const Arithmetic & lhs, const Arithmetic2 & rhs) #undef CT } +#pragma clang attribute pop namespace std { diff --git a/base/base/wide_integer_to_string.cpp b/base/base/wide_integer_to_string.cpp new file mode 100644 index 00000000000..0504bd08368 --- /dev/null +++ b/base/base/wide_integer_to_string.cpp @@ -0,0 +1,57 @@ +#include +#include + +#include +#include + + +namespace wide +{ + +template +inline std::string to_string(const integer & n) +{ + std::string res; + if (integer::_impl::operator_eq(n, 0U)) + return "0"; + + integer t; + bool is_neg = integer::_impl::is_negative(n); + if (is_neg) + t = integer::_impl::operator_unary_minus(n); + else + t = n; + + while (!integer::_impl::operator_eq(t, 0U)) + { + res.insert(res.begin(), '0' + char(integer::_impl::operator_percent(t, 10U))); + t = integer::_impl::operator_slash(t, 10U); + } + + if (is_neg) + res.insert(res.begin(), '-'); + return res; +} + +template std::string to_string(const integer<128, signed> & n); +template std::string to_string(const integer<128, unsigned> & n); +template std::string to_string(const integer<256, signed> & n); +template std::string to_string(const integer<256, unsigned> & n); + +} + +template +std::ostream & operator<<(std::ostream & out, const wide::integer & value) +{ + return out << to_string(value); +} + +std::ostream & operator<<(std::ostream & out, const wide::integer<128, signed> & value); +std::ostream & operator<<(std::ostream & out, const wide::integer<128, unsigned> & value); +std::ostream & operator<<(std::ostream & out, const wide::integer<256, signed> & value); +std::ostream & operator<<(std::ostream & out, const wide::integer<256, unsigned> & value); + +template struct fmt::formatter>; +template struct fmt::formatter>; +template struct fmt::formatter>; +template struct fmt::formatter>; diff --git a/base/base/wide_integer_to_string.h b/base/base/wide_integer_to_string.h index f703a722afa..2c63e964dd6 100644 --- a/base/base/wide_integer_to_string.h +++ b/base/base/wide_integer_to_string.h @@ -1,49 +1,38 @@ #pragma once #include -#include #include -#include "wide_integer.h" - - namespace wide { - template -inline std::string to_string(const integer & n) -{ - std::string res; - if (integer::_impl::operator_eq(n, 0U)) - return "0"; +class integer; +} - integer t; - bool is_neg = integer::_impl::is_negative(n); - if (is_neg) - t = integer::_impl::operator_unary_minus(n); - else - t = n; +using Int128 = wide::integer<128, signed>; +using UInt128 = wide::integer<128, unsigned>; +using Int256 = wide::integer<256, signed>; +using UInt256 = wide::integer<256, unsigned>; - while (!integer::_impl::operator_eq(t, 0U)) - { - res.insert(res.begin(), '0' + char(integer::_impl::operator_percent(t, 10U))); - t = integer::_impl::operator_slash(t, 10U); - } +namespace wide +{ - if (is_neg) - res.insert(res.begin(), '-'); - return res; -} +template +std::string to_string(const integer & n); +extern template std::string to_string(const Int128 & n); +extern template std::string to_string(const UInt128 & n); +extern template std::string to_string(const Int256 & n); +extern template std::string to_string(const UInt256 & n); } - template -std::ostream & operator<<(std::ostream & out, const wide::integer & value) -{ - return out << to_string(value); -} +std::ostream & operator<<(std::ostream & out, const wide::integer & value); +extern std::ostream & operator<<(std::ostream & out, const Int128 & value); +extern std::ostream & operator<<(std::ostream & out, const UInt128 & value); +extern std::ostream & operator<<(std::ostream & out, const Int256 & value); +extern std::ostream & operator<<(std::ostream & out, const UInt256 & value); /// See https://fmt.dev/latest/api.html#formatting-user-defined-types template @@ -67,3 +56,8 @@ struct fmt::formatter> return fmt::format_to(ctx.out(), "{}", to_string(value)); } }; + +extern template struct fmt::formatter; +extern template struct fmt::formatter; +extern template struct fmt::formatter; +extern template struct fmt::formatter; diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt index 8948e25cb8e..a14f66ce22b 100644 --- a/base/glibc-compatibility/CMakeLists.txt +++ b/base/glibc-compatibility/CMakeLists.txt @@ -11,6 +11,9 @@ if (GLIBC_COMPATIBILITY) if (ARCH_AARCH64) list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s) set (musl_arch_include_dir musl/aarch64) + # Disable getauxval in aarch64. ARM glibc minimum requirement for the project is 2.18 and getauxval is present + # in 2.16. Having a custom one introduces issues with sanitizers + list (REMOVE_ITEM glibc_compatibility_sources musl/getauxval.c) elseif (ARCH_AMD64) list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s) set (musl_arch_include_dir musl/x86_64) @@ -18,7 +21,7 @@ if (GLIBC_COMPATIBILITY) message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.") endif () - if (SANITIZE STREQUAL thread) + if (SANITIZE STREQUAL thread AND ARCH_AMD64) # Disable TSAN instrumentation that conflicts with re-exec due to high ASLR entropy using getauxval # See longer comment in __auxv_init_procfs # In the case of tsan we need to make sure getauxval is not instrumented as that would introduce tsan diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c index ec2cce1e4aa..cc0cdf25b03 100644 --- a/base/glibc-compatibility/musl/getauxval.c +++ b/base/glibc-compatibility/musl/getauxval.c @@ -25,9 +25,10 @@ // We don't have libc struct available here. // Compute aux vector manually (from /proc/self/auxv). // -// Right now there is only 51 AT_* constants, -// so 64 should be enough until this implementation will be replaced with musl. -static unsigned long __auxv_procfs[64]; +// Right now there are 51 AT_* constants. Custom kernels have been encountered +// making use of up to 71. 128 should be enough until this implementation is +// replaced with musl. +static unsigned long __auxv_procfs[128]; static unsigned long __auxv_secure = 0; // Common static unsigned long * __auxv_environ = NULL; diff --git a/base/harmful/harmful.c b/base/harmful/harmful.c index 54b552a84ea..3b20b7f946f 100644 --- a/base/harmful/harmful.c +++ b/base/harmful/harmful.c @@ -65,14 +65,11 @@ TRAP(gethostbyaddr) TRAP(gethostbyname) TRAP(gethostbyname2) TRAP(gethostent) -TRAP(getlogin) -TRAP(getmntent) TRAP(getnetbyaddr) TRAP(getnetbyname) TRAP(getnetent) TRAP(getnetgrent) TRAP(getnetgrent_r) -TRAP(getopt) TRAP(getopt_long) TRAP(getopt_long_only) TRAP(getpass) @@ -133,7 +130,6 @@ TRAP(nrand48) TRAP(__ppc_get_timebase_freq) TRAP(ptsname) TRAP(putchar_unlocked) -TRAP(putenv) TRAP(pututline) TRAP(pututxline) TRAP(putwchar_unlocked) @@ -148,7 +144,6 @@ TRAP(sethostent) TRAP(sethostid) TRAP(setkey) //TRAP(setlocale) // Used by replxx at startup -TRAP(setlogmask) TRAP(setnetent) TRAP(setnetgrent) TRAP(setprotoent) @@ -203,7 +198,6 @@ TRAP(lgammal) TRAP(nftw) TRAP(nl_langinfo) TRAP(putc_unlocked) -TRAP(rand) /** In the current POSIX.1 specification (POSIX.1-2008), readdir() is not required to be thread-safe. However, in modern * implementations (including the glibc implementation), concurrent calls to readdir() that specify different directory streams * are thread-safe. In cases where multiple threads must read from the same directory stream, using readdir() with external @@ -288,4 +282,15 @@ TRAP(tss_get) TRAP(tss_set) TRAP(tss_delete) +#ifndef USE_MUSL +/// These produce duplicate symbol errors when statically linking with musl. +/// Maybe we can remove them from the musl fork. +TRAP(getopt) +TRAP(putenv) +TRAP(setlogmask) +TRAP(rand) +TRAP(getmntent) +TRAP(getlogin) +#endif + #endif diff --git a/base/poco/CMakeLists.txt b/base/poco/CMakeLists.txt index 82c48b5b622..bf477dc6788 100644 --- a/base/poco/CMakeLists.txt +++ b/base/poco/CMakeLists.txt @@ -1,9 +1,5 @@ -add_subdirectory (Crypto) -add_subdirectory (Data) -add_subdirectory (Data/ODBC) add_subdirectory (Foundation) add_subdirectory (JSON) -add_subdirectory (MongoDB) add_subdirectory (Net) add_subdirectory (NetSSL_OpenSSL) add_subdirectory (Redis) diff --git a/base/poco/Crypto/CMakeLists.txt b/base/poco/Crypto/CMakeLists.txt deleted file mode 100644 index 67171afadfb..00000000000 --- a/base/poco/Crypto/CMakeLists.txt +++ /dev/null @@ -1,35 +0,0 @@ -if (ENABLE_SSL) - file (GLOB SRCS src/*.cpp) - - add_library (_poco_crypto ${SRCS}) - add_library (Poco::Crypto ALIAS _poco_crypto) - - # TODO: remove these warning exclusions - target_compile_options (_poco_crypto - PRIVATE - -Wno-covered-switch-default - -Wno-deprecated-dynamic-exception-spec - -Wno-extra-semi-stmt - -Wno-missing-noreturn - -Wno-newline-eof - -Wno-old-style-cast - -Wno-shadow - -Wno-shorten-64-to-32 - -Wno-sign-compare - -Wno-suggest-destructor-override - -Wno-suggest-override - -Wno-unreachable-code-return - -Wno-unused-parameter - -Wno-zero-as-null-pointer-constant - -Wno-used-but-marked-unused - ) - target_include_directories (_poco_crypto SYSTEM PUBLIC "include") - target_link_libraries (_poco_crypto PUBLIC Poco::Foundation OpenSSL::SSL OpenSSL::Crypto) - - message (STATUS "Using Poco::Crypto") -else () - add_library (_poco_crypto INTERFACE) - add_library (Poco::Crypto ALIAS _poco_crypto) - - message (STATUS "Not using Poco::Crypto") -endif () diff --git a/base/poco/Crypto/include/Poco/Crypto/Cipher.h b/base/poco/Crypto/include/Poco/Crypto/Cipher.h deleted file mode 100644 index ffd993280c1..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/Cipher.h +++ /dev/null @@ -1,141 +0,0 @@ -// -// Cipher.h -// -// Library: Crypto -// Package: Cipher -// Module: Cipher -// -// Definition of the Cipher class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_Cipher_INCLUDED -#define Crypto_Cipher_INCLUDED - - -#include -#include -#include -#include "Poco/AutoPtr.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/RefCountedObject.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class CryptoTransform; - - - class Crypto_API Cipher : public Poco::RefCountedObject - /// Represents the abstract base class from which all implementations of - /// symmetric/asymmetric encryption algorithms must inherit. Use the CipherFactory - /// class to obtain an instance of this class: - /// - /// CipherFactory& factory = CipherFactory::defaultFactory(); - /// // Creates a 256-bit AES cipher - /// Cipher* pCipher = factory.createCipher(CipherKey("aes-256")); - /// Cipher* pRSACipher = factory.createCipher(RSAKey(RSAKey::KL_1024, RSAKey::EXP_SMALL)); - /// - /// Check the different Key constructors on how to initialize/create - /// a key. The above example auto-generates random keys. - /// - /// Note that you won't be able to decrypt data encrypted with a random key - /// once the Cipher is destroyed unless you persist the generated key and IV. - /// An example usage for random keys is to encrypt data saved in a temporary - /// file. - /// - /// Once your key is set up, you can use the Cipher object to encrypt or - /// decrypt strings or, in conjunction with a CryptoInputStream or a - /// CryptoOutputStream, to encrypt streams of data. - /// - /// Since encrypted strings will contain arbitrary binary data that will cause - /// problems in applications that are not binary-safe (eg., when sending - /// encrypted data in e-mails), the encryptString() and decryptString() can - /// encode (or decode, respectively) encrypted data using a "transport encoding". - /// Supported encodings are Base64 and BinHex. - /// - /// The following example encrypts and decrypts a string utilizing Base64 - /// encoding: - /// - /// std::string plainText = "This is my secret information"; - /// std::string encrypted = pCipher->encryptString(plainText, Cipher::ENC_BASE64); - /// std::string decrypted = pCipher->decryptString(encrypted, Cipher::ENC_BASE64); - /// - /// In order to encrypt a stream of data (eg. to encrypt files), you can use - /// a CryptoStream: - /// - /// // Create an output stream that will encrypt all data going through it - /// // and write pass it to the underlying file stream. - /// Poco::FileOutputStream sink("encrypted.dat"); - /// CryptoOutputStream encryptor(sink, pCipher->createEncryptor()); - /// - /// Poco::FileInputStream source("source.txt"); - /// Poco::StreamCopier::copyStream(source, encryptor); - /// - /// // Always close output streams to flush all internal buffers - /// encryptor.close(); - /// sink.close(); - { - public: - typedef Poco::AutoPtr Ptr; - typedef std::vector ByteVec; - - enum Encoding - /// Transport encoding to use for encryptString() and decryptString(). - { - ENC_NONE = 0x00, /// Plain binary output - ENC_BASE64 = 0x01, /// Base64-encoded output - ENC_BINHEX = 0x02, /// BinHex-encoded output - ENC_BASE64_NO_LF = 0x81, /// Base64-encoded output, no linefeeds - ENC_BINHEX_NO_LF = 0x82 /// BinHex-encoded output, no linefeeds - - }; - - virtual ~Cipher(); - /// Destroys the Cipher. - - virtual const std::string & name() const = 0; - /// Returns the name of the Cipher. - - virtual CryptoTransform * createEncryptor() = 0; - /// Creates an encryptor object to be used with a CryptoStream. - - virtual CryptoTransform * createDecryptor() = 0; - /// Creates a decryptor object to be used with a CryptoStream. - - virtual std::string encryptString(const std::string & str, Encoding encoding = ENC_NONE); - /// Directly encrypt a string and encode it using the given encoding. - - virtual std::string decryptString(const std::string & str, Encoding encoding = ENC_NONE); - /// Directly decrypt a string that is encoded with the given encoding. - - virtual void encrypt(std::istream & source, std::ostream & sink, Encoding encoding = ENC_NONE); - /// Directly encrypts an input stream and encodes it using the given encoding. - - virtual void decrypt(std::istream & source, std::ostream & sink, Encoding encoding = ENC_NONE); - /// Directly decrypt an input stream that is encoded with the given encoding. - - protected: - Cipher(); - /// Creates a new Cipher object. - - private: - Cipher(const Cipher &); - Cipher & operator=(const Cipher &); - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_Cipher_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherFactory.h b/base/poco/Crypto/include/Poco/Crypto/CipherFactory.h deleted file mode 100644 index f32865e3461..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CipherFactory.h +++ /dev/null @@ -1,78 +0,0 @@ -// -// CipherFactory.h -// -// Library: Crypto -// Package: Cipher -// Module: CipherFactory -// -// Definition of the CipherFactory class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CipherFactory_INCLUDED -#define Crypto_CipherFactory_INCLUDED - - -#include "Poco/Crypto/Crypto.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Cipher; - class CipherKey; - class RSAKey; - - - class Crypto_API CipherFactory - /// A factory for Cipher objects. See the Cipher class for examples on how to - /// use the CipherFactory. - { - public: - CipherFactory(); - /// Creates a new CipherFactory object. - - virtual ~CipherFactory(); - /// Destroys the CipherFactory. - - Cipher * createCipher(const CipherKey & key); - /// Creates a Cipher object for the given Cipher name. Valid cipher - /// names depend on the OpenSSL version the library is linked with; - /// see the output of - /// - /// openssl enc --help - /// - /// for a list of supported block and stream ciphers. - /// - /// Common examples are: - /// - /// * AES: "aes-128", "aes-256" - /// * DES: "des", "des3" - /// * Blowfish: "bf" - - Cipher * createCipher(const RSAKey & key, RSAPaddingMode paddingMode = RSA_PADDING_PKCS1); - /// Creates a RSACipher using the given RSA key and padding mode - /// for public key encryption/private key decryption. - - static CipherFactory & defaultFactory(); - /// Returns the default CipherFactory. - - private: - CipherFactory(const CipherFactory &); - CipherFactory & operator=(const CipherFactory &); - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CipherFactory_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherImpl.h b/base/poco/Crypto/include/Poco/Crypto/CipherImpl.h deleted file mode 100644 index 057e77a4cd9..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CipherImpl.h +++ /dev/null @@ -1,72 +0,0 @@ -// -// CipherImpl.h -// -// Library: Crypto -// Package: Cipher -// Module: CipherImpl -// -// Definition of the CipherImpl class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CipherImpl_INCLUDED -#define Crypto_CipherImpl_INCLUDED - - -#include -#include "Poco/Crypto/Cipher.h" -#include "Poco/Crypto/CipherKey.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/OpenSSLInitializer.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class CipherImpl : public Cipher - /// An implementation of the Cipher class for OpenSSL's crypto library. - { - public: - CipherImpl(const CipherKey & key); - /// Creates a new CipherImpl object for the given CipherKey. - - virtual ~CipherImpl(); - /// Destroys the CipherImpl. - - const std::string & name() const; - /// Returns the name of the cipher. - - CryptoTransform * createEncryptor(); - /// Creates an encryptor object. - - CryptoTransform * createDecryptor(); - /// Creates a decryptor object. - - private: - CipherKey _key; - OpenSSLInitializer _openSSLInitializer; - }; - - - // - // Inlines - // - inline const std::string & CipherImpl::name() const - { - return _key.name(); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CipherImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherKey.h b/base/poco/Crypto/include/Poco/Crypto/CipherKey.h deleted file mode 100644 index b39cb1a9294..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CipherKey.h +++ /dev/null @@ -1,203 +0,0 @@ -// -// CipherKey.h -// -// Library: Crypto -// Package: Cipher -// Module: CipherKey -// -// Definition of the CipherKey class. -// -// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CipherKey_INCLUDED -#define Crypto_CipherKey_INCLUDED - - -#include "Poco/Crypto/CipherKeyImpl.h" -#include "Poco/Crypto/Crypto.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API CipherKey - /// CipherKey stores the key information for decryption/encryption of data. - /// To create a random key, using the following code: - /// - /// CipherKey key("aes-256"); - /// - /// Note that you won't be able to decrypt data encrypted with a random key - /// once the Cipher is destroyed unless you persist the generated key and IV. - /// An example usage for random keys is to encrypt data saved in a temporary - /// file. - /// - /// To create a key using a human-readable password - /// string, use the following code. We create a AES Cipher and - /// use a salt value to make the key more robust: - /// - /// std::string password = "secret"; - /// std::string salt("asdff8723lasdf(**923412"); - /// CipherKey key("aes-256", password, salt); - /// - /// You may also control the digest and the number of iterations used to generate the key - /// by specifying the specific values. Here we create a key with the same data as before, - /// except that we use 100 iterations instead of DEFAULT_ITERATION_COUNT, and sha1 instead of - /// the default md5: - /// - /// std::string password = "secret"; - /// std::string salt("asdff8723lasdf(**923412"); - /// std::string digest ("sha1"); - /// CipherKey key("aes-256", password, salt, 100, digest); - /// - { - public: - typedef CipherKeyImpl::Mode Mode; - typedef CipherKeyImpl::ByteVec ByteVec; - - enum - { - DEFAULT_ITERATION_COUNT = 2000 - /// Default iteration count to use with - /// generateKey(). RSA security recommends - /// an iteration count of at least 1000. - }; - - CipherKey( - const std::string & name, - const std::string & passphrase, - const std::string & salt = "", - int iterationCount = DEFAULT_ITERATION_COUNT, - const std::string & digest = "md5"); - /// Creates a new CipherKeyImpl object using the given - /// cipher name, passphrase, salt value, iteration count and digest. - - CipherKey(const std::string & name, const ByteVec & key, const ByteVec & iv); - /// Creates a new CipherKeyImpl object using the given cipher - /// name, key and initialization vector (IV). - /// - /// The size of the IV must match the cipher's expected - /// IV size (see ivSize()), except for GCM mode, which allows - /// a custom IV size. - - CipherKey(const std::string & name); - /// Creates a new CipherKeyImpl object. Autoinitializes key and - /// initialization vector. - - ~CipherKey(); - /// Destroys the CipherKeyImpl. - - const std::string & name() const; - /// Returns the name of the Cipher. - - int keySize() const; - /// Returns the key size of the Cipher. - - int blockSize() const; - /// Returns the block size of the Cipher. - - int ivSize() const; - /// Returns the IV size of the Cipher. - - Mode mode() const; - /// Returns the Cipher's mode of operation. - - const ByteVec & getKey() const; - /// Returns the key for the Cipher. - - void setKey(const ByteVec & key); - /// Sets the key for the Cipher. - - const ByteVec & getIV() const; - /// Returns the initialization vector (IV) for the Cipher. - - void setIV(const ByteVec & iv); - /// Sets the initialization vector (IV) for the Cipher. - /// - /// The size of the vector must match the cipher's expected - /// IV size (see ivSize()), except for GCM mode, which allows - /// a custom IV size. - - CipherKeyImpl::Ptr impl(); - /// Returns the impl object - - private: - CipherKeyImpl::Ptr _pImpl; - }; - - - // - // inlines - // - inline const std::string & CipherKey::name() const - { - return _pImpl->name(); - } - - - inline int CipherKey::keySize() const - { - return _pImpl->keySize(); - } - - - inline int CipherKey::blockSize() const - { - return _pImpl->blockSize(); - } - - - inline int CipherKey::ivSize() const - { - return _pImpl->ivSize(); - } - - - inline CipherKey::Mode CipherKey::mode() const - { - return _pImpl->mode(); - } - - - inline const CipherKey::ByteVec & CipherKey::getKey() const - { - return _pImpl->getKey(); - } - - - inline void CipherKey::setKey(const CipherKey::ByteVec & key) - { - _pImpl->setKey(key); - } - - - inline const CipherKey::ByteVec & CipherKey::getIV() const - { - return _pImpl->getIV(); - } - - - inline void CipherKey::setIV(const CipherKey::ByteVec & iv) - { - _pImpl->setIV(iv); - } - - - inline CipherKeyImpl::Ptr CipherKey::impl() - { - return _pImpl; - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CipherKey_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h b/base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h deleted file mode 100644 index 700ae95c892..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CipherKeyImpl.h +++ /dev/null @@ -1,168 +0,0 @@ -// -// CipherKeyImpl.h -// -// Library: Crypto -// Package: Cipher -// Module: CipherKeyImpl -// -// Definition of the CipherKeyImpl class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CipherKeyImpl_INCLUDED -#define Crypto_CipherKeyImpl_INCLUDED - - -#include -#include "Poco/AutoPtr.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/RefCountedObject.h" - - -struct evp_cipher_st; -typedef struct evp_cipher_st EVP_CIPHER; - - -namespace Poco -{ -namespace Crypto -{ - - - class CipherKeyImpl : public RefCountedObject - /// An implementation of the CipherKey class for OpenSSL's crypto library. - { - public: - typedef std::vector ByteVec; - typedef Poco::AutoPtr Ptr; - - enum Mode - /// Cipher mode of operation. This mode determines how multiple blocks - /// are connected; this is essential to improve security. - { - MODE_STREAM_CIPHER, /// Stream cipher - MODE_ECB, /// Electronic codebook (plain concatenation) - MODE_CBC, /// Cipher block chaining (default) - MODE_CFB, /// Cipher feedback - MODE_OFB, /// Output feedback - MODE_CTR, /// Counter mode - MODE_GCM, /// Galois/Counter mode - MODE_CCM /// Counter with CBC-MAC - }; - - CipherKeyImpl( - const std::string & name, - const std::string & passphrase, - const std::string & salt, - int iterationCount, - const std::string & digest); - /// Creates a new CipherKeyImpl object, using - /// the given cipher name, passphrase, salt value - /// and iteration count. - - CipherKeyImpl(const std::string & name, const ByteVec & key, const ByteVec & iv); - /// Creates a new CipherKeyImpl object, using the - /// given cipher name, key and initialization vector. - - CipherKeyImpl(const std::string & name); - /// Creates a new CipherKeyImpl object. Autoinitializes key - /// and initialization vector. - - virtual ~CipherKeyImpl(); - /// Destroys the CipherKeyImpl. - - const std::string & name() const; - /// Returns the name of the Cipher. - - int keySize() const; - /// Returns the key size of the Cipher. - - int blockSize() const; - /// Returns the block size of the Cipher. - - int ivSize() const; - /// Returns the IV size of the Cipher. - - Mode mode() const; - /// Returns the Cipher's mode of operation. - - const ByteVec & getKey() const; - /// Returns the key for the Cipher. - - void setKey(const ByteVec & key); - /// Sets the key for the Cipher. - - const ByteVec & getIV() const; - /// Returns the initialization vector (IV) for the Cipher. - - void setIV(const ByteVec & iv); - /// Sets the initialization vector (IV) for the Cipher. - - const EVP_CIPHER * cipher(); - /// Returns the cipher object - - private: - void generateKey(const std::string & passphrase, const std::string & salt, int iterationCount); - /// Generates key and IV from a password and optional salt string. - - void generateKey(); - /// Generates key and IV from random data. - - void getRandomBytes(ByteVec & vec, std::size_t count); - /// Stores random bytes in vec. - - private: - const EVP_CIPHER * _pCipher; - const EVP_MD * _pDigest; - std::string _name; - ByteVec _key; - ByteVec _iv; - OpenSSLInitializer _openSSLInitializer; - }; - - - // - // Inlines - // - inline const std::string & CipherKeyImpl::name() const - { - return _name; - } - - - inline const CipherKeyImpl::ByteVec & CipherKeyImpl::getKey() const - { - return _key; - } - - - inline void CipherKeyImpl::setKey(const ByteVec & key) - { - poco_assert(key.size() == static_cast(keySize())); - _key = key; - } - - - inline const CipherKeyImpl::ByteVec & CipherKeyImpl::getIV() const - { - return _iv; - } - - - inline const EVP_CIPHER * CipherKeyImpl::cipher() - { - return _pCipher; - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CipherKeyImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/Crypto.h b/base/poco/Crypto/include/Poco/Crypto/Crypto.h deleted file mode 100644 index 42dc8cb137c..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/Crypto.h +++ /dev/null @@ -1,116 +0,0 @@ -// -// Crypto.h -// -// Library: Crypto -// Package: CryptoCore -// Module: Crypto -// -// Basic definitions for the Poco Crypto library. -// This file must be the first file included by every other Crypto -// header file. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_Crypto_INCLUDED -#define Crypto_Crypto_INCLUDED - - -#define POCO_EXTERNAL_OPENSSL_DEFAULT 1 -#define POCO_EXTERNAL_OPENSSL_SLPRO 2 - - -#include -#include "Poco/Foundation.h" - - -#ifndef OPENSSL_VERSION_PREREQ -# if defined(OPENSSL_VERSION_MAJOR) && defined(OPENSSL_VERSION_MINOR) -# define OPENSSL_VERSION_PREREQ(maj, min) ((OPENSSL_VERSION_MAJOR << 16) + OPENSSL_VERSION_MINOR >= ((maj) << 16) + (min)) -# else -# define OPENSSL_VERSION_PREREQ(maj, min) (OPENSSL_VERSION_NUMBER >= (((maj) << 28) | ((min) << 20))) -# endif -#endif - - -enum RSAPaddingMode -/// The padding mode used for RSA public key encryption. -{ - RSA_PADDING_PKCS1, - /// PKCS #1 v1.5 padding. This currently is the most widely used mode. - - RSA_PADDING_PKCS1_OAEP, - /// EME-OAEP as defined in PKCS #1 v2.0 with SHA-1, MGF1 and an empty - /// encoding parameter. This mode is recommended for all new applications. - - RSA_PADDING_SSLV23, - /// PKCS #1 v1.5 padding with an SSL-specific modification that denotes - /// that the server is SSL3 capable. - - RSA_PADDING_NONE - /// Raw RSA encryption. This mode should only be used to implement cryptographically - /// sound padding modes in the application code. Encrypting user data directly with RSA - /// is insecure. -}; - - -// -// The following block is the standard way of creating macros which make exporting -// from a DLL simpler. All files within this DLL are compiled with the Crypto_EXPORTS -// symbol defined on the command line. this symbol should not be defined on any project -// that uses this DLL. This way any other project whose source files include this file see -// Crypto_API functions as being imported from a DLL, whereas this DLL sees symbols -// defined with this macro as being exported. -// - - -#if !defined(Crypto_API) -# if !defined(POCO_NO_GCC_API_ATTRIBUTE) && defined(__GNUC__) && (__GNUC__ >= 4) -# define Crypto_API __attribute__((visibility("default"))) -# else -# define Crypto_API -# endif -#endif - - -// -// Automatically link Crypto and OpenSSL libraries. -// - - -namespace Poco -{ -namespace Crypto -{ - - - void Crypto_API initializeCrypto(); - /// Initialize the Crypto library, as well as the underlying OpenSSL - /// libraries, by calling OpenSSLInitializer::initialize(). - /// - /// Should be called before using any class from the Crypto library. - /// The Crypto library will be initialized automatically, through - /// OpenSSLInitializer instances held by various Crypto classes - /// (Cipher, CipherKey, RSAKey, X509Certificate). - /// However, it is recommended to call initializeCrypto() - /// in any case at application startup. - /// - /// Can be called multiple times; however, for every call to - /// initializeCrypto(), a matching call to uninitializeCrypto() - /// must be performed. - - - void Crypto_API uninitializeCrypto(); - /// Uninitializes the Crypto library by calling - /// OpenSSLInitializer::uninitialize(). - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_Crypto_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CryptoException.h b/base/poco/Crypto/include/Poco/Crypto/CryptoException.h deleted file mode 100644 index 8a2ddd27405..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CryptoException.h +++ /dev/null @@ -1,59 +0,0 @@ -// -// CryptoException.h -// -// -// Library: Crypto -// Package: Crypto -// Module: CryptoException -// -// Definition of the CryptoException class. -// -// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CryptoException_INCLUDED -#define Crypto_CryptoException_INCLUDED - - -#include "Poco/Crypto/Crypto.h" -#include "Poco/Exception.h" - - -namespace Poco -{ -namespace Crypto -{ - - - POCO_DECLARE_EXCEPTION(Crypto_API, CryptoException, Poco::Exception) - - - class Crypto_API OpenSSLException : public CryptoException - { - public: - OpenSSLException(int code = 0); - OpenSSLException(const std::string & msg, int code = 0); - OpenSSLException(const std::string & msg, const std::string & arg, int code = 0); - OpenSSLException(const std::string & msg, const Poco::Exception & exc, int code = 0); - OpenSSLException(const OpenSSLException & exc); - ~OpenSSLException() throw(); - OpenSSLException & operator=(const OpenSSLException & exc); - const char * name() const throw(); - const char * className() const throw(); - Poco::Exception * clone() const; - void rethrow() const; - - private: - void setExtMessage(); - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CryptoException_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CryptoStream.h b/base/poco/Crypto/include/Poco/Crypto/CryptoStream.h deleted file mode 100644 index 788b0ab57ce..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CryptoStream.h +++ /dev/null @@ -1,195 +0,0 @@ -// -// CryptoStream.h -// -// Library: Crypto -// Package: Cipher -// Module: CryptoStream -// -// Definition of the CryptoStreamBuf, CryptoInputStream and CryptoOutputStream -// classes. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CryptoStream_INCLUDED -#define Crypto_CryptoStream_INCLUDED - - -#include -#include "Poco/Buffer.h" -#include "Poco/BufferedStreamBuf.h" -#include "Poco/Crypto/Crypto.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class CryptoTransform; - class Cipher; - - - class Crypto_API CryptoStreamBuf : public Poco::BufferedStreamBuf - /// This stream buffer performs cryptographic transformation on the data - /// going through it. - { - public: - CryptoStreamBuf(std::istream & istr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192); - CryptoStreamBuf(std::ostream & ostr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192); - - virtual ~CryptoStreamBuf(); - - void close(); - /// Flushes all buffers and finishes the encryption. - - protected: - int readFromDevice(char * buffer, std::streamsize length); - int writeToDevice(const char * buffer, std::streamsize length); - - private: - CryptoTransform * _pTransform; - std::istream * _pIstr; - std::ostream * _pOstr; - bool _eof; - - Poco::Buffer _buffer; - - CryptoStreamBuf(const CryptoStreamBuf &); - CryptoStreamBuf & operator=(const CryptoStreamBuf &); - }; - - - class Crypto_API CryptoIOS : public virtual std::ios - /// The base class for CryptoInputStream and CryptoOutputStream. - /// - /// This class is needed to ensure correct initialization order of the - /// stream buffer and base classes. - { - public: - CryptoIOS(std::istream & istr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192); - CryptoIOS(std::ostream & ostr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192); - ~CryptoIOS(); - CryptoStreamBuf * rdbuf(); - - protected: - CryptoStreamBuf _buf; - }; - - - class Crypto_API CryptoInputStream : public CryptoIOS, public std::istream - /// This stream transforms all data passing through it using the given - /// CryptoTransform. - /// - /// Use a CryptoTransform object provided by Cipher::createEncrytor() or - /// Cipher::createDecryptor() to create an encrypting or decrypting stream, - /// respectively. - { - public: - CryptoInputStream(std::istream & istr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192); - /// Create a new CryptoInputStream object. The CryptoInputStream takes the - /// ownership of the given CryptoTransform object. - - CryptoInputStream(std::istream & istr, Cipher & cipher, std::streamsize bufferSize = 8192); - /// Create a new encrypting CryptoInputStream object using the given cipher. - - ~CryptoInputStream(); - /// Destroys the CryptoInputStream. - }; - - - class Crypto_API CryptoOutputStream : public CryptoIOS, public std::ostream - /// This stream transforms all data passing through it using the given - /// CryptoTransform. - /// - /// Use a CryptoTransform object provided by Cipher::createEncrytor() or - /// Cipher::createDecryptor() to create an encrypting or decrypting stream, - /// respectively. - /// - /// After all data has been passed through the stream, close() must be called - /// to ensure completion of cryptographic transformation. - { - public: - CryptoOutputStream(std::ostream & ostr, CryptoTransform * pTransform, std::streamsize bufferSize = 8192); - /// Create a new CryptoOutputStream object. The CryptoOutputStream takes the - /// ownership of the given CryptoTransform object. - - CryptoOutputStream(std::ostream & ostr, Cipher & cipher, std::streamsize bufferSize = 8192); - /// Create a new decrypting CryptoOutputStream object using the given cipher. - - ~CryptoOutputStream(); - /// Destroys the CryptoOutputStream. - - void close(); - /// Flushes all buffers and finishes the encryption. - }; - - - class Crypto_API DecryptingInputStream : public CryptoIOS, public std::istream - /// This stream decrypts all data passing through it using the given - /// Cipher. - { - public: - DecryptingInputStream(std::istream & istr, Cipher & cipher, std::streamsize bufferSize = 8192); - /// Create a new DecryptingInputStream object using the given cipher. - - ~DecryptingInputStream(); - /// Destroys the DecryptingInputStream. - }; - - - class Crypto_API DecryptingOutputStream : public CryptoIOS, public std::ostream - /// This stream decrypts all data passing through it using the given - /// Cipher. - { - public: - DecryptingOutputStream(std::ostream & ostr, Cipher & cipher, std::streamsize bufferSize = 8192); - /// Create a new DecryptingOutputStream object using the given cipher. - - ~DecryptingOutputStream(); - /// Destroys the DecryptingOutputStream. - - void close(); - /// Flushes all buffers and finishes the decryption. - }; - - - class Crypto_API EncryptingInputStream : public CryptoIOS, public std::istream - /// This stream encrypts all data passing through it using the given - /// Cipher. - { - public: - EncryptingInputStream(std::istream & istr, Cipher & cipher, std::streamsize bufferSize = 8192); - /// Create a new EncryptingInputStream object using the given cipher. - - ~EncryptingInputStream(); - /// Destroys the EncryptingInputStream. - }; - - - class Crypto_API EncryptingOutputStream : public CryptoIOS, public std::ostream - /// This stream encrypts all data passing through it using the given - /// Cipher. - { - public: - EncryptingOutputStream(std::ostream & ostr, Cipher & cipher, std::streamsize bufferSize = 8192); - /// Create a new EncryptingOutputStream object using the given cipher. - - ~EncryptingOutputStream(); - /// Destroys the EncryptingOutputStream. - - void close(); - /// Flushes all buffers and finishes the encryption. - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CryptoStream_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h b/base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h deleted file mode 100644 index 7fbcff2b5c3..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/CryptoTransform.h +++ /dev/null @@ -1,88 +0,0 @@ -// -// CryptoTransform.h -// -// Library: Crypto -// Package: Cipher -// Module: CryptoTransform -// -// Definition of the CryptoTransform class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_CryptoTransform_INCLUDED -#define Crypto_CryptoTransform_INCLUDED - - -#include -#include "Poco/Crypto/Crypto.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API CryptoTransform - /// This interface represents the basic operations for cryptographic - /// transformations to be used with a CryptoInputStream or a - /// CryptoOutputStream. - /// - /// Implementations of this class are returned by the Cipher class to - /// perform encryption or decryption of data. - { - public: - CryptoTransform(); - /// Creates a new CryptoTransform object. - - virtual ~CryptoTransform(); - /// Destroys the CryptoTransform. - - virtual std::size_t blockSize() const = 0; - /// Returns the block size for this CryptoTransform. - - virtual int setPadding(int padding); - /// Enables or disables padding. By default encryption operations are padded using standard block - /// padding and the padding is checked and removed when decrypting. If the padding parameter is zero then - /// no padding is performed, the total amount of data encrypted or decrypted must then be a multiple of - /// the block size or an error will occur. - - virtual std::string getTag(std::size_t tagSize = 16) = 0; - /// Returns the GCM tag after encrypting using GCM mode. - /// - /// Must be called after finalize(). - - virtual void setTag(const std::string & tag) = 0; - /// Sets the GCM tag for authenticated decryption using GCM mode. - /// - /// Must be set before finalize() is called, otherwise - /// decryption will fail. - - virtual std::streamsize - transform(const unsigned char * input, std::streamsize inputLength, unsigned char * output, std::streamsize outputLength) - = 0; - /// Transforms a chunk of data. The inputLength is arbitrary and does not - /// need to be a multiple of the block size. The output buffer has a maximum - /// capacity of the given outputLength that must be at least - /// inputLength + blockSize() - 1 - /// Returns the number of bytes written to the output buffer. - - virtual std::streamsize finalize(unsigned char * output, std::streamsize length) = 0; - /// Finalizes the transformation. The output buffer must contain enough - /// space for at least two blocks, ie. - /// length >= 2*blockSize() - /// must be true. Returns the number of bytes written to the output - /// buffer. - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_CryptoTransform_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/DigestEngine.h b/base/poco/Crypto/include/Poco/Crypto/DigestEngine.h deleted file mode 100644 index 77a1349f8e2..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/DigestEngine.h +++ /dev/null @@ -1,83 +0,0 @@ -// -// DigestEngine.h -// -// Library: Crypto -// Package: Digest -// Module: DigestEngine -// -// Definition of the DigestEngine class. -// -// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_DigestEngine_INCLUDED -#define Crypto_DigestEngine_INCLUDED - - -#include -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/DigestEngine.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API DigestEngine : public Poco::DigestEngine - /// This class implements a Poco::DigestEngine for all - /// digest algorithms supported by OpenSSL. - { - public: - DigestEngine(const std::string & name); - /// Creates a DigestEngine using the digest with the given name - /// (e.g., "MD5", "SHA1", "SHA256", "SHA512", etc.). - /// See the OpenSSL documentation for a list of supported digest algorithms. - /// - /// Throws a Poco::NotFoundException if no algorithm with the given name exists. - - ~DigestEngine(); - /// Destroys the DigestEngine. - - const std::string & algorithm() const; - /// Returns the name of the digest algorithm. - - int nid() const; - /// Returns the NID (OpenSSL object identifier) of the digest algorithm. - - // DigestEngine - std::size_t digestLength() const; - void reset(); - const Poco::DigestEngine::Digest & digest(); - - protected: - void updateImpl(const void * data, std::size_t length); - - private: - std::string _name; - EVP_MD_CTX * _pContext; - Poco::DigestEngine::Digest _digest; - OpenSSLInitializer _openSSLInitializer; - }; - - - // - // inlines - // - inline const std::string & DigestEngine::algorithm() const - { - return _name; - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_DigestEngine_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h b/base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h deleted file mode 100644 index 2d0c31ba09c..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/ECDSADigestEngine.h +++ /dev/null @@ -1,103 +0,0 @@ -// -// ECDSADigestEngine.h -// -// -// Library: Crypto -// Package: ECDSA -// Module: ECDSADigestEngine -// -// Definition of the ECDSADigestEngine class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_ECDSADigestEngine_INCLUDED -#define Crypto_ECDSADigestEngine_INCLUDED - - -#include -#include -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/DigestEngine.h" -#include "Poco/Crypto/ECKey.h" -#include "Poco/DigestEngine.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API ECDSADigestEngine : public Poco::DigestEngine - /// This class implements a Poco::DigestEngine that can be - /// used to compute a secure digital signature. - /// - /// First another Poco::Crypto::DigestEngine is created and - /// used to compute a cryptographic hash of the data to be - /// signed. Then, the hash value is encrypted, using - /// the ECDSA private key. - /// - /// To verify a signature, pass it to the verify() - /// member function. It will decrypt the signature - /// using the ECDSA public key and compare the resulting - /// hash with the actual hash of the data. - { - public: - ECDSADigestEngine(const ECKey & key, const std::string & name); - /// Creates the ECDSADigestEngine with the given ECDSA key, - /// using the hash algorithm with the given name - /// (e.g., "SHA1", "SHA256", "SHA512", etc.). - /// See the OpenSSL documentation for a list of supported digest algorithms. - /// - /// Throws a Poco::NotFoundException if no algorithm with the given name exists. - - ~ECDSADigestEngine(); - /// Destroys the ECDSADigestEngine. - - std::size_t digestLength() const; - /// Returns the length of the digest in bytes. - - void reset(); - /// Resets the engine so that a new - /// digest can be computed. - - const DigestEngine::Digest & digest(); - /// Finishes the computation of the digest - /// (the first time it's called) and - /// returns the message digest. - /// - /// Can be called multiple times. - - const DigestEngine::Digest & signature(); - /// Signs the digest using the ECDSADSA algorithm - /// and the private key (the first time it's - /// called) and returns the result. - /// - /// Can be called multiple times. - - bool verify(const DigestEngine::Digest & signature); - /// Verifies the data against the signature. - /// - /// Returns true if the signature can be verified, false otherwise. - - protected: - void updateImpl(const void * data, std::size_t length); - - private: - ECKey _key; - Poco::Crypto::DigestEngine _engine; - Poco::DigestEngine::Digest _digest; - Poco::DigestEngine::Digest _signature; - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_ECDSADigestEngine_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/ECKey.h b/base/poco/Crypto/include/Poco/Crypto/ECKey.h deleted file mode 100644 index babcd814ec9..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/ECKey.h +++ /dev/null @@ -1,138 +0,0 @@ -// -// ECKey.h -// -// -// Library: Crypto -// Package: EC -// Module: ECKey -// -// Definition of the ECKey class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_ECKey_INCLUDED -#define Crypto_ECKey_INCLUDED - - -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/ECKeyImpl.h" -#include "Poco/Crypto/KeyPair.h" - -namespace Poco -{ -namespace Crypto -{ - - - class X509Certificate; - class PKCS12Container; - - - class Crypto_API ECKey : public KeyPair - /// This class stores an EC key pair, consisting - /// of private and public key. Storage of the private - /// key is optional. - /// - /// If a private key is available, the ECKey can be - /// used for decrypting data (encrypted with the public key) - /// or computing secure digital signatures. - { - public: - ECKey(const EVPPKey & key); - /// Constructs ECKeyImpl by extracting the EC key. - - ECKey(const X509Certificate & cert); - /// Extracts the EC public key from the given certificate. - - ECKey(const PKCS12Container & cert); - /// Extracts the EC private key from the given certificate. - - ECKey(const std::string & eccGroup); - /// Creates the ECKey. Creates a new public/private key pair using the given parameters. - /// Can be used to sign data and verify signatures. - - ECKey(const std::string & publicKeyFile, const std::string & privateKeyFile, const std::string & privateKeyPassphrase = ""); - /// Creates the ECKey, by reading public and private key from the given files and - /// using the given passphrase for the private key. - /// - /// Cannot be used for signing or decryption unless a private key is available. - /// - /// If a private key is specified, you don't need to specify a public key file. - /// OpenSSL will auto-create the public key from the private key. - - ECKey(std::istream * pPublicKeyStream, std::istream * pPrivateKeyStream = 0, const std::string & privateKeyPassphrase = ""); - /// Creates the ECKey, by reading public and private key from the given streams and - /// using the given passphrase for the private key. - /// - /// Cannot be used for signing or decryption unless a private key is available. - /// - /// If a private key is specified, you don't need to specify a public key file. - /// OpenSSL will auto-create the public key from the private key. - - ~ECKey(); - /// Destroys the ECKey. - - ECKeyImpl::Ptr impl() const; - /// Returns the impl object. - - static std::string getCurveName(int nid = -1); - /// Returns elliptical curve name corresponding to - /// the given nid; if nid is not found, returns - /// empty string. - /// - /// If nid is -1, returns first curve name. - /// - /// If no curves are found, returns empty string; - - static int getCurveNID(std::string & name); - /// Returns the NID of the specified curve. - /// - /// If name is empty, returns the first curve NID - /// and updates the name accordingly. - - static bool hasCurve(const std::string & name); - /// Returns true if the named curve is found, - /// false otherwise. - - private: - ECKeyImpl::Ptr _pImpl; - }; - - - // - // inlines - // - inline ECKeyImpl::Ptr ECKey::impl() const - { - return _pImpl; - } - - - inline std::string ECKey::getCurveName(int nid) - { - return ECKeyImpl::getCurveName(nid); - } - - - inline int ECKey::getCurveNID(std::string & name) - { - return ECKeyImpl::getCurveNID(name); - } - - - inline bool ECKey::hasCurve(const std::string & name) - { - return ECKeyImpl::hasCurve(name); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_ECKey_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h b/base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h deleted file mode 100644 index d051ef1b768..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/ECKeyImpl.h +++ /dev/null @@ -1,143 +0,0 @@ -// -// ECKeyImpl.h -// -// -// Library: Crypto -// Package: EC -// Module: ECKeyImpl -// -// Definition of the ECKeyImpl class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_ECKeyImplImpl_INCLUDED -#define Crypto_ECKeyImplImpl_INCLUDED - - -#include -#include -#include -#include -#include -#include "Poco/AutoPtr.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/EVPPKey.h" -#include "Poco/Crypto/KeyPairImpl.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/RefCountedObject.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class X509Certificate; - class PKCS12Container; - - - class ECKeyImpl : public KeyPairImpl - /// Elliptic Curve key clas implementation. - { - public: - typedef Poco::AutoPtr Ptr; - typedef std::vector ByteVec; - - ECKeyImpl(const EVPPKey & key); - /// Constructs ECKeyImpl by extracting the EC key. - - ECKeyImpl(const X509Certificate & cert); - /// Constructs ECKeyImpl by extracting the EC public key from the given certificate. - - ECKeyImpl(const PKCS12Container & cert); - /// Constructs ECKeyImpl by extracting the EC private key from the given certificate. - - ECKeyImpl(int eccGroup); - /// Creates the ECKey of the specified group. Creates a new public/private keypair using the given parameters. - /// Can be used to sign data and verify signatures. - - ECKeyImpl(const std::string & publicKeyFile, const std::string & privateKeyFile, const std::string & privateKeyPassphrase); - /// Creates the ECKey, by reading public and private key from the given files and - /// using the given passphrase for the private key. Can only by used for signing if - /// a private key is available. - - ECKeyImpl(std::istream * pPublicKeyStream, std::istream * pPrivateKeyStream, const std::string & privateKeyPassphrase); - /// Creates the ECKey. Can only by used for signing if pPrivKey - /// is not null. If a private key file is specified, you don't need to - /// specify a public key file. OpenSSL will auto-create it from the private key. - - ~ECKeyImpl(); - /// Destroys the ECKeyImpl. - - EC_KEY * getECKey(); - /// Returns the OpenSSL EC key. - - const EC_KEY * getECKey() const; - /// Returns the OpenSSL EC key. - - int size() const; - /// Returns the EC key length in bits. - - int groupId() const; - /// Returns the EC key group integer Id. - - std::string groupName() const; - /// Returns the EC key group name. - - static std::string getCurveName(int nid = -1); - /// Returns elliptical curve name corresponding to - /// the given nid; if nid is not found, returns - /// empty string. - /// - /// If nid is -1, returns first curve name. - /// - /// If no curves are found, returns empty string; - - static int getCurveNID(std::string & name); - /// Returns the NID of the specified curve. - /// - /// If name is empty, returns the first curve NID - /// and updates the name accordingly. - - static bool hasCurve(const std::string & name); - /// Returns true if the named curve is found, - /// false otherwise. - - private: - void checkEC(const std::string & method, const std::string & func) const; - void freeEC(); - - EC_KEY * _pEC; - }; - - - // - // inlines - // - inline EC_KEY * ECKeyImpl::getECKey() - { - return _pEC; - } - - - inline const EC_KEY * ECKeyImpl::getECKey() const - { - return _pEC; - } - - - inline std::string ECKeyImpl::groupName() const - { - return OBJ_nid2sn(groupId()); - } -} -} // namespace Poco::Crypto - - -#endif // Crypto_ECKeyImplImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h b/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h deleted file mode 100644 index acc79ec92b2..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/EVPPKey.h +++ /dev/null @@ -1,369 +0,0 @@ -// -// EVPPKey.h -// -// -// Library: Crypto -// Package: CryptoCore -// Module: EVPPKey -// -// Definition of the EVPPKey class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_EVPPKeyImpl_INCLUDED -#define Crypto_EVPPKeyImpl_INCLUDED - - -#include -#include -#include -#include -#include -#include -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/CryptoException.h" -#include "Poco/StreamCopier.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class ECKey; - class RSAKey; - - - class Crypto_API EVPPKey - /// Utility class for conversion of native keys to EVP. - /// Currently, only RSA and EC keys are supported. - { - public: - explicit EVPPKey(const std::string & ecCurveName); - /// Constructs EVPPKey from ECC curve name. - /// - /// Only EC keys can be wrapped by an EVPPKey - /// created using this constructor. - - explicit EVPPKey(const char * ecCurveName); - /// Constructs EVPPKey from ECC curve name. - /// - /// Only EC keys can be wrapped by an EVPPKey - /// created using this constructor. - - explicit EVPPKey(EVP_PKEY * pEVPPKey); - /// Constructs EVPPKey from EVP_PKEY pointer. - /// The content behind the supplied pointer is internally duplicated. - - template - explicit EVPPKey(K * pKey) : _pEVPPKey(EVP_PKEY_new()) - /// Constructs EVPPKey from a "native" OpenSSL (RSA or EC_KEY), - /// or a Poco wrapper (RSAKey, ECKey) key pointer. - { - if (!_pEVPPKey) - throw OpenSSLException(); - setKey(pKey); - } - - EVPPKey(const std::string & publicKeyFile, const std::string & privateKeyFile, const std::string & privateKeyPassphrase = ""); - /// Creates the EVPPKey, by reading public and private key from the given files and - /// using the given passphrase for the private key. Can only by used for signing if - /// a private key is available. - - EVPPKey(std::istream * pPublicKeyStream, std::istream * pPrivateKeyStream, const std::string & privateKeyPassphrase = ""); - /// Creates the EVPPKey. Can only by used for signing if pPrivKey - /// is not null. If a private key file is specified, you don't need to - /// specify a public key file. OpenSSL will auto-create it from the private key. - - EVPPKey(const EVPPKey & other); - /// Copy constructor. - - EVPPKey & operator=(const EVPPKey & other); - /// Assignment operator. - -#ifdef POCO_ENABLE_CPP11 - - EVPPKey(EVPPKey && other); - /// Move constructor. - - EVPPKey & operator=(EVPPKey && other); - /// Assignment move operator. - -#endif // POCO_ENABLE_CPP11 - - ~EVPPKey(); - /// Destroys the EVPPKey. - - bool operator==(const EVPPKey & other) const; - /// Comparison operator. - /// Returns true if public key components and parameters - /// of the other key are equal to this key. - /// - /// Works as expected when one key contains only public key, - /// while the other one contains private (thus also public) key. - - bool operator!=(const EVPPKey & other) const; - /// Comparison operator. - /// Returns true if public key components and parameters - /// of the other key are different from this key. - /// - /// Works as expected when one key contains only public key, - /// while the other one contains private (thus also public) key. - - void save(const std::string & publicKeyFile, const std::string & privateKeyFile = "", const std::string & privateKeyPassphrase = "") - const; - /// Exports the public and/or private keys to the given files. - /// - /// If an empty filename is specified, the corresponding key - /// is not exported. - - void - save(std::ostream * pPublicKeyStream, std::ostream * pPrivateKeyStream = 0, const std::string & privateKeyPassphrase = "") const; - /// Exports the public and/or private key to the given streams. - /// - /// If a null pointer is passed for a stream, the corresponding - /// key is not exported. - - int type() const; - /// Returns the EVPPKey type NID. - - bool isSupported(int type) const; - /// Returns true if OpenSSL type is supported - - operator const EVP_PKEY *() const; - /// Returns const pointer to the OpenSSL EVP_PKEY structure. - - operator EVP_PKEY *(); - /// Returns pointer to the OpenSSL EVP_PKEY structure. - - static EVP_PKEY * duplicate(const EVP_PKEY * pFromKey, EVP_PKEY ** pToKey); - /// Duplicates pFromKey into *pToKey and returns - // the pointer to duplicated EVP_PKEY. - - private: - EVPPKey(); - - static int type(const EVP_PKEY * pEVPPKey); - void newECKey(const char * group); - void duplicate(EVP_PKEY * pEVPPKey); - - void setKey(ECKey * pKey); - void setKey(RSAKey * pKey); - void setKey(EC_KEY * pKey); - void setKey(RSA * pKey); - static int passCB(char * buf, int size, int, void * pass); - - typedef EVP_PKEY * (*PEM_read_FILE_Key_fn)(FILE *, EVP_PKEY **, pem_password_cb *, void *); - typedef EVP_PKEY * (*PEM_read_BIO_Key_fn)(BIO *, EVP_PKEY **, pem_password_cb *, void *); - typedef void * (*EVP_PKEY_get_Key_fn)(EVP_PKEY *); - - // The following load*() functions are used by both native and EVP_PKEY type key - // loading from BIO/FILE. - // When used for EVP key loading, getFunc is null (ie. native key is not extracted - // from the loaded EVP_PKEY). - template - static bool - loadKey(K ** ppKey, PEM_read_FILE_Key_fn readFunc, F getFunc, const std::string & keyFile, const std::string & pass = "") - { - poco_assert_dbg( - ((typeid(K *) == typeid(RSA *) || typeid(K *) == typeid(EC_KEY *)) && getFunc) - || ((typeid(K *) == typeid(EVP_PKEY *)) && !getFunc)); - poco_check_ptr(ppKey); - poco_assert_dbg(!*ppKey); - - FILE * pFile = 0; - if (!keyFile.empty()) - { - if (!getFunc) - *ppKey = (K *)EVP_PKEY_new(); - EVP_PKEY * pKey = getFunc ? EVP_PKEY_new() : (EVP_PKEY *)*ppKey; - if (pKey) - { - pFile = fopen(keyFile.c_str(), "r"); - if (pFile) - { - pem_password_cb * pCB = pass.empty() ? (pem_password_cb *)0 : &passCB; - void * pPassword = pass.empty() ? (void *)0 : (void *)pass.c_str(); - if (readFunc(pFile, &pKey, pCB, pPassword)) - { - fclose(pFile); - pFile = 0; - if (getFunc) - { - *ppKey = (K *)getFunc(pKey); - EVP_PKEY_free(pKey); - } - else - { - poco_assert_dbg(typeid(K *) == typeid(EVP_PKEY *)); - *ppKey = (K *)pKey; - } - if (!*ppKey) - goto error; - return true; - } - goto error; - } - else - { - if (getFunc) - EVP_PKEY_free(pKey); - throw IOException("ECKeyImpl, cannot open file", keyFile); - } - } - else - goto error; - } - return false; - - error: - if (pFile) - fclose(pFile); - throw OpenSSLException("EVPKey::loadKey(string)"); - } - - template - static bool loadKey(K ** ppKey, PEM_read_BIO_Key_fn readFunc, F getFunc, std::istream * pIstr, const std::string & pass = "") - { - poco_assert_dbg( - ((typeid(K *) == typeid(RSA *) || typeid(K *) == typeid(EC_KEY *)) && getFunc) - || ((typeid(K *) == typeid(EVP_PKEY *)) && !getFunc)); - poco_check_ptr(ppKey); - poco_assert_dbg(!*ppKey); - - BIO * pBIO = 0; - if (pIstr) - { - std::ostringstream ostr; - Poco::StreamCopier::copyStream(*pIstr, ostr); - std::string key = ostr.str(); - pBIO = BIO_new_mem_buf(const_cast(key.data()), static_cast(key.size())); - if (pBIO) - { - if (!getFunc) - *ppKey = (K *)EVP_PKEY_new(); - EVP_PKEY * pKey = getFunc ? EVP_PKEY_new() : (EVP_PKEY *)*ppKey; - if (pKey) - { - pem_password_cb * pCB = pass.empty() ? (pem_password_cb *)0 : &passCB; - void * pPassword = pass.empty() ? (void *)0 : (void *)pass.c_str(); - if (readFunc(pBIO, &pKey, pCB, pPassword)) - { - BIO_free(pBIO); - pBIO = 0; - if (getFunc) - { - *ppKey = (K *)getFunc(pKey); - EVP_PKEY_free(pKey); - } - else - { - poco_assert_dbg(typeid(K *) == typeid(EVP_PKEY *)); - *ppKey = (K *)pKey; - } - if (!*ppKey) - goto error; - return true; - } - if (getFunc) - EVP_PKEY_free(pKey); - goto error; - } - else - goto error; - } - else - goto error; - } - return false; - - error: - if (pBIO) - BIO_free(pBIO); - throw OpenSSLException("EVPKey::loadKey(stream)"); - } - - EVP_PKEY * _pEVPPKey; - - friend class ECKeyImpl; - friend class RSAKeyImpl; - }; - - - // - // inlines - // - - - inline bool EVPPKey::operator==(const EVPPKey & other) const - { - poco_check_ptr(other._pEVPPKey); - poco_check_ptr(_pEVPPKey); - return (1 == EVP_PKEY_cmp(_pEVPPKey, other._pEVPPKey)); - } - - - inline bool EVPPKey::operator!=(const EVPPKey & other) const - { - return !(other == *this); - } - - - inline int EVPPKey::type(const EVP_PKEY * pEVPPKey) - { - if (!pEVPPKey) - return NID_undef; - - return EVP_PKEY_type(EVP_PKEY_id(pEVPPKey)); - } - - - inline int EVPPKey::type() const - { - return type(_pEVPPKey); - } - - - inline bool EVPPKey::isSupported(int type) const - { - return type == EVP_PKEY_EC || type == EVP_PKEY_RSA; - } - - - inline EVPPKey::operator const EVP_PKEY *() const - { - return _pEVPPKey; - } - - - inline EVPPKey::operator EVP_PKEY *() - { - return _pEVPPKey; - } - - - inline void EVPPKey::setKey(EC_KEY * pKey) - { - if (!EVP_PKEY_set1_EC_KEY(_pEVPPKey, pKey)) - throw OpenSSLException(); - } - - - inline void EVPPKey::setKey(RSA * pKey) - { - if (!EVP_PKEY_set1_RSA(_pEVPPKey, pKey)) - throw OpenSSLException(); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_EVPPKeyImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/KeyPair.h b/base/poco/Crypto/include/Poco/Crypto/KeyPair.h deleted file mode 100644 index 291a0f8b749..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/KeyPair.h +++ /dev/null @@ -1,103 +0,0 @@ -// -// KeyPair.h -// -// -// Library: Crypto -// Package: CryptoCore -// Module: KeyPair -// -// Definition of the KeyPair class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_KeyPair_INCLUDED -#define Crypto_KeyPair_INCLUDED - - -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/KeyPairImpl.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class X509Certificate; - - - class Crypto_API KeyPair - /// This is a parent class for classes storing a key pair, consisting - /// of private and public key. Storage of the private key is optional. - /// - /// If a private key is available, the KeyPair can be - /// used for decrypting data (encrypted with the public key) - /// or computing secure digital signatures. - { - public: - enum Type - { - KT_RSA = KeyPairImpl::KT_RSA_IMPL, - KT_EC = KeyPairImpl::KT_EC_IMPL - }; - - explicit KeyPair(KeyPairImpl::Ptr pKeyPairImpl = 0); - /// Extracts the RSA public key from the given certificate. - - virtual ~KeyPair(); - /// Destroys the KeyPair. - - virtual int size() const; - /// Returns the RSA modulus size. - - KeyPairImpl::Ptr impl() const; - /// Returns the impl object. - - const std::string & name() const; - /// Returns key pair name - - Type type() const; - /// Returns key pair type - - private: - KeyPairImpl::Ptr _pImpl; - }; - - - // - // inlines - // - - inline int KeyPair::size() const - { - return _pImpl->size(); - } - - inline const std::string & KeyPair::name() const - { - return _pImpl->name(); - } - - inline KeyPairImpl::Ptr KeyPair::impl() const - { - return _pImpl; - } - - - inline KeyPair::Type KeyPair::type() const - { - return (KeyPair::Type)impl()->type(); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_KeyPair_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h b/base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h deleted file mode 100644 index ecafbef0241..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/KeyPairImpl.h +++ /dev/null @@ -1,94 +0,0 @@ -// -// KeyPairImpl.h -// -// -// Library: Crypto -// Package: CryptoCore -// Module: KeyPairImpl -// -// Definition of the KeyPairImpl class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_KeyPairImplImpl_INCLUDED -#define Crypto_KeyPairImplImpl_INCLUDED - - -#include -#include -#include "Poco/AutoPtr.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/RefCountedObject.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class KeyPairImpl : public Poco::RefCountedObject - /// Class KeyPairImpl - { - public: - enum Type - { - KT_RSA_IMPL = 0, - KT_EC_IMPL - }; - - typedef Poco::AutoPtr Ptr; - typedef std::vector ByteVec; - - KeyPairImpl(const std::string & name, Type type); - /// Create KeyPairImpl with specified type and name. - - virtual ~KeyPairImpl(); - /// Destroys the KeyPairImpl. - - virtual int size() const = 0; - /// Returns the key size. - - const std::string & name() const; - /// Returns key pair name - - Type type() const; - /// Returns key pair type - - private: - KeyPairImpl(); - - std::string _name; - Type _type; - OpenSSLInitializer _openSSLInitializer; - }; - - - // - // inlines - // - - - inline const std::string & KeyPairImpl::name() const - { - return _name; - } - - - inline KeyPairImpl::Type KeyPairImpl::type() const - { - return _type; - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_KeyPairImplImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h b/base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h deleted file mode 100644 index 147cfaeefca..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/OpenSSLInitializer.h +++ /dev/null @@ -1,117 +0,0 @@ -// -// OpenSSLInitializer.h -// -// Library: Crypto -// Package: CryptoCore -// Module: OpenSSLInitializer -// -// Definition of the OpenSSLInitializer class. -// -// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_OpenSSLInitializer_INCLUDED -#define Crypto_OpenSSLInitializer_INCLUDED - - -#include -#include "Poco/AtomicCounter.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Mutex.h" - -#if defined(OPENSSL_FIPS) && OPENSSL_VERSION_NUMBER < 0x010001000L -# include -#endif - - -extern "C" { -struct CRYPTO_dynlock_value -{ - Poco::FastMutex _mutex; -}; -} - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API OpenSSLInitializer - /// Initializes the OpenSSL library. - /// - /// The class ensures the earliest initialization and the - /// latest shutdown of the OpenSSL library. - { - public: - OpenSSLInitializer(); - /// Automatically initialize OpenSSL on startup. - - ~OpenSSLInitializer(); - /// Automatically shut down OpenSSL on exit. - - static void initialize(); - /// Initializes the OpenSSL machinery. - - static void uninitialize(); - /// Shuts down the OpenSSL machinery. - - static bool isFIPSEnabled(); - // Returns true if FIPS mode is enabled, false otherwise. - - static void enableFIPSMode(bool enabled); - // Enable or disable FIPS mode. If FIPS is not available, this method doesn't do anything. - - protected: - enum - { - SEEDSIZE = 256 - }; - - // OpenSSL multithreading support - static void lock(int mode, int n, const char * file, int line); - static unsigned long id(); - static struct CRYPTO_dynlock_value * dynlockCreate(const char * file, int line); - static void dynlock(int mode, struct CRYPTO_dynlock_value * lock, const char * file, int line); - static void dynlockDestroy(struct CRYPTO_dynlock_value * lock, const char * file, int line); - - private: - static Poco::FastMutex * _mutexes; - static Poco::AtomicCounter _rc; - }; - - - // - // inlines - // - inline bool OpenSSLInitializer::isFIPSEnabled() - { -#ifdef OPENSSL_FIPS - return FIPS_mode() ? true : false; -#else - return false; -#endif - } - -#ifdef OPENSSL_FIPS - inline void OpenSSLInitializer::enableFIPSMode(bool enabled) - { - FIPS_mode_set(enabled); - } -#else - inline void OpenSSLInitializer::enableFIPSMode(bool /*enabled*/) - { - } -#endif - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_OpenSSLInitializer_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h b/base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h deleted file mode 100644 index 40baa496827..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/PKCS12Container.h +++ /dev/null @@ -1,162 +0,0 @@ -// -// PKCS12Container.h -// -// Library: Crypto -// Package: Certificate -// Module: PKCS12Container -// -// Definition of the PKCS12Container class. -// -// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_PKCS12Container_INCLUDED -#define Crypto_PKCS12Container_INCLUDED - - -#include -#include -#include -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/EVPPKey.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/Crypto/X509Certificate.h" -#include "Poco/Path.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API PKCS12Container - /// This class implements PKCS#12 container functionality. - { - public: - typedef X509Certificate::List CAList; - typedef std::vector CANameList; - - explicit PKCS12Container(std::istream & istr, const std::string & password = ""); - /// Creates the PKCS12Container object from a stream. - - explicit PKCS12Container(const std::string & path, const std::string & password = ""); - /// Creates the PKCS12Container object from a file. - - PKCS12Container(const PKCS12Container & cont); - /// Copy constructor. - - PKCS12Container & operator=(const PKCS12Container & cont); - /// Assignment operator. - -#ifdef POCO_ENABLE_CPP11 - - PKCS12Container(PKCS12Container && cont); - /// Move constructor. - - PKCS12Container & operator=(PKCS12Container && cont); - /// Move assignment operator. - -#endif // POCO_ENABLE_CPP11 - - ~PKCS12Container(); - /// Destroys the PKCS12Container. - - bool hasKey() const; - /// Returns true if container contains the key. - - EVPPKey getKey() const; - /// Return key as openssl EVP_PKEY wrapper object. - - bool hasX509Certificate() const; - /// Returns true if container has X509 certificate. - - const X509Certificate & getX509Certificate() const; - /// Returns the X509 certificate. - /// Throws NotFoundException if there is no certificate. - - const CAList & getCACerts() const; - /// Returns the list of CA certificates in this container. - - const std::string & getFriendlyName() const; - /// Returns the friendly name of the certificate bag. - - const CANameList & getFriendlyNamesCA() const; - /// Returns a list of CA certificates friendly names. - - private: - void load(PKCS12 * pPKCS12, const std::string & password = ""); - std::string extractFriendlyName(X509 * pCert); - -#ifdef POCO_ENABLE_CPP11 - typedef std::unique_ptr CertPtr; -#else - typedef std::auto_ptr CertPtr; -#endif // #ifdef POCO_ENABLE_CPP11 - - OpenSSLInitializer _openSSLInitializer; - EVP_PKEY * _pKey; - CertPtr _pX509Cert; - CAList _caCertList; - CANameList _caCertNames; - std::string _pkcsFriendlyName; - }; - - - // - // inlines - // - - inline bool PKCS12Container::hasX509Certificate() const - { - return _pX509Cert.get() != 0; - } - - - inline const X509Certificate & PKCS12Container::getX509Certificate() const - { - if (!hasX509Certificate()) - throw NotFoundException("PKCS12Container X509 certificate"); - return *_pX509Cert; - } - - - inline const std::string & PKCS12Container::getFriendlyName() const - { - return _pkcsFriendlyName; - } - - - inline const PKCS12Container::CAList & PKCS12Container::getCACerts() const - { - return _caCertList; - } - - - inline const PKCS12Container::CANameList & PKCS12Container::getFriendlyNamesCA() const - { - return _caCertNames; - } - - - inline bool PKCS12Container::hasKey() const - { - return _pKey != 0; - } - - - inline EVPPKey PKCS12Container::getKey() const - { - return EVPPKey(_pKey); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_PKCS12Container_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h b/base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h deleted file mode 100644 index c0920b4099a..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/RSACipherImpl.h +++ /dev/null @@ -1,80 +0,0 @@ -// -// RSACipherImpl.h -// -// Library: Crypto -// Package: RSA -// Module: RSACipherImpl -// -// Definition of the RSACipherImpl class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_RSACipherImpl_INCLUDED -#define Crypto_RSACipherImpl_INCLUDED - - -#include -#include "Poco/Crypto/Cipher.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/Crypto/RSAKey.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class RSACipherImpl : public Cipher - /// An implementation of the Cipher class for - /// asymmetric (public-private key) encryption - /// based on the the RSA algorithm in OpenSSL's - /// crypto library. - /// - /// Encryption is using the public key, decryption - /// requires the private key. - { - public: - RSACipherImpl(const RSAKey & key, RSAPaddingMode paddingMode); - /// Creates a new RSACipherImpl object for the given RSAKey - /// and using the given padding mode. - - virtual ~RSACipherImpl(); - /// Destroys the RSACipherImpl. - - const std::string & name() const; - /// Returns the name of the Cipher. - - CryptoTransform * createEncryptor(); - /// Creates an encryptor object. - - CryptoTransform * createDecryptor(); - /// Creates a decryptor object. - - private: - RSAKey _key; - RSAPaddingMode _paddingMode; - OpenSSLInitializer _openSSLInitializer; - }; - - - // - // Inlines - // - inline const std::string & RSACipherImpl::name() const - { - return _key.name(); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_RSACipherImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h b/base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h deleted file mode 100644 index 980fae1f0f0..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/RSADigestEngine.h +++ /dev/null @@ -1,114 +0,0 @@ -// -// RSADigestEngine.h -// -// Library: Crypto -// Package: RSA -// Module: RSADigestEngine -// -// Definition of the RSADigestEngine class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_RSADigestEngine_INCLUDED -#define Crypto_RSADigestEngine_INCLUDED - - -#include -#include -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/DigestEngine.h" -#include "Poco/Crypto/RSAKey.h" -#include "Poco/DigestEngine.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API RSADigestEngine : public Poco::DigestEngine - /// This class implements a Poco::DigestEngine that can be - /// used to compute a secure digital signature. - /// - /// First another Poco::Crypto::DigestEngine is created and - /// used to compute a cryptographic hash of the data to be - /// signed. Then, the hash value is encrypted, using - /// the RSA private key. - /// - /// To verify a signature, pass it to the verify() - /// member function. It will decrypt the signature - /// using the RSA public key and compare the resulting - /// hash with the actual hash of the data. - { - public: - enum DigestType - { - DIGEST_MD5, - DIGEST_SHA1 - }; - - //@ deprecated - RSADigestEngine(const RSAKey & key, DigestType digestType = DIGEST_SHA1); - /// Creates the RSADigestEngine with the given RSA key, - /// using the MD5 or SHA-1 hash algorithm. - /// Kept for backward compatibility - - RSADigestEngine(const RSAKey & key, const std::string & name); - /// Creates the RSADigestEngine with the given RSA key, - /// using the hash algorithm with the given name - /// (e.g., "MD5", "SHA1", "SHA256", "SHA512", etc.). - /// See the OpenSSL documentation for a list of supported digest algorithms. - /// - /// Throws a Poco::NotFoundException if no algorithm with the given name exists. - - ~RSADigestEngine(); - /// Destroys the RSADigestEngine. - - std::size_t digestLength() const; - /// Returns the length of the digest in bytes. - - void reset(); - /// Resets the engine so that a new - /// digest can be computed. - - const DigestEngine::Digest & digest(); - /// Finishes the computation of the digest - /// (the first time it's called) and - /// returns the message digest. - /// - /// Can be called multiple times. - - const DigestEngine::Digest & signature(); - /// Signs the digest using the RSA algorithm - /// and the private key (the first time it's - /// called) and returns the result. - /// - /// Can be called multiple times. - - bool verify(const DigestEngine::Digest & signature); - /// Verifies the data against the signature. - /// - /// Returns true if the signature can be verified, false otherwise. - - protected: - void updateImpl(const void * data, std::size_t length); - - private: - RSAKey _key; - Poco::Crypto::DigestEngine _engine; - Poco::DigestEngine::Digest _digest; - Poco::DigestEngine::Digest _signature; - }; - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_RSADigestEngine_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/RSAKey.h b/base/poco/Crypto/include/Poco/Crypto/RSAKey.h deleted file mode 100644 index 47a01e15746..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/RSAKey.h +++ /dev/null @@ -1,124 +0,0 @@ -// -// RSAKey.h -// -// Library: Crypto -// Package: RSA -// Module: RSAKey -// -// Definition of the RSAKey class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_RSAKey_INCLUDED -#define Crypto_RSAKey_INCLUDED - - -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/KeyPair.h" -#include "Poco/Crypto/RSAKeyImpl.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class X509Certificate; - class PKCS12Container; - - - class Crypto_API RSAKey : public KeyPair - /// This class stores an RSA key pair, consisting - /// of private and public key. Storage of the private - /// key is optional. - /// - /// If a private key is available, the RSAKey can be - /// used for decrypting data (encrypted with the public key) - /// or computing secure digital signatures. - { - public: - enum KeyLength - { - KL_512 = 512, - KL_1024 = 1024, - KL_2048 = 2048, - KL_4096 = 4096 - }; - - enum Exponent - { - EXP_SMALL = 0, - EXP_LARGE - }; - - RSAKey(const EVPPKey & key); - /// Constructs ECKeyImpl by extracting the EC key. - - RSAKey(const X509Certificate & cert); - /// Extracts the RSA public key from the given certificate. - - RSAKey(const PKCS12Container & cert); - /// Extracts the RSA private key from the given certificate. - - RSAKey(KeyLength keyLength, Exponent exp); - /// Creates the RSAKey. Creates a new public/private keypair using the given parameters. - /// Can be used to sign data and verify signatures. - - RSAKey(const std::string & publicKeyFile, const std::string & privateKeyFile = "", const std::string & privateKeyPassphrase = ""); - /// Creates the RSAKey, by reading public and private key from the given files and - /// using the given passphrase for the private key. - /// - /// Cannot be used for signing or decryption unless a private key is available. - /// - /// If a private key is specified, you don't need to specify a public key file. - /// OpenSSL will auto-create the public key from the private key. - - RSAKey(std::istream * pPublicKeyStream, std::istream * pPrivateKeyStream = 0, const std::string & privateKeyPassphrase = ""); - /// Creates the RSAKey, by reading public and private key from the given streams and - /// using the given passphrase for the private key. - /// - /// Cannot be used for signing or decryption unless a private key is available. - /// - /// If a private key is specified, you don't need to specify a public key file. - /// OpenSSL will auto-create the public key from the private key. - - ~RSAKey(); - /// Destroys the RSAKey. - - RSAKeyImpl::ByteVec modulus() const; - /// Returns the RSA modulus. - - RSAKeyImpl::ByteVec encryptionExponent() const; - /// Returns the RSA encryption exponent. - - RSAKeyImpl::ByteVec decryptionExponent() const; - /// Returns the RSA decryption exponent. - - RSAKeyImpl::Ptr impl() const; - /// Returns the impl object. - - private: - RSAKeyImpl::Ptr _pImpl; - }; - - - // - // inlines - // - inline RSAKeyImpl::Ptr RSAKey::impl() const - { - return _pImpl; - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_RSAKey_INCLUDED \ No newline at end of file diff --git a/base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h b/base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h deleted file mode 100644 index 010c68bacd7..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/RSAKeyImpl.h +++ /dev/null @@ -1,128 +0,0 @@ -// -// RSAKeyImpl.h -// -// Library: Crypto -// Package: RSA -// Module: RSAKeyImpl -// -// Definition of the RSAKeyImpl class. -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_RSAKeyImplImpl_INCLUDED -#define Crypto_RSAKeyImplImpl_INCLUDED - - -#include -#include -#include -#include "Poco/AutoPtr.h" -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/EVPPKey.h" -#include "Poco/Crypto/KeyPairImpl.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/RefCountedObject.h" - - -struct bignum_st; -struct rsa_st; -typedef struct bignum_st BIGNUM; -typedef struct rsa_st RSA; - - -namespace Poco -{ -namespace Crypto -{ - - - class X509Certificate; - class PKCS12Container; - - - class RSAKeyImpl : public KeyPairImpl - /// class RSAKeyImpl - { - public: - typedef Poco::AutoPtr Ptr; - typedef std::vector ByteVec; - - RSAKeyImpl(const EVPPKey & key); - /// Constructs ECKeyImpl by extracting the EC key. - - RSAKeyImpl(const X509Certificate & cert); - /// Extracts the RSA public key from the given certificate. - - RSAKeyImpl(const PKCS12Container & cert); - /// Extracts the EC private key from the given certificate. - - RSAKeyImpl(int keyLength, unsigned long exponent); - /// Creates the RSAKey. Creates a new public/private keypair using the given parameters. - /// Can be used to sign data and verify signatures. - - RSAKeyImpl(const std::string & publicKeyFile, const std::string & privateKeyFile, const std::string & privateKeyPassphrase); - /// Creates the RSAKey, by reading public and private key from the given files and - /// using the given passphrase for the private key. Can only by used for signing if - /// a private key is available. - - RSAKeyImpl(std::istream * pPublicKeyStream, std::istream * pPrivateKeyStream, const std::string & privateKeyPassphrase); - /// Creates the RSAKey. Can only by used for signing if pPrivKey - /// is not null. If a private key file is specified, you don't need to - /// specify a public key file. OpenSSL will auto-create it from the private key. - - ~RSAKeyImpl(); - /// Destroys the RSAKeyImpl. - - RSA * getRSA(); - /// Returns the OpenSSL RSA object. - - const RSA * getRSA() const; - /// Returns the OpenSSL RSA object. - - int size() const; - /// Returns the RSA modulus size. - - ByteVec modulus() const; - /// Returns the RSA modulus. - - ByteVec encryptionExponent() const; - /// Returns the RSA encryption exponent. - - ByteVec decryptionExponent() const; - /// Returns the RSA decryption exponent. - - private: - RSAKeyImpl(); - - void freeRSA(); - static ByteVec convertToByteVec(const BIGNUM * bn); - - RSA * _pRSA; - }; - - - // - // inlines - // - inline RSA * RSAKeyImpl::getRSA() - { - return _pRSA; - } - - - inline const RSA * RSAKeyImpl::getRSA() const - { - return _pRSA; - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_RSAKeyImplImpl_INCLUDED diff --git a/base/poco/Crypto/include/Poco/Crypto/X509Certificate.h b/base/poco/Crypto/include/Poco/Crypto/X509Certificate.h deleted file mode 100644 index 30147623035..00000000000 --- a/base/poco/Crypto/include/Poco/Crypto/X509Certificate.h +++ /dev/null @@ -1,248 +0,0 @@ -// -// X509Certificate.h -// -// Library: Crypto -// Package: Certificate -// Module: X509Certificate -// -// Definition of the X509Certificate class. -// -// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Crypto_X509Certificate_INCLUDED -#define Crypto_X509Certificate_INCLUDED - - -#include -#include -#include -#include -#include "Poco/Crypto/Crypto.h" -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/DateTime.h" -#include "Poco/SharedPtr.h" - - -namespace Poco -{ -namespace Crypto -{ - - - class Crypto_API X509Certificate - /// This class represents a X509 Certificate. - { - public: - typedef std::vector List; - - enum NID - /// Name identifier for extracting information from - /// a certificate subject's or issuer's distinguished name. - { - NID_COMMON_NAME = 13, - NID_COUNTRY = 14, - NID_LOCALITY_NAME = 15, - NID_STATE_OR_PROVINCE = 16, - NID_ORGANIZATION_NAME = 17, - NID_ORGANIZATION_UNIT_NAME = 18, - NID_PKCS9_EMAIL_ADDRESS = 48, - NID_SERIAL_NUMBER = 105 - }; - - explicit X509Certificate(std::istream & istr); - /// Creates the X509Certificate object by reading - /// a certificate in PEM format from a stream. - - explicit X509Certificate(const std::string & path); - /// Creates the X509Certificate object by reading - /// a certificate in PEM format from a file. - - explicit X509Certificate(X509 * pCert); - /// Creates the X509Certificate from an existing - /// OpenSSL certificate. Ownership is taken of - /// the certificate. - - X509Certificate(X509 * pCert, bool shared); - /// Creates the X509Certificate from an existing - /// OpenSSL certificate. Ownership is taken of - /// the certificate. If shared is true, the - /// certificate's reference count is incremented. - - X509Certificate(const X509Certificate & cert); - /// Creates the certificate by copying another one. - - X509Certificate & operator=(const X509Certificate & cert); - /// Assigns a certificate. - - void swap(X509Certificate & cert); - /// Exchanges the certificate with another one. - - ~X509Certificate(); - /// Destroys the X509Certificate. - - long version() const; - /// Returns the version of the certificate. - - const std::string & serialNumber() const; - /// Returns the certificate serial number as a - /// string in decimal encoding. - - const std::string & issuerName() const; - /// Returns the certificate issuer's distinguished name. - - std::string issuerName(NID nid) const; - /// Extracts the information specified by the given - /// NID (name identifier) from the certificate issuer's - /// distinguished name. - - const std::string & subjectName() const; - /// Returns the certificate subject's distinguished name. - - std::string subjectName(NID nid) const; - /// Extracts the information specified by the given - /// NID (name identifier) from the certificate subject's - /// distinguished name. - - std::string commonName() const; - /// Returns the common name stored in the certificate - /// subject's distinguished name. - - void extractNames(std::string & commonName, std::set & domainNames) const; - /// Extracts the common name and the alias domain names from the - /// certificate. - - Poco::DateTime validFrom() const; - /// Returns the date and time the certificate is valid from. - - Poco::DateTime expiresOn() const; - /// Returns the date and time the certificate expires. - - void save(std::ostream & stream) const; - /// Writes the certificate to the given stream. - /// The certificate is written in PEM format. - - void save(const std::string & path) const; - /// Writes the certificate to the file given by path. - /// The certificate is written in PEM format. - - bool issuedBy(const X509Certificate & issuerCertificate) const; - /// Checks whether the certificate has been issued by - /// the issuer given by issuerCertificate. This can be - /// used to validate a certificate chain. - /// - /// Verifies if the certificate has been signed with the - /// issuer's private key, using the public key from the issuer - /// certificate. - /// - /// Returns true if verification against the issuer certificate - /// was successful, false otherwise. - - bool equals(const X509Certificate & otherCertificate) const; - /// Checks whether the certificate is equal to - /// the other certificate, by comparing the hashes - /// of both certificates. - /// - /// Returns true if both certificates are identical, - /// otherwise false. - - const X509 * certificate() const; - /// Returns the underlying OpenSSL certificate. - - X509 * dup() const; - /// Duplicates and returns the underlying OpenSSL certificate. Note that - /// the caller assumes responsibility for the lifecycle of the created - /// certificate. - - std::string signatureAlgorithm() const; - /// Returns the certificate signature algorithm long name. - - void print(std::ostream & out) const; - /// Prints the certificate information to ostream. - - static List readPEM(const std::string & pemFileName); - /// Reads and returns a list of certificates from - /// the specified PEM file. - - static void writePEM(const std::string & pemFileName, const List & list); - /// Writes the list of certificates to the specified PEM file. - - protected: - void load(std::istream & stream); - /// Loads the certificate from the given stream. The - /// certificate must be in PEM format. - - void load(const std::string & path); - /// Loads the certificate from the given file. The - /// certificate must be in PEM format. - - void init(); - /// Extracts issuer and subject name from the certificate. - - private: - enum - { - NAME_BUFFER_SIZE = 256 - }; - - std::string _issuerName; - std::string _subjectName; - std::string _serialNumber; - X509 * _pCert; - OpenSSLInitializer _openSSLInitializer; - }; - - - // - // inlines - // - - - inline long X509Certificate::version() const - { - // This is defined by standards (X.509 et al) to be - // one less than the certificate version. - // So, eg. a version 3 certificate will return 2. - return X509_get_version(_pCert) + 1; - } - - - inline const std::string & X509Certificate::serialNumber() const - { - return _serialNumber; - } - - - inline const std::string & X509Certificate::issuerName() const - { - return _issuerName; - } - - - inline const std::string & X509Certificate::subjectName() const - { - return _subjectName; - } - - - inline const X509 * X509Certificate::certificate() const - { - return _pCert; - } - - - inline X509 * X509Certificate::dup() const - { - return X509_dup(_pCert); - } - - -} -} // namespace Poco::Crypto - - -#endif // Crypto_X509Certificate_INCLUDED diff --git a/base/poco/Crypto/src/Cipher.cpp b/base/poco/Crypto/src/Cipher.cpp deleted file mode 100644 index 8b4d57964f5..00000000000 --- a/base/poco/Crypto/src/Cipher.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// -// Cipher.cpp -// -// Library: Crypto -// Package: Cipher -// Module: Cipher -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/Cipher.h" -#include "Poco/Crypto/CryptoStream.h" -#include "Poco/Crypto/CryptoTransform.h" -#include "Poco/Base64Encoder.h" -#include "Poco/Base64Decoder.h" -#include "Poco/HexBinaryEncoder.h" -#include "Poco/HexBinaryDecoder.h" -#include "Poco/StreamCopier.h" -#include "Poco/Exception.h" -#include -#include - - -namespace Poco { -namespace Crypto { - - -Cipher::Cipher() -{ -} - - -Cipher::~Cipher() -{ -} - - -std::string Cipher::encryptString(const std::string& str, Encoding encoding) -{ - std::istringstream source(str); - std::ostringstream sink; - - encrypt(source, sink, encoding); - - return sink.str(); -} - - -std::string Cipher::decryptString(const std::string& str, Encoding encoding) -{ - std::istringstream source(str); - std::ostringstream sink; - - decrypt(source, sink, encoding); - return sink.str(); -} - - -void Cipher::encrypt(std::istream& source, std::ostream& sink, Encoding encoding) -{ - CryptoInputStream encryptor(source, createEncryptor()); - - switch (encoding) - { - case ENC_NONE: - StreamCopier::copyStream(encryptor, sink); - break; - - case ENC_BASE64: - case ENC_BASE64_NO_LF: - { - Poco::Base64Encoder encoder(sink); - if (encoding == ENC_BASE64_NO_LF) - { - encoder.rdbuf()->setLineLength(0); - } - StreamCopier::copyStream(encryptor, encoder); - encoder.close(); - } - break; - - case ENC_BINHEX: - case ENC_BINHEX_NO_LF: - { - Poco::HexBinaryEncoder encoder(sink); - if (encoding == ENC_BINHEX_NO_LF) - { - encoder.rdbuf()->setLineLength(0); - } - StreamCopier::copyStream(encryptor, encoder); - encoder.close(); - } - break; - - default: - throw Poco::InvalidArgumentException("Invalid argument", "encoding"); - } -} - - -void Cipher::decrypt(std::istream& source, std::ostream& sink, Encoding encoding) -{ - CryptoOutputStream decryptor(sink, createDecryptor()); - - switch (encoding) - { - case ENC_NONE: - StreamCopier::copyStream(source, decryptor); - decryptor.close(); - break; - - case ENC_BASE64: - case ENC_BASE64_NO_LF: - { - Poco::Base64Decoder decoder(source); - StreamCopier::copyStream(decoder, decryptor); - decryptor.close(); - } - break; - - case ENC_BINHEX: - case ENC_BINHEX_NO_LF: - { - Poco::HexBinaryDecoder decoder(source); - StreamCopier::copyStream(decoder, decryptor); - decryptor.close(); - } - break; - - default: - throw Poco::InvalidArgumentException("Invalid argument", "encoding"); - } -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherFactory.cpp b/base/poco/Crypto/src/CipherFactory.cpp deleted file mode 100644 index 28ba3748643..00000000000 --- a/base/poco/Crypto/src/CipherFactory.cpp +++ /dev/null @@ -1,65 +0,0 @@ -// -// CipherFactory.cpp -// -// Library: Crypto -// Package: Cipher -// Module: CipherFactory -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CipherFactory.h" -#include "Poco/Crypto/Cipher.h" -#include "Poco/Crypto/CipherKey.h" -#include "Poco/Crypto/RSAKey.h" -#include "Poco/Crypto/CipherImpl.h" -#include "Poco/Crypto/RSACipherImpl.h" -#include "Poco/Exception.h" -#include "Poco/SingletonHolder.h" -#include -#include - - -namespace Poco { -namespace Crypto { - - -CipherFactory::CipherFactory() -{ -} - - -CipherFactory::~CipherFactory() -{ -} - - -namespace -{ - static Poco::SingletonHolder holder; -} - - -CipherFactory& CipherFactory::defaultFactory() -{ - return *holder.get(); -} - - -Cipher* CipherFactory::createCipher(const CipherKey& key) -{ - return new CipherImpl(key); -} - - -Cipher* CipherFactory::createCipher(const RSAKey& key, RSAPaddingMode paddingMode) -{ - return new RSACipherImpl(key, paddingMode); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherImpl.cpp b/base/poco/Crypto/src/CipherImpl.cpp deleted file mode 100644 index ae574775957..00000000000 --- a/base/poco/Crypto/src/CipherImpl.cpp +++ /dev/null @@ -1,272 +0,0 @@ -// -// CipherImpl.cpp -// -// Library: Crypto -// Package: Cipher -// Module: CipherImpl -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CipherImpl.h" -#include "Poco/Crypto/CryptoTransform.h" -#include "Poco/Exception.h" -#include "Poco/Buffer.h" -#include - - -namespace Poco { -namespace Crypto { - - -namespace -{ - void throwError() - { - unsigned long err; - std::string msg; - - while ((err = ERR_get_error())) - { - if (!msg.empty()) - msg.append("; "); - msg.append(ERR_error_string(err, 0)); - } - - throw Poco::IOException(msg); - } - - - class CryptoTransformImpl: public CryptoTransform - { - public: - typedef Cipher::ByteVec ByteVec; - - enum Direction - { - DIR_ENCRYPT, - DIR_DECRYPT - }; - - CryptoTransformImpl( - const EVP_CIPHER* pCipher, - const ByteVec& key, - const ByteVec& iv, - Direction dir); - - ~CryptoTransformImpl(); - - std::size_t blockSize() const; - int setPadding(int padding); - std::string getTag(std::size_t tagSize); - void setTag(const std::string& tag); - - std::streamsize transform( - const unsigned char* input, - std::streamsize inputLength, - unsigned char* output, - std::streamsize outputLength); - - std::streamsize finalize( - unsigned char* output, - std::streamsize length); - - private: - const EVP_CIPHER* _pCipher; -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - EVP_CIPHER_CTX* _pContext; -#else - EVP_CIPHER_CTX _context; -#endif - ByteVec _key; - ByteVec _iv; - }; - - - CryptoTransformImpl::CryptoTransformImpl( - const EVP_CIPHER* pCipher, - const ByteVec& key, - const ByteVec& iv, - Direction dir): - _pCipher(pCipher), - _key(key), - _iv(iv) - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - _pContext = EVP_CIPHER_CTX_new(); - EVP_CipherInit( - _pContext, - _pCipher, - &_key[0], - _iv.empty() ? 0 : &_iv[0], - (dir == DIR_ENCRYPT) ? 1 : 0); -#else - EVP_CipherInit( - &_context, - _pCipher, - &_key[0], - _iv.empty() ? 0 : &_iv[0], - (dir == DIR_ENCRYPT) ? 1 : 0); -#endif - -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - if (_iv.size() != EVP_CIPHER_iv_length(_pCipher) && EVP_CIPHER_mode(_pCipher) == EVP_CIPH_GCM_MODE) - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - int rc = EVP_CIPHER_CTX_ctrl(_pContext, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL); -#else - int rc = EVP_CIPHER_CTX_ctrl(&_context, EVP_CTRL_GCM_SET_IVLEN, _iv.size(), NULL); -#endif - if (rc == 0) throwError(); - } -#endif - } - - - CryptoTransformImpl::~CryptoTransformImpl() - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - EVP_CIPHER_CTX_cleanup(_pContext); - EVP_CIPHER_CTX_free(_pContext); -#else - EVP_CIPHER_CTX_cleanup(&_context); -#endif - } - - - std::size_t CryptoTransformImpl::blockSize() const - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - return EVP_CIPHER_CTX_block_size(_pContext); -#else - return EVP_CIPHER_CTX_block_size(&_context); -#endif - } - - - int CryptoTransformImpl::setPadding(int padding) - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - return EVP_CIPHER_CTX_block_size(_pContext); -#else - return EVP_CIPHER_CTX_set_padding(&_context, padding); -#endif - } - - - std::string CryptoTransformImpl::getTag(std::size_t tagSize) - { - std::string tag; -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - Poco::Buffer buffer(tagSize); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - int rc = EVP_CIPHER_CTX_ctrl(_pContext, EVP_CTRL_GCM_GET_TAG, tagSize, buffer.begin()); -#else - int rc = EVP_CIPHER_CTX_ctrl(&_context, EVP_CTRL_GCM_GET_TAG, tagSize, buffer.begin()); -#endif - if (rc == 0) throwError(); - tag.assign(buffer.begin(), tagSize); -#endif - return tag; - } - - - void CryptoTransformImpl::setTag(const std::string& tag) - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - int rc = EVP_CIPHER_CTX_ctrl(_pContext, EVP_CTRL_GCM_SET_TAG, tag.size(), const_cast(tag.data())); -#elif OPENSSL_VERSION_NUMBER >= 0x10001000L - int rc = EVP_CIPHER_CTX_ctrl(&_context, EVP_CTRL_GCM_SET_TAG, tag.size(), const_cast(tag.data())); -#else - int rc = 0; -#endif - if (rc == 0) throwError(); - } - - - std::streamsize CryptoTransformImpl::transform( - const unsigned char* input, - std::streamsize inputLength, - unsigned char* output, - std::streamsize outputLength) - { - poco_assert (outputLength >= (inputLength + blockSize() - 1)); - - int outLen = static_cast(outputLength); -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - int rc = EVP_CipherUpdate( - _pContext, - output, - &outLen, - input, - static_cast(inputLength)); -#else - int rc = EVP_CipherUpdate( - &_context, - output, - &outLen, - input, - static_cast(inputLength)); -#endif - if (rc == 0) - throwError(); - - return static_cast(outLen); - } - - - std::streamsize CryptoTransformImpl::finalize( - unsigned char* output, - std::streamsize length) - { - poco_assert (length >= blockSize()); - - int len = static_cast(length); - - // Use the '_ex' version that does not perform implicit cleanup since we - // will call EVP_CIPHER_CTX_cleanup() from the dtor as there is no - // guarantee that finalize() will be called if an error occurred. -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - int rc = EVP_CipherFinal_ex(_pContext, output, &len); -#else - int rc = EVP_CipherFinal_ex(&_context, output, &len); -#endif - - if (rc == 0) - throwError(); - - return static_cast(len); - } -} - - -CipherImpl::CipherImpl(const CipherKey& key): - _key(key) -{ -} - - -CipherImpl::~CipherImpl() -{ -} - - -CryptoTransform* CipherImpl::createEncryptor() -{ - CipherKeyImpl::Ptr p = _key.impl(); - return new CryptoTransformImpl(p->cipher(), p->getKey(), p->getIV(), CryptoTransformImpl::DIR_ENCRYPT); -} - - -CryptoTransform* CipherImpl::createDecryptor() -{ - CipherKeyImpl::Ptr p = _key.impl(); - return new CryptoTransformImpl(p->cipher(), p->getKey(), p->getIV(), CryptoTransformImpl::DIR_DECRYPT); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherKey.cpp b/base/poco/Crypto/src/CipherKey.cpp deleted file mode 100644 index 275edd1c883..00000000000 --- a/base/poco/Crypto/src/CipherKey.cpp +++ /dev/null @@ -1,49 +0,0 @@ -// -// CipherKey.cpp -// -// Library: Crypto -// Package: Cipher -// Module: CipherKey -// -// Copyright (c) 2007, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CipherKey.h" - - -namespace Poco { -namespace Crypto { - - -CipherKey::CipherKey(const std::string& name, - const std::string& passphrase, - const std::string& salt, - int iterationCount, - const std::string &digest): - _pImpl(new CipherKeyImpl(name, passphrase, salt, iterationCount, digest)) -{ -} - - -CipherKey::CipherKey(const std::string& name, const ByteVec& key, const ByteVec& iv): - _pImpl(new CipherKeyImpl(name, key, iv)) -{ -} - - -CipherKey::CipherKey(const std::string& name): - _pImpl(new CipherKeyImpl(name)) -{ -} - - -CipherKey::~CipherKey() -{ -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CipherKeyImpl.cpp b/base/poco/Crypto/src/CipherKeyImpl.cpp deleted file mode 100644 index d5ebc5e3b17..00000000000 --- a/base/poco/Crypto/src/CipherKeyImpl.cpp +++ /dev/null @@ -1,222 +0,0 @@ -// -// CipherKeyImpl.cpp -// -// Library: Crypto -// Package: Cipher -// Module: CipherKeyImpl -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CipherKeyImpl.h" -#include "Poco/Crypto/CryptoTransform.h" -#include "Poco/Crypto/CipherFactory.h" -#include "Poco/Exception.h" -#include "Poco/RandomStream.h" -#include -#include - - -namespace Poco { -namespace Crypto { - - -CipherKeyImpl::CipherKeyImpl(const std::string& name, - const std::string& passphrase, - const std::string& salt, - int iterationCount, - const std::string& digest): - _pCipher(0), - _pDigest(0), - _name(name), - _key(), - _iv() -{ - // dummy access to Cipherfactory so that the EVP lib is initilaized - CipherFactory::defaultFactory(); - _pCipher = EVP_get_cipherbyname(name.c_str()); - - if (!_pCipher) - throw Poco::NotFoundException("Cipher " + name + " was not found"); - - _pDigest = EVP_get_digestbyname(digest.c_str()); - - if (!_pDigest) - throw Poco::NotFoundException("Digest " + name + " was not found"); - - _key = ByteVec(keySize()); - _iv = ByteVec(ivSize()); - generateKey(passphrase, salt, iterationCount); -} - - -CipherKeyImpl::CipherKeyImpl(const std::string& name, - const ByteVec& key, - const ByteVec& iv): - _pCipher(0), - _pDigest(0), - _name(name), - _key(key), - _iv(iv) -{ - // dummy access to Cipherfactory so that the EVP lib is initialized - CipherFactory::defaultFactory(); - _pCipher = EVP_get_cipherbyname(name.c_str()); - - if (!_pCipher) - throw Poco::NotFoundException("Cipher " + name + " was not found"); -} - - -CipherKeyImpl::CipherKeyImpl(const std::string& name): - _pCipher(0), - _pDigest(0), - _name(name), - _key(), - _iv() -{ - // dummy access to Cipherfactory so that the EVP lib is initilaized - CipherFactory::defaultFactory(); - _pCipher = EVP_get_cipherbyname(name.c_str()); - - if (!_pCipher) - throw Poco::NotFoundException("Cipher " + name + " was not found"); - _key = ByteVec(keySize()); - _iv = ByteVec(ivSize()); - generateKey(); -} - - -CipherKeyImpl::~CipherKeyImpl() -{ -} - - -CipherKeyImpl::Mode CipherKeyImpl::mode() const -{ - switch (EVP_CIPHER_mode(_pCipher)) - { - case EVP_CIPH_STREAM_CIPHER: - return MODE_STREAM_CIPHER; - - case EVP_CIPH_ECB_MODE: - return MODE_ECB; - - case EVP_CIPH_CBC_MODE: - return MODE_CBC; - - case EVP_CIPH_CFB_MODE: - return MODE_CFB; - - case EVP_CIPH_OFB_MODE: - return MODE_OFB; - -#if OPENSSL_VERSION_NUMBER >= 0x10001000L - case EVP_CIPH_CTR_MODE: - return MODE_CTR; - - case EVP_CIPH_GCM_MODE: - return MODE_GCM; - -#endif - } - throw Poco::IllegalStateException("Unexpected value of EVP_CIPHER_mode()"); -} - - -void CipherKeyImpl::generateKey() -{ - ByteVec vec; - - getRandomBytes(vec, keySize()); - setKey(vec); - - getRandomBytes(vec, ivSize()); - setIV(vec); -} - - -void CipherKeyImpl::getRandomBytes(ByteVec& vec, std::size_t count) -{ - Poco::RandomInputStream random; - - vec.clear(); - vec.reserve(count); - - for (int i = 0; i < count; ++i) - vec.push_back(static_cast(random.get())); -} - - -void CipherKeyImpl::generateKey( - const std::string& password, - const std::string& salt, - int iterationCount) -{ - unsigned char keyBytes[EVP_MAX_KEY_LENGTH]; - unsigned char ivBytes[EVP_MAX_IV_LENGTH]; - - // OpenSSL documentation specifies that the salt must be an 8-byte array. - unsigned char saltBytes[8]; - - if (!salt.empty()) - { - int len = static_cast(salt.size()); - // Create the salt array from the salt string - for (int i = 0; i < 8; ++i) - saltBytes[i] = salt.at(i % len); - for (int i = 8; i < len; ++i) - saltBytes[i % 8] ^= salt.at(i); - } - - // Now create the key and IV, using the MD5 digest algorithm. - int keySize = EVP_BytesToKey( - _pCipher, - _pDigest ? _pDigest : EVP_md5(), - (salt.empty() ? 0 : saltBytes), - reinterpret_cast(password.data()), - static_cast(password.size()), - iterationCount, - keyBytes, - ivBytes); - - // Copy the buffers to our member byte vectors. - _key.assign(keyBytes, keyBytes + keySize); - - if (ivSize() == 0) - _iv.clear(); - else - _iv.assign(ivBytes, ivBytes + ivSize()); -} - - -int CipherKeyImpl::keySize() const -{ - return EVP_CIPHER_key_length(_pCipher); -} - - -int CipherKeyImpl::blockSize() const -{ - return EVP_CIPHER_block_size(_pCipher); -} - - -int CipherKeyImpl::ivSize() const -{ - return EVP_CIPHER_iv_length(_pCipher); -} - - -void CipherKeyImpl::setIV(const ByteVec& iv) -{ - poco_assert(mode() == MODE_GCM || iv.size() == static_cast(ivSize())); - _iv = iv; -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CryptoException.cpp b/base/poco/Crypto/src/CryptoException.cpp deleted file mode 100644 index a4450af9bf0..00000000000 --- a/base/poco/Crypto/src/CryptoException.cpp +++ /dev/null @@ -1,108 +0,0 @@ -// -// CryptoException.cpp -// -// -// Library: Crypto -// Package: Crypto -// Module: CryptoException -// -// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CryptoException.h" -#include "Poco/NumberFormatter.h" -#include -#include - - -namespace Poco { -namespace Crypto { - - -POCO_IMPLEMENT_EXCEPTION(CryptoException, Exception, "Crypto Exception") - - -OpenSSLException::OpenSSLException(int otherCode): CryptoException(otherCode) -{ - setExtMessage(); -} - - -OpenSSLException::OpenSSLException(const std::string& msg, int otherCode): CryptoException(msg, otherCode) -{ - setExtMessage(); -} - - -OpenSSLException::OpenSSLException(const std::string& msg, const std::string& arg, int otherCode): CryptoException(msg, arg, otherCode) -{ - setExtMessage(); -} - - -OpenSSLException::OpenSSLException(const std::string& msg, const Poco::Exception& exc, int otherCode): CryptoException(msg, exc, otherCode) -{ - setExtMessage(); -} - - -OpenSSLException::OpenSSLException(const OpenSSLException& exc): CryptoException(exc) -{ - setExtMessage(); -} - - -OpenSSLException::~OpenSSLException() throw() -{ -} - - -OpenSSLException& OpenSSLException::operator = (const OpenSSLException& exc) -{ - CryptoException::operator = (exc); - return *this; -} - - -const char* OpenSSLException::name() const throw() -{ - return "OpenSSLException"; -} - - -const char* OpenSSLException::className() const throw() -{ - return typeid(*this).name(); -} - - -Poco::Exception* OpenSSLException::clone() const -{ - return new OpenSSLException(*this); -} - - -void OpenSSLException::setExtMessage() -{ - Poco::UInt64 e = static_cast(ERR_get_error()); - char buf[128] = { 0 }; - char* pErr = ERR_error_string(static_cast(e), buf); - std::string err; - if (pErr) err = pErr; - else err = NumberFormatter::format(e); - - extendedMessage(err); -} - - -void OpenSSLException::rethrow() const -{ - throw *this; -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CryptoStream.cpp b/base/poco/Crypto/src/CryptoStream.cpp deleted file mode 100644 index 9cb693990a3..00000000000 --- a/base/poco/Crypto/src/CryptoStream.cpp +++ /dev/null @@ -1,355 +0,0 @@ -// -// CryptoStream.cpp -// -// Library: Crypto -// Package: Cipher -// Module: CryptoStream -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CryptoStream.h" -#include "Poco/Crypto/CryptoTransform.h" -#include "Poco/Crypto/Cipher.h" -#include "Poco/Exception.h" -#include - - -#undef min -#undef max - - -namespace Poco { -namespace Crypto { - - -// -// CryptoStreamBuf -// - - -CryptoStreamBuf::CryptoStreamBuf(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize): - Poco::BufferedStreamBuf(bufferSize, std::ios::in), - _pTransform(pTransform), - _pIstr(&istr), - _pOstr(0), - _eof(false), - _buffer(static_cast(bufferSize)) -{ - poco_check_ptr (pTransform); - poco_assert (bufferSize > 2 * pTransform->blockSize()); -} - - -CryptoStreamBuf::CryptoStreamBuf(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize): - Poco::BufferedStreamBuf(bufferSize, std::ios::out), - _pTransform(pTransform), - _pIstr(0), - _pOstr(&ostr), - _eof(false), - _buffer(static_cast(bufferSize)) -{ - poco_check_ptr (pTransform); - poco_assert (bufferSize > 2 * pTransform->blockSize()); -} - - -CryptoStreamBuf::~CryptoStreamBuf() -{ - try - { - close(); - } - catch (...) - { - } - delete _pTransform; -} - - -void CryptoStreamBuf::close() -{ - sync(); - - if (_pIstr) - { - _pIstr = 0; - } - else if (_pOstr) - { - // Close can be called multiple times. By zeroing the pointer we make - // sure that we call finalize() only once, even if an exception is - // thrown. - std::ostream* pOstr = _pOstr; - _pOstr = 0; - - // Finalize transformation. - std::streamsize n = _pTransform->finalize(_buffer.begin(), static_cast(_buffer.size())); - - if (n > 0) - { - pOstr->write(reinterpret_cast(_buffer.begin()), n); - if (!pOstr->good()) - throw Poco::IOException("Output stream failure"); - } - } -} - - -int CryptoStreamBuf::readFromDevice(char* buffer, std::streamsize length) -{ - if (!_pIstr) - return 0; - - int count = 0; - - while (!_eof) - { - int m = (static_cast(length) - count)/2 - static_cast(_pTransform->blockSize()); - - // Make sure we can read at least one more block. Explicitly check - // for m < 0 since blockSize() returns an unsigned int and the - // comparison might give false results for m < 0. - if (m <= 0) - break; - - int n = 0; - - if (_pIstr->good()) - { - _pIstr->read(reinterpret_cast(_buffer.begin()), m); - n = static_cast(_pIstr->gcount()); - } - - if (n == 0) - { - _eof = true; - - // No more data, finalize transformation - count += static_cast(_pTransform->finalize( - reinterpret_cast(buffer + count), - static_cast(length) - count)); - } - else - { - // Transform next chunk of data - count += static_cast(_pTransform->transform( - _buffer.begin(), - n, - reinterpret_cast(buffer + count), - static_cast(length) - count)); - } - } - - return count; -} - - -int CryptoStreamBuf::writeToDevice(const char* buffer, std::streamsize length) -{ - if (!_pOstr) - return 0; - - std::size_t maxChunkSize = _buffer.size()/2; - std::size_t count = 0; - - while (count < length) - { - // Truncate chunk size so that the maximum output fits into _buffer. - std::size_t n = static_cast(length) - count; - if (n > maxChunkSize) - n = maxChunkSize; - - // Transform next chunk of data - std::streamsize k = _pTransform->transform( - reinterpret_cast(buffer + count), - static_cast(n), - _buffer.begin(), - static_cast(_buffer.size())); - - // Attention: (n != k) might be true. In count, we have to track how - // many bytes from buffer have been consumed, not how many bytes have - // been written to _pOstr! - count += n; - - if (k > 0) - { - _pOstr->write(reinterpret_cast(_buffer.begin()), k); - if (!_pOstr->good()) - throw Poco::IOException("Output stream failure"); - } - } - - return static_cast(count); -} - - -// -// CryptoIOS -// - - -CryptoIOS::CryptoIOS(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize): - _buf(istr, pTransform, bufferSize) -{ - poco_ios_init(&_buf); -} - - -CryptoIOS::CryptoIOS(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize): - _buf(ostr, pTransform, bufferSize) -{ - poco_ios_init(&_buf); -} - - -CryptoIOS::~CryptoIOS() -{ -} - - -CryptoStreamBuf* CryptoIOS::rdbuf() -{ - return &_buf; -} - - -// -// CryptoInputStream -// - - -CryptoInputStream::CryptoInputStream(std::istream& istr, CryptoTransform* pTransform, std::streamsize bufferSize): - CryptoIOS(istr, pTransform, bufferSize), - std::istream(&_buf) -{ -} - - -CryptoInputStream::CryptoInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize): - CryptoIOS(istr, cipher.createEncryptor(), bufferSize), - std::istream(&_buf) -{ -} - - -CryptoInputStream::~CryptoInputStream() -{ -} - - -// -// CryptoOutputStream -// - - -CryptoOutputStream::CryptoOutputStream(std::ostream& ostr, CryptoTransform* pTransform, std::streamsize bufferSize): - CryptoIOS(ostr, pTransform, bufferSize), - std::ostream(&_buf) -{ -} - - -CryptoOutputStream::CryptoOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize): - CryptoIOS(ostr, cipher.createDecryptor(), bufferSize), - std::ostream(&_buf) -{ -} - - -CryptoOutputStream::~CryptoOutputStream() -{ -} - - -void CryptoOutputStream::close() -{ - _buf.close(); -} - - -// -// EncryptingInputStream -// - - -EncryptingInputStream::EncryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize): - CryptoIOS(istr, cipher.createEncryptor(), bufferSize), - std::istream(&_buf) -{ -} - - -EncryptingInputStream::~EncryptingInputStream() -{ -} - - -// -// EncryptingOuputStream -// - - -EncryptingOutputStream::EncryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize): - CryptoIOS(ostr, cipher.createEncryptor(), bufferSize), - std::ostream(&_buf) -{ -} - - -EncryptingOutputStream::~EncryptingOutputStream() -{ -} - - -void EncryptingOutputStream::close() -{ - _buf.close(); -} - - -// -// DecryptingInputStream -// - - -DecryptingInputStream::DecryptingInputStream(std::istream& istr, Cipher& cipher, std::streamsize bufferSize): - CryptoIOS(istr, cipher.createDecryptor(), bufferSize), - std::istream(&_buf) -{ -} - - -DecryptingInputStream::~DecryptingInputStream() -{ -} - - -// -// DecryptingOuputStream -// - - -DecryptingOutputStream::DecryptingOutputStream(std::ostream& ostr, Cipher& cipher, std::streamsize bufferSize): - CryptoIOS(ostr, cipher.createDecryptor(), bufferSize), - std::ostream(&_buf) -{ -} - - -DecryptingOutputStream::~DecryptingOutputStream() -{ -} - - -void DecryptingOutputStream::close() -{ - _buf.close(); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/CryptoTransform.cpp b/base/poco/Crypto/src/CryptoTransform.cpp deleted file mode 100644 index e4e58fd6e64..00000000000 --- a/base/poco/Crypto/src/CryptoTransform.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// -// CryptoTransform.cpp -// -// Library: Crypto -// Package: Cipher -// Module: CryptoTransform -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/CryptoTransform.h" - - -namespace Poco { -namespace Crypto { - - -CryptoTransform::CryptoTransform() -{ -} - - -CryptoTransform::~CryptoTransform() -{ -} - - -int CryptoTransform::setPadding(int padding) -{ - return 1; -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/DigestEngine.cpp b/base/poco/Crypto/src/DigestEngine.cpp deleted file mode 100644 index a09879fe732..00000000000 --- a/base/poco/Crypto/src/DigestEngine.cpp +++ /dev/null @@ -1,80 +0,0 @@ -// -// DigestEngine.cpp -// -// Library: Crypto -// Package: Digest -// Module: DigestEngine -// -// Copyright (c) 2012, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/DigestEngine.h" -#include "Poco/Exception.h" - - -namespace Poco { -namespace Crypto { - - -DigestEngine::DigestEngine(const std::string& name): - _name(name), - _pContext(EVP_MD_CTX_create()) -{ - const EVP_MD* md = EVP_get_digestbyname(_name.c_str()); - if (!md) throw Poco::NotFoundException(_name); - EVP_DigestInit_ex(_pContext, md, NULL); -} - - -DigestEngine::~DigestEngine() -{ - EVP_MD_CTX_destroy(_pContext); -} - -int DigestEngine::nid() const -{ - return EVP_MD_type(EVP_MD_CTX_md(_pContext)); -} - -std::size_t DigestEngine::digestLength() const -{ - return EVP_MD_CTX_size(_pContext); -} - - -void DigestEngine::reset() -{ -#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) - EVP_MD_CTX_free(_pContext); - _pContext = EVP_MD_CTX_create(); -#else - EVP_MD_CTX_cleanup(_pContext); -#endif - const EVP_MD* md = EVP_get_digestbyname(_name.c_str()); - if (!md) throw Poco::NotFoundException(_name); - EVP_DigestInit_ex(_pContext, md, NULL); -} - - -const Poco::DigestEngine::Digest& DigestEngine::digest() -{ - _digest.clear(); - unsigned len = EVP_MD_CTX_size(_pContext); - _digest.resize(len); - EVP_DigestFinal_ex(_pContext, &_digest[0], &len); - reset(); - return _digest; -} - - -void DigestEngine::updateImpl(const void* data, std::size_t length) -{ - EVP_DigestUpdate(_pContext, data, length); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/ECDSADigestEngine.cpp b/base/poco/Crypto/src/ECDSADigestEngine.cpp deleted file mode 100644 index 59512770cd1..00000000000 --- a/base/poco/Crypto/src/ECDSADigestEngine.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// -// ECDSADigestEngine.cpp -// -// -// Library: Crypto -// Package: ECDSA -// Module: ECDSADigestEngine -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/ECDSADigestEngine.h" -#include - - -namespace Poco { -namespace Crypto { - - -ECDSADigestEngine::ECDSADigestEngine(const ECKey& key, const std::string &name): - _key(key), - _engine(name) -{ -} - - -ECDSADigestEngine::~ECDSADigestEngine() -{ -} - - -std::size_t ECDSADigestEngine::digestLength() const -{ - return _engine.digestLength(); -} - - -void ECDSADigestEngine::reset() -{ - _engine.reset(); - _digest.clear(); - _signature.clear(); -} - - -const DigestEngine::Digest& ECDSADigestEngine::digest() -{ - if (_digest.empty()) - { - _digest = _engine.digest(); - } - return _digest; -} - - -const DigestEngine::Digest& ECDSADigestEngine::signature() -{ - if (_signature.empty()) - { - digest(); - _signature.resize(_key.size()); - unsigned sigLen = static_cast(_signature.size()); - if (!ECDSA_sign(0, &_digest[0], static_cast(_digest.size()), - &_signature[0], &sigLen, _key.impl()->getECKey())) - { - throw OpenSSLException(); - } - if (sigLen < _signature.size()) _signature.resize(sigLen); - } - return _signature; -} - - -bool ECDSADigestEngine::verify(const DigestEngine::Digest& sig) -{ - digest(); - EC_KEY* pKey = _key.impl()->getECKey(); - if (pKey) - { - int ret = ECDSA_verify(0, &_digest[0], static_cast(_digest.size()), - &sig[0], static_cast(sig.size()), - pKey); - if (1 == ret) return true; - else if (0 == ret) return false; - } - throw OpenSSLException(); -} - - -void ECDSADigestEngine::updateImpl(const void* data, std::size_t length) -{ - _engine.update(data, length); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/ECKey.cpp b/base/poco/Crypto/src/ECKey.cpp deleted file mode 100644 index 56b8663776e..00000000000 --- a/base/poco/Crypto/src/ECKey.cpp +++ /dev/null @@ -1,75 +0,0 @@ -// -// ECKey.cpp -// -// -// Library: Crypto -// Package: EC -// Module: ECKey -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/ECKey.h" -#include - - -namespace Poco { -namespace Crypto { - - -ECKey::ECKey(const EVPPKey& key): - KeyPair(new ECKeyImpl(key)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -ECKey::ECKey(const X509Certificate& cert): - KeyPair(new ECKeyImpl(cert)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -ECKey::ECKey(const PKCS12Container& cont): - KeyPair(new ECKeyImpl(cont)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -ECKey::ECKey(const std::string& eccGroup): - KeyPair(new ECKeyImpl(OBJ_txt2nid(eccGroup.c_str()))), - _pImpl(KeyPair::impl().cast()) -{ -} - - -ECKey::ECKey(const std::string& publicKeyFile, - const std::string& privateKeyFile, - const std::string& privateKeyPassphrase): - KeyPair(new ECKeyImpl(publicKeyFile, privateKeyFile, privateKeyPassphrase)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -ECKey::ECKey(std::istream* pPublicKeyStream, - std::istream* pPrivateKeyStream, - const std::string& privateKeyPassphrase): - KeyPair(new ECKeyImpl(pPublicKeyStream, pPrivateKeyStream, privateKeyPassphrase)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -ECKey::~ECKey() -{ -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/ECKeyImpl.cpp b/base/poco/Crypto/src/ECKeyImpl.cpp deleted file mode 100644 index 99ffae76067..00000000000 --- a/base/poco/Crypto/src/ECKeyImpl.cpp +++ /dev/null @@ -1,258 +0,0 @@ -// -// ECKeyImpl.cpp -// -// -// Library: Crypto -// Package: EC -// Module: ECKeyImpl -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/ECKeyImpl.h" -#include "Poco/Crypto/X509Certificate.h" -#include "Poco/Crypto/PKCS12Container.h" -#include "Poco/FileStream.h" -#include "Poco/Format.h" -#include "Poco/StreamCopier.h" -#include -#include -#if OPENSSL_VERSION_NUMBER >= 0x00908000L -#include -#endif - - -namespace Poco { -namespace Crypto { - - -ECKeyImpl::ECKeyImpl(const EVPPKey& key): - KeyPairImpl("ec", KT_EC_IMPL), - _pEC(EVP_PKEY_get1_EC_KEY(const_cast((const EVP_PKEY*)key))) -{ - checkEC("ECKeyImpl(const EVPPKey&)", "EVP_PKEY_get1_EC_KEY()"); -} - - -ECKeyImpl::ECKeyImpl(const X509Certificate& cert): - KeyPairImpl("ec", KT_EC_IMPL), - _pEC(0) -{ - const X509* pCert = cert.certificate(); - if (pCert) - { - EVP_PKEY* pKey = X509_get_pubkey(const_cast(pCert)); - if (pKey) - { - _pEC = EVP_PKEY_get1_EC_KEY(pKey); - EVP_PKEY_free(pKey); - checkEC("ECKeyImpl(const const X509Certificate&)", "EVP_PKEY_get1_EC_KEY()"); - return; - } - } - throw OpenSSLException("ECKeyImpl(const X509Certificate&)"); -} - - -ECKeyImpl::ECKeyImpl(const PKCS12Container& cont): - KeyPairImpl("ec", KT_EC_IMPL), - _pEC(EVP_PKEY_get1_EC_KEY(cont.getKey())) -{ - checkEC("ECKeyImpl(const PKCS12Container&)", "EVP_PKEY_get1_EC_KEY()"); -} - - -ECKeyImpl::ECKeyImpl(int curve): - KeyPairImpl("ec", KT_EC_IMPL), - _pEC(EC_KEY_new_by_curve_name(curve)) -{ - poco_check_ptr(_pEC); - EC_KEY_set_asn1_flag(_pEC, OPENSSL_EC_NAMED_CURVE); - if (!(EC_KEY_generate_key(_pEC))) - throw OpenSSLException("ECKeyImpl(int curve): EC_KEY_generate_key()"); - checkEC("ECKeyImpl(int curve)", "EC_KEY_generate_key()"); -} - - -ECKeyImpl::ECKeyImpl(const std::string& publicKeyFile, - const std::string& privateKeyFile, - const std::string& privateKeyPassphrase): KeyPairImpl("ec", KT_EC_IMPL), _pEC(0) -{ - if (EVPPKey::loadKey(&_pEC, PEM_read_PrivateKey, EVP_PKEY_get1_EC_KEY, privateKeyFile, privateKeyPassphrase)) - { - checkEC(Poco::format("ECKeyImpl(%s, %s, %s)", - publicKeyFile, privateKeyFile, privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), - "PEM_read_PrivateKey() or EVP_PKEY_get1_EC_KEY()"); - return; // private key is enough - } - - // no private key, this must be public key only, otherwise throw - if (!EVPPKey::loadKey(&_pEC, PEM_read_PUBKEY, EVP_PKEY_get1_EC_KEY, publicKeyFile)) - { - throw OpenSSLException("ECKeyImpl(const string&, const string&, const string&"); - } - checkEC(Poco::format("ECKeyImpl(%s, %s, %s)", - publicKeyFile, privateKeyFile, privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), - "PEM_read_PUBKEY() or EVP_PKEY_get1_EC_KEY()"); -} - - -ECKeyImpl::ECKeyImpl(std::istream* pPublicKeyStream, - std::istream* pPrivateKeyStream, - const std::string& privateKeyPassphrase): KeyPairImpl("ec", KT_EC_IMPL), _pEC(0) -{ - if (EVPPKey::loadKey(&_pEC, PEM_read_bio_PrivateKey, EVP_PKEY_get1_EC_KEY, pPrivateKeyStream, privateKeyPassphrase)) - { - checkEC(Poco::format("ECKeyImpl(stream, stream, %s)", - privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), - "PEM_read_bio_PrivateKey() or EVP_PKEY_get1_EC_KEY()"); - return; // private key is enough - } - - // no private key, this must be public key only, otherwise throw - if (!EVPPKey::loadKey(&_pEC, PEM_read_bio_PUBKEY, EVP_PKEY_get1_EC_KEY, pPublicKeyStream)) - { - throw OpenSSLException("ECKeyImpl(istream*, istream*, const string&"); - } - checkEC(Poco::format("ECKeyImpl(stream, stream, %s)", - privateKeyPassphrase.empty() ? privateKeyPassphrase : std::string("***")), - "PEM_read_bio_PUBKEY() or EVP_PKEY_get1_EC_KEY()"); -} - - -ECKeyImpl::~ECKeyImpl() -{ - freeEC(); -} - - -void ECKeyImpl::checkEC(const std::string& method, const std::string& func) const -{ - if (!_pEC) throw OpenSSLException(Poco::format("%s: %s", method, func)); - if (!EC_KEY_check_key(_pEC)) - throw OpenSSLException(Poco::format("%s: EC_KEY_check_key()", method)); -} - - -void ECKeyImpl::freeEC() -{ - if (_pEC) - { - EC_KEY_free(_pEC); - _pEC = 0; - } -} - - -int ECKeyImpl::size() const -{ - int sz = -1; - EVP_PKEY* pKey = EVP_PKEY_new(); - if (pKey && EVP_PKEY_set1_EC_KEY(pKey, _pEC)) - { - sz = EVP_PKEY_bits(pKey); - EVP_PKEY_free(pKey); - return sz; - } - throw OpenSSLException("ECKeyImpl::size()"); -} - - -int ECKeyImpl::groupId() const -{ - if (_pEC) - { - const EC_GROUP* ecGroup = EC_KEY_get0_group(_pEC); - if (ecGroup) - { - return EC_GROUP_get_curve_name(ecGroup); - } - else - { - throw OpenSSLException("ECKeyImpl::groupName()"); - } - } - throw NullPointerException("ECKeyImpl::groupName() => _pEC"); -} - - -std::string ECKeyImpl::getCurveName(int nid) -{ - std::string curveName; - size_t len = EC_get_builtin_curves(NULL, 0); - EC_builtin_curve* pCurves = - (EC_builtin_curve*) OPENSSL_malloc(sizeof(EC_builtin_curve) * len); - if (!pCurves) return curveName; - - if (!EC_get_builtin_curves(pCurves, len)) - { - OPENSSL_free(pCurves); - return curveName; - } - - if (-1 == nid) nid = pCurves[0].nid; - const int bufLen = 128; - char buf[bufLen]; - std::memset(buf, 0, bufLen); - OBJ_obj2txt(buf, bufLen, OBJ_nid2obj(nid), 0); - curveName = buf; - OPENSSL_free(pCurves); - return curveName; -} - - -int ECKeyImpl::getCurveNID(std::string& name) -{ - std::string curveName; - size_t len = EC_get_builtin_curves(NULL, 0); - EC_builtin_curve* pCurves = - (EC_builtin_curve*)OPENSSL_malloc(static_cast(sizeof(EC_builtin_curve) * len)); - if (!pCurves) return -1; - - if (!EC_get_builtin_curves(pCurves, len)) - { - OPENSSL_free(pCurves); - return -1; - } - - int nid = -1; - const int bufLen = 128; - char buf[bufLen]; - if (name.empty()) - { - std::memset(buf, 0, bufLen); - OBJ_obj2txt(buf, bufLen, OBJ_nid2obj(nid), 0); - name = buf; - nid = pCurves[0].nid; - } - else - { - for (int i = 0; i < len; ++i) - { - std::memset(buf, 0, bufLen); - OBJ_obj2txt(buf, bufLen, OBJ_nid2obj(pCurves[i].nid), 0); - if (strncmp(name.c_str(), buf, name.size() > bufLen ? bufLen : name.size()) == 0) - { - nid = pCurves[i].nid; - break; - } - } - } - - OPENSSL_free(pCurves); - return nid; -} - - -bool ECKeyImpl::hasCurve(const std::string& name) -{ - std::string tmp(name); - return (-1 != getCurveNID(tmp)); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/EVPPKey.cpp b/base/poco/Crypto/src/EVPPKey.cpp deleted file mode 100644 index ffa0a78c070..00000000000 --- a/base/poco/Crypto/src/EVPPKey.cpp +++ /dev/null @@ -1,321 +0,0 @@ -// -// EVPPKey.cpp -// -// -// Library: Crypto -// Package: CryptoCore -// Module: EVPPKey -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/EVPPKey.h" -#include "Poco/Crypto/ECKey.h" -#include "Poco/Crypto/RSAKey.h" -#include "Poco/NumberFormatter.h" - - -namespace Poco { -namespace Crypto { - - -EVPPKey::EVPPKey(const std::string& ecCurveName): _pEVPPKey(0) -{ - newECKey(ecCurveName.c_str()); - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey::EVPPKey(const char* ecCurveName): _pEVPPKey(0) -{ - newECKey(ecCurveName); - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey::EVPPKey(EVP_PKEY* pEVPPKey): _pEVPPKey(0) -{ - duplicate(pEVPPKey, &_pEVPPKey); - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey::EVPPKey(const std::string& publicKeyFile, - const std::string& privateKeyFile, - const std::string& privateKeyPassphrase): _pEVPPKey(0) -{ - if (loadKey(&_pEVPPKey, PEM_read_PrivateKey, (EVP_PKEY_get_Key_fn)0, privateKeyFile, privateKeyPassphrase)) - { - poco_check_ptr(_pEVPPKey); - return; // private key is enough - } - - // no private key, this must be public key only, otherwise throw - if (!loadKey(&_pEVPPKey, PEM_read_PUBKEY, (EVP_PKEY_get_Key_fn)0, publicKeyFile)) - { - throw OpenSSLException("ECKeyImpl(const string&, const string&, const string&"); - } - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey::EVPPKey(std::istream* pPublicKeyStream, - std::istream* pPrivateKeyStream, - const std::string& privateKeyPassphrase): _pEVPPKey(0) -{ - if (loadKey(&_pEVPPKey, PEM_read_bio_PrivateKey, (EVP_PKEY_get_Key_fn)0, pPrivateKeyStream, privateKeyPassphrase)) - { - poco_check_ptr(_pEVPPKey); - return; // private key is enough - } - - // no private key, this must be public key only, otherwise throw - if (!loadKey(&_pEVPPKey, PEM_read_bio_PUBKEY, (EVP_PKEY_get_Key_fn)0, pPublicKeyStream)) - { - throw OpenSSLException("ECKeyImpl(istream*, istream*, const string&"); - } - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey::EVPPKey(const EVPPKey& other) -{ - duplicate(other._pEVPPKey, &_pEVPPKey); - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey& EVPPKey::operator=(const EVPPKey& other) -{ - duplicate(other._pEVPPKey, &_pEVPPKey); - poco_check_ptr(_pEVPPKey); - return *this; -} - - -#ifdef POCO_ENABLE_CPP11 - -EVPPKey::EVPPKey(EVPPKey&& other): _pEVPPKey(other._pEVPPKey) -{ - other._pEVPPKey = nullptr; - poco_check_ptr(_pEVPPKey); -} - - -EVPPKey& EVPPKey::operator=(EVPPKey&& other) -{ - _pEVPPKey = other._pEVPPKey; - other._pEVPPKey = nullptr; - poco_check_ptr(_pEVPPKey); - return *this; -} - -#endif // POCO_ENABLE_CPP11 - -EVPPKey::~EVPPKey() -{ - if (_pEVPPKey) EVP_PKEY_free(_pEVPPKey); -} - - -void EVPPKey::save(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase) const -{ - if (!publicKeyFile.empty() && (publicKeyFile != privateKeyFile)) - { - BIO* bio = BIO_new(BIO_s_file()); - if (!bio) throw Poco::IOException("Cannot create BIO for writing public key file", publicKeyFile); - try - { - if (BIO_write_filename(bio, const_cast(publicKeyFile.c_str()))) - { - if (!PEM_write_bio_PUBKEY(bio, _pEVPPKey)) - { - throw Poco::WriteFileException("Failed to write public key to file", publicKeyFile); - } - } - else throw Poco::CreateFileException("Cannot create public key file"); - } - catch (...) - { - BIO_free(bio); - throw; - } - BIO_free(bio); - } - - if (!privateKeyFile.empty()) - { - BIO* bio = BIO_new(BIO_s_file()); - if (!bio) throw Poco::IOException("Cannot create BIO for writing private key file", privateKeyFile); - try - { - if (BIO_write_filename(bio, const_cast(privateKeyFile.c_str()))) - { - int rc = 0; - if (privateKeyPassphrase.empty()) - { - rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, 0, 0, 0, 0, 0); - } - else - { - rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, EVP_des_ede3_cbc(), - reinterpret_cast(const_cast(privateKeyPassphrase.c_str())), - static_cast(privateKeyPassphrase.length()), 0, 0); - } - if (!rc) - throw Poco::FileException("Failed to write private key to file", privateKeyFile); - } - else throw Poco::CreateFileException("Cannot create private key file", privateKeyFile); - } - catch (...) - { - BIO_free(bio); - throw; - } - BIO_free(bio); - } -} - - -void EVPPKey::save(std::ostream* pPublicKeyStream, std::ostream* pPrivateKeyStream, const std::string& privateKeyPassphrase) const -{ - if (pPublicKeyStream && (pPublicKeyStream != pPrivateKeyStream)) - { - BIO* bio = BIO_new(BIO_s_mem()); - if (!bio) throw Poco::IOException("Cannot create BIO for writing public key"); - if (!PEM_write_bio_PUBKEY(bio, _pEVPPKey)) - { - BIO_free(bio); - throw Poco::WriteFileException("Failed to write public key to stream"); - } - char* pData; - long size = BIO_get_mem_data(bio, &pData); - pPublicKeyStream->write(pData, static_cast(size)); - BIO_free(bio); - } - - if (pPrivateKeyStream) - { - BIO* bio = BIO_new(BIO_s_mem()); - if (!bio) throw Poco::IOException("Cannot create BIO for writing public key"); - int rc = 0; - if (privateKeyPassphrase.empty()) - rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, 0, 0, 0, 0, 0); - else - rc = PEM_write_bio_PrivateKey(bio, _pEVPPKey, EVP_des_ede3_cbc(), - reinterpret_cast(const_cast(privateKeyPassphrase.c_str())), - static_cast(privateKeyPassphrase.length()), 0, 0); - if (!rc) - { - BIO_free(bio); - throw Poco::FileException("Failed to write private key to stream"); - } - char* pData; - long size = BIO_get_mem_data(bio, &pData); - pPrivateKeyStream->write(pData, static_cast(size)); - BIO_free(bio); - } -} - - -EVP_PKEY* EVPPKey::duplicate(const EVP_PKEY* pFromKey, EVP_PKEY** pToKey) -{ - if (!pFromKey) throw NullPointerException("EVPPKey::duplicate(): " - "provided key pointer is null."); - - *pToKey = EVP_PKEY_new(); - if (!*pToKey) throw NullPointerException("EVPPKey::duplicate(): " - "EVP_PKEY_new() returned null."); - - int keyType = type(pFromKey); - switch (keyType) - { - case EVP_PKEY_RSA: - { - RSA* pRSA = EVP_PKEY_get1_RSA(const_cast(pFromKey)); - if (pRSA) - { - EVP_PKEY_set1_RSA(*pToKey, pRSA); - RSA_free(pRSA); - } - else throw OpenSSLException("EVPPKey::duplicate(): EVP_PKEY_get1_RSA()"); - break; - } - case EVP_PKEY_EC: - { - EC_KEY* pEC = EVP_PKEY_get1_EC_KEY(const_cast(pFromKey)); - if (pEC) - { - EVP_PKEY_set1_EC_KEY(*pToKey, pEC); - EC_KEY_free(pEC); - int cmp = EVP_PKEY_cmp_parameters(*pToKey, pFromKey); - if (cmp < 0) - throw OpenSSLException("EVPPKey::duplicate(): EVP_PKEY_cmp_parameters()"); - if (0 == cmp) - { - if(!EVP_PKEY_copy_parameters(*pToKey, pFromKey)) - throw OpenSSLException("EVPPKey::duplicate(): EVP_PKEY_copy_parameters()"); - } - } - else throw OpenSSLException(); - break; - } - default: - throw NotImplementedException("EVPPKey:duplicate(); Key type: " + - NumberFormatter::format(keyType)); - } - - return *pToKey; -} - - -void EVPPKey::newECKey(const char* ecCurveName) -{ - int curveID = OBJ_txt2nid(ecCurveName); - EC_KEY* pEC = EC_KEY_new_by_curve_name(curveID); - if (!pEC) goto err; - if (!EC_KEY_generate_key(pEC)) goto err; - _pEVPPKey = EVP_PKEY_new(); - if (!_pEVPPKey) goto err; - if (!EVP_PKEY_set1_EC_KEY(_pEVPPKey, pEC)) goto err; - EC_KEY_free(pEC); - return; -err: - throw OpenSSLException("EVPPKey:newECKey()"); -} - - -void EVPPKey::setKey(ECKey* pKey) -{ - poco_check_ptr(pKey); - poco_check_ptr(pKey->impl()); - setKey(pKey->impl()->getECKey()); -} - - -void EVPPKey::setKey(RSAKey* pKey) -{ - poco_check_ptr(pKey); - poco_check_ptr(pKey->impl()); - setKey(pKey->impl()->getRSA()); -} - - -int EVPPKey::passCB(char* buf, int size, int, void* pass) -{ - if (pass) - { - int len = (int)std::strlen((char*)pass); - if(len > size) len = size; - std::memcpy(buf, pass, len); - return len; - } - return 0; -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/KeyPair.cpp b/base/poco/Crypto/src/KeyPair.cpp deleted file mode 100644 index 1c650806b37..00000000000 --- a/base/poco/Crypto/src/KeyPair.cpp +++ /dev/null @@ -1,34 +0,0 @@ -// -// KeyPair.cpp -// -// -// Library: Crypto -// Package: CryptoCore -// Module: KeyPair -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/KeyPair.h" -#include - - -namespace Poco { -namespace Crypto { - - -KeyPair::KeyPair(KeyPairImpl::Ptr pKeyPairImpl): _pImpl(pKeyPairImpl) -{ -} - - -KeyPair::~KeyPair() -{ -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/KeyPairImpl.cpp b/base/poco/Crypto/src/KeyPairImpl.cpp deleted file mode 100644 index c782f04270d..00000000000 --- a/base/poco/Crypto/src/KeyPairImpl.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// -// KeyPairImpl.cpp -// -// -// Library: Crypto -// Package: CryptoCore -// Module: KeyPairImpl -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/KeyPairImpl.h" - - -namespace Poco { -namespace Crypto { - - -KeyPairImpl::KeyPairImpl(const std::string& name, Type type): - _name(name), - _type(type) -{ -} - - -KeyPairImpl::~KeyPairImpl() -{ -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/OpenSSLInitializer.cpp b/base/poco/Crypto/src/OpenSSLInitializer.cpp deleted file mode 100644 index 31798e8dd7e..00000000000 --- a/base/poco/Crypto/src/OpenSSLInitializer.cpp +++ /dev/null @@ -1,162 +0,0 @@ -// -// OpenSSLInitializer.cpp -// -// Library: Crypto -// Package: CryptoCore -// Module: OpenSSLInitializer -// -// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/OpenSSLInitializer.h" -#include "Poco/RandomStream.h" -#include "Poco/Thread.h" -#include -#include -#include -#include -#if OPENSSL_VERSION_NUMBER >= 0x0907000L -#include -#endif - - -using Poco::RandomInputStream; -using Poco::Thread; - - - - -namespace Poco { -namespace Crypto { - - -Poco::FastMutex* OpenSSLInitializer::_mutexes(0); -Poco::AtomicCounter OpenSSLInitializer::_rc; - - -OpenSSLInitializer::OpenSSLInitializer() -{ - initialize(); -} - - -OpenSSLInitializer::~OpenSSLInitializer() -{ - try - { - uninitialize(); - } - catch (...) - { - poco_unexpected(); - } -} - - -void OpenSSLInitializer::initialize() -{ - if (++_rc == 1) - { -#if OPENSSL_VERSION_NUMBER >= 0x0907000L - OPENSSL_config(NULL); -#endif - SSL_library_init(); - SSL_load_error_strings(); - OpenSSL_add_all_algorithms(); - - char seed[SEEDSIZE]; - RandomInputStream rnd; - rnd.read(seed, sizeof(seed)); - RAND_seed(seed, SEEDSIZE); - - int nMutexes = CRYPTO_num_locks(); - _mutexes = new Poco::FastMutex[nMutexes]; - CRYPTO_set_locking_callback(&OpenSSLInitializer::lock); -// Not needed on Windows (see SF #110: random unhandled exceptions when linking with ssl). -// https://sourceforge.net/p/poco/bugs/110/ -// -// From http://www.openssl.org/docs/crypto/threads.html : -// "If the application does not register such a callback using CRYPTO_THREADID_set_callback(), -// then a default implementation is used - on Windows and BeOS this uses the system's -// default thread identifying APIs" - CRYPTO_set_id_callback(&OpenSSLInitializer::id); - CRYPTO_set_dynlock_create_callback(&OpenSSLInitializer::dynlockCreate); - CRYPTO_set_dynlock_lock_callback(&OpenSSLInitializer::dynlock); - CRYPTO_set_dynlock_destroy_callback(&OpenSSLInitializer::dynlockDestroy); - } -} - - -void OpenSSLInitializer::uninitialize() -{ - if (--_rc == 0) - { - EVP_cleanup(); - ERR_free_strings(); - CRYPTO_set_locking_callback(0); - CRYPTO_set_id_callback(0); - delete [] _mutexes; - - CONF_modules_free(); - } -} - - -void OpenSSLInitializer::lock(int mode, int n, const char* file, int line) -{ - if (mode & CRYPTO_LOCK) - _mutexes[n].lock(); - else - _mutexes[n].unlock(); -} - - -unsigned long OpenSSLInitializer::id() -{ - // Note: we use an old-style C cast here because - // neither static_cast<> nor reinterpret_cast<> - // work uniformly across all platforms. - return (unsigned long) Poco::Thread::currentTid(); -} - - -struct CRYPTO_dynlock_value* OpenSSLInitializer::dynlockCreate(const char* file, int line) -{ - return new CRYPTO_dynlock_value; -} - - -void OpenSSLInitializer::dynlock(int mode, struct CRYPTO_dynlock_value* lock, const char* file, int line) -{ - poco_check_ptr (lock); - - if (mode & CRYPTO_LOCK) - lock->_mutex.lock(); - else - lock->_mutex.unlock(); -} - - -void OpenSSLInitializer::dynlockDestroy(struct CRYPTO_dynlock_value* lock, const char* file, int line) -{ - delete lock; -} - - -void initializeCrypto() -{ - OpenSSLInitializer::initialize(); -} - - -void uninitializeCrypto() -{ - OpenSSLInitializer::uninitialize(); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/PKCS12Container.cpp b/base/poco/Crypto/src/PKCS12Container.cpp deleted file mode 100644 index def89bf0898..00000000000 --- a/base/poco/Crypto/src/PKCS12Container.cpp +++ /dev/null @@ -1,191 +0,0 @@ -// -// PKCS12Container.cpp -// -// -// Library: Crypto -// Package: Certificate -// Module: PKCS12Container -// -// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/PKCS12Container.h" -#include "Poco/NumberFormatter.h" -#include "Poco/StreamCopier.h" -#include -#include - - -namespace Poco { -namespace Crypto { - - -PKCS12Container::PKCS12Container(std::istream& istr, const std::string& password): _pKey(0) -{ - std::ostringstream ostr; - Poco::StreamCopier::copyStream(istr, ostr); - const std::string& cont = ostr.str(); - - BIO *pBIO = BIO_new_mem_buf(const_cast(cont.data()), static_cast(cont.size())); - if (pBIO) - { - PKCS12* pPKCS12 = 0; - d2i_PKCS12_bio(pBIO, &pPKCS12); - BIO_free(pBIO); - if (!pPKCS12) throw OpenSSLException("PKCS12Container(istream&, const string&)"); - load(pPKCS12, password); - } - else - { - throw Poco::NullPointerException("PKCS12Container(istream&, const string&)"); - } -} - - -PKCS12Container::PKCS12Container(const std::string& path, const std::string& password): _pKey(0) -{ - FILE* pFile = fopen(path.c_str(), "rb"); - if (pFile) - { - PKCS12* pPKCS12 = d2i_PKCS12_fp(pFile, NULL); - fclose (pFile); - if (!pPKCS12) throw OpenSSLException("PKCS12Container(const string&, const string&)"); - load(pPKCS12, password); - } - else - { - throw Poco::OpenFileException("PKCS12Container: " + path); - } -} - - -PKCS12Container::PKCS12Container(const PKCS12Container& other): - _pKey(EVPPKey::duplicate(other._pKey, &_pKey)), - _pX509Cert(new X509Certificate(*other._pX509Cert)), - _caCertList(other._caCertList), - _caCertNames(other._caCertNames), - _pkcsFriendlyName(other._pkcsFriendlyName) -{ -} - - -PKCS12Container& PKCS12Container::operator = (const PKCS12Container& other) -{ - if (&other != this) - { - if (_pKey) EVP_PKEY_free(_pKey); - _pKey = EVPPKey::duplicate(other._pKey, &_pKey); - _pX509Cert.reset(new X509Certificate(*other._pX509Cert)); - _caCertList = other._caCertList; - _caCertNames = other._caCertNames; - _pkcsFriendlyName = other._pkcsFriendlyName; - } - return *this; -} - - -#ifdef POCO_ENABLE_CPP11 - - -PKCS12Container::PKCS12Container(PKCS12Container&& other): - _pKey(other._pKey), - _pX509Cert(std::move(other._pX509Cert)), - _caCertList(std::move(other._caCertList)), - _caCertNames(std::move(other._caCertNames)), - _pkcsFriendlyName(std::move(other._pkcsFriendlyName)) -{ - other._pKey = 0; -} - - -PKCS12Container& PKCS12Container::operator = (PKCS12Container&& other) -{ - if (&other != this) - { - if (_pKey) EVP_PKEY_free(_pKey); - _pKey = other._pKey; other._pKey = 0; - _pX509Cert = std::move(other._pX509Cert); - _caCertList = std::move(other._caCertList); - _caCertNames = std::move(other._caCertNames); - _pkcsFriendlyName = std::move(other._pkcsFriendlyName); - } - return *this; -} - - -#endif // POCO_ENABLE_CPP11 - - -PKCS12Container::~PKCS12Container() -{ - if (_pKey) EVP_PKEY_free(_pKey); -} - - -std::string PKCS12Container::extractFriendlyName(X509* pCert) -{ - std::string friendlyName; - if(!pCert) throw NullPointerException("PKCS12Container::extractFriendlyName(X509)"); - - // This is how PKCS12_add_cert() sets friendlyName (via PKCS12_add_friendlyname()) - int namelen = 0; - char *name = (char *)X509_alias_get0(pCert, &namelen); - - friendlyName = std::string(name, namelen); - return friendlyName; -} - - -void PKCS12Container::load(PKCS12* pPKCS12, const std::string& password) -{ - if (pPKCS12) - { - X509* pCert = 0; - STACK_OF(X509)* pCA = 0; - if (PKCS12_parse(pPKCS12, password.c_str(), &_pKey, &pCert, &pCA)) - { - if (pCert) - { - _pX509Cert.reset(new X509Certificate(pCert, true)); - _pkcsFriendlyName = extractFriendlyName(pCert); - } - else _pX509Cert.reset(); - - _caCertList.clear(); - _caCertNames.clear(); - if (pCA) - { - int certCount = sk_X509_num(pCA); - for (int i = 0; i < certCount; ++i) - { - X509* pX509 = sk_X509_value(pCA, i); - if (pX509) - { - _caCertList.push_back(X509Certificate(pX509, true)); - _caCertNames.push_back(extractFriendlyName(pX509)); - } - else throw OpenSSLException("PKCS12Container::load()"); - } - } - } - else - { - throw OpenSSLException(); - } - PKCS12_free(pPKCS12); - sk_X509_pop_free(pCA, X509_free); - if (pCert) X509_free(pCert); - poco_assert_dbg (_caCertList.size() == _caCertNames.size()); - } - else - { - throw NullPointerException("PKCS12Container: struct PKCS12"); - } -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/RSACipherImpl.cpp b/base/poco/Crypto/src/RSACipherImpl.cpp deleted file mode 100644 index 4f9ddcf32ad..00000000000 --- a/base/poco/Crypto/src/RSACipherImpl.cpp +++ /dev/null @@ -1,342 +0,0 @@ -// -// RSACipherImpl.cpp -// -// Library: Crypto -// Package: RSA -// Module: RSACipherImpl -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/RSACipherImpl.h" -#include "Poco/Crypto/CryptoTransform.h" -#include "Poco/Exception.h" -#include -#include -#include - - -namespace Poco { -namespace Crypto { - - -namespace -{ - void throwError() - { - unsigned long err; - std::string msg; - - while ((err = ERR_get_error())) - { - if (!msg.empty()) - msg.append("; "); - msg.append(ERR_error_string(err, 0)); - } - - throw Poco::IOException(msg); - } - - - int mapPaddingMode(RSAPaddingMode paddingMode) - { - switch (paddingMode) - { - case RSA_PADDING_PKCS1: - return RSA_PKCS1_PADDING; - case RSA_PADDING_PKCS1_OAEP: - return RSA_PKCS1_OAEP_PADDING; - case RSA_PADDING_NONE: - return RSA_NO_PADDING; - default: - poco_bugcheck(); - return RSA_NO_PADDING; - } - } - - - class RSAEncryptImpl: public CryptoTransform - { - public: - RSAEncryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode); - ~RSAEncryptImpl(); - - std::size_t blockSize() const; - std::size_t maxDataSize() const; - std::string getTag(std::size_t); - void setTag(const std::string&); - - std::streamsize transform( - const unsigned char* input, - std::streamsize inputLength, - unsigned char* output, - std::streamsize outputLength); - - std::streamsize finalize(unsigned char* output, std::streamsize length); - - private: - const RSA* _pRSA; - RSAPaddingMode _paddingMode; - std::streamsize _pos; - unsigned char* _pBuf; - }; - - - RSAEncryptImpl::RSAEncryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode): - _pRSA(pRSA), - _paddingMode(paddingMode), - _pos(0), - _pBuf(0) - { - _pBuf = new unsigned char[blockSize()]; - } - - - RSAEncryptImpl::~RSAEncryptImpl() - { - delete [] _pBuf; - } - - - std::size_t RSAEncryptImpl::blockSize() const - { - return RSA_size(_pRSA); - } - - - std::size_t RSAEncryptImpl::maxDataSize() const - { - std::size_t size = blockSize(); - switch (_paddingMode) - { - case RSA_PADDING_PKCS1: - case RSA_PADDING_SSLV23: - size -= 11; - break; - case RSA_PADDING_PKCS1_OAEP: - size -= 41; - break; - default: - break; - } - return size; - } - - - std::string RSAEncryptImpl::getTag(std::size_t) - { - return std::string(); - } - - - void RSAEncryptImpl::setTag(const std::string&) - { - } - - - std::streamsize RSAEncryptImpl::transform( - const unsigned char* input, - std::streamsize inputLength, - unsigned char* output, - std::streamsize outputLength) - { - // always fill up the buffer before writing! - std::streamsize maxSize = static_cast(maxDataSize()); - std::streamsize rsaSize = static_cast(blockSize()); - poco_assert_dbg(_pos <= maxSize); - poco_assert (outputLength >= rsaSize); - int rc = 0; - while (inputLength > 0) - { - // check how many data bytes we are missing to get the buffer full - poco_assert_dbg (maxSize >= _pos); - std::streamsize missing = maxSize - _pos; - if (missing == 0) - { - poco_assert (outputLength >= rsaSize); - int n = RSA_public_encrypt(static_cast(maxSize), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); - if (n == -1) - throwError(); - rc += n; - output += n; - outputLength -= n; - _pos = 0; - - } - else - { - if (missing > inputLength) - missing = inputLength; - - std::memcpy(_pBuf + _pos, input, static_cast(missing)); - input += missing; - _pos += missing; - inputLength -= missing; - } - } - return rc; - } - - - std::streamsize RSAEncryptImpl::finalize(unsigned char* output, std::streamsize length) - { - poco_assert (length >= blockSize()); - poco_assert (_pos <= maxDataSize()); - int rc = 0; - if (_pos > 0) - { - rc = RSA_public_encrypt(static_cast(_pos), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); - if (rc == -1) throwError(); - } - return rc; - } - - - class RSADecryptImpl: public CryptoTransform - { - public: - RSADecryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode); - ~RSADecryptImpl(); - - std::size_t blockSize() const; - std::string getTag(std::size_t); - void setTag(const std::string&); - - std::streamsize transform( - const unsigned char* input, - std::streamsize inputLength, - unsigned char* output, - std::streamsize outputLength); - - std::streamsize finalize( - unsigned char* output, - std::streamsize length); - - private: - const RSA* _pRSA; - RSAPaddingMode _paddingMode; - std::streamsize _pos; - unsigned char* _pBuf; - }; - - - RSADecryptImpl::RSADecryptImpl(const RSA* pRSA, RSAPaddingMode paddingMode): - _pRSA(pRSA), - _paddingMode(paddingMode), - _pos(0), - _pBuf(0) - { - _pBuf = new unsigned char[blockSize()]; - } - - - RSADecryptImpl::~RSADecryptImpl() - { - delete [] _pBuf; - } - - - std::size_t RSADecryptImpl::blockSize() const - { - return RSA_size(_pRSA); - } - - - std::string RSADecryptImpl::getTag(std::size_t) - { - return std::string(); - } - - - void RSADecryptImpl::setTag(const std::string&) - { - } - - - std::streamsize RSADecryptImpl::transform( - const unsigned char* input, - std::streamsize inputLength, - unsigned char* output, - std::streamsize outputLength) - { - - // always fill up the buffer before decrypting! - std::streamsize rsaSize = static_cast(blockSize()); - poco_assert_dbg(_pos <= rsaSize); - poco_assert (outputLength >= rsaSize); - int rc = 0; - while (inputLength > 0) - { - // check how many data bytes we are missing to get the buffer full - poco_assert_dbg (rsaSize >= _pos); - std::streamsize missing = rsaSize - _pos; - if (missing == 0) - { - int tmp = RSA_private_decrypt(static_cast(rsaSize), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); - if (tmp == -1) - throwError(); - rc += tmp; - output += tmp; - outputLength -= tmp; - _pos = 0; - - } - else - { - if (missing > inputLength) - missing = inputLength; - - std::memcpy(_pBuf + _pos, input, static_cast(missing)); - input += missing; - _pos += missing; - inputLength -= missing; - } - } - return rc; - } - - - std::streamsize RSADecryptImpl::finalize(unsigned char* output, std::streamsize length) - { - poco_assert (length >= blockSize()); - int rc = 0; - if (_pos > 0) - { - rc = RSA_private_decrypt(static_cast(_pos), _pBuf, output, const_cast(_pRSA), mapPaddingMode(_paddingMode)); - if (rc == -1) - throwError(); - } - return rc; - } -} - - -RSACipherImpl::RSACipherImpl(const RSAKey& key, RSAPaddingMode paddingMode): - _key(key), - _paddingMode(paddingMode) -{ -} - - -RSACipherImpl::~RSACipherImpl() -{ -} - - -CryptoTransform* RSACipherImpl::createEncryptor() -{ - return new RSAEncryptImpl(_key.impl()->getRSA(), _paddingMode); -} - - -CryptoTransform* RSACipherImpl::createDecryptor() -{ - return new RSADecryptImpl(_key.impl()->getRSA(), _paddingMode); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/RSADigestEngine.cpp b/base/poco/Crypto/src/RSADigestEngine.cpp deleted file mode 100644 index 948aa25ac6c..00000000000 --- a/base/poco/Crypto/src/RSADigestEngine.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// -// RSADigestEngine.cpp -// -// Library: Crypto -// Package: RSA -// Module: RSADigestEngine -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/RSADigestEngine.h" -#include - - -namespace Poco { -namespace Crypto { - - -RSADigestEngine::RSADigestEngine(const RSAKey& key, DigestType digestType): - _key(key), - _engine(digestType == DIGEST_MD5 ? "MD5" : "SHA1") -{ -} - -RSADigestEngine::RSADigestEngine(const RSAKey& key, const std::string &name): - _key(key), - _engine(name) -{ -} - - -RSADigestEngine::~RSADigestEngine() -{ -} - - -std::size_t RSADigestEngine::digestLength() const -{ - return _engine.digestLength(); -} - - -void RSADigestEngine::reset() -{ - _engine.reset(); - _digest.clear(); - _signature.clear(); -} - - -const DigestEngine::Digest& RSADigestEngine::digest() -{ - if (_digest.empty()) - { - _digest = _engine.digest(); - } - return _digest; -} - - -const DigestEngine::Digest& RSADigestEngine::signature() -{ - if (_signature.empty()) - { - digest(); - _signature.resize(_key.size()); - unsigned sigLen = static_cast(_signature.size()); - RSA_sign(_engine.nid(), &_digest[0], static_cast(_digest.size()), &_signature[0], &sigLen, _key.impl()->getRSA()); - // truncate _sig to sigLen - if (sigLen < _signature.size()) - _signature.resize(sigLen); - } - return _signature; -} - - -bool RSADigestEngine::verify(const DigestEngine::Digest& sig) -{ - digest(); - DigestEngine::Digest sigCpy = sig; // copy becausse RSA_verify can modify sigCpy - int ret = RSA_verify(_engine.nid(), &_digest[0], static_cast(_digest.size()), &sigCpy[0], static_cast(sigCpy.size()), _key.impl()->getRSA()); - return ret != 0; -} - - -void RSADigestEngine::updateImpl(const void* data, std::size_t length) -{ - _engine.update(data, length); -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/RSAKey.cpp b/base/poco/Crypto/src/RSAKey.cpp deleted file mode 100644 index b81a0281bf5..00000000000 --- a/base/poco/Crypto/src/RSAKey.cpp +++ /dev/null @@ -1,87 +0,0 @@ -// -// RSAKey.cpp -// -// Library: Crypto -// Package: RSA -// Module: RSAKey -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/RSAKey.h" -#include - - -namespace Poco { -namespace Crypto { - - -RSAKey::RSAKey(const EVPPKey& key): - KeyPair(new RSAKeyImpl(key)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -RSAKey::RSAKey(const X509Certificate& cert): - KeyPair(new RSAKeyImpl(cert)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -RSAKey::RSAKey(const PKCS12Container& cont): - KeyPair(new RSAKeyImpl(cont)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -RSAKey::RSAKey(KeyLength keyLength, Exponent exp): - KeyPair(new RSAKeyImpl(keyLength, (exp == EXP_LARGE) ? RSA_F4 : RSA_3)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -RSAKey::RSAKey(const std::string& publicKeyFile, const std::string& privateKeyFile, const std::string& privateKeyPassphrase): - KeyPair(new RSAKeyImpl(publicKeyFile, privateKeyFile, privateKeyPassphrase)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -RSAKey::RSAKey(std::istream* pPublicKeyStream, std::istream* pPrivateKeyStream, const std::string& privateKeyPassphrase): - KeyPair(new RSAKeyImpl(pPublicKeyStream, pPrivateKeyStream, privateKeyPassphrase)), - _pImpl(KeyPair::impl().cast()) -{ -} - - -RSAKey::~RSAKey() -{ -} - -RSAKeyImpl::ByteVec RSAKey::modulus() const -{ - return _pImpl->modulus(); -} - - -RSAKeyImpl::ByteVec RSAKey::encryptionExponent() const -{ - return _pImpl->encryptionExponent(); -} - - -RSAKeyImpl::ByteVec RSAKey::decryptionExponent() const -{ - return _pImpl->decryptionExponent(); -} - - -} } // namespace Poco::Crypto \ No newline at end of file diff --git a/base/poco/Crypto/src/RSAKeyImpl.cpp b/base/poco/Crypto/src/RSAKeyImpl.cpp deleted file mode 100644 index 229a3bce828..00000000000 --- a/base/poco/Crypto/src/RSAKeyImpl.cpp +++ /dev/null @@ -1,289 +0,0 @@ -// -// RSAKeyImpl.cpp -// -// Library: Crypto -// Package: RSA -// Module: RSAKeyImpl -// -// Copyright (c) 2008, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/RSAKeyImpl.h" -#include "Poco/Crypto/X509Certificate.h" -#include "Poco/Crypto/PKCS12Container.h" -#include "Poco/FileStream.h" -#include "Poco/StreamCopier.h" -#include -#include -#include -#include -#if OPENSSL_VERSION_NUMBER >= 0x00908000L -#include -#endif - - -namespace Poco { -namespace Crypto { - - -RSAKeyImpl::RSAKeyImpl(const EVPPKey& key): - KeyPairImpl("rsa", KT_RSA_IMPL), - _pRSA(EVP_PKEY_get1_RSA(const_cast((const EVP_PKEY*)key))) -{ - if (!_pRSA) throw OpenSSLException(); -} - - -RSAKeyImpl::RSAKeyImpl(const X509Certificate& cert): - KeyPairImpl("rsa", KT_RSA_IMPL), - _pRSA(0) -{ - const X509* pCert = cert.certificate(); - EVP_PKEY* pKey = X509_get_pubkey(const_cast(pCert)); - if (pKey) - { - _pRSA = EVP_PKEY_get1_RSA(pKey); - EVP_PKEY_free(pKey); - } - else - throw OpenSSLException("RSAKeyImpl(const X509Certificate&)"); -} - - -RSAKeyImpl::RSAKeyImpl(const PKCS12Container& cont): - KeyPairImpl("rsa", KT_RSA_IMPL), - _pRSA(0) -{ - EVPPKey key = cont.getKey(); - _pRSA = EVP_PKEY_get1_RSA(key); -} - - -RSAKeyImpl::RSAKeyImpl(int keyLength, unsigned long exponent): KeyPairImpl("rsa", KT_RSA_IMPL), - _pRSA(0) -{ -#if OPENSSL_VERSION_NUMBER >= 0x00908000L - _pRSA = RSA_new(); - int ret = 0; - BIGNUM* bn = 0; - try - { - bn = BN_new(); - BN_set_word(bn, exponent); - ret = RSA_generate_key_ex(_pRSA, keyLength, bn, 0); - BN_free(bn); - } - catch (...) - { - BN_free(bn); - throw; - } - if (!ret) throw Poco::InvalidArgumentException("Failed to create RSA context"); -#else - _pRSA = RSA_generate_key(keyLength, exponent, 0, 0); - if (!_pRSA) throw Poco::InvalidArgumentException("Failed to create RSA context"); -#endif -} - - -RSAKeyImpl::RSAKeyImpl(const std::string& publicKeyFile, - const std::string& privateKeyFile, - const std::string& privateKeyPassphrase): KeyPairImpl("rsa", KT_RSA_IMPL), - _pRSA(0) -{ - poco_assert_dbg(_pRSA == 0); - - _pRSA = RSA_new(); - if (!publicKeyFile.empty()) - { - BIO* bio = BIO_new(BIO_s_file()); - if (!bio) throw Poco::IOException("Cannot create BIO for reading public key", publicKeyFile); - int rc = BIO_read_filename(bio, const_cast(publicKeyFile.c_str())); - if (rc) - { - RSA* pubKey = PEM_read_bio_RSAPublicKey(bio, &_pRSA, 0, 0); - if (!pubKey) - { - int rc = BIO_reset(bio); - // BIO_reset() normally returns 1 for success and 0 or -1 for failure. - // File BIOs are an exception, they return 0 for success and -1 for failure. - if (rc != 0) throw Poco::FileException("Failed to load public key", publicKeyFile); - pubKey = PEM_read_bio_RSA_PUBKEY(bio, &_pRSA, 0, 0); - } - BIO_free(bio); - if (!pubKey) - { - freeRSA(); - throw Poco::FileException("Failed to load public key", publicKeyFile); - } - } - else - { - freeRSA(); - throw Poco::FileNotFoundException("Public key file", publicKeyFile); - } - } - - if (!privateKeyFile.empty()) - { - BIO* bio = BIO_new(BIO_s_file()); - if (!bio) throw Poco::IOException("Cannot create BIO for reading private key", privateKeyFile); - int rc = BIO_read_filename(bio, const_cast(privateKeyFile.c_str())); - if (rc) - { - RSA* privKey = 0; - if (privateKeyPassphrase.empty()) - privKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, 0); - else - privKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, const_cast(privateKeyPassphrase.c_str())); - BIO_free(bio); - if (!privKey) - { - freeRSA(); - throw Poco::FileException("Failed to load private key", privateKeyFile); - } - } - else - { - freeRSA(); - throw Poco::FileNotFoundException("Private key file", privateKeyFile); - } - } -} - - -RSAKeyImpl::RSAKeyImpl(std::istream* pPublicKeyStream, - std::istream* pPrivateKeyStream, - const std::string& privateKeyPassphrase): KeyPairImpl("rsa", KT_RSA_IMPL), - _pRSA(0) -{ - poco_assert_dbg(_pRSA == 0); - - _pRSA = RSA_new(); - if (pPublicKeyStream) - { - std::string publicKeyData; - Poco::StreamCopier::copyToString(*pPublicKeyStream, publicKeyData); - BIO* bio = BIO_new_mem_buf(const_cast(publicKeyData.data()), static_cast(publicKeyData.size())); - if (!bio) throw Poco::IOException("Cannot create BIO for reading public key"); - RSA* publicKey = PEM_read_bio_RSAPublicKey(bio, &_pRSA, 0, 0); - if (!publicKey) - { - int rc = BIO_reset(bio); - // BIO_reset() normally returns 1 for success and 0 or -1 for failure. - // File BIOs are an exception, they return 0 for success and -1 for failure. - if (rc != 1) throw Poco::FileException("Failed to load public key"); - publicKey = PEM_read_bio_RSA_PUBKEY(bio, &_pRSA, 0, 0); - } - BIO_free(bio); - if (!publicKey) - { - freeRSA(); - throw Poco::FileException("Failed to load public key"); - } - } - - if (pPrivateKeyStream) - { - std::string privateKeyData; - Poco::StreamCopier::copyToString(*pPrivateKeyStream, privateKeyData); - BIO* bio = BIO_new_mem_buf(const_cast(privateKeyData.data()), static_cast(privateKeyData.size())); - if (!bio) throw Poco::IOException("Cannot create BIO for reading private key"); - RSA* privateKey = 0; - if (privateKeyPassphrase.empty()) - privateKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, 0); - else - privateKey = PEM_read_bio_RSAPrivateKey(bio, &_pRSA, 0, const_cast(privateKeyPassphrase.c_str())); - BIO_free(bio); - if (!privateKey) - { - freeRSA(); - throw Poco::FileException("Failed to load private key"); - } - } -} - - -RSAKeyImpl::~RSAKeyImpl() -{ - freeRSA(); -} - - -void RSAKeyImpl::freeRSA() -{ - if (_pRSA) RSA_free(_pRSA); - _pRSA = 0; -} - - -int RSAKeyImpl::size() const -{ - return RSA_size(_pRSA); -} - - -RSAKeyImpl::ByteVec RSAKeyImpl::modulus() const -{ -#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) - const BIGNUM* n = 0; - const BIGNUM* e = 0; - const BIGNUM* d = 0; - RSA_get0_key(_pRSA, &n, &e, &d); - return convertToByteVec(n); -#else - return convertToByteVec(_pRSA->n); -#endif -} - - -RSAKeyImpl::ByteVec RSAKeyImpl::encryptionExponent() const -{ -#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) - const BIGNUM* n = 0; - const BIGNUM* e = 0; - const BIGNUM* d = 0; - RSA_get0_key(_pRSA, &n, &e, &d); - return convertToByteVec(e); -#else - return convertToByteVec(_pRSA->e); -#endif -} - - -RSAKeyImpl::ByteVec RSAKeyImpl::decryptionExponent() const -{ -#if OPENSSL_VERSION_NUMBER >= 0x10100000L && !defined(LIBRESSL_VERSION_NUMBER) - const BIGNUM* n = 0; - const BIGNUM* e = 0; - const BIGNUM* d = 0; - RSA_get0_key(_pRSA, &n, &e, &d); - return convertToByteVec(d); -#else - return convertToByteVec(_pRSA->d); -#endif -} - - -RSAKeyImpl::ByteVec RSAKeyImpl::convertToByteVec(const BIGNUM* bn) -{ - int numBytes = BN_num_bytes(bn); - ByteVec byteVector(numBytes); - - ByteVec::value_type* buffer = new ByteVec::value_type[numBytes]; - BN_bn2bin(bn, buffer); - - for (int i = 0; i < numBytes; ++i) - byteVector[i] = buffer[i]; - - delete [] buffer; - - return byteVector; -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Crypto/src/X509Certificate.cpp b/base/poco/Crypto/src/X509Certificate.cpp deleted file mode 100644 index 16458d0477b..00000000000 --- a/base/poco/Crypto/src/X509Certificate.cpp +++ /dev/null @@ -1,382 +0,0 @@ -// -// X509Certificate.cpp -// -// Library: Crypto -// Package: Certificate -// Module: X509Certificate -// -// Copyright (c) 2006-2009, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#include "Poco/Crypto/X509Certificate.h" -#include "Poco/Crypto/CryptoException.h" -#include "Poco/StreamCopier.h" -#include "Poco/String.h" -#include "Poco/DateTimeParser.h" -#include "Poco/Format.h" -#include -#include -#include -#include -#include - - -namespace Poco { -namespace Crypto { - - -X509Certificate::X509Certificate(std::istream& istr): - _pCert(0) -{ - load(istr); -} - - -X509Certificate::X509Certificate(const std::string& path): - _pCert(0) -{ - load(path); -} - - -X509Certificate::X509Certificate(X509* pCert): - _pCert(pCert) -{ - poco_check_ptr(_pCert); - - init(); -} - - -X509Certificate::X509Certificate(X509* pCert, bool shared): - _pCert(pCert) -{ - poco_check_ptr(_pCert); - - if (shared) - { -#if OPENSSL_VERSION_NUMBER >= 0x10100000L - X509_up_ref(_pCert); -#else - _pCert->references++; -#endif - } - - init(); -} - - -X509Certificate::X509Certificate(const X509Certificate& cert): - _issuerName(cert._issuerName), - _subjectName(cert._subjectName), - _serialNumber(cert._serialNumber), - _pCert(cert._pCert) -{ - _pCert = X509_dup(_pCert); -} - - -X509Certificate& X509Certificate::operator = (const X509Certificate& cert) -{ - X509Certificate tmp(cert); - swap(tmp); - return *this; -} - - -void X509Certificate::swap(X509Certificate& cert) -{ - using std::swap; - swap(cert._issuerName, _issuerName); - swap(cert._subjectName, _subjectName); - swap(cert._serialNumber, _serialNumber); - swap(cert._pCert, _pCert); -} - - -X509Certificate::~X509Certificate() -{ - X509_free(_pCert); -} - - -void X509Certificate::load(std::istream& istr) -{ - poco_assert (!_pCert); - - std::stringstream certStream; - Poco::StreamCopier::copyStream(istr, certStream); - std::string cert = certStream.str(); - - BIO *pBIO = BIO_new_mem_buf(const_cast(cert.data()), static_cast(cert.size())); - if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate"); - _pCert = PEM_read_bio_X509(pBIO, 0, 0, 0); - BIO_free(pBIO); - - if (!_pCert) throw Poco::IOException("Failed to load certificate from stream"); - - init(); -} - - -void X509Certificate::load(const std::string& path) -{ - poco_assert (!_pCert); - - BIO *pBIO = BIO_new(BIO_s_file()); - if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate file", path); - if (!BIO_read_filename(pBIO, const_cast(path.c_str()))) - { - BIO_free(pBIO); - throw Poco::OpenFileException("Cannot open certificate file for reading", path); - } - - _pCert = PEM_read_bio_X509(pBIO, 0, 0, 0); - BIO_free(pBIO); - - if (!_pCert) throw Poco::ReadFileException("Failed to load certificate from", path); - - init(); -} - - -void X509Certificate::save(std::ostream& stream) const -{ - BIO *pBIO = BIO_new(BIO_s_mem()); - if (!pBIO) throw Poco::IOException("Cannot create BIO for writing certificate"); - try - { - if (!PEM_write_bio_X509(pBIO, _pCert)) - throw Poco::IOException("Failed to write certificate to stream"); - - char *pData; - long size; - size = BIO_get_mem_data(pBIO, &pData); - stream.write(pData, size); - } - catch (...) - { - BIO_free(pBIO); - throw; - } - BIO_free(pBIO); -} - - -void X509Certificate::save(const std::string& path) const -{ - BIO *pBIO = BIO_new(BIO_s_file()); - if (!pBIO) throw Poco::IOException("Cannot create BIO for reading certificate file", path); - if (!BIO_write_filename(pBIO, const_cast(path.c_str()))) - { - BIO_free(pBIO); - throw Poco::CreateFileException("Cannot create certificate file", path); - } - try - { - if (!PEM_write_bio_X509(pBIO, _pCert)) - throw Poco::WriteFileException("Failed to write certificate to file", path); - } - catch (...) - { - BIO_free(pBIO); - throw; - } - BIO_free(pBIO); -} - - -void X509Certificate::init() -{ - char buffer[NAME_BUFFER_SIZE]; - X509_NAME_oneline(X509_get_issuer_name(_pCert), buffer, sizeof(buffer)); - _issuerName = buffer; - X509_NAME_oneline(X509_get_subject_name(_pCert), buffer, sizeof(buffer)); - _subjectName = buffer; - BIGNUM* pBN = ASN1_INTEGER_to_BN(X509_get_serialNumber(const_cast(_pCert)), 0); - if (pBN) - { - char* pSN = BN_bn2hex(pBN); - if (pSN) - { - _serialNumber = pSN; - OPENSSL_free(pSN); - } - BN_free(pBN); - } -} - - -std::string X509Certificate::commonName() const -{ - return subjectName(NID_COMMON_NAME); -} - - -std::string X509Certificate::issuerName(NID nid) const -{ - if (X509_NAME* issuer = X509_get_issuer_name(_pCert)) - { - char buffer[NAME_BUFFER_SIZE]; - if (X509_NAME_get_text_by_NID(issuer, nid, buffer, sizeof(buffer)) >= 0) - return std::string(buffer); - } - return std::string(); -} - - -std::string X509Certificate::subjectName(NID nid) const -{ - if (X509_NAME* subj = X509_get_subject_name(_pCert)) - { - char buffer[NAME_BUFFER_SIZE]; - if (X509_NAME_get_text_by_NID(subj, nid, buffer, sizeof(buffer)) >= 0) - return std::string(buffer); - } - return std::string(); -} - - -void X509Certificate::extractNames(std::string& cmnName, std::set& domainNames) const -{ - domainNames.clear(); - if (STACK_OF(GENERAL_NAME)* names = static_cast(X509_get_ext_d2i(_pCert, NID_subject_alt_name, 0, 0))) - { - for (int i = 0; i < sk_GENERAL_NAME_num(names); ++i) - { - const GENERAL_NAME* name = sk_GENERAL_NAME_value(names, i); - if (name->type == GEN_DNS) - { - const char* data = reinterpret_cast(ASN1_STRING_data(name->d.ia5)); - std::size_t len = ASN1_STRING_length(name->d.ia5); - domainNames.insert(std::string(data, len)); - } - } - GENERAL_NAMES_free(names); - } - - cmnName = commonName(); - if (!cmnName.empty() && domainNames.empty()) - { - domainNames.insert(cmnName); - } -} - - -Poco::DateTime X509Certificate::validFrom() const -{ - ASN1_TIME* certTime = X509_get_notBefore(_pCert); - std::string dateTime(reinterpret_cast(certTime->data)); - int tzd; - return DateTimeParser::parse("%y%m%d%H%M%S", dateTime, tzd); -} - - -Poco::DateTime X509Certificate::expiresOn() const -{ - ASN1_TIME* certTime = X509_get_notAfter(_pCert); - std::string dateTime(reinterpret_cast(certTime->data)); - int tzd; - return DateTimeParser::parse("%y%m%d%H%M%S", dateTime, tzd); -} - - -bool X509Certificate::issuedBy(const X509Certificate& issuerCertificate) const -{ - X509* pCert = const_cast(_pCert); - X509* pIssuerCert = const_cast(issuerCertificate.certificate()); - EVP_PKEY* pIssuerPublicKey = X509_get_pubkey(pIssuerCert); - if (!pIssuerPublicKey) throw Poco::InvalidArgumentException("Issuer certificate has no public key"); - int rc = X509_verify(pCert, pIssuerPublicKey); - EVP_PKEY_free(pIssuerPublicKey); - return rc == 1; -} - - -bool X509Certificate::equals(const X509Certificate& otherCertificate) const -{ - X509* pCert = const_cast(_pCert); - X509* pOtherCert = const_cast(otherCertificate.certificate()); - return X509_cmp(pCert, pOtherCert) == 0; -} - - -std::string X509Certificate::signatureAlgorithm() const -{ - int sigNID = NID_undef; - -#if (OPENSSL_VERSION_NUMBER >= 0x1010000fL) && !defined(LIBRESSL_VERSION_NUMBER) - sigNID = X509_get_signature_nid(_pCert); -#else - poco_check_ptr(_pCert->sig_alg); - sigNID = OBJ_obj2nid(_pCert->sig_alg->algorithm); -#endif - - if (sigNID != NID_undef) - { - const char* pAlgName = OBJ_nid2ln(sigNID); - if (pAlgName) return std::string(pAlgName); - else throw OpenSSLException(Poco::format("X509Certificate::" - "signatureAlgorithm(): OBJ_nid2ln(%d)", sigNID)); - } - else - throw NotFoundException("X509Certificate::signatureAlgorithm()"); - - return ""; -} - - -X509Certificate::List X509Certificate::readPEM(const std::string& pemFileName) -{ - List caCertList; - BIO* pBIO = BIO_new_file(pemFileName.c_str(), "r"); - if (pBIO == NULL) throw OpenFileException("X509Certificate::readPEM()"); - X509* x = PEM_read_bio_X509(pBIO, NULL, 0, NULL); - if (!x) throw OpenSSLException(Poco::format("X509Certificate::readPEM(%s)", pemFileName)); - while(x) - { - caCertList.push_back(X509Certificate(x)); - x = PEM_read_bio_X509(pBIO, NULL, 0, NULL); - } - BIO_free(pBIO); - return caCertList; -} - - -void X509Certificate::writePEM(const std::string& pemFileName, const List& list) -{ - BIO* pBIO = BIO_new_file(pemFileName.c_str(), "a"); - if (pBIO == NULL) throw OpenFileException("X509Certificate::writePEM()"); - List::const_iterator it = list.begin(); - List::const_iterator end = list.end(); - for (; it != end; ++it) - { - if (!PEM_write_bio_X509(pBIO, const_cast(it->certificate()))) - { - throw OpenSSLException("X509Certificate::writePEM()"); - } - } - BIO_free(pBIO); -} - - -void X509Certificate::print(std::ostream& out) const -{ - out << "subjectName: " << subjectName() << std::endl; - out << "issuerName: " << issuerName() << std::endl; - out << "commonName: " << commonName() << std::endl; - out << "country: " << subjectName(X509Certificate::NID_COUNTRY) << std::endl; - out << "localityName: " << subjectName(X509Certificate::NID_LOCALITY_NAME) << std::endl; - out << "stateOrProvince: " << subjectName(X509Certificate::NID_STATE_OR_PROVINCE) << std::endl; - out << "organizationName: " << subjectName(X509Certificate::NID_ORGANIZATION_NAME) << std::endl; - out << "organizationUnitName: " << subjectName(X509Certificate::NID_ORGANIZATION_UNIT_NAME) << std::endl; - out << "emailAddress: " << subjectName(X509Certificate::NID_PKCS9_EMAIL_ADDRESS) << std::endl; - out << "serialNumber: " << subjectName(X509Certificate::NID_SERIAL_NUMBER) << std::endl; -} - - -} } // namespace Poco::Crypto diff --git a/base/poco/Data/CMakeLists.txt b/base/poco/Data/CMakeLists.txt deleted file mode 100644 index dfdf3194f3c..00000000000 --- a/base/poco/Data/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -file (GLOB SRCS src/*.cpp) - -add_library (_poco_data ${SRCS}) -add_library (Poco::Data ALIAS _poco_data) - -# TODO: remove these warning exclusions -target_compile_options (_poco_data - PRIVATE - -Wno-comma - -Wno-covered-switch-default - -Wno-deprecated-dynamic-exception-spec - -Wno-extra-semi-stmt - -Wno-old-style-cast - -Wno-shorten-64-to-32 - -Wno-sign-compare - -Wno-unused-parameter - -Wno-zero-as-null-pointer-constant -) -target_include_directories (_poco_data SYSTEM PUBLIC "include") -target_link_libraries (_poco_data PUBLIC Poco::Foundation) diff --git a/base/poco/Data/ODBC/CMakeLists.txt b/base/poco/Data/ODBC/CMakeLists.txt deleted file mode 100644 index 73f2c81ea6c..00000000000 --- a/base/poco/Data/ODBC/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -if (ENABLE_ODBC) - if (NOT TARGET ch_contrib::unixodbc) - message(FATAL_ERROR "Configuration error: unixodbc is not a target") - endif() - - set (SRCS - src/Binder.cpp - src/ConnectionHandle.cpp - src/Connector.cpp - src/EnvironmentHandle.cpp - src/Extractor.cpp - src/ODBCException.cpp - src/ODBCMetaColumn.cpp - src/ODBCStatementImpl.cpp - src/Parameter.cpp - src/Preparator.cpp - src/SessionImpl.cpp - src/TypeInfo.cpp - src/Unicode.cpp - src/Utility.cpp - ) - - add_library (_poco_data_odbc ${SRCS}) - add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) - - # TODO: remove these warning exclusions - target_compile_options (_poco_data_odbc - PRIVATE - -Wno-cast-qual - -Wno-deprecated-dynamic-exception-spec - -Wno-extra-semi-stmt - -Wno-old-style-cast - -Wno-sign-compare - -Wno-tautological-constant-out-of-range-compare - -Wno-tautological-unsigned-zero-compare - -Wno-unused-parameter - -Wno-unused-variable - -Wno-zero-as-null-pointer-constant - ) - target_include_directories (_poco_data_odbc SYSTEM PUBLIC "include") - target_link_libraries (_poco_data_odbc PUBLIC Poco::Data ch_contrib::unixodbc) - - message (STATUS "Using Poco::Data::ODBC") -else () - add_library (_poco_data_odbc INTERFACE) - add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) - if (TARGET ch_contrib::unixodbc) - target_link_libraries (_poco_data_odbc INTERFACE ch_contrib::unixodbc) - endif() - - message (STATUS "Not using Poco::Data::ODBC") -endif () diff --git a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h b/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h deleted file mode 100644 index 3169563879c..00000000000 --- a/base/poco/Data/ODBC/include/Poco/Data/ODBC/Binder.h +++ /dev/null @@ -1,1534 +0,0 @@ -// -// Binder.h -// -// Library: Data/ODBC -// Package: ODBC -// Module: Binder -// -// Definition of the Binder class. -// -// Copyright (c) 2006, Applied Informatics Software Engineering GmbH. -// and Contributors. -// -// SPDX-License-Identifier: BSL-1.0 -// - - -#ifndef Data_ODBC_Binder_INCLUDED -#define Data_ODBC_Binder_INCLUDED - - -#include -#include -#include -#include -#include "Poco/Data/AbstractBinder.h" -#include "Poco/Data/LOB.h" -#include "Poco/Data/ODBC/Handle.h" -#include "Poco/Data/ODBC/ODBC.h" -#include "Poco/Data/ODBC/ODBCMetaColumn.h" -#include "Poco/Data/ODBC/Parameter.h" -#include "Poco/Data/ODBC/TypeInfo.h" -#include "Poco/Data/ODBC/Utility.h" -#include "Poco/Exception.h" -#include - - -namespace Poco -{ - - -class DateTime; - - -namespace Data -{ - - - class Date; - class Time; - - - namespace ODBC - { - - - class ODBC_API Binder : public Poco::Data::AbstractBinder - /// Binds placeholders in the sql query to the provided values. Performs data types mapping. - { - public: - typedef AbstractBinder::Direction Direction; - typedef std::map ParamMap; - - static const size_t DEFAULT_PARAM_SIZE = 1024; - - enum ParameterBinding - { - PB_IMMEDIATE, - PB_AT_EXEC - }; - - Binder( - const StatementHandle & rStmt, - std::size_t maxFieldSize, - ParameterBinding dataBinding = PB_IMMEDIATE, - TypeInfo * pDataTypes = 0); - /// Creates the Binder. - - ~Binder(); - /// Destroys the Binder. - - void bind(std::size_t pos, const Poco::Int8 & val, Direction dir); - /// Binds an Int8. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an Int8 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an Int8 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an Int8 list. - - void bind(std::size_t pos, const Poco::UInt8 & val, Direction dir); - /// Binds an UInt8. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an UInt8 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an UInt8 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an UInt8 list. - - void bind(std::size_t pos, const Poco::Int16 & val, Direction dir); - /// Binds an Int16. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an Int16 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an Int16 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an Int16 list. - - void bind(std::size_t pos, const Poco::UInt16 & val, Direction dir); - /// Binds an UInt16. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an UInt16 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an UInt16 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an UInt16 list. - - void bind(std::size_t pos, const Poco::Int32 & val, Direction dir); - /// Binds an Int32. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an Int32 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an Int32 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an Int32 list. - - void bind(std::size_t pos, const Poco::UInt32 & val, Direction dir); - /// Binds an UInt32. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an UInt32 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an UInt32 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an UInt32 list. - - void bind(std::size_t pos, const Poco::Int64 & val, Direction dir); - /// Binds an Int64. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an Int64 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an Int64 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an Int64 list. - - void bind(std::size_t pos, const Poco::UInt64 & val, Direction dir); - /// Binds an UInt64. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds an UInt64 vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds an UInt64 deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds an UInt64 list. - -#ifndef POCO_LONG_IS_64_BIT - void bind(std::size_t pos, const long & val, Direction dir); - /// Binds a long. - - void bind(std::size_t pos, const unsigned long & val, Direction dir); - /// Binds an unsigned long. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a long vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a long deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a long list. -#endif - - void bind(std::size_t pos, const bool & val, Direction dir); - /// Binds a boolean. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a boolean vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a boolean deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a boolean list. - - void bind(std::size_t pos, const float & val, Direction dir); - /// Binds a float. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a float vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a float deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a float list. - - void bind(std::size_t pos, const double & val, Direction dir); - /// Binds a double. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a double vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a double deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a double list. - - void bind(std::size_t pos, const char & val, Direction dir); - /// Binds a single character. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a character vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a character deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a character list. - - void bind(std::size_t pos, const std::string & val, Direction dir); - /// Binds a string. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a string vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a string deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a string list. - - void bind(std::size_t pos, const UTF16String & val, Direction dir); - /// Binds a string. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a string vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a string deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a string list. - - void bind(std::size_t pos, const BLOB & val, Direction dir); - /// Binds a BLOB. In-bound only. - - void bind(std::size_t pos, const CLOB & val, Direction dir); - /// Binds a CLOB. In-bound only. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a BLOB vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a BLOB deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a BLOB list. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a CLOB vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a CLOB deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a CLOB list. - - void bind(std::size_t pos, const Date & val, Direction dir); - /// Binds a Date. - - void bind(std::size_t pos, const std::vector & val, Direction dir); - /// Binds a Date vector. - - void bind(std::size_t pos, const std::deque & val, Direction dir); - /// Binds a Date deque. - - void bind(std::size_t pos, const std::list & val, Direction dir); - /// Binds a Date list. - - void bind(std::size_t pos, const Time & val, Direction dir); - /// Binds a Time. - - void bind(std::size_t pos, const std::vector - + 0 2 + fair_round_robin 1000 @@ -445,6 +493,9 @@ + + 106700800 + - - + /var/lib/clickhouse/tmp/ @@ -529,7 +579,9 @@ 12 - + @@ -882,6 +935,8 @@ + + @@ -925,6 +980,8 @@ Optional. If you don't use replicated tables, you could omit that. See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ + + Note: ClickHouse Cloud https://clickhouse.com/cloud has ClickHouse Keeper automatically configured for every service. --> false + true + + system + latency_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false + + system @@ -1180,6 +1250,19 @@ false + + + system + query_metric_log
+ 7500 + 1048576 + 8192 + 524288 + 1000 + false + + + + + + a Base64-encoded key to the stdout. + + Note: ClickHouse Cloud https://clickhouse.com/cloud supports encryption with customer-managed keys. + --> @@ -1418,10 +1512,14 @@ + Works only if ZooKeeper is enabled. Comment it if such functionality isn't required. + Note: ClickHouse Cloud https://clickhouse.com/cloud always runs DDL queries on cluster. + --> /clickhouse/task_queue/ddl + + /clickhouse/task_queue/replicas @@ -1452,6 +1550,7 @@ @@ -1461,7 +1560,9 @@ --> - + - + - false - - false - - + true + true + + true https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 - - false @@ -1680,6 +1778,7 @@ --> + backups diff --git a/programs/server/config.yaml.example b/programs/server/config.yaml.example index 5d5499f876c..5bfdb185d91 100644 --- a/programs/server/config.yaml.example +++ b/programs/server/config.yaml.example @@ -1,6 +1,6 @@ # This is an example of a configuration file "config.xml" rewritten in YAML # You can read this documentation for detailed information about YAML configuration: -# https://clickhouse.com/docs/en/operations/configuration-files/ +# https://clickhouse.com/docs/operations/configuration-files/ # NOTE: User and query level settings are set up in "users.yaml" file. # If you have accidentally specified user-level settings here, server won't start. @@ -136,11 +136,12 @@ interserver_http_port: 9009 # Allow multiple servers to listen on the same address:port. This is not recommended. # listen_reuse_port: 0 -# listen_backlog: 64 -max_connections: 4096 +# listen_backlog: 4096 + +# max_connections: 4096 # For 'Connection: keep-alive' in HTTP 1.1 -keep_alive_timeout: 3 +keep_alive_timeout: 10 # gRPC protocol (see src/Server/grpc_protos/clickhouse_grpc.proto for the API) # grpc_port: 9100 @@ -522,7 +523,7 @@ remap_executable: false # port: 9019 # Configuration of clusters that could be used in Distributed tables. -# https://clickhouse.com/docs/en/operations/table_engines/distributed/ +# https://clickhouse.com/docs/operations/table_engines/distributed/ remote_servers: # Test only shard config for testing distributed storage default: @@ -581,7 +582,7 @@ remote_servers: # ZooKeeper is used to store metadata about replicas, when using Replicated tables. # Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/ +# See https://clickhouse.com/docs/engines/table-engines/mergetree-family/replication/ # zookeeper: # - node: @@ -596,7 +597,7 @@ remote_servers: # Substitutions for parameters of replicated tables. # Optional. If you don't use replicated tables, you could omit that. -# See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/#creating-replicated-tables +# See https://clickhouse.com/docs/engines/table-engines/mergetree-family/replication/#creating-replicated-tables # macros: # shard: 01 # replica: example01-01-1 @@ -666,7 +667,7 @@ query_log: database: system table: query_log - # PARTITION BY expr: https://clickhouse.com/docs/en/table_engines/mergetree-family/custom_partitioning_key/ + # PARTITION BY expr: https://clickhouse.com/docs/table_engines/mergetree-family/custom_partitioning_key/ # Example: # event_date # toMonday(event_date) @@ -674,7 +675,7 @@ query_log: # toStartOfHour(event_time) partition_by: toYYYYMM(event_date) - # Table TTL specification: https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl + # Table TTL specification: https://clickhouse.com/docs/engines/table-engines/mergetree-family/mergetree/#mergetree-table-ttl # Example: # event_date + INTERVAL 1 WEEK # event_date + INTERVAL 7 DAY DELETE @@ -736,6 +737,13 @@ metric_log: flush_interval_milliseconds: 7500 collect_interval_milliseconds: 1000 +# Latency log contains rows with current values of latency buckets collected with "collect_interval_milliseconds" interval. +latency_log: + database: system + table: latency_log + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 + # Error log contains rows with current values of errors collected with "collect_interval_milliseconds" interval. error_log: database: system @@ -743,6 +751,13 @@ error_log: flush_interval_milliseconds: 7500 collect_interval_milliseconds: 1000 +# Query metric log contains history of memory and metric values from table system.events for individual queries, periodically flushed to disk. +query_metric_log: + database: system + table: query_metric_log + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 + # Asynchronous metric log contains values of metrics from # system.asynchronous_metrics. asynchronous_metric_log: @@ -791,7 +806,7 @@ top_level_domains_lists: '' # public_suffix_list: /path/to/public_suffix_list.dat # Configuration of external dictionaries. See: -# https://clickhouse.com/docs/en/sql-reference/dictionaries/external-dictionaries/external-dicts +# https://clickhouse.com/docs/sql-reference/dictionaries/external-dictionaries/external-dicts dictionaries_config: '*_dictionary.xml' # Uncomment if you want data to be compressed 30-100% better. diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index 0b099b15536..d1865d45531 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -476,7 +476,7 @@ - +
@@ -532,9 +532,15 @@ } ] +/// Dashboard selector +const dashboardSearchQuery = (dashboard_name) => `SELECT title, query FROM system.dashboards WHERE dashboard = '${dashboard_name}'`; +let dashboard_queries = { + "Overview": dashboardSearchQuery("Overview"), +}; +const default_dashboard = 'Overview'; /// Query to fill `queries` list for the dashboard -let search_query = `SELECT title, query FROM system.dashboards WHERE dashboard = 'Overview'`; +let search_query = dashboardSearchQuery(default_dashboard); let customized = false; let queries = []; @@ -547,7 +553,7 @@ let params = default_params; /// Palette generation for charts -function generatePalette(numColors) { +function generateColor(l, c, h) { // oklch() does not work in firefox<=125 inside element so we convert it back to rgb for now. // Based on https://github.com/color-js/color.js/blob/main/src/spaces/oklch.js const multiplyMatrices = (A, B) => { @@ -592,13 +598,17 @@ ], xyz) } - const oklch2rgb = lch => srgbLinear2rgb(xyz2rgbLinear(oklab2xyz(oklch2oklab(lch)))) + const oklch2rgb = lch => srgbLinear2rgb(xyz2rgbLinear(oklab2xyz(oklch2oklab(lch)))) + + let rgb = oklch2rgb([l, c, h]); + return `rgb(${rgb[0] * 255}, ${rgb[1] * 255}, ${rgb[2] * 255})`; +} + +function generatePalette(numColors) { palette = []; for (let i = 0; i < numColors; i++) { - //palette.push(`oklch(${theme != 'dark' ? 0.75 : 0.5}, 0.15, ${360 * i / numColors})`); - let rgb = oklch2rgb([theme != 'dark' ? 0.75 : 0.5, 0.15, 360 * i / numColors]); - palette.push(`rgb(${rgb[0] * 255}, ${rgb[1] * 255}, ${rgb[2] * 255})`); + palette.push(generateColor(theme != 'dark' ? 0.75 : 0.5, 0.15, 360 * i / numColors)); } return palette; } @@ -1110,7 +1120,7 @@ let response, reply, error; try { - response = await fetch(url + url_params, { method: "POST", body: query }); + response = await fetch(url + url_params, { method: "POST", body: query, headers: { 'Authorization': 'never' } }); reply = await response.text(); if (response.ok) { reply = JSON.parse(reply); @@ -1140,6 +1150,15 @@ return {reply, error}; } +function stringHash(str) { + let hash = 0; + for (let i = 0; i < str.length; i++) { + hash = ((hash << 5) - hash) + str.charCodeAt(i); + hash = hash & hash; + } + return hash; +} + async function draw(idx, chart, url_params, query) { if (plots[idx]) { plots[idx].destroy(); @@ -1232,9 +1251,12 @@ title_div.style.display = 'block'; } - const [line_color, fill_color, grid_color, axes_color] = theme != 'dark' - ? ["#ff8888", "#ffeeee", "#eeeedd", "#2c3235"] - : ["#886644", "#004455", "#2c3235", "#c7d0d9"]; + const color_rotate = Math.abs(stringHash(query)); + + const line_color = theme != 'dark' ? generateColor(0.75, 0.14, (21 + color_rotate) % 360) : generateColor(0.53, 0.07, (56 + color_rotate) % 360); + const fill_color = theme != 'dark' ? generateColor(0.96, 0.02, (21 + color_rotate) % 360) : generateColor(0.36, 0.07, (56 + color_rotate) % 360); + const grid_color = theme != 'dark' ? "#eeeedd" : "#2c3235"; + const axes_color = theme != 'dark' ? "#2c3235" : "#c7d0d9"; let sync = uPlot.sync("sync"); @@ -1439,7 +1461,7 @@ try { updateParams(); if (do_search) { - search_query = document.getElementById('search-query').value; + search_query = toSearchQuery(document.getElementById('search-query').value); queries = []; refreshCustomized(false); } @@ -1504,7 +1526,7 @@ document.getElementById('url').value = host; document.getElementById('user').value = user; document.getElementById('password').value = password; - document.getElementById('search-query').value = search_query; + document.getElementById('search-query').value = fromSearchQuery(search_query); refreshCustomized(); } @@ -1543,6 +1565,44 @@ } catch {} } +function fromSearchQuery(query) { + for (const dashboard_name in dashboard_queries) { + if (query == dashboard_queries[dashboard_name]) + return dashboard_name; + } + return query; +} + +function toSearchQuery(value) { + if (value in dashboard_queries) + return dashboard_queries[value]; + else + return value; +} + +async function populateSearchOptions() { + let {reply, error} = await doFetch("SELECT dashboard FROM system.dashboards GROUP BY dashboard ORDER BY ALL"); + if (error) { + throw new Error(error); + } + let data = reply.data; + if (data.dashboard.length == 0) { + console.log("Unable to fetch dashboards list"); + return; + } + dashboard_queries = {}; + for (let i = 0; i < data.dashboard.length; i++) { + const dashboard = data.dashboard[i]; + dashboard_queries[dashboard] = dashboardSearchQuery(dashboard); + } + const searchOptions = document.getElementById('search-options'); + for (const dashboard in dashboard_queries) { + const opt = document.createElement('option'); + opt.value = dashboard; + searchOptions.appendChild(opt); + } +} + async function start() { try { updateFromState(); @@ -1558,6 +1618,7 @@ } else { drawAll(); } + await populateSearchOptions(); } catch (e) { showError(e.message); } diff --git a/programs/server/merges.html b/programs/server/merges.html new file mode 100644 index 00000000000..119fb058b0b --- /dev/null +++ b/programs/server/merges.html @@ -0,0 +1,441 @@ + + + + + ClickHouse Merges Visualizer + + + + +
+
+
+ + +
+ + + 10x + 0000-00-00 00:00:00 + +
+
+
+
+ + + diff --git a/programs/server/play.html b/programs/server/play.html index 0d76a01cf7e..f465ddaa257 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -2,7 +2,7 @@ - + ClickHouse Query