diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6db56e583ee4..b54a5c32de63 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -2322,11 +2322,11 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_tsan, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2341,7 +2341,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -2371,13 +2371,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2392,7 +2392,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -2422,13 +2422,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2443,7 +2443,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -2473,13 +2473,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2494,7 +2494,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -2524,13 +2524,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2545,7 +2545,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -2575,13 +2575,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2596,7 +2596,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -2626,13 +2626,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2647,7 +2647,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -2677,13 +2677,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2698,7 +2698,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -2728,13 +2728,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2749,7 +2749,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -2779,13 +2779,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "MasterCI" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_msan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2800,7 +2800,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -2830,7 +2830,7 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "MasterCI" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "MasterCI" --ci --timestamp stateless_tests_amd_debug_distributed_plan_s3_storage_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] @@ -6753,7 +6753,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_8, stateless_tests_amd_msan_wasmedge_parallel_2_8, stateless_tests_amd_msan_wasmedge_parallel_3_8, stateless_tests_amd_msan_wasmedge_parallel_4_8, stateless_tests_amd_msan_wasmedge_parallel_5_8, stateless_tests_amd_msan_wasmedge_parallel_6_8, stateless_tests_amd_msan_wasmedge_parallel_7_8, stateless_tests_amd_msan_wasmedge_parallel_8_8, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_llvm_coverage_per_test, build_amd_msan, build_amd_release, build_amd_release_pr_cache_warmup, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_release_pr_cache_warmup, build_arm_tsan, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sign_release_amd_release, sign_release_arm_release, source_upload, sqllogic_test, sqlstorm_test, sqltest, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_2_2, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_1_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_2_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_3_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_4_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_5_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_6_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_7_8, stateless_tests_amd_llvm_coverage_per_test_per_test_coverage_8_8, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_parallel_1_4, stateless_tests_amd_tsan_parallel_2_4, stateless_tests_amd_tsan_parallel_3_4, stateless_tests_amd_tsan_parallel_4_4, stateless_tests_amd_tsan_s3_storage_parallel_1_3, stateless_tests_amd_tsan_s3_storage_parallel_2_3, stateless_tests_amd_tsan_s3_storage_parallel_3_3, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6896,16 +6896,16 @@ jobs: - stateless_tests_amd_tsan_parallel_4_4 - stateless_tests_amd_tsan_sequential_1_2 - stateless_tests_amd_tsan_sequential_2_2 - - stateless_tests_amd_msan_wasmedge_parallel_1_8 - - stateless_tests_amd_msan_wasmedge_parallel_2_8 - - stateless_tests_amd_msan_wasmedge_parallel_3_8 - - stateless_tests_amd_msan_wasmedge_parallel_4_8 - - stateless_tests_amd_msan_wasmedge_parallel_5_8 - - stateless_tests_amd_msan_wasmedge_parallel_6_8 - - stateless_tests_amd_msan_wasmedge_parallel_7_8 - - stateless_tests_amd_msan_wasmedge_parallel_8_8 - - stateless_tests_amd_msan_wasmedge_sequential_1_2 - - stateless_tests_amd_msan_wasmedge_sequential_2_2 + - stateless_tests_amd_msan_wasmedge_parallel_1_6 + - stateless_tests_amd_msan_wasmedge_parallel_2_6 + - stateless_tests_amd_msan_wasmedge_parallel_3_6 + - stateless_tests_amd_msan_wasmedge_parallel_4_6 + - stateless_tests_amd_msan_wasmedge_parallel_5_6 + - stateless_tests_amd_msan_wasmedge_parallel_6_6 + - stateless_tests_amd_msan_wasmedge_sequential_1_4 + - stateless_tests_amd_msan_wasmedge_sequential_2_4 + - stateless_tests_amd_msan_wasmedge_sequential_3_4 + - stateless_tests_amd_msan_wasmedge_sequential_4_4 - stateless_tests_amd_debug_distributed_plan_s3_storage_parallel - stateless_tests_amd_debug_distributed_plan_s3_storage_sequential - stateless_tests_amd_tsan_s3_storage_parallel_1_3 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 734e8779091b..208ddf97e3e8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1357,11 +1357,11 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_debug, sequential)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1376,7 +1376,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -1404,13 +1404,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1425,7 +1425,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -1453,13 +1453,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1474,7 +1474,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -1502,13 +1502,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1523,7 +1523,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -1551,13 +1551,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1572,7 +1572,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -1600,13 +1600,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1621,7 +1621,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -1649,13 +1649,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1670,7 +1670,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -1698,13 +1698,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1719,7 +1719,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -1747,13 +1747,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1768,7 +1768,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -1796,13 +1796,13 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "PR" --ci --timestamp - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1817,7 +1817,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -1845,7 +1845,7 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "PR" --ci --timestamp stateless_tests_amd_debug_distributed_plan_s3_storage_parallel: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] @@ -5571,48 +5571,6 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'BuzzHouse (amd_msan)' --workflow "PR" --ci --timestamp - parser_memory_check: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester-aarch64] - needs: [build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'UGFyc2VyIG1lbW9yeSBjaGVjaw==') }} - name: "Parser memory check" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Setup - uses: ./.github/actions/runner_setup - - name: Docker setup - uses: ./.github/actions/docker_setup - with: - test_name: "Parser memory check" - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Parser memory check' --workflow "PR" --ci --timestamp - promql_compliance: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] @@ -5918,7 +5876,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, parser_memory_check, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_8, stateless_tests_amd_msan_wasmedge_parallel_2_8, stateless_tests_amd_msan_wasmedge_parallel_3_8, stateless_tests_amd_msan_wasmedge_parallel_4_8, stateless_tests_amd_msan_wasmedge_parallel_5_8, stateless_tests_amd_msan_wasmedge_parallel_6_8, stateless_tests_amd_msan_wasmedge_parallel_7_8, stateless_tests_amd_msan_wasmedge_parallel_8_8, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_arm_asan_ubsan, bugfix_validation_unit_tests, build_amd_asan_ubsan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_binary, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, build_wasm_parser, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_arm_asan_ubsan, ci_tests, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_ubsan_targeted, integration_tests_amd_msan_10_10, integration_tests_amd_msan_1_10, integration_tests_amd_msan_2_10, integration_tests_amd_msan_3_10, integration_tests_amd_msan_4_10, integration_tests_amd_msan_5_10, integration_tests_amd_msan_6_10, integration_tests_amd_msan_7_10, integration_tests_amd_msan_8_10, integration_tests_amd_msan_9_10, integration_tests_amd_tsan_1_8, integration_tests_amd_tsan_2_8, integration_tests_amd_tsan_3_8, integration_tests_amd_tsan_4_8, integration_tests_amd_tsan_5_8, integration_tests_amd_tsan_6_8, integration_tests_amd_tsan_7_8, integration_tests_amd_tsan_8_8, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, keeper_stress_tests_pr, promql_compliance, quick_functional_tests, source_upload, sqllogic_test, sqlstorm_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_6, stateless_tests_amd_msan_wasmedge_parallel_2_6, stateless_tests_amd_msan_wasmedge_parallel_3_6, stateless_tests_amd_msan_wasmedge_parallel_4_6, stateless_tests_amd_msan_wasmedge_parallel_5_6, stateless_tests_amd_msan_wasmedge_parallel_6_6, stateless_tests_amd_msan_wasmedge_sequential_1_4, stateless_tests_amd_msan_wasmedge_sequential_2_4, stateless_tests_amd_msan_wasmedge_sequential_3_4, stateless_tests_amd_msan_wasmedge_sequential_4_4, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_parallel_selected_tests, stateless_tests_amd_tsan_s3_storage_sequential_selected_tests, stateless_tests_amd_tsan_sequential_selected_tests, stateless_tests_arm_asan_ubsan_azure_parallel_1_8, stateless_tests_arm_asan_ubsan_azure_parallel_2_8, stateless_tests_arm_asan_ubsan_azure_parallel_3_8, stateless_tests_arm_asan_ubsan_azure_parallel_4_8, stateless_tests_arm_asan_ubsan_azure_parallel_5_8, stateless_tests_arm_asan_ubsan_azure_parallel_6_8, stateless_tests_arm_asan_ubsan_azure_parallel_7_8, stateless_tests_arm_asan_ubsan_azure_parallel_8_8, stateless_tests_arm_asan_ubsan_azure_sequential_1_2, stateless_tests_arm_asan_ubsan_azure_sequential_2_2, stateless_tests_arm_asan_ubsan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_msan, unit_tests_msan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ !cancelled() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -6039,16 +5997,16 @@ jobs: - bugfix_validation_unit_tests - stateless_tests_amd_debug_parallel - stateless_tests_amd_debug_sequential - - stateless_tests_amd_msan_wasmedge_parallel_1_8 - - stateless_tests_amd_msan_wasmedge_parallel_2_8 - - stateless_tests_amd_msan_wasmedge_parallel_3_8 - - stateless_tests_amd_msan_wasmedge_parallel_4_8 - - stateless_tests_amd_msan_wasmedge_parallel_5_8 - - stateless_tests_amd_msan_wasmedge_parallel_6_8 - - stateless_tests_amd_msan_wasmedge_parallel_7_8 - - stateless_tests_amd_msan_wasmedge_parallel_8_8 - - stateless_tests_amd_msan_wasmedge_sequential_1_2 - - stateless_tests_amd_msan_wasmedge_sequential_2_2 + - stateless_tests_amd_msan_wasmedge_parallel_1_6 + - stateless_tests_amd_msan_wasmedge_parallel_2_6 + - stateless_tests_amd_msan_wasmedge_parallel_3_6 + - stateless_tests_amd_msan_wasmedge_parallel_4_6 + - stateless_tests_amd_msan_wasmedge_parallel_5_6 + - stateless_tests_amd_msan_wasmedge_parallel_6_6 + - stateless_tests_amd_msan_wasmedge_sequential_1_4 + - stateless_tests_amd_msan_wasmedge_sequential_2_4 + - stateless_tests_amd_msan_wasmedge_sequential_3_4 + - stateless_tests_amd_msan_wasmedge_sequential_4_4 - stateless_tests_amd_debug_distributed_plan_s3_storage_parallel - stateless_tests_amd_debug_distributed_plan_s3_storage_sequential - stateless_tests_arm_binary_parallel @@ -6129,7 +6087,6 @@ jobs: - buzzhouse_arm_asan_ubsan - buzzhouse_amd_tsan - buzzhouse_amd_msan - - parser_memory_check - promql_compliance - sqllogic_test - sqlstorm_test diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 9968fd9cc635..47411981ce29 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -1192,11 +1192,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_1_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_1_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAxLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1211,7 +1211,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 1/6)" - name: Prepare env script run: | @@ -1238,14 +1238,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 1/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_1_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_1_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1256,11 +1256,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_2_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_2_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAyLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1275,7 +1275,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 2/6)" - name: Prepare env script run: | @@ -1302,14 +1302,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 2/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_2_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_2_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1320,11 +1320,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_3_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_3_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCAzLzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1339,7 +1339,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 3/6)" - name: Prepare env script run: | @@ -1366,14 +1366,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 3/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_3_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_3_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1384,11 +1384,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_4_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_4_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA0LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1403,7 +1403,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 4/6)" - name: Prepare env script run: | @@ -1430,14 +1430,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 4/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_4_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_4_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1448,11 +1448,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_5_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_5_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA1LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1467,7 +1467,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 5/6)" - name: Prepare env script run: | @@ -1494,14 +1494,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 5/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_5_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_5_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1512,11 +1512,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_6_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_parallel_6_6: + runs-on: [self-hosted, altinity-on-demand, altinity-builder, 64g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA2LzYp') }} + name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1531,7 +1531,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 6/6)" - name: Prepare env script run: | @@ -1558,14 +1558,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 6/6)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_6_8 + name: failure-stateless_tests_amd_msan_wasmedge_parallel_6_6 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1576,11 +1576,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_7_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_1_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA3Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1595,7 +1595,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 7/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/4)" - name: Prepare env script run: | @@ -1622,14 +1622,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 7/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_7_8 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_1_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1640,11 +1640,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_parallel_8_8: - runs-on: [self-hosted, altinity-on-demand, altinity-func-tester] + stateless_tests_amd_msan_wasmedge_sequential_2_4: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHBhcmFsbGVsLCA4Lzgp') }} - name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1659,7 +1659,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, parallel, 8/8)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/4)" - name: Prepare env script run: | @@ -1686,14 +1686,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, parallel, 8/8)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_parallel_8_8 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_2_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1704,11 +1704,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_sequential_1_2: + stateless_tests_amd_msan_wasmedge_sequential_3_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDEvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDMvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1723,7 +1723,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 1/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 3/4)" - name: Prepare env script run: | @@ -1750,14 +1750,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 1/2)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 3/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_sequential_1_2 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_3_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt @@ -1768,11 +1768,11 @@ jobs: if-no-files-found: ignore retention-days: 14 - stateless_tests_amd_msan_wasmedge_sequential_2_2: + stateless_tests_amd_msan_wasmedge_sequential_4_4: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 32g] needs: [build_amd_asan_ubsan, build_amd_debug, build_amd_msan, build_amd_tsan, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_selected_tests, stateless_tests_amd_asan_ubsan_distributed_plan_parallel_selected_tests, stateless_tests_amd_debug_parallel, stateless_tests_amd_tsan_parallel_selected_tests, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDIvMik=') }} - name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhbWRfbXNhbiwgV2FzbUVkZ2UsIHNlcXVlbnRpYWwsIDQvNCk=') }} + name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1787,7 +1787,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 2/2)" + test_name: "Stateless tests (amd_msan, WasmEdge, sequential, 4/4)" - name: Prepare env script run: | @@ -1814,14 +1814,14 @@ jobs: id: run run: | . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 2/2)' --workflow "Community PR" --ci --timestamp + PYTHONUNBUFFERED=1 python3 -m praktika run 'Stateless tests (amd_msan, WasmEdge, sequential, 4/4)' --workflow "Community PR" --ci --timestamp - name: Upload failure report artifact if: failure() uses: actions/upload-artifact@v7 continue-on-error: true with: - name: failure-stateless_tests_amd_msan_wasmedge_sequential_2_2 + name: failure-stateless_tests_amd_msan_wasmedge_sequential_4_4 path: | ci/tmp/result_*.json ci/tmp/test_result.txt diff --git a/.gitmodules b/.gitmodules index 2f3dd2930a0a..ea6e27a86628 100644 --- a/.gitmodules +++ b/.gitmodules @@ -365,7 +365,7 @@ url = https://github.com/ClickHouse/ai-sdk-cpp [submodule "contrib/nlohmann-json"] path = contrib/nlohmann-json - url = https://github.com/nlohmann/json.git + url = https://github.com/ClickHouse/json [submodule "contrib/crc32c"] path = contrib/crc32c url = https://github.com/ClickHouse/crc32c @@ -416,7 +416,7 @@ url = https://github.com/ClickHouse/geometry.hpp [submodule "contrib/zxc"] path = contrib/zxc - url = https://github.com/ClickHouse/zxc + url = https://github.com/hellobertrand/zxc [submodule "contrib/libucontext"] path = contrib/libucontext url = https://github.com/kaniini/libucontext diff --git a/CMakeLists.txt b/CMakeLists.txt index 7860b6cd2ed8..68b194ad629c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,8 @@ if (OS_LINUX) if (LINKER_NAME MATCHES "lld" AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT WITH_COVERAGE) # Fold identical code and data sections to reduce binary size. - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=all") + # `safe` keeps address-taken functions distinct, as C and C++ require. + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--icf=safe") endif () # In Release builds, strip residual debug sections that may come from contrib @@ -345,7 +346,7 @@ if (NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT WITH_COVERAGE) set(COMPILER_FLAGS "${COMPILER_FLAGS} -ffunction-sections -fdata-sections") # -faddrsig emits an address-significance table that lets lld's ICF # distinguish address-taken symbols from non-address-taken ones. - # Only useful on Linux where we use lld with --icf=all; Darwin's ld64 + # Only useful on Linux where we use lld with --icf=safe; Darwin's ld64 # does not consume these sections. if (OS_LINUX) set(COMPILER_FLAGS "${COMPILER_FLAGS} -faddrsig") diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h index 475226652f5a..9e295ce5d299 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h @@ -149,6 +149,10 @@ namespace Net /// The object is normally guarded by the socket's mutex; a caller that uses it /// directly must ensure the socket is not accessed concurrently. + void markFatalError(); + /// Records that an external operation on the underlying `SSL` object failed fatally. + /// An orderly SSL shutdown must not be attempted afterwards. + X509 * peerCertificate() const; /// Returns the peer's certificate. @@ -233,7 +237,7 @@ namespace Net /// Returns true iff the given host name is the local host /// (either "localhost" or "127.0.0.1"). - bool mustRetry(int rc, Poco::Timespan & remaining_time); + bool mustRetry(int rc, int sslError, int socketError, Poco::Timespan & remaining_time); /// Returns true if the last operation should be retried, /// otherwise false. /// @@ -247,7 +251,7 @@ namespace Net /// not become readable or writable within the sockets /// receive or send timeout. - int handleError(int rc); + int handleError(int rc, int sslError, int socketError, unsigned long errorCode); /// Handles an SSL error by throwing an appropriate exception. void reset(); @@ -281,6 +285,10 @@ namespace Net Poco::AutoPtr _pSocket; Context::Ptr _pContext; bool _needHandshake; + bool _fatalError; + bool _pendingWrite = false; + /// Whether the last `SSL_write` returned `SSL_ERROR_WANT_WRITE`. OpenSSL keeps that record pending + /// until `SSL_write` is retried, even after other operations change what `SSL_get_error` reports. std::string _peerHostName; Session::Ptr _pSession; const BIO_METHOD * _bioMethod = nullptr; diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h index a222ee01914c..c5420492ec66 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h @@ -183,6 +183,9 @@ namespace Net /// Returns the underlying OpenSSL SSL object, or null if the SSL handshake /// has not been performed yet. + void markFatalError(); + /// Records that an external operation on the underlying `SSL` object failed fatally. + void setLazyHandshake(bool flag = true); /// Enable lazy SSL handshake. If enabled, the SSL handshake /// will be performed the first time date is sent or @@ -301,6 +304,12 @@ namespace Net } + inline void SecureStreamSocketImpl::markFatalError() + { + _impl.markFatalError(); + } + + inline Session::Ptr SecureStreamSocketImpl::currentSession() { return _impl.currentSession(); diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp index 60a55fec88bb..61e0a720ba61 100644 --- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp @@ -25,6 +25,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/NumberParser.h" #include "Poco/Format.h" +#include #include #include @@ -60,11 +61,52 @@ struct RemainingTimeCounter Poco::Timestamp start; }; +struct SSLOperationResult +{ + int rc = 0; + int sslError = SSL_ERROR_NONE; + int socketError = 0; + unsigned long errorCode = 0; +}; + + +template +SSLOperationResult performSSLOperation(SSL * ssl, Operation && operation, bool zeroIsError = true) +{ + /// The error queue contract for TLS I/O and `SSL_get_error`: + /// https://docs.openssl.org/3.5/man3/SSL_get_error/ + /// The queue is cleared with `ERR_clear_error`: + /// https://docs.openssl.org/3.5/man3/ERR_clear_error/ + ERR_clear_error(); + + SSLOperationResult result; + /// `errno` is only meaningful if it was set by this operation. In particular, a custom `BIO` + /// can report `SSL_ERROR_SYSCALL` without changing it, so do not inherit another syscall result. + errno = 0; + result.rc = operation(); + + if (result.rc < 0 || (zeroIsError && result.rc == 0)) + { + /// Save `errno` before calling another function. `SSL_get_error` must be the first + /// OpenSSL call after the operation and must observe the queue created by that operation. + result.socketError = errno; + result.sslError = SSL_get_error(ssl, result.rc); + result.errorCode = ERR_get_error(); + } + + /// `errorCode` above preserves the diagnostic used by `handleError`. Do not leave any + /// additional entries in the thread-local queue for another connection on this thread. + ERR_clear_error(); + return result; +} + + SecureSocketImpl::SecureSocketImpl(Poco::AutoPtr pSocketImpl, Context::Ptr pContext): _pSSL(nullptr), _pSocket(pSocketImpl), _pContext(pContext), - _needHandshake(false) + _needHandshake(false), + _fatalError(false) { poco_check_ptr (_pSocket); poco_check_ptr (_pContext); @@ -111,6 +153,13 @@ void SecureSocketImpl::setMutex(std::unique_ptr mutex) } +void SecureSocketImpl::markFatalError() +{ + ScopedLock lock(*_mutex); + _fatalError = true; +} + + const BIO_METHOD * SecureSocketImpl::getBioMethod() const { return _bioMethod ? _bioMethod : BIO_s_socket(); @@ -121,6 +170,8 @@ void SecureSocketImpl::acceptSSL() { ScopedLock lock(*_mutex); poco_assert (!_pSSL); + _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create BIO object"); @@ -190,6 +241,8 @@ void SecureSocketImpl::connectSSL(bool performHandshake) ScopedLock lock(*_mutex); poco_assert (!_pSSL); poco_assert (_pSocket->initialized()); + _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(getBioMethod()); if (!pBIO) throw SSLException("Cannot create SSL BIO object"); @@ -218,31 +271,39 @@ void SecureSocketImpl::connectSSL(bool performHandshake) SSL_set_session(_pSSL, _pSession->sslSession()); } + SSL_set_connect_state(_pSSL); + _needHandshake = true; + try { if (performHandshake && _pSocket->getBlocking()) { - int ret; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - ret = SSL_connect(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_connect(_pSSL); + }); + } + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc <= 0) + { + if (handleError(result.rc, result.sslError, result.socketError, result.errorCode) < 0) + throw Poco::TimeoutException("SSL handshake timed out"); + throw SSLConnectionUnexpectedlyClosedException(); } - while (mustRetry(ret, remaining_time)); - handleError(ret); + _needHandshake = false; verifyPeerCertificate(); } - else - { - SSL_set_connect_state(_pSSL); - _needHandshake = true; - } } catch (...) { SSL_free(_pSSL); _pSSL = 0; + _fatalError = false; throw; } } @@ -271,6 +332,16 @@ void SecureSocketImpl::shutdown() ScopedLock lock(*_mutex); if (_pSSL) { + if (_fatalError) + { + /// OpenSSL forbids `SSL_shutdown` after a fatal TLS or syscall error. + /// https://docs.openssl.org/3.5/man3/SSL_shutdown/ + /// Close the underlying transport without attempting an orderly TLS shutdown. + if (_pSocket->getBlocking()) + _pSocket->shutdown(); + return; + } + // Don't shut down the socket more than once. int shutdownState = SSL_get_shutdown(_pSSL); bool shutdownSent = (shutdownState & SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN; @@ -283,8 +354,24 @@ void SecureSocketImpl::shutdown() // most web browsers, so we just set the shutdown // flag by calling SSL_shutdown() once and be // done with it. - int rc = SSL_shutdown(_pSSL); - if (rc < 0) handleError(rc); + /// A zero result is not an error for `SSL_shutdown` and must not be passed to + /// `SSL_get_error`; it means that `close_notify` was sent but not received yet. + SSLOperationResult result; + Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); + do + { + RemainingTimeCounter counter(remaining_time); + result = performSSLOperation(_pSSL, [this] + { + return SSL_shutdown(_pSSL); + }, false); + } + /// OpenSSL does not dispatch the `close_notify` alert while a record write from an + /// earlier `SSL_write` is still pending, so retrying cannot make progress then. + while (!_pendingWrite && result.rc < 0 + && mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc < 0) + handleError(result.rc, result.sslError, result.socketError, result.errorCode); if (_pSocket->getBlocking()) { _pSocket->shutdown(); @@ -328,24 +415,22 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_write(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_write(_pSSL, buffer, length); + }); + _pendingWrite = result.sslError == SSL_ERROR_WANT_WRITE; } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - // At this stage we still can have last not yet received SSL message containing SSL error - // so make a read to force SSL to process possible SSL error - if (SSL_get_error(_pSSL, rc) == SSL_ERROR_SYSCALL && SocketImpl::lastError() == POCO_ECONNRESET) - { - char c = 0; - SSL_read(_pSSL, &c, 1); - } - - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc == 0) throw SSLConnectionUnexpectedlyClosedException(); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_write timed out"); @@ -379,6 +464,7 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { @@ -386,12 +472,16 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) /// so thread can be blocked on recv/send and epoll_wait several times /// until SSL_read will return rc > 0. Let's use our own time counter. RemainingTimeCounter counter(remaining_time); - rc = SSL_read(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_read(_pSSL, buffer, length); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_read timed out"); return rc; @@ -419,16 +509,21 @@ int SecureSocketImpl::completeHandshake() poco_check_ptr (_pSSL); int rc; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_do_handshake(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_do_handshake(_pSSL); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL handshake timed out"); return rc; @@ -536,15 +631,13 @@ Poco::Timespan SecureSocketImpl::getMaxTimeoutOrLimit() return remaining_time; } -bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) +bool SecureSocketImpl::mustRetry(int rc, int sslError, int socketError, Poco::Timespan& remaining_time) { if (remaining_time == 0) return false; ScopedLock lock(*_mutex); if (rc <= 0) { - int sslError = SSL_get_error(_pSSL, rc); - int socketError = _pSocket->lastError(); switch (sslError) { case SSL_ERROR_WANT_READ: @@ -571,21 +664,20 @@ bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) case SSL_ERROR_SYSCALL: return socketError == POCO_EAGAIN || socketError == POCO_EINTR; default: - return socketError == POCO_EINTR; + /// `errno` is only meaningful for `SSL_ERROR_SYSCALL`; other errors must not + /// be retried because of a leftover `EINTR`, especially `SSL_ERROR_SSL`. + return false; } } return false; } -int SecureSocketImpl::handleError(int rc) +int SecureSocketImpl::handleError(int rc, int sslError, int error, unsigned long errorCode) { ScopedLock lock(*_mutex); if (rc > 0) return rc; - int sslError = SSL_get_error(_pSSL, rc); - int error = SocketImpl::lastError(); - switch (sslError) { case SSL_ERROR_ZERO_RETURN: @@ -598,18 +690,21 @@ int SecureSocketImpl::handleError(int rc) case SSL_ERROR_WANT_ACCEPT: case SSL_ERROR_WANT_X509_LOOKUP: // these should not occur + _fatalError = true; poco_bugcheck(); return rc; case SSL_ERROR_SYSCALL: + _fatalError = true; if (error != 0) { SocketImpl::error(error); } - // fallthrough + [[fallthrough]]; + case SSL_ERROR_SSL: default: { - long lastError = ERR_get_error(); - if (lastError == 0) + _fatalError = true; + if (errorCode == 0) { if (rc == 0) { @@ -631,7 +726,7 @@ int SecureSocketImpl::handleError(int rc) else { char buffer[256]; - ERR_error_string_n(lastError, buffer, sizeof(buffer)); + ERR_error_string_n(errorCode, buffer, sizeof(buffer)); std::string msg(buffer); throw SSLException(msg); } @@ -658,6 +753,7 @@ void SecureSocketImpl::reset() SSL_free(_pSSL); _pSSL = nullptr; } + _fatalError = false; } diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 0d96a727b19f..eaec79a77d78 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -975,25 +975,25 @@ class JobConfigs: for total_batches in (2,) for batch in range(1, total_batches + 1) ], - # One shard can lose worker concurrency mid-suite under MSan and - # overrun the shared 3.5h functional-test budget. - *[ + # NOTE (strtgbb): 64g keeps MSan at 6 workers, Lower mem can cause workers to be killed + *[ Job.ParamSet( parameter=f"amd_msan, WasmEdge, parallel, {batch}/{total_batches}", - runs_on=RunnerLabels.FUNC_TESTER_AMD, + runs_on=RunnerLabels.AMD_LARGE, requires=[ArtifactNames.CH_AMD_MSAN_GH], timeout=3600 * 4, ) - for total_batches in (8,) + for total_batches in (6,) for batch in range(1, total_batches + 1) ], + # NOTE (strtgbb): 2/2 was ~3.3h of sequential work and overran 3.5h in teardown. *[ Job.ParamSet( parameter=f"amd_msan, WasmEdge, sequential, {batch}/{total_batches}", runs_on=RunnerLabels.AMD_SMALL_MEM, requires=[ArtifactNames.CH_AMD_MSAN_GH], ) - for total_batches in (2,) + for total_batches in (4,) for batch in range(1, total_batches + 1) ], Job.ParamSet( diff --git a/ci/jobs/check_style.py b/ci/jobs/check_style.py index 26767e7eb84d..2015f1d6ecd2 100644 --- a/ci/jobs/check_style.py +++ b/ci/jobs/check_style.py @@ -3,6 +3,7 @@ import math import multiprocessing import os +import pathlib import re import shlex from concurrent.futures import ProcessPoolExecutor @@ -878,6 +879,125 @@ def check_catch_all(files) -> str: return "\n".join(violations) +# Storage classes whose tables can be deferred behind `StorageTableProxy`, which means a pointer +# taken from `DatabaseCatalog` may be the proxy rather than the engine. +DEFERRABLE_STORAGE_CLASSES = ( + "MergeTreeData", + "StorageMergeTree", + "StorageReplicatedMergeTree", + "StorageSharedMergeTree", + "StorageSetOrJoinBase", + "StorageSet", + "StorageJoin", + "StorageSharedSet", + "StorageSharedJoin", + "StorageEmbeddedRocksDB", + "IKeyValueEntity", + "IStorageURLBase", + "IBackgroundOperation", + "StorageWithCommonVirtualColumns", + "StorageLog", + "StorageStripeLog", + "StorageURL", + "StorageObjectStorage", + "StorageKeeperMap", + "StorageMySQL", + "StoragePostgreSQL", + "StorageMongoDB", + "StorageRedis", + "StorageSQLite", + "StorageXDBC", + "StorageHive", + "StorageArrowFlight", + "StorageYTsaurus", + "StorageBigQuery", + "StorageKafka", + "StorageKafka2", + "StorageFileLog", + "StorageRabbitMQ", + "StorageNATS", + "StorageObjectStorageQueue", + "IStreamingStorage", +) + +# Casts on an operand that cannot be a catalog pointer, so no proxy can be in the way. +_NOT_A_CATALOG_POINTER = re.compile( + r"^(?:\*?this\b" + r"|shared_from_this\(\)" + r"|&?\w*(?:snapshot|storage_snapshot)->storage\b" + r"|&?\w*reading->getMergeTreeData\(\)" + r")" +) + + +def _without_comments(text): + """Blanks out comments, keeping the line layout so offsets and line numbers still match.""" + return re.sub( + r"//[^\n]*|/\*.*?\*/", lambda m: re.sub(r"[^\n]", " ", m.group(0)), text, flags=re.S + ) + + +def _cast_operand(text, open_paren): + """The argument of a cast whose '(' is at `open_paren`, or None when unbalanced.""" + depth = 0 + for i in range(open_paren, min(open_paren + 2000, len(text))): + depth += (text[i] == "(") - (text[i] == ")") + if not depth: + return " ".join(text[open_paren + 1 : i].split()) + return None + + +def check_storage_casts(files) -> str: + """Require `castStorage` for casts to an engine that supports deferred loading. + + Such a table lives behind `StorageTableProxy` until its first access and the catalog keeps + handing out that proxy afterwards, so a direct cast fails for the whole life of the table. + """ + types = "|".join(DEFERRABLE_STORAGE_CLASSES) + cast_head = re.compile( + r"\b(?Pdynamic_cast|typeid_cast|dynamic_pointer_cast|static_pointer_cast)\s*<\s*" + r"(?:const\s+)?(?:" + types + r")\s*[*&]?\s*>\s*\(" + ) + # `IStorage::as()` is a `typeid_cast` on the receiver, so the operand is what precedes it. + as_cast = re.compile( + r"(?P[\w.\[\]()]+)(?:->|\.)as\s*<\s*(?:const\s+)?(?:" + types + r")\s*>\s*\(\s*\)" + ) + resolvers = ("castStorage", "resolveStorageProxy", "resolveStorageProxyLoading") + + violations = [] + for path in files: + # The helpers and the proxy itself have to reach the nested storage directly. + if not path.endswith((".cpp", ".h")) or path.endswith( + ("StorageProxy.h", "StorageTableProxy.h", "StorageTableFunction.h") + ): + continue + try: + lines = pathlib.Path(path).read_text(errors="replace").splitlines() + except OSError: + continue + + text = _without_comments("\n".join(lines)) + casts = [(m, m.group("cast"), _cast_operand(text, m.end() - 1)) for m in cast_head.finditer(text)] + casts += [(m, "as", m.group("operand")) for m in as_cast.finditer(text)] + + for match, cast, operand in casts: + if not operand or _NOT_A_CATALOG_POINTER.match(operand) or any(r in operand for r in resolvers): + continue + line = text.count("\n", 0, match.start()) + 1 + # The marker goes on the cast or, when the line is long, the one above it. + if any("NOLINT(storage-cast)" in lines[i] for i in (line - 1, line - 2) if i >= 0): + continue + violations.append( + f"{path}:{line}: {cast} to a deferrable storage engine on `{operand[:60]}`. Such a table " + "is reached through StorageTableProxy, so this cast fails for the whole life of the table " + "and whatever it guards is silently skipped. Use castStorage(ptr, " + "DeferredTable::Load) when the query names this table, or DeferredTable::Skip " + "when this walks every table and must not load one. If the pointer cannot come from " + "DatabaseCatalog, say why in a `/// NOLINT(storage-cast)` comment." + ) + return "\n".join(violations) + + def check_file_names(files): files_set = set() for file in files: @@ -1371,6 +1491,15 @@ def parse_args(): files=cpp_files, ) ) + testname = "storage_casts" + if testpattern.lower() in testname.lower(): + results.append( + run_check_concurrent( + check_name=testname, + check_function=check_storage_casts, + files=cpp_files, + ) + ) testname = "compose_images_from_dockerhub" if testpattern.lower() in testname.lower(): results.append( diff --git a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt index 4c4a0ec48a10..0b63ce3eb10b 100644 --- a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt +++ b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt @@ -14,6 +14,7 @@ allow_deprecated_database_ordinary allow_deprecated_snowflake_conversion_functions allow_distributed_ddl allow_drop_detached +allow_executable_tables allow_execute_multiif_columnar allow_experimental_ai_functions allow_experimental_alter_materialized_view_structure diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index ab1c92dc9763..fd402def0eaf 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -215,7 +215,8 @@ # job.set_run_after(CORE_BLOCKING_JOB_NAMES) # for job in JobConfigs.performance_comparison_with_master_head_jobs # ], # NOTE (strtgbb): failed previously due to GH secrets not being handled properly, try again later - JobConfigs.parser_memory_check_job, + # NOTE (strtgbb): ooks up master clickhouse-examples on upstream S3 + # JobConfigs.parser_memory_check_job, # ClickBench runs on PRs only when files in its digest change # (see `clickbench_jobs.digest_config`), so the cost is bounded. # *[ diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 95803bd73fbb..2579d47569b9 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,13 +2,13 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54523) +SET(VERSION_REVISION 54533) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 11) -SET(VERSION_GITHASH bedf2ab54b8a0c34afbf2d907eaf933324f43cd6) -SET(VERSION_DESCRIBE v26.8.11.10001.altinitytest) -SET(VERSION_STRING 26.8.11.10001.altinitytest) +SET(VERSION_PATCH 21) +SET(VERSION_GITHASH 68170c6e7043e16504bb80174ee032a196a1aa63) +SET(VERSION_DESCRIBE v26.8.21.10001.altinitytest) +SET(VERSION_STRING 26.8.21.10001.altinitytest) # end of autochange SET(VERSION_TWEAK 10001) diff --git a/cmake/strip_rust_symbols.sh b/cmake/strip_rust_symbols.sh index cb91889088bf..4987b189590f 100755 --- a/cmake/strip_rust_symbols.sh +++ b/cmake/strip_rust_symbols.sh @@ -50,7 +50,8 @@ for sym in "$@"; do done # Localize all symbols except the public ones, then strip unneeded locals -"$OBJCOPY" $KEEP_FLAGS --strip-unneeded "$WORK_DIR/combined.o" "$WORK_DIR/stripped.o" +# Drop .llvm_addrsig: after ld -r and objcopy its sh_link is 0, so lld --icf=safe ignores it with a warning +"$OBJCOPY" $KEEP_FLAGS --strip-unneeded --remove-section=.llvm_addrsig "$WORK_DIR/combined.o" "$WORK_DIR/stripped.o" # Repackage as .a (replace original) rm -f "$LIB_PATH" diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4253bf5fad42..537a9f057984 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -266,8 +266,12 @@ option(ENABLE_GOOGLE_CLOUD_CPP "Enable Google Cloud Cpp" ${ENABLE_GOOGLE_CLOUD_C # `crc32c` is a small portable library used by both `google-cloud-cpp` and the snappy framing format # in `SnappyWriteBuffer`/`SnappyFramedReadBuffer`, so it is always built when libraries are enabled. add_contrib (crc32c-cmake crc32c) +# `nlohmann::json` is header-only and is used by both `google-cloud-cpp` and `ai-sdk-cpp`, which are +# enabled independently of each other. It is added for everyone so that there is one copy of it in +# the build: two copies of the same version define the same symbols with different layouts, which +# the linker resolves silently - see `contrib/ai-sdk-cpp-cmake/CMakeLists.txt`. +add_contrib (nlohmann-json-cmake nlohmann-json) if(ENABLE_GOOGLE_CLOUD_CPP) - add_contrib (nlohmann-json-cmake nlohmann-json) add_contrib (google-cloud-cpp-cmake google-cloud-cpp) # requires grpc, protobuf, absl, nlohmann's json, crc32c else() message(STATUS "Not using Google Cloud Cpp") diff --git a/contrib/NuRaft b/contrib/NuRaft index a0eda0069d2d..884294e9dbee 160000 --- a/contrib/NuRaft +++ b/contrib/NuRaft @@ -1 +1 @@ -Subproject commit a0eda0069d2dac0852c453cbe7b0285eda218479 +Subproject commit 884294e9dbee183b53601bbe2c8c5753cec1a3db diff --git a/contrib/ai-sdk-cpp-cmake/CMakeLists.txt b/contrib/ai-sdk-cpp-cmake/CMakeLists.txt index 6bcf697023ab..72230124d7e1 100644 --- a/contrib/ai-sdk-cpp-cmake/CMakeLists.txt +++ b/contrib/ai-sdk-cpp-cmake/CMakeLists.txt @@ -38,13 +38,16 @@ set(AI_SDK_ANTHROPIC_SOURCES "${AI_SDK_SOURCE_DIR}/src/providers/anthropic/anthropic_factory.cpp" ) -# Add nlohmann_json from submodule -set(NLOHMANN_JSON_SOURCE_DIR "${AI_SDK_THIRD_PARTY_DIR}/nlohmann_json_patched") -add_library(_ai_sdk_nlohmann_json INTERFACE) -target_include_directories(_ai_sdk_nlohmann_json SYSTEM INTERFACE - "${NLOHMANN_JSON_SOURCE_DIR}/include" -) -add_library(ai_sdk_nlohmann_json::ai_sdk_nlohmann_json ALIAS _ai_sdk_nlohmann_json) +# `nlohmann::json` comes from `contrib/nlohmann-json`, not from the `third_party/nlohmann_json_patched` +# copy this library vendors. Both are version 3.12.0, so both define the same +# `nlohmann::json_abi_v3_12_0` symbols - and the two definitions differ: the `contrib` copy is the +# ClickHouse fork that reads numbers without consulting the locale, whose `lexer` holds a +# `number_buffer` member that the vendored copy has no field for. Linking both is an ODR violation +# that the linker resolves by keeping one definition of each symbol, so a `parser` laid out by one +# header ends up in a `parse` compiled from the other, which reads and destroys members past the +# end of it: `AddressSanitizer` reports a stack-buffer-overflow and the client aborts on the first +# response of the model. The vendored copy exists to keep the number parsing away from the locale, +# which the fork does properly, so there is nothing left to keep it for. # Add httplib from submodule (header-only) set(HTTPLIB_SOURCE_DIR "${AI_SDK_THIRD_PARTY_DIR}/httplib-header-only") @@ -79,7 +82,7 @@ target_include_directories(_ai-sdk-cpp-core SYSTEM target_link_libraries(_ai-sdk-cpp-core PUBLIC - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue @@ -105,7 +108,7 @@ target_include_directories(_ai-sdk-cpp-openai SYSTEM target_link_libraries(_ai-sdk-cpp-openai PUBLIC _ai-sdk-cpp-core - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue @@ -133,7 +136,7 @@ target_include_directories(_ai-sdk-cpp-anthropic SYSTEM target_link_libraries(_ai-sdk-cpp-anthropic PUBLIC _ai-sdk-cpp-core - ai_sdk_nlohmann_json::ai_sdk_nlohmann_json + ch_contrib::nlohmann_json PRIVATE httplib::httplib concurrentqueue diff --git a/contrib/cctz b/contrib/cctz index 8e694da054a9..bfd13df99e26 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit 8e694da054a9a31d98392bf03ee188b04f810d0a +Subproject commit bfd13df99e26583c1a00edc5032964b01bab551b diff --git a/contrib/croaring b/contrib/croaring index 025ae3f7add1..342463b31b90 160000 --- a/contrib/croaring +++ b/contrib/croaring @@ -1 +1 @@ -Subproject commit 025ae3f7add169bc820dcfd46fa9304f382ec40a +Subproject commit 342463b31b909737bd6295c69b2f8e0ed9497424 diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 57ccb00086b0..b2945e968d8d 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -32,7 +32,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cert.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" +# "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" # compiled via the rdkafka_conf_sensitive.c wrapper below "${RDKAFKA_SOURCE_DIR}/rdkafka_coord.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_error.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_event.c" @@ -84,6 +84,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_txnmgr.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_zstd.c" # WITH_ZSTD + "${CMAKE_CURRENT_SOURCE_DIR}/rdkafka_conf_sensitive.c" # ClickHouse wrapper compiling rdkafka_conf.c, see the comment in the file "${RDKAFKA_SOURCE_DIR}/rdlist.c" "${RDKAFKA_SOURCE_DIR}/rdlog.c" "${RDKAFKA_SOURCE_DIR}/rdmap.c" diff --git a/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h new file mode 100644 index 000000000000..83b99dce955c --- /dev/null +++ b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h @@ -0,0 +1,14 @@ +/// See contrib/librdkafka-cmake/rdkafka_conf_sensitive.c +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/// The names of the configuration properties that librdkafka marks with the _RK_SENSITIVE flag, +/// i.e. whose values must not appear in logs, as a NULL-terminated array of static strings. +const char * const * chrd_kafka_conf_sensitive_properties(void); + +#ifdef __cplusplus +} +#endif diff --git a/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c new file mode 100644 index 000000000000..740f444ad0a1 --- /dev/null +++ b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c @@ -0,0 +1,31 @@ +/// Wrapper around librdkafka's rdkafka_conf.c: compiles the file (this is the only place it is +/// compiled, see the SRCS list in contrib/librdkafka-cmake/CMakeLists.txt) and adds an accessor +/// for the static rd_kafka_properties table, which is the source of truth for the _RK_SENSITIVE +/// flag. ClickHouse uses the flag to hide the values of sensitive properties (e.g. sasl.password) +/// in its logs. No public librdkafka API exposes the flag: the redacting dump +/// (rd_kafka_anyconf_dump with redact_sensitive) is static, and the public rd_kafka_conf_dump +/// does not redact. + +#include "rdkafka_conf.c" + +#include + +/* The table ends with a terminator entry, so the count is an upper bound and the + * array is always NULL-terminated. The names point into the static table. */ +static const char * chrd_sensitive_names[sizeof(rd_kafka_properties) / sizeof(*rd_kafka_properties)]; +static pthread_once_t chrd_sensitive_names_once = PTHREAD_ONCE_INIT; + +static void chrd_fill_sensitive_names(void) +{ + const struct rd_kafka_property * prop = NULL; + size_t n = 0; + for (prop = rd_kafka_properties; prop->name; prop++) + if (prop->scope & _RK_SENSITIVE) + chrd_sensitive_names[n++] = prop->name; +} + +const char * const * chrd_kafka_conf_sensitive_properties(void) +{ + pthread_once(&chrd_sensitive_names_once, chrd_fill_sensitive_names); + return chrd_sensitive_names; +} diff --git a/contrib/nats-io b/contrib/nats-io index 3e3a3f10572e..b88704dfeb55 160000 --- a/contrib/nats-io +++ b/contrib/nats-io @@ -1 +1 @@ -Subproject commit 3e3a3f10572ed9ff1b7ad17873881877cb62f301 +Subproject commit b88704dfeb55e0f1dee165b9c1daccb90d5816d4 diff --git a/contrib/nlohmann-json b/contrib/nlohmann-json index 55f93686c015..4438a2a0cda7 160000 --- a/contrib/nlohmann-json +++ b/contrib/nlohmann-json @@ -1 +1 @@ -Subproject commit 55f93686c01528224f448c19128836e7df245f72 +Subproject commit 4438a2a0cda77bf4cd74652423e03363e003a0cc diff --git a/contrib/nlohmann-json-cmake/CMakeLists.txt b/contrib/nlohmann-json-cmake/CMakeLists.txt index 752b845bc819..863c35363cd1 100644 --- a/contrib/nlohmann-json-cmake/CMakeLists.txt +++ b/contrib/nlohmann-json-cmake/CMakeLists.txt @@ -1,8 +1,4 @@ set(JSON_DIR ${ClickHouse_SOURCE_DIR}/contrib/nlohmann-json) add_library(_nlohmann_json INTERFACE) -set_property( - TARGET _nlohmann_json - APPEND - PROPERTY INTERFACE_INCLUDE_DIRECTORIES - ${JSON_DIR}/single_include) +target_include_directories(_nlohmann_json SYSTEM INTERFACE ${JSON_DIR}/single_include) add_library(ch_contrib::nlohmann_json ALIAS _nlohmann_json) diff --git a/contrib/replxx b/contrib/replxx index c2de583a3cd4..4ab25d428dd0 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit c2de583a3cd41f7b476cb625e954c191b6fcc448 +Subproject commit 4ab25d428dd0bfc0590cac0d15e1fb872794dd62 diff --git a/contrib/zxc b/contrib/zxc index b9890cfe3466..e568d1967309 160000 --- a/contrib/zxc +++ b/contrib/zxc @@ -1 +1 @@ -Subproject commit b9890cfe3466b19d0eb5005874ce81728efb9565 +Subproject commit e568d1967309fd5d160f45f2fdabcbff6ab3f058 diff --git a/contrib/zxc-cmake/CMakeLists.txt b/contrib/zxc-cmake/CMakeLists.txt index 98c52cb1111b..70451492f30d 100644 --- a/contrib/zxc-cmake/CMakeLists.txt +++ b/contrib/zxc-cmake/CMakeLists.txt @@ -33,9 +33,14 @@ set(ZXC_CORE_SOURCES # emitted symbols (e.g. zxc_compress_chunk_wrapper_avx2). zxc_dispatch.c declares # these suffixed symbols unconditionally per architecture, so the exact set of # variants below must match what the dispatcher expects for each target: -# x86-64 -> _default, _sse2, _avx2, _avx512 -# aarch64 -> _default, _neon +# x86-64 -> _default, _avx2, _avx512 +# aarch64 -> _default only (NEON is the AArch64 baseline) # other -> _default only +# SSE2 is the x86-64 baseline and NEON the AArch64 one, so `_default` already +# compiles those code paths; zxc dropped the dedicated `_sse2` and `_neon` +# variants in v0.14.0 and the dispatcher no longer references them. The +# remaining `_neon32` variant is for 32-bit ARM, which ClickHouse does not +# target. set(ZXC_VARIANT_OBJECTS "") macro(zxc_add_variant suffix) @@ -54,11 +59,14 @@ macro(zxc_add_variant suffix) endmacro() if (OS_DARWIN AND ARCH_AMD64) - # zxc's x86-64 runtime dispatch calls `__builtin_cpu_init` / + # Historically zxc's x86-64 runtime dispatch called `__builtin_cpu_init` / # `__builtin_cpu_supports`, which reference compiler-rt's `__cpu_model`. That - # symbol is not linked into ClickHouse's macOS cross-build, so linking fails - # with "undefined ___cpu_model". macOS builds are for local development only, - # so restrict zxc to its portable scalar core there. + # symbol is not linked into ClickHouse's macOS cross-build, so linking failed + # with "undefined ___cpu_model". Since v0.14.0 the dispatcher issues `CPUID` + # and `XGETBV` directly and no longer needs `__cpu_model`, so this is now a + # conservative restriction rather than a required one: macOS builds are for + # local development only and are never test-run in CI, so there is nothing + # to gain from the x86 SIMD variants there. Lifting it is a separate change. set(ZXC_FORCE_SCALAR TRUE) elseif (SANITIZE STREQUAL "memory") # clang's MemorySanitizer has no precise shadow model for the x86 byte-shuffle @@ -67,7 +75,7 @@ elseif (SANITIZE STREQUAL "memory") # that the shuffle discards still poisons the logical result. zxc's SIMD # kernels deliberately do speculative 16/32-byte loads whose tail lanes fall # into malloc'ed scratch buffers or not-yet-written output and are then - # discarded by such shuffles (e.g. `zxc_decode_copy_overlap_run`, + # discarded by such shuffles (e.g. `zxc_decode_copy_overlap_run32`, # `zxc_pivco_merge`), which triggers false `use-of-uninitialized-value` # reports. The scalar core branches only on initialized bytes, so build only # that under MSan. (AArch64 NEON `tbl` gets a precise shadow, but keep all @@ -90,16 +98,16 @@ if (ZXC_FORCE_SCALAR) target_compile_definitions(${_tgt} PRIVATE ZXC_DISABLE_SIMD) endforeach() elseif (ARCH_AMD64) - # SSE2/AVX2 flags are redundant with the x86-64-v3 baseline but harmless; + # The AVX2 flags are redundant with the x86-64-v3 baseline but harmless; # AVX512 is NOT in the baseline, so those flags are required for that variant - # (only ever entered at runtime on AVX-512 capable CPUs). - zxc_add_variant(_sse2 -msse2) - zxc_add_variant(_avx2 -mavx2 -mfma -mbmi -mbmi2 -mlzcnt) + # (only ever entered at runtime on AVX-512 capable CPUs). The flag sets must + # stay in sync with `zxc_detect_cpu_features`, which admits a CPU to the AVX2 + # and AVX-512 tiers only after proving BMI1, BMI2 and LZCNT as well. Note + # that it does not probe FMA, so `-mfma` must not be added here: the + # compiler would be free to emit FMA instructions into a variant that runs + # on any AVX2 CPU. + zxc_add_variant(_avx2 -mavx2 -mbmi -mbmi2 -mlzcnt) zxc_add_variant(_avx512 -mavx512f -mavx512bw -mavx512vbmi -mavx512vbmi2 -mbmi -mbmi2 -mlzcnt) -elseif (ARCH_AARCH64) - # NEON is guaranteed by ClickHouse's armv8.2-a+simd baseline; no extra flags - # (adding -march=armv8-a would downgrade below the baseline). - zxc_add_variant(_neon) endif () add_library(_zxc ${ZXC_CORE_SOURCES} ${ZXC_VARIANT_OBJECTS}) diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index b8c0961ea8bb..7a64e4c5abc1 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -16,7 +16,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac -FROM alpine:3.21 +FROM alpine ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ @@ -121,6 +121,9 @@ RUN clickhouse-keeper --version \ && chown root:clickhouse "${DEFAULT_LOG_DIR}" \ && chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" +RUN apk update +RUN apk add --upgrade libssl3 libcrypto3 + # /var/lib/clickhouse is necessary due to the current default configuration for Keeper VOLUME "${DEFAULT_DATA_DIR}" /var/lib/clickhouse EXPOSE 2181 10181 44444 9181 diff --git a/docs/concepts/features/interfaces/http.mdx b/docs/concepts/features/interfaces/http.mdx index ff580401ef9c..dbadd8c4a7c9 100644 --- a/docs/concepts/features/interfaces/http.mdx +++ b/docs/concepts/features/interfaces/http.mdx @@ -190,7 +190,7 @@ wget -nv -O- 'http://localhost:8123/?query=SELECT 1, 2, 3 FORMAT JSON' } ``` -You can use the `default_format` URL parameter to specify a default format other than `TabSeparated`. The `X-ClickHouse-Format` header selects the format of the response explicitly: it is an alias for the `output_format` setting, so it also overrides a `FORMAT` clause in the query. It never changes how the request body of an `INSERT` is parsed — use `input_format` or `format` for that. +You can use the `default_format` URL parameter to specify a default format other than `TabSeparated`. The `X-ClickHouse-Format` header selects the format of the response explicitly: it is an alias for the `output_format` setting, so it also overrides a `FORMAT` clause in the query. It never changes how the request body of an `INSERT` is parsed — use `input_format` or `format` for that. To restore the behavior of versions before 26.8, where the header was an alias for `default_format` and did not override a `FORMAT` clause, disable the `http_x_clickhouse_format_overrides_output_format` setting (in a user profile or as a URL parameter; like the format settings themselves, it can be changed in read-only mode). ```bash $ echo 'SELECT 1 FORMAT Pretty' | curl 'http://localhost:8123/?' --data-binary @- @@ -316,7 +316,7 @@ INSERT INTO t SELECT number FROM numbers(10) SETTINGS limit = 2; | Setting | Effect | |---------|--------| -| `output_format` | Overrides the output format. It takes precedence over the query's `FORMAT` clause, the path extension, `format`, and `default_format`. It can also be set using the `X-ClickHouse-Format` header. | +| `output_format` | Overrides the output format. It takes precedence over the query's `FORMAT` clause, the path extension, `format`, and `default_format`. It can also be set using the `X-ClickHouse-Format` header. With `http_x_clickhouse_format_overrides_output_format = 0`, the header sets `default_format` instead, as in versions before 26.8. | | `input_format` | Overrides the input format for `INSERT`. It takes precedence over the query's `FORMAT` clause and `format`. | | `format` | Overrides the format in both directions unless a direction-specific setting is provided. | | `default_format` | Sets the output format when the query has no `FORMAT` clause, no path extension, and no other format override. | diff --git a/docs/concepts/features/performance/allocation-profiling.mdx b/docs/concepts/features/performance/allocation-profiling.mdx index 7c51dfc01f93..383f8bca8cb8 100644 --- a/docs/concepts/features/performance/allocation-profiling.mdx +++ b/docs/concepts/features/performance/allocation-profiling.mdx @@ -354,7 +354,7 @@ The same query can split the waste of each size class between its backtraces, in WITH 60e9 AS min_age_ns SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), - toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), b.waste)))) + toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), toFloat64(b.waste))))) FROM system.jemalloc_arena_bins AS b LEFT JOIN ( diff --git a/docs/concepts/features/security/tls/configuring-tls.mdx b/docs/concepts/features/security/tls/configuring-tls.mdx index be54610c4cdc..ca80687a4e25 100644 --- a/docs/concepts/features/security/tls/configuring-tls.mdx +++ b/docs/concepts/features/security/tls/configuring-tls.mdx @@ -645,6 +645,19 @@ For `clickhouse-client`, you can also use the `--accept-invalid-certificate` CLI ``` +## Rotating certificates and CA certificates without a restart {#rotating-certificates-without-restart} + +ClickHouse server and ClickHouse Keeper watch the files referenced by `certificateFile`, `privateKeyFile` and `caConfig` +in the `openSSL.server` and `openSSL.client` sections (and in `protocols.*` for composable protocols), or the files in the +directory if `caConfig` is a directory. When one of these files changes, or when `SYSTEM RELOAD CONFIG` is executed, the +certificates are reloaded and used for all new TLS connections, including HTTPS, the secure native protocol, interserver +connections, connections to Keeper and the Raft connections between Keeper nodes. Established connections keep using the +certificates they were opened with. + +To rotate a CA without downtime, first replace the `caConfig` file with a bundle that contains both the old and the new +CA certificate, then switch the node and client certificates to ones issued by the new CA, and finally replace the bundle +with the new CA certificate only. + ## Summary {#summary} This article focused on getting a ClickHouse environment configured with TLS. The settings will differ for different requirements in production environments; for example, certificate verification levels, protocols, ciphers, etc. But you should now have a good understanding of the steps involved in configuring and implementing secure connections. diff --git a/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx b/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx index 730c453a6957..2bf316f458c8 100644 --- a/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx +++ b/docs/guides/clickhouse/performance-and-monitoring/analyzer.mdx @@ -277,22 +277,31 @@ SELECT category, sum(value) FROM t WHERE service = 'svc1' GROUP BY category; Error: `CTE with name ... already exists (MULTIPLE_EXPRESSIONS_FOR_ALIAS)`. Exception code: 179 -Cause: The old analyzer permitted defining multiple Common Table Expressions (WITH ...) with the same name shadowing the earlier one. The analyzer forbids this ambiguity. +Cause: The old analyzer permitted defining multiple Common Table Expressions (WITH ...) with the same name, a later definition shadowing the earlier one. The analyzer rejects this ambiguity by default. -Solution: Rename duplicate CTEs to be unique. +Solution: Rename duplicate CTEs to be unique. As a migration aid, enable `analyzer_compatibility_allow_cte_redefinition = 1` (available since ClickHouse `26.10`) to restore the legacy behavior: a reference binds to the latest definition of the name that is not being resolved at that moment, so a redefinition can read the previous definition and the query body reads the last one. + +Limitations: a CTE declared as `MATERIALIZED` and a CTE in a `WITH RECURSIVE` clause cannot be redefined even with the setting enabled. One shape differs from the old analyzer: a CTE declared between two definitions of a name also binds to the last definition, where the old analyzer bound it to the definition visible at its declaration point. ```sql /* ORIGINAL QUERY */ -WITH - data AS (SELECT 1 AS id), - data AS (SELECT 2 AS id) -- Redefined +WITH + data AS (SELECT 1 AS id), + data AS (SELECT id + 1 AS id FROM data) -- Redefined, reads the previous definition SELECT * FROM data; /* FIXED QUERY */ -WITH - raw_data AS (SELECT 1 AS id), - processed_data AS (SELECT 2 AS id) +WITH + raw_data AS (SELECT 1 AS id), + processed_data AS (SELECT id + 1 AS id FROM raw_data) SELECT * FROM processed_data; + +/* LEGACY BEHAVIOR AS A MIGRATION AID */ +WITH + data AS (SELECT 1 AS id), + data AS (SELECT id + 1 AS id FROM data) +SELECT * FROM data +SETTINGS analyzer_compatibility_allow_cte_redefinition = 1; ``` ### Ambiguous column identifiers {#ambiguous-column-identifiers} diff --git a/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx b/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx index d8002f31a5f2..5c13a3cb8a13 100644 --- a/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx +++ b/docs/products/cloud/guides/data-sources/accessing-s3-data-securely.mdx @@ -141,6 +141,7 @@ DESCRIBE TABLE s3('https://s3.amazonaws.com/BUCKETNAME/BUCKETOBJECT.csv','CSVWit Below is an example query that uses the `role_session_name` as a shared secret to query data from a bucket. If the `role_session_name` isn't correct, this operation will fail. +Because it can act as a shared secret, ClickHouse masks the value of `role_session_name` as `[HIDDEN]` in `system.query_log`, `SHOW CREATE TABLE`, and everywhere else the query text is shown, the same way as `external_id`. ```sql DESCRIBE TABLE s3('https://s3.amazonaws.com/BUCKETNAME/BUCKETOBJECT.csv','CSVWithNames',extra_credentials(role_arn = 'arn:aws:iam::111111111111:role/ClickHouseAccessRole-001', role_session_name = 'secret-role-name')) diff --git a/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx b/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx index 2866642128ba..29a4dbc5d465 100644 --- a/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx +++ b/docs/reference/engines/table-engines/mergetree-family/textindexes.mdx @@ -528,8 +528,8 @@ Search tokens that the postprocessor maps to an empty string are ignored, i.e. t | [hasAnyTokens(col, arr)](/reference/functions/regular-functions/string-search-functions#hasAnyTokens) | no (array elements are tokens as-is) | all | yes | | [hasAllTokens(col, arr)](/reference/functions/regular-functions/string-search-functions#hasAllTokens) | no (array elements are tokens as-is) | all | yes | | [hasPhrase](/reference/functions/regular-functions/string-search-functions#hasPhrase) | yes | `splitByNonAlpha`, `splitByString`, `splitByRegexp`³, `ngrams`, `asciiCJK`, `icu` | yes³ | -| [startsWith](/reference/functions/regular-functions/string-functions#startsWith) | yes | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK` | yes | -| [endsWith](/reference/functions/regular-functions/string-functions#endsWith) | yes | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK` | yes | +| [startsWith](/reference/functions/regular-functions/string-functions#startsWith) | yes⁴ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`, `array`⁴ | yes⁴ | +| [endsWith](/reference/functions/regular-functions/string-functions#endsWith) | yes⁴ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`, `array`⁴ | yes⁴ | | [like](/reference/functions/regular-functions/string-search-functions#like) | yes¹ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`¹ | yes¹ | | [match](/reference/functions/regular-functions/string-search-functions#match) | yes¹ | `splitByNonAlpha`, `ngrams`, `sparseGrams`, `asciiCJK`¹ | yes¹ | | [ilike](/reference/functions/regular-functions/string-search-functions#like) | yes² (`lower`/`upper` only) | `splitByNonAlpha`, `array`² | no² | @@ -542,14 +542,19 @@ Search tokens that the postprocessor maps to an empty string are ignored, i.e. t | [hasAll](/reference/functions/regular-functions/array-functions#hasAll) | yes | `array` | yes | ¹ `LIKE` and `match` use direct read as a hint for the listed tokenizers, otherwise they fall back to brute-force scan. -`LIKE` additionally supports a *direct read (without hint)* (enabled via `use_text_index_like_evaluation_by_dictionary_scan`) for `splitByNonAlpha` and `array` tokenizers without preprocessor or postprocessor. +`LIKE` additionally supports evaluation by a dictionary scan (enabled via `use_text_index_like_evaluation_by_dictionary_scan`) for `splitByNonAlpha` and `array` tokenizers without preprocessor or postprocessor. +A `%value%` pattern is then a *direct read (without hint)*, while `value%` and `%value` patterns remain a hint, see [LIKE/ILIKE queries](#like-ilike-queries-perf). -² `ILIKE` is only supported via direct read (without hint) (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer). -There is no fallback to using the index as a hint: if the setting is disabled or the tokenizer is not in the supported set, the index is not used for `ILIKE`. +² `ILIKE` is only supported via evaluation by a dictionary scan (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer). +There is no fallback to using the index as a hint for patterns the dictionary scan does not support: if the setting is disabled or the tokenizer is not in the supported set, the index is not used for `ILIKE`. The preprocessor, if present, must be `lower` or `upper`; postprocessors are not supported. ³ `hasPhrase` on a `splitByRegexp` text index does **not** support a postprocessor: the combination is rejected with an exception, because the postprocessor row-level rewrite assumes whitespace-splitting `splitByNonAlpha`-style tokens. Without a postprocessor, `splitByRegexp` is fully supported by `hasPhrase`. +⁴ `startsWith` and `endsWith` search the complete tokens of the needle, and the token at the open end of the needle is incomplete because the value continues there: `startsWith(col, 'ClickHouse is')` searches the token `ClickHouse`, while `startsWith(col, 'ClickHouse')` has no complete token to search. +The latter is instead evaluated by a dictionary scan (`use_text_index_like_evaluation_by_dictionary_scan = 1`, `splitByNonAlpha` or `array` tokenizer, no preprocessor or postprocessor), which is also the path taken by `col LIKE 'ClickHouse%'` because the analyzer pass [optimize_rewrite_like_perfect_affix](/reference/settings/session-settings/optimize-rewrite#optimize_rewrite_like_perfect_affix) rewrites it into `startsWith`. +See [LIKE/ILIKE queries](#like-ilike-queries-perf). + **Experimental: Support phrase search argument (optional)**. Experimental parameter `support_phrase_search` (default: `0`) controls whether the index stores token positions. @@ -1393,10 +1398,20 @@ This ordering enables skipping even more data granules than the granules skipped ### LIKE/ILIKE queries {#like-ilike-queries-perf} -When a LIKE/ILIKE query pattern is `%%` and the text index tokenizer is `splitByNonAlpha` or `array`, ClickHouse leverages the inverted index to speed up LIKE/ILIKE queries significantly. To achieve that, ClickHouse scans the inverted index dictionary instead of a full-table scan to find the matching pattern. +When a LIKE/ILIKE query pattern is `%%`, `%` or `%` and the text index tokenizer is `splitByNonAlpha` or `array`, ClickHouse leverages the inverted index to speed up LIKE/ILIKE queries significantly. To achieve that, ClickHouse scans the inverted index dictionary instead of a full-table scan to find the matching pattern. + +How the result of the dictionary scan is used depends on where the needle is anchored: +- `%value%` matches a row if and only if it matches one of the row's tokens, so the index decides the query on its own: a [direct read (without hint)](#direct-read) that removes the original predicate. +- `value%` and `%value` anchor the needle at the whole value, whereas the dictionary scan can only anchor it at a token, so the scan returns a superset of the matching rows and is used as a [direct read as a hint](#direct-read). + +The same dictionary scan serves `startsWith(col, 'value')` and `endsWith(col, 'value')` when the needle has no complete token to search. +This is the path most `value%` and `%value` patterns actually take, because the analyzer pass [optimize_rewrite_like_perfect_affix](/reference/settings/session-settings/optimize-rewrite#optimize_rewrite_like_perfect_affix) (enabled by default) rewrites `col LIKE 'value%'` into `startsWith(col, 'value')` and `col LIKE '%value'` into `endsWith(col, 'value')`. +Needles that span several tokens keep using the complete tokens of the needle and do not need a dictionary scan. When the optimization is enabled, LIKE/ILIKE queries should be significantly faster than a full-table scan. However, when the pattern matches most dictionary tokens, the performance can be worse compared to a full-table scan. Luckily, there is a fallback mechanism to prevent that. +The speed-up of a `value%` or `%value` pattern comes from skipping granules, so it depends on how the matching rows are laid out. Needles that match rows in every granule prune nothing, and the query pays for the dictionary scan on top of the full scan it would have done anyway. Disable [use_text_index_like_evaluation_by_dictionary_scan](/reference/settings/session-settings/use-text#use_text_index_like_evaluation_by_dictionary_scan) for such workloads. + The optimization is controlled by a setting: - [use_text_index_like_evaluation_by_dictionary_scan](/reference/settings/session-settings/use-text#use_text_index_like_evaluation_by_dictionary_scan) @@ -1404,7 +1419,7 @@ The fallback mechanism is controlled by two settings: - [text_index_like_min_pattern_length](/reference/settings/session-settings/text-index#text_index_like_min_pattern_length) - [text_index_like_max_postings_to_read](/reference/settings/session-settings/text-index#text_index_like_max_postings_to_read) -This optimization supports only functions `like` and `ilike`. +This optimization supports only functions `like`, `ilike`, `startsWith`, and `endsWith`. ### Trivial count queries {#count-queries-perf} diff --git a/docs/reference/interfaces/specs/NativeFormat.mdx b/docs/reference/interfaces/specs/NativeFormat.mdx index 9a5cf1bfe687..301aaa82b489 100644 --- a/docs/reference/interfaces/specs/NativeFormat.mdx +++ b/docs/reference/interfaces/specs/NativeFormat.mdx @@ -1108,14 +1108,11 @@ Header blocks (rows = 0) and empty blocks therefore emit nothing, and a decoder | | | `1` | `STRING` | Native-format compatibility mode — Object transmitted as a single `String` column containing JSON text. | | | | `2` | `V2` | V1 layout minus the `max_dynamic_paths` parameter. | | | | `3` | `FLATTENED` | Native-format compatibility mode — flattened path representation. | -| | | `4` | `V3` | V2 plus a shared-data serialization version sub-field and a statistics flag. | -| **Object shared data** (sub-stream used in Object `V3`) | VarUInt | `0` | `MAP` | Shared data encoded as `Map(String, String)`. | -| | | `1` | `MAP_WITH_BUCKETS` | Same as `MAP` but split into N buckets for scan efficiency. | -| | | `2` | `ADVANCED` | Compact granule format with separate streams for paths / marks / metadata. | +| | | `4` | `V3` | Written into MergeTree parts only, never onto the wire; the layout is not specified here. | | **Dynamic** | UInt64 LE | `1` | `V1` | Original encoding. Includes `max_dynamic_types` and a list of runtime variant types. | | | | `2` | `V2` | V1 minus the `max_dynamic_types` parameter. | | | | `3` | `FLATTENED` | Native-format compatibility mode. | -| | | `4` | `V3` | V2 plus binary-encoded variant type names and empty-statistics support. | +| | | `4` | `V3` | Written into MergeTree parts only, never onto the wire; the layout is not specified here. | | **Variant** discriminators mode | UInt64 LE | `0` | `BASIC` | Every row's discriminator is written literally. | | | | `1` | `COMPACT` | If all rows in a granule share one discriminator, only a single value + granule marker is written. | | **Variant** granule format (when mode is `COMPACT`) | UInt8 | `0` | `PLAIN` | Granule has heterogeneous discriminators. | @@ -1127,7 +1124,7 @@ A few things worth noting about the table: - **The values are not contiguous.** `Dynamic` uses `1`, `2`, `3`, `4` with `V3` at `4` and `FLATTENED` at `3`. A higher number is not necessarily newer. - **Some values are native-format-only.** `Object::STRING`, `Object::FLATTENED`, and `Dynamic::FLATTENED` exist for native-protocol compatibility with clients that do not implement full Object/Dynamic. They do not appear in MergeTree on-disk storage. -- **`V3` is primarily on-disk.** Clients consuming the native TCP protocol typically see `FLATTENED` (value `3`) rather than `V3` (value `4`). +- **`V3` is on-disk only.** `Object::V3` and `Dynamic::V3` are listed because the values are assigned and must not be reused, not because a decoder can meet them: ClickHouse never sends them over `Native` and rejects them there with `INCORRECT_DATA`. Everything they carry — the shared-data sub-version of `Object::V3` and its bucket and granule layouts — describes a part rather than a block, and is therefore out of scope for this page. Clients consuming the native TCP protocol see `V1`, `V2`, `STRING` or `FLATTENED`. #### LowCardinality(T) {#lowcardinality} @@ -1313,11 +1310,17 @@ The state prefix (version + type list) is read at the start of every block with Do **not**, however, bound the **flattened** `num_types` by `ColumnDynamic::MAX_DYNAMIC_TYPES_LIMIT` (`254` in ClickHouse): the flattened type list carries *every* distinct runtime type, including those that had overflowed into the shared variant, so a valid flattened block can legitimately list far more than that. The `MAX_DYNAMIC_TYPES_LIMIT` bound applies only to the `num_dynamic_types` count in the non-flat `V1`/`V2`/`V3` prefixes, which counts regular-variant slots and is capped by the limit (ClickHouse validates it there before the `+ 1` for the shared variant). + +**Truncated streams** + +The discriminator run holds exactly `num_rows` entries, and a decoder must verify that it read that many rather than accepting whatever the stream contained. A shorter run does not contradict anything that follows it: the per-type value counts are derived from the discriminators actually read, so they stay consistent with the value runs, and a run made entirely of NULL markers demands no value bytes at all. ClickHouse rejects a short discriminator run with `INCORRECT_DATA` ("Mismatch in flattened `Dynamic` column: N rows are expected, but the indexes stream contains only M rows") before any per-type check, because the reconstructed column is then used for `num_rows` rows regardless of its actual size. + + Runtime types whose serialization is stateful (`LowCardinality`, `Variant`, `Dynamic`, `JSON`) carry nested state prefixes after the type-name list. -The runtime type list normally follows the `Variant` canonicalization — the regular variant slots are written in `DataTypeVariant` (type-name) order, so the wire order does not follow insertion order. It is **not always** globally sorted, however: types that overflowed into the shared variant (for example under `Dynamic(max_types=N)`) are appended after the regular slots in first-seen order, so the tail of the list can break type-name order. A decoder must therefore treat the transmitted type list as authoritative for discriminator assignment and must not re-sort it itself. For rows `[42::UInt64, "hi", NULL]` the two types are `String` and `UInt64`, and `"String"` sorts before `"UInt64"`, so the discriminators are `0` = String, `1` = UInt64, `2` = NULL: +The runtime type list normally follows the `Variant` canonicalization — the regular variant slots are written in `DataTypeVariant` (type-name) order, so the wire order does not follow insertion order. It is **not always** globally sorted, however: types that overflowed into the shared variant (for example under `Dynamic(max_types=N)`) are appended after the regular slots in first-seen order, so the tail of the list can break type-name order. A decoder must therefore treat the transmitted type list as authoritative for discriminator assignment and must not re-sort it itself. What the list may not contain is a repeated type or `Nothing`: two equal entries map onto one discriminator and `Nothing` is not a variant at all, so in both cases part of the list has nowhere to be reconstructed into. ClickHouse rejects either with `INCORRECT_DATA`. For rows `[42::UInt64, "hi", NULL]` the two types are `String` and `UInt64`, and `"String"` sorts before `"UInt64"`, so the discriminators are `0` = String, `1` = UInt64, `2` = NULL: ```text 03 00 00 00 00 00 00 00 state prefix: UInt64 version = 3 (FLATTENED) @@ -1355,14 +1358,18 @@ In FLATTENED mode there is **no shared-data column** (that overflow store belong [for each dynamic path: its Dynamic data] ← num_rows values (discriminators + runs) ``` -Note the two-phase shape: **all** path state prefixes come first, then **all** path data. A dynamic path's `Dynamic` prefix (in the prefix phase) is therefore separated from its data (in the data phase). The state prefix is read at the start of every block with rows > 0, and every path column (typed or dynamic) holds exactly `num_rows` values. Row `r`'s object is assembled by reading each path's value at index `r`; a dynamic path whose `Dynamic` discriminator is NULL for that row contributes no key. +Note the two-phase shape: **all** path state prefixes come first, then **all** path data. A dynamic path's `Dynamic` prefix (in the prefix phase) is therefore separated from its data (in the data phase). The state prefix is read at the start of every block with rows > 0, and every path column (typed or dynamic) holds exactly `num_rows` values. Row `r`'s object is assembled by reading each path's value at index `r`; a dynamic path whose `Dynamic` discriminator is NULL for that row contributes no key. The dynamic path names must be distinct and none of them may repeat a path that the type string already declares as typed — either would give the assembled object two values under one key, so ClickHouse rejects both with `INCORRECT_DATA`. The same applies to the dynamic path list of the non-flat encodings. **Malformed counts** -`num_dynamic_paths` in the FLATTENED layout documented here — and the dynamic-paths count in the non-flat `V1`/`V2`/`V3` encodings — is read from the stream before the path names. (There is no separate flattened-paths field in the non-flat prefixes: `V1`/`V2`/`V3` carry only the dynamic-paths count, plus a `max_dynamic_paths` value in `V1` that is read and discarded, and the `V3` shared-data metadata below.) As with [`Dynamic`](#dynamic), a decoder must treat these counts as untrusted and must not size an allocation on them directly — neither a `SIZE_MAX`-family count nor a large-but-representable one. ClickHouse reads the path names one entry at a time with only a capped pre-allocation hint, so a corrupted count is rejected either as `INCORRECT_DATA` ("JSON/Object column has too many paths", when it exceeds what the container can hold) or as an ordinary read error once the stream runs out of path names. +`num_dynamic_paths` in the FLATTENED layout documented here — and the dynamic-paths count in the non-flat `V1`/`V2` encodings — is read from the stream before the path names. (There is no separate flattened-paths field in the non-flat prefixes: they carry only the dynamic-paths count, plus a `max_dynamic_paths` value in `V1` that is read and discarded.) As with [`Dynamic`](#dynamic), a decoder must treat these counts as untrusted and must not size an allocation on them directly — neither a `SIZE_MAX`-family count nor a large-but-representable one. ClickHouse reads the path names one entry at a time with only a capped pre-allocation hint, so a corrupted count is rejected either as `INCORRECT_DATA` ("JSON/Object column has too many paths", when it exceeds what the container can hold) or as an ordinary read error once the stream runs out of path names. + + + +**Truncated streams** -The non-flat `V3` prefix additionally carries a `shared_data_buckets` count (present when the shared-data serialization version is `MAP_WITH_BUCKETS` or `ADVANCED`). It sizes the per-bucket reader state and column vectors directly (not via a grow-on-demand loop), so a decoder must reject an implausible bucket count up front. Unlike the path and type counts, this count has a tight writer-side invariant: the number of buckets is chosen from small MergeTree settings (`object_shared_data_buckets_for_compact_part` / `object_shared_data_buckets_for_wide_part`) that are non-zero and capped at `256`, so the only valid on-wire range is `1 … 256`. ClickHouse rejects any value outside that range — including a large-but-representable count such as `100000`, which is far below the container's `max_size()` — with `INCORRECT_DATA` ("JSON/Object column has an invalid number of shared data buckets"). +The `num_rows` values per path column are a hard requirement, not an upper bound: a decoder must check every path column, typed and dynamic, against the block's row count instead of trusting the stream to end where it should. The assembled object has no independent size to compare against — its row count is whatever the unflattening produced — so a path column that came up short is not detectable later, and reading row `r` of it goes past its end. ClickHouse rejects it with `INCORRECT_DATA` ("Unexpected size of typed path / flattened path *p* in flattened Object column"). For a dynamic path this is on top of the equivalent check inside its [`Dynamic`](#dynamic) column. `JSON` value `{"a": 42, "b": "hi"}` (one row, both paths dynamic). A JSON integer is inferred as `Int64`: @@ -1378,9 +1385,9 @@ The non-flat `V3` prefix additionally carries a `shared_data_buckets` count (pre 00 02 68 69 "b" data: discriminator 0, String "hi" ``` -#### JSON non-flat (V2/V3) {#json-non-flat} +#### JSON non-flat (V1/V2) {#json-non-flat} -The non-flattened `Object` encodings (`V1`/`V2`/`V3`) are used by MergeTree on-disk storage and are what the server emits over the wire when the flattened flag is off — `V1` over `clickhouse-client` / HTTP `FORMAT Native` (revision `0`), `V2` over the native TCP protocol. They carry a shared-data column and are **not** specified on this page. Note that they do **not** carry per-path statistics over the Native wire: `NativeWriter` leaves statistics disabled, so the `Object` structure prefix has no statistics section and the bytes after it are the typed/dynamic/shared-data prefixes and data directly. Statistics appear only on the MergeTree on-disk paths that enable them. To decode a `JSON` column with this page, a client must select one of the documented tiers: set `output_format_native_write_json_as_string = 1` for the [String fallback](#json-tier-1-string-fallback), or `output_format_native_use_flattened_dynamic_and_json_serialization = 1` (with `output_format_native_write_json_as_string = 0`) for the [FLATTENED Object](#json-tier-2-flattened-object) layout. +The non-flattened `Object` encodings are what the server emits over the wire when the flattened flag is off — `V1` over `clickhouse-client` / HTTP `FORMAT Native` (revision `0`), `V2` over the native TCP protocol. (`V3`, the third non-flat encoding, is written into MergeTree parts only and is rejected on the wire.) They carry a shared-data column and are **not** specified on this page. Note that they do **not** carry per-path statistics over the Native wire: `NativeWriter` leaves statistics disabled, so the `Object` structure prefix has no statistics section and the bytes after it are the typed/dynamic/shared-data prefixes and data directly. Statistics appear only on the MergeTree on-disk paths that enable them. To decode a `JSON` column with this page, a client must select one of the documented tiers: set `output_format_native_write_json_as_string = 1` for the [String fallback](#json-tier-1-string-fallback), or `output_format_native_use_flattened_dynamic_and_json_serialization = 1` (with `output_format_native_write_json_as_string = 0`) for the [FLATTENED Object](#json-tier-2-flattened-object) layout. ## Compression frame {#compression-frame} diff --git a/docs/reference/interfaces/specs/NativeProtocol.mdx b/docs/reference/interfaces/specs/NativeProtocol.mdx index 6fd620ad09c3..6d165d1b21e8 100644 --- a/docs/reference/interfaces/specs/NativeProtocol.mdx +++ b/docs/reference/interfaces/specs/NativeProtocol.mdx @@ -144,7 +144,7 @@ When a feature is active, its fields **must** be present on the wire. The protoc | SERVER_SETTINGS | 54474 | ServerHello | Server broadcasts its non-default settings as a list at the tail of ServerHello, after `nonce`. Format: `(key, flags, value)` triples terminated by an empty key — same as the Query packet's settings list. | | QUERY_AND_LINE_NUMBERS | 54475 | ClientInfo | Adds `script_query_number` (VarUInt) and `script_line_number` (VarUInt) at the tail of ClientInfo. Used by clickhouse-client for multi-statement script error attribution; external clients send `0, 0`. | | JWT_IN_INTERSERVER | 54476 | ClientInfo | Adds a JWT-presence UInt8 + optional `String jwt` at the tail of ClientInfo. External clients (no JWT) send byte `0x00`. (Spelled `DBMS_MIN_REVISON_WITH_JWT_IN_INTERSERVER` in C++ — note the typo in the constant name.) | -| QUERY_PLAN_SERIALIZATION | 54477 | ServerHello, QueryPlan packet | ServerHello appends `VarUInt query_plan_serialization_version` after server settings. Also introduces `ClientPacket::QueryPlan` (code `13`) for inter-server delivery of pre-built query plans — external clients never send. | +| QUERY_PLAN_SERIALIZATION | 54477 | ServerHello, QueryPlan packet | ServerHello appends `VarUInt query_plan_serialization_version` after server settings. Also introduces `ClientPacket::QueryPlan` (code `13`) for inter-server delivery of pre-built query plans — external clients never send. Query-plan serialization version `10` prefixes the serialized plan with `VarUInt max_threads` and `Bool concurrency_control`; when either is non-default, initiators use SQL shipping for peers below version `10` so the peer builds its plan from the transmitted query settings. | | PARALLEL_BLOCK_MARSHALLING | 54478 | Block (Column) | Server may wrap columns in `ColumnBLOB` (compressed inline) for parallel processing. Gated on the query having compression enabled AND `rows > 1`; otherwise the regular column wire format applies. Clients that never enable compression on outgoing Query packets see no wire change. | | VERSIONED_CLUSTER_FUNCTION_PROTOCOL | 54479 | ServerHello | Adds `VarUInt cluster_function_protocol_version` at the tail of ServerHello. Used for `*Cluster` table functions (`s3Cluster`, etc.). Current value: `8` (`DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION`); version `7` is reserved for a private-repository feature (Iceberg compaction), and `8` adds an optional `read_source_index` to the inter-server cluster read-task payload (the `ReadTaskResponse` body, which stays unspecified here — see below). External clients decode and ignore. | | OUT_OF_ORDER_BUCKETS_IN_AGGREGATION | 54480 | BlockInfo | Adds field 3 (`out_of_order_buckets: Vec`) to BlockInfo's field-tagged stream. Decoded as `[VarUInt count][Int32]*count`. External clients don't emit this themselves; the decoder reads any non-empty list the server sends. | @@ -908,7 +908,7 @@ Below version `8` the announcement is fire-and-forget regardless of mode, and th | 10 | MergeTreeReadTaskResponse | not specified | Parallel read task response | | 11 | SSHChallengeRequest | [SSH auth](#ssh-authentication) | SSH auth challenge request | | 12 | SSHChallengeResponse | [SSH auth](#ssh-authentication) | SSH auth challenge response | -| 13 | QueryPlan | not specified | Query plan | +| 13 | QueryPlan | [QueryPlan](#queryplan) | Query plan | | 14 | MergeTreeAllRangesAnnouncementResponse | [MergeTreeAllRangesAnnouncementResponse](#mergetreeallrangesannouncementresponse) | Initiator's reply to a follower's [`MergeTreeAllRangesAnnouncement`](#packet-type-reference) (gated on `parallel_replicas_protocol_version ≥ 8` — see [VERSIONED_PARALLEL_REPLICAS_PROTOCOL](#feature-table)). Inter-server only — external clients never send. | ### Server → Client {#server-to-client} @@ -935,6 +935,19 @@ Below version `8` the announcement is fire-and-forget regardless of mode, and th | 17 | TimezoneUpdate | [TimezoneUpdate](#timezoneupdate) | Server timezone update | | 18 | SSHChallenge | [SSH auth](#ssh-authentication) | SSH auth challenge | +### QueryPlan {#queryplan} + +`QueryPlan` is an inter-server-only client packet. It follows a [`Query`](#query) whose `stage` is `QueryPlan` (`7`) and contains one serialized plan stream. + +| # | Field | Type | Gate | Description | +|---|---|---|---|---| +| 1 | serialization_version | VarUInt | universal | Query-plan serialization version, no greater than the receiver's `query_plan_serialization_version` advertised in `ServerHello`. | +| 2 | max_threads | VarUInt | serialization version ≥ `10` | Plan-level thread limit. `0` means no plan-specific limit. | +| 3 | concurrency_control | Bool | serialization version ≥ `10` | Whether the plan participates in concurrency control. | +| 4 | plan | versioned query-plan stream | universal | Serialized plan tree, step settings, headers, and prepared sets. Its internal layout is owned by the query-plan serialization version. | + +Before version `10`, fields 2 and 3 are absent. An initiator with either non-default execution limit must not send that older layout: it ships the original SQL query instead, so the older receiver reconstructs the limits from the query settings. + ## Configuration {#configuration} This section covers the tunables that shape native protocol connections: diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 03357b3470b0..23c3fedd9bc7 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1159,12 +1159,14 @@ SSL client/server configuration. Support for SSL is provided by the `libpoco` library. The available configuration options are explained in [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h). Default values can be found in [SSLManager.cpp](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/src/SSLManager.cpp). +The files referenced by `certificateFile`, `privateKeyFile` and `caConfig` are reloaded without a restart when they change or on `SYSTEM RELOAD CONFIG`. New connections use the reloaded certificates, established connections are not affected. + Keys for server/client settings: | Option | Description | Default Value | |-------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------| | `cacheSessions` | Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`. | `false` | -| `caConfig` | Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). | | +| `caConfig` | Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html). Like `certificateFile` and `privateKeyFile`, the CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`. | | | `certificateFile` | Path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate. | | | `cipherList` | Supported OpenSSL encryptions. | `ALL:!ADH:!LOW:!EXP:!MD5:!3DES:@STRENGTH` | | `disableProtocols` | Protocols that are not allowed to be used. | | diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index fe530bd9d2f6..f86cb3a01521 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -626,14 +626,14 @@ try Coordination::EventPtr unused_event = std::make_shared(); - const std::string cert_path = config().getString("openSSL.server.certificateFile", ""); - const std::string key_path = config().getString("openSSL.server.privateKeyFile", ""); - + /// TLS certificates, keys and CA certificates are reloaded by CertificateReloader when these files change. std::vector extra_paths = {include_from_path}; - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); + for (const auto * key : {"openSSL.server.certificateFile", "openSSL.server.privateKeyFile", "openSSL.server.caConfig", + "openSSL.client.certificateFile", "openSSL.client.privateKeyFile", "openSSL.client.caConfig"}) + { + if (auto file_path = config().getString(key, ""); !file_path.empty()) + extra_paths.emplace_back(std::move(file_path)); + } /// ConfigReloader have to strict parameters which are redundant in our case auto main_config_reloader = std::make_unique( diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index ad78a0d204a2..9884fc325c09 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -487,6 +487,10 @@ namespace ServerSetting extern const ServerSettingsString logger_shutdown_level; extern const ServerSettingsString openssl_server_certificate_file; extern const ServerSettingsString openssl_server_private_key_file; + extern const ServerSettingsString openssl_server_ca_config; + extern const ServerSettingsString openssl_client_certificate_file; + extern const ServerSettingsString openssl_client_private_key_file; + extern const ServerSettingsString openssl_client_ca_config; extern const ServerSettingsString distributed_ddl_path; extern const ServerSettingsString distributed_ddl_replicas_path; extern const ServerSettingsInt32 distributed_ddl_pool_size; @@ -1132,7 +1136,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto condition_read_buffer = ReadBufferFromString(condition); auto condition_write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Checking startup query condition `{}`", condition); + LOG_DEBUG(log, "Checking startup query condition `{}`", formatQueryForLogging(condition, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -1166,7 +1170,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto read_buffer = ReadBufferFromString(query); auto write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Executing query `{}`", query); + LOG_DEBUG(log, "Executing query `{}`", formatQueryForLogging(query, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -2668,26 +2672,25 @@ try tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization"); } - std::string cert_path = server_settings[ServerSetting::openssl_server_certificate_file]; - std::string key_path = server_settings[ServerSetting::openssl_server_private_key_file]; - + /// TLS certificates, keys and CA certificates are reloaded by CertificateReloader when these files change. std::vector extra_paths = {include_from_path}; - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); + auto watch_path = [&](const std::string & file_path) + { + if (!file_path.empty()) + extra_paths.emplace_back(file_path); + }; + watch_path(server_settings[ServerSetting::openssl_server_certificate_file]); + watch_path(server_settings[ServerSetting::openssl_server_private_key_file]); + watch_path(server_settings[ServerSetting::openssl_server_ca_config]); + watch_path(server_settings[ServerSetting::openssl_client_certificate_file]); + watch_path(server_settings[ServerSetting::openssl_client_private_key_file]); + watch_path(server_settings[ServerSetting::openssl_client_ca_config]); Poco::Util::AbstractConfiguration::Keys protocols; config().keys("protocols", protocols); for (const auto & protocol : protocols) - { - cert_path = config().getString("protocols." + protocol + ".certificateFile", ""); - key_path = config().getString("protocols." + protocol + ".privateKeyFile", ""); - if (!cert_path.empty()) - extra_paths.emplace_back(cert_path); - if (!key_path.empty()) - extra_paths.emplace_back(key_path); - } + for (const auto * key : {"certificateFile", "privateKeyFile", "caConfig"}) + watch_path(config().getString("protocols." + protocol + "." + key, "")); DNSResolver::instance().setFilterSettings(server_settings[ServerSetting::dns_allow_resolve_names_to_ipv4], server_settings[ServerSetting::dns_allow_resolve_names_to_ipv6]); /// DNSCacheUpdater uses BackgroundSchedulePool which lives in shared context diff --git a/programs/server/jemalloc.html b/programs/server/jemalloc.html index fd785fa66f28..8505ae77e245 100644 --- a/programs/server/jemalloc.html +++ b/programs/server/jemalloc.html @@ -2922,7 +2922,7 @@

Mutex Statistics

/// Both branches must come from the same scan of the sampled allocations: every /// scan flushes a fresh profile dump, so two independent subqueries could observe /// different dumps and double-count or drop a cell. - const query = `SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), b.waste)))) FROM system.jemalloc_arena_bins AS b LEFT JOIN (${fragmentationPinnersSubquery(minAgeNs)}) AS s ON s.size_class = b.index AND s.arena = b.arena WHERE b.large = 0 AND b.waste >= ${minWaste}${purposeCond} SETTINGS allow_introspection_functions = 1`; + const query = `SELECT format('{} {}', if(s.est_old_objects > 0, s.stack, format('[unattributed];arena_{}_class_{}', toString(b.arena), toString(b.size))), toString(toUInt64(if(s.est_old_objects > 0, b.waste * s.est_old_objects / sum(s.est_old_objects) OVER (PARTITION BY b.arena, b.index), toFloat64(b.waste))))) FROM system.jemalloc_arena_bins AS b LEFT JOIN (${fragmentationPinnersSubquery(minAgeNs)}) AS s ON s.size_class = b.index AND s.arena = b.arena WHERE b.large = 0 AND b.waste >= ${minWaste}${purposeCond} SETTINGS allow_introspection_functions = 1`; const response = await fetch(buildUrl(query), fetchOptions); if (!response.ok) { throw new Error(`HTTP ${response.status}: ${response.statusText}`); diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index 3b5e4424239a..a0bcd3479bec 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -298,6 +298,11 @@ namespace for (const auto & certificate_subject : ssl_certificate_credentials->getSSLCertificateSubjects().at(type)) { + // Subjects are extracted with their exact bytes, so an embedded NUL byte survives. No valid + // hostname or URI contains one, and '*' must not match a span like "evil\0" in + // "evil\0.corp.example.com", so such a subject never matches a wildcard. + if (certificate_subject.contains('\0')) + continue; // Checked before the substr below so its length cannot underflow when prefix and suffix overlap. if (certificate_subject.size() < prefix.size() + suffix.size()) continue; diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index fd68f0c6da25..fcf380632efe 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -77,6 +77,10 @@ bool isAlwaysChangeableInReadonly(std::string_view name) /// HTTP routing / session. if (name == "database" || name == "default_format") return true; + /// Selects which of `output_format` / `default_format` the `X-ClickHouse-Format` header aliases; + /// both targets are changeable here, so the switch between them must be too. + if (name == "http_x_clickhouse_format_overrides_output_format") + return true; /// Output format selection and response compression. if (name == "format" || name == "input_format" || name == "output_format" || name == "compression") return true; diff --git a/src/AggregateFunctions/AggregateFunctionFactory.h b/src/AggregateFunctions/AggregateFunctionFactory.h index 53248dae079f..c3323b1a2177 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.h +++ b/src/AggregateFunctions/AggregateFunctionFactory.h @@ -31,6 +31,8 @@ class ASTFunction; * The invoker has arguments: name of aggregate function, types of arguments, values of parameters. * Parameters are for "parametric" aggregate functions. * For example, in quantileWeighted(0.9)(x, weight), 0.9 is "parameter" and x, weight are "arguments". + * `Settings` is null when the function is constructed outside a query, e.g. while a background + * thread parses an `AggregateFunction(...)` type name. */ using AggregateFunctionCreator = std::function; diff --git a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp index eb5f188de545..b84aaa5daf12 100644 --- a/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp +++ b/src/AggregateFunctions/AggregateFunctionFlameGraph.cpp @@ -691,7 +691,9 @@ static void check(const std::string & name, const DataTypes & argument_types, co static AggregateFunctionPtr createAggregateFunctionFlameGraph(const std::string & name, const DataTypes & argument_types, const Array & params, const Settings * settings) { - if (!(*settings)[Setting::allow_introspection_functions]) + /// The factory passes null when there is no query context to take settings from, and with no + /// principal there is nothing to authorize. + if (settings && !(*settings)[Setting::allow_introspection_functions]) throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Introspection functions are disabled, because setting 'allow_introspection_functions' is set to 0"); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h index 614bf5aa1470..a98b076c532f 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionArray.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionArray.h @@ -231,6 +231,11 @@ class AggregateFunctionArray final : public IAggregateFunctionHelperinsertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp index 2dd9f6bfe11f..f73eed07a9ab 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionCombinatorsArgMinArgMax.cpp @@ -222,6 +222,11 @@ class AggregateFunctionCombinatorArgMinArgMax final : public IAggregateFunctionH nested_function->insertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_function->rollbackInsertResult(place, to); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h b/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h index 69c2af03fef1..036884376681 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionDistinct.h @@ -291,6 +291,11 @@ class AggregateFunctionDistinct final : public IAggregateFunctionDataHelperinsertMergeResultInto(getNestedPlace(place), to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(getNestedPlace(place), to); + } + size_t sizeOfData() const override { return prefix_size + nested_func->sizeOfData(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h b/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h index 219c94ec9cd3..17b499acd3bf 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionForEach.h @@ -342,17 +342,15 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper - void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + void transferElements(AggregateDataPtr __restrict place, ColumnArray & arr_to, size_t & transferred, Arena * arena) const { AggregateFunctionForEachData & state = data(place); - - ColumnArray & arr_to = assert_cast(to); - ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); IColumn & elems_to = arr_to.getData(); - char * nested_state = state.array_of_aggregate_datas; - for (size_t i = 0; i < state.dynamic_array_size; ++i) + char * nested_state = state.array_of_aggregate_datas + transferred * nested_size_of_data; + for (; transferred < state.dynamic_array_size; ++transferred) { if constexpr (merge) nested_func->insertMergeResultInto(nested_state, elems_to, arena); @@ -361,9 +359,43 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper + void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + { + ColumnArray & arr_to = assert_cast(to); + size_t transferred = 0; + + if constexpr (!merge) + { + /// A nested function that is not a state aliases nothing and need not be atomic. + if (nested_func->isState()) + { + const size_t offsets_before = arr_to.getOffsets().size(); + + try + { + transferElements(place, arr_to, transferred, arena); + } + catch (...) + { + arr_to.getOffsets().resize_assume_reserved(offsets_before); + const char * nested_state = data(place).array_of_aggregate_datas; + for (size_t i = transferred; i-- > 0;) + nested_func->rollbackInsertResult(nested_state + i * nested_size_of_data, arr_to.getData()); + throw; + } + + return; + } + } + + transferElements(place, arr_to, transferred, arena); + } + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override { insertResultIntoImpl(place, to, arena); @@ -374,6 +406,19 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + const AggregateFunctionForEachData & state = data(place); + + ColumnArray & arr_to = assert_cast(to); + ColumnArray::Offsets & offsets_to = arr_to.getOffsets(); + + offsets_to.resize_assume_reserved(offsets_to.size() - 1); + const char * nested_state = state.array_of_aggregate_datas; + for (size_t i = state.dynamic_array_size; i-- > 0;) + nested_func->rollbackInsertResult(nested_state + i * nested_size_of_data, arr_to.getData()); + } + bool allocatesMemoryInArena() const override { return true; diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h index ed6c8e84f8a6..f5cdb6b30e84 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionIf.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionIf.h @@ -240,6 +240,11 @@ class AggregateFunctionIf final : public IAggregateFunctionHelperinsertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp index b711b1288624..1cdc6b1881f2 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -154,6 +155,23 @@ class AggregateFunctionMap final return map_type->getKeyType(); } + /// Reads one key out of a key column in the representation `merged_maps` is keyed by. + typename Data::SearchType keyAt(const IColumn & key_column, size_t row) const + { + if constexpr (std::is_same_v) + { + if (key_type->getTypeId() == TypeIndex::FixedString) + return assert_cast(key_column).getDataAt(row); + if (key_type->getTypeId() == TypeIndex::IPv6) + return assert_cast(key_column).getDataAt(row); + return assert_cast(key_column).getDataAt(row); + } + else + { + return assert_cast &>(key_column).getData()[row]; + } + } + void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override { const auto & map_column = assert_cast(*columns[0]); @@ -170,24 +188,7 @@ class AggregateFunctionMap final for (size_t i = 0; i < size; ++i) { - typename Data::SearchType key; - - if constexpr (std::is_same_v) - { - std::string_view key_ref; - if (key_type->getTypeId() == TypeIndex::FixedString) - key_ref = assert_cast(key_column).getDataAt(offset + i); - else if (key_type->getTypeId() == TypeIndex::IPv6) - key_ref = assert_cast(key_column).getDataAt(offset + i); - else - key_ref = assert_cast(key_column).getDataAt(offset + i); - - key = key_ref; - } - else - { - key = assert_cast &>(key_column).getData()[offset + i]; - } + const typename Data::SearchType key = keyAt(key_column, offset + i); auto it = merged_maps.find(key); @@ -339,13 +340,40 @@ class AggregateFunctionMap final } } + /// `transferred` counts the values whose transfer returned, so a caller that catches can undo those. + template + void transferValues( + AggregateDataPtr __restrict place, + ColumnMap & map_column, + const VectorWithMemoryTracking & keys, + size_t & transferred, + Arena * arena) const + { + auto & nested_data_column = map_column.getNestedData(); + auto & key_column = nested_data_column.getColumn(0); + auto & val_column = nested_data_column.getColumn(1); + + auto & merged_maps = this->data(place).merged_maps; + + // insert using sorted keys to result column + for (; transferred < keys.size(); ++transferred) + { + key_column.insert(keys[transferred]); + if constexpr (merge) + nested_func->insertMergeResultInto(merged_maps[keys[transferred]], val_column, arena); + else + nested_func->insertResultInto(merged_maps[keys[transferred]], val_column, arena); + } + + IColumn::Offsets & res_offsets = map_column.getNestedColumn().getOffsets(); + res_offsets.push_back(val_column.size()); + } + template void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const { auto & map_column = assert_cast(to); - auto & nested_column = map_column.getNestedColumn(); auto & nested_data_column = map_column.getNestedData(); - auto & key_column = nested_data_column.getColumn(0); auto & val_column = nested_data_column.getColumn(1); @@ -360,18 +388,37 @@ class AggregateFunctionMap final } ::sort(keys.begin(), keys.end()); - // insert using sorted keys to result column - for (auto & key : keys) + size_t transferred = 0; + + if constexpr (!merge) { - key_column.insert(key); - if constexpr (merge) - nested_func->insertMergeResultInto(merged_maps[key], val_column, arena); - else - nested_func->insertResultInto(merged_maps[key], val_column, arena); + /// A nested function that is not a state aliases nothing and need not be atomic. + if (nested_func->isState()) + { + /// `ColumnString::insert` grows `chars` before it appends the offset, so a row count + /// cannot undo an interrupted key insert but a checkpoint, which restores both, can. + const auto keys_checkpoint = key_column.getCheckpoint(); + IColumn::Offsets & res_offsets = map_column.getNestedColumn().getOffsets(); + const size_t offsets_before = res_offsets.size(); + + try + { + transferValues(place, map_column, keys, transferred, arena); + } + catch (...) + { + res_offsets.resize_assume_reserved(offsets_before); + for (size_t i = transferred; i-- > 0;) + nested_func->rollbackInsertResult(merged_maps[keys[i]], val_column); + key_column.rollback(*keys_checkpoint); + throw; + } + + return; + } } - IColumn::Offsets & res_offsets = nested_column.getOffsets(); - res_offsets.push_back(val_column.size()); + transferValues(place, map_column, keys, transferred, arena); } void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override @@ -384,6 +431,33 @@ class AggregateFunctionMap final insertResultIntoImpl(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + auto & map_column = assert_cast(to); + auto & nested_data_column = map_column.getNestedData(); + auto & key_column = nested_data_column.getColumn(0); + auto & val_column = nested_data_column.getColumn(1); + + auto & merged_maps = this->data(place).merged_maps; + const size_t appended = merged_maps.size(); + + IColumn::Offsets & res_offsets = map_column.getNestedColumn().getOffsets(); + res_offsets.resize_assume_reserved(res_offsets.size() - 1); + + /// The values were appended in sorted key order while `merged_maps` is unordered, and a row's + /// undo depends on which place produced it, so recover that order from the keys themselves. + const size_t appended_end = key_column.size(); + for (size_t pos = appended_end; pos-- > appended_end - appended;) + { + auto it = merged_maps.find(keyAt(key_column, pos)); + if (it == merged_maps.end()) + abortOnFailedAssertion("AggregateFunctionMap::rollbackInsertResult: appended key is missing from merged_maps"); + nested_func->rollbackInsertResult(it->second, val_column); + } + + key_column.popBack(appended); + } + bool allocatesMemoryInArena() const override { return true; } AggregateFunctionPtr getNestedFunction() const override { return nested_func; } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h index b80eb99657f7..113cb24963d0 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h @@ -139,6 +139,11 @@ class AggregateFunctionMerge final : public IAggregateFunctionHelperinsertMergeResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h index 232fe5d8fac6..18cd17272fa4 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionNull.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionNull.h @@ -248,10 +248,25 @@ class AggregateFunctionNullBase : public IAggregateFunctionHelper if (getFlag(place)) { if constexpr (merge) + { nested_function->insertMergeResultInto(nestedPlace(place), to_concrete.getNestedColumn(), arena); + to_concrete.getNullMapData().push_back(false); + } else + { nested_function->insertResultInto(nestedPlace(place), to_concrete.getNestedColumn(), arena); - to_concrete.getNullMapData().push_back(false); + + /// A nested call that threw has already restored the nested column itself. + try + { + to_concrete.getNullMapData().push_back(false); + } + catch (...) + { + nested_function->rollbackInsertResult(nestedPlace(place), to_concrete.getNestedColumn()); + throw; + } + } } else { @@ -277,6 +292,28 @@ class AggregateFunctionNullBase : public IAggregateFunctionHelper insertResultIntoImpl(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + if constexpr (result_is_nullable) + { + ColumnNullable & to_concrete = assert_cast(to); + if (getFlag(place)) + { + to_concrete.getNullMapData().pop_back(); + nested_function->rollbackInsertResult(nestedPlace(place), to_concrete.getNestedColumn()); + } + else + { + /// insertResultInto appended a state the column itself owns, so this pop must destroy it. + to_concrete.popBack(1); + } + } + else + { + nested_function->rollbackInsertResult(nestedPlace(place), to); + } + } + bool allocatesMemoryInArena() const override { return nested_function->allocatesMemoryInArena(); diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h b/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h index 3f51d7b5723e..1bf5cbc176dd 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionOrFill.h @@ -319,10 +319,20 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper(to); col.getNullMapColumn().insertDefault(); - if constexpr (merge) - nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); - else - nested_function->insertResultInto(place, col.getNestedColumn(), arena); + /// The null map entry is this call's own, and the nested transfer restores the nested + /// column itself when it throws, so only the entry has to go. + try + { + if constexpr (merge) + nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); + else + nested_function->insertResultInto(place, col.getNestedColumn(), arena); + } + catch (...) + { + col.getNullMapColumn().getData().pop_back(); + throw; + } } } else @@ -356,10 +366,18 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper(to); col.getNullMapColumn().getData().push_back(static_cast(1)); - if constexpr (merge) - nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); - else - nested_function->insertResultInto(place, col.getNestedColumn(), arena); + try + { + if constexpr (merge) + nested_function->insertMergeResultInto(place, col.getNestedColumn(), arena); + else + nested_function->insertResultInto(place, col.getNestedColumn(), arena); + } + catch (...) + { + col.getNullMapColumn().getData().pop_back(); + throw; + } } } else @@ -384,6 +402,35 @@ class AggregateFunctionOrFill final : public IAggregateFunctionHelper(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + /// The flag-unset branches are indistinguishable from the appended row alone: with a state-nested + /// function they alias like the flag-set branch, otherwise they append a default `to` owns. + if (!place[size_of_data] && !nested_function->isState()) + { + to.popBack(1); + return; + } + + if constexpr (UseNull) + { + if (!result_is_nullable || inner_nullable) + { + nested_function->rollbackInsertResult(place, to); + } + else + { + ColumnNullable & col = typeid_cast(to); + col.getNullMapColumn().getData().pop_back(); + nested_function->rollbackInsertResult(place, col.getNestedColumn()); + } + } + else + { + nested_function->rollbackInsertResult(place, to); + } + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } /// After `Nullable(Tuple)` was introduced, Tuple's `canBeInsideNullable` now returns true, diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionResample.h b/src/AggregateFunctions/Combinators/AggregateFunctionResample.h index b418906d739a..21defecebde7 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionResample.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionResample.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,12 @@ struct Settings; namespace ErrorCodes { extern const int ARGUMENT_OUT_OF_BOUND; + extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ +extern const char aggregate_function_state_transfer_throw[]; } template @@ -229,23 +236,63 @@ class AggregateFunctionResample final : public IAggregateFunctionHelper(nested_function_->getResultType()); } + /// `transferred` counts the buckets whose transfer returned, so a caller that catches can undo those. template - void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + void transferBuckets(AggregateDataPtr __restrict place, ColumnArray & col, size_t & transferred, Arena * arena) const { - auto & col = assert_cast(to); - auto & col_offsets = assert_cast(col.getOffsetsColumn()); - - for (size_t i = 0; i < total; ++i) + for (; transferred < total; ++transferred) { if constexpr (merge) - nested_function->insertMergeResultInto(place + i * size_of_data, col.getData(), arena); + nested_function->insertMergeResultInto(place + transferred * size_of_data, col.getData(), arena); else - nested_function->insertResultInto(place + i * size_of_data, col.getData(), arena); + nested_function->insertResultInto(place + transferred * size_of_data, col.getData(), arena); } + if constexpr (!merge) + { + fiu_do_on(FailPoints::aggregate_function_state_transfer_throw, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure in AggregateFunctionResample::insertResultInto"); + }); + } + + auto & col_offsets = assert_cast(col.getOffsetsColumn()); col_offsets.getData().push_back(col.getData().size()); } + template + void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + { + auto & col = assert_cast(to); + size_t transferred = 0; + + if constexpr (!merge) + { + /// A nested function that is not a state aliases nothing and need not be atomic. + if (nested_function->isState()) + { + const size_t offsets_before = assert_cast(col.getOffsetsColumn()).size(); + + try + { + transferBuckets(place, col, transferred, arena); + } + catch (...) + { + auto & col_offsets = assert_cast(col.getOffsetsColumn()); + col_offsets.getData().resize_assume_reserved(offsets_before); + for (size_t i = transferred; i-- > 0;) + nested_function->rollbackInsertResult(place + i * size_of_data, col.getData()); + throw; + } + + return; + } + } + + transferBuckets(place, col, transferred, arena); + } + void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const override { insertResultIntoImpl(place, to, arena); @@ -256,6 +303,16 @@ class AggregateFunctionResample final : public IAggregateFunctionHelper(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + auto & col = assert_cast(to); + auto & col_offsets = assert_cast(col.getOffsetsColumn()); + + col_offsets.getData().resize_assume_reserved(col_offsets.size() - 1); + for (size_t i = total; i-- > 0;) + nested_function->rollbackInsertResult(place + i * size_of_data, col.getData()); + } + AggregateFunctionPtr getNestedFunction() const override { return nested_function; } }; diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h b/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h index 19a786db1f20..7385a3e944db 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionSimpleState.h @@ -103,6 +103,11 @@ class AggregateFunctionSimpleState final : public IAggregateFunctionHelperinsertResultInto(place, to, arena); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept override + { + nested_func->rollbackInsertResult(place, to); + } + bool allocatesMemoryInArena() const override { return nested_func->allocatesMemoryInArena(); } AggregateFunctionPtr getNestedFunction() const override { return nested_func; } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionState.h b/src/AggregateFunctions/Combinators/AggregateFunctionState.h index 5cb41d1d8716..7318c51022c0 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionState.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionState.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -10,6 +11,16 @@ namespace DB { struct Settings; +namespace ErrorCodes +{ +extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ +extern const char aggregate_function_state_transfer_throw[]; +} + /** Not an aggregate function, but an adapter of aggregate functions, * Aggregate functions with the `State` suffix differ from the corresponding ones in that their states are not finalized. @@ -144,7 +155,18 @@ class AggregateFunctionState final : public IAggregateFunctionHelper(to).getData().push_back(place); + auto & column = assert_cast(to); + + /// Only once the column holds an aliased state, which is the partial transfer to undo. + if (unlikely(!column.empty())) + { + fiu_do_on(FailPoints::aggregate_function_state_transfer_throw, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure in AggregateFunctionState::insertResultInto"); + }); + } + + column.getData().push_back(place); } void insertMergeResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override @@ -152,6 +174,13 @@ class AggregateFunctionState final : public IAggregateFunctionHelper(to).insertFrom(place); } + void rollbackInsertResult(ConstAggregateDataPtr __restrict, IColumn & to) const noexcept override + { + /// insertResultInto aliased a state that `place` still owns, so the row must go without the + /// destroy that ColumnAggregateFunction::popBack performs. + assert_cast(to).popBackWithoutDestroy(1); + } + /// Aggregate function or aggregate function state. bool isState() const override { return true; } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp index 4001af145fa1..13b9d034887e 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp @@ -499,6 +499,13 @@ void AggregateFunctionTuple::insertMergeResultInto(AggregateDataPtr __restrict p insertResultIntoImpl(place, to, arena); } +void AggregateFunctionTuple::rollbackInsertResult(ConstAggregateDataPtr __restrict place, IColumn & to) const noexcept +{ + auto & tuple_to = assert_cast(to); + for (size_t i = nested_functions.size(); i-- > 0;) + nested_functions[i]->rollbackInsertResult(place + state_offsets[i], tuple_to.getColumn(i)); +} + bool AggregateFunctionTuple::allocatesMemoryInArena() const { for (const auto & func : nested_functions) diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h index 4cefe838a21b..3c03a82531c3 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -11,6 +12,16 @@ namespace DB { struct Settings; +namespace ErrorCodes +{ +extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ +extern const char aggregate_function_state_transfer_throw_after_child[]; +} + /** Adaptor for aggregate functions. * Adding -Tuple suffix to aggregate function * will convert that aggregate function to a function, accepting Tuples, @@ -130,6 +141,7 @@ class AggregateFunctionTuple final : public IAggregateFunctionHelper - void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + void transferElements(AggregateDataPtr __restrict place, ColumnTuple & tuple_to, size_t & transferred, Arena * arena) const { - auto & tuple_to = assert_cast(to); - for (size_t i = 0; i < nested_functions.size(); ++i) + for (; transferred < nested_functions.size(); ++transferred) { + if constexpr (!for_merge) + { + /// Only past the first element, whose transfer returned, is there a completed child to undo. + if (unlikely(transferred > 0)) + { + fiu_do_on(FailPoints::aggregate_function_state_transfer_throw_after_child, + { + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure in AggregateFunctionTuple::insertResultInto"); + }); + } + } + if constexpr (for_merge) - nested_functions[i]->insertMergeResultInto(place + state_offsets[i], tuple_to.getColumn(i), arena); + nested_functions[transferred]->insertMergeResultInto( + place + state_offsets[transferred], tuple_to.getColumn(transferred), arena); else - nested_functions[i]->insertResultInto(place + state_offsets[i], tuple_to.getColumn(i), arena); + nested_functions[transferred]->insertResultInto( + place + state_offsets[transferred], tuple_to.getColumn(transferred), arena); } } + template + void insertResultIntoImpl(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const + { + auto & tuple_to = assert_cast(to); + size_t transferred = 0; + + if constexpr (!for_merge) + { + /// An element that is not a state aliases nothing and need not be atomic. + if (isState()) + { + try + { + transferElements(place, tuple_to, transferred, arena); + } + catch (...) + { + for (size_t i = transferred; i-- > 0;) + nested_functions[i]->rollbackInsertResult(place + state_offsets[i], tuple_to.getColumn(i)); + throw; + } + + return; + } + } + + transferElements(place, tuple_to, transferred, arena); + } + /// Shared implementation of the batch add overrides. Hoists the per-element column pointers, so /// no per-row unwrapping work remains in the row loop. /// `get_place` returns the aggregation state for a row, or nullptr when the row has none. diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 7f7e321d3ad0..ffa802107efe 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -294,6 +294,15 @@ class IAggregateFunction : public std::enable_shared_from_this 2) + return header_size; + return 16; + default: + return header_size; + } + } + /// You can only call for an empty object. void read(DB::ReadBuffer & in) { @@ -178,6 +217,28 @@ class ThetaSketchData : private boost::noncopyable if (bytes.empty()) return; + /** `compact_theta_sketch_parser::parse` verifies that the buffer holds 8 bytes and then reads + * header fields that lie beyond them before it checks the size again: `num_entries` at offset + * 8 for serial versions 1, 2 and 3, and `theta` at offset 16 for serial versions 1 and 2. A + * state shorter than that - which any `CAST` from a string can produce - is therefore read + * past its end, and the out-of-bounds value decides the size the parser then demands. + * + * Require up front the bytes the parser reads before it validates anything, exactly as the + * `std::istream` deserializer of `datasketches` has to consume them before it can decide + * anything about the sketch. A state that stops short of them is refused the same way the + * parser refuses one that stops short of its entries, and the parser then never reads past + * the buffer. The upstream fix is to move each `check_memory_size` before the field it + * guards; this keeps the read in bounds for every version of `datasketches-cpp` this + * repository pulls in. + */ + const size_t required_size = headerBytesReadBeforeSizeCheck(bytes.data(), bytes.size()); + if (bytes.size() < required_size) + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "Cannot deserialize Theta sketch state: at least {} bytes expected, actual {}", + required_size, + bytes.size()); + try { auto sk = datasketches::compact_theta_sketch::deserialize(bytes.data(), bytes.size()); diff --git a/src/Analyzer/Passes/CNF.cpp b/src/Analyzer/Passes/CNF.cpp index 7107ccddbb7d..0c656b33e56f 100644 --- a/src/Analyzer/Passes/CNF.cpp +++ b/src/Analyzer/Passes/CNF.cpp @@ -135,10 +135,13 @@ class PushNotVisitor { if (add_negation) { - if (function_name == "and") - function_node->resolveAsFunction(or_function_resolver); - else - function_node->resolveAsFunction(and_function_resolver); + /// Negate a new node: the formula can reference this one from several places. + const auto & resolver = function_name == "and" ? or_function_resolver : and_function_resolver; + auto negated_node = std::make_shared(resolver->getName()); + negated_node->getArguments().getNodes() = function_node->getArguments().getNodes(); + negated_node->resolveAsFunction(resolver); + node = std::move(negated_node); + function_node = node->as(); } auto & arguments = function_node->getArguments().getNodes(); @@ -212,12 +215,12 @@ class PushOrVisitor return true; auto & other_node = arguments[1 - and_node_id]; - auto & and_function_arguments = arguments[and_node_id]->as().getArguments().getNodes(); + const auto & and_function_arguments = arguments[and_node_id]->as().getArguments().getNodes(); - auto lhs = createFunctionNode(or_resolver, other_node->clone(), std::move(and_function_arguments[0])); + auto lhs = createFunctionNode(or_resolver, other_node->clone(), and_function_arguments[0]); num_atoms += countAtoms(other_node); - auto rhs = createFunctionNode(or_resolver, std::move(other_node), std::move(and_function_arguments[1])); + auto rhs = createFunctionNode(or_resolver, other_node, and_function_arguments[1]); node = createFunctionNode(and_resolver, std::move(lhs), std::move(rhs)); return visit(node, num_atoms); @@ -236,7 +239,7 @@ class PushOrVisitor class CollectGroupsVisitor { public: - void visit(QueryTreeNodePtr & node) + void visit(const QueryTreeNodePtr & node) { CNF::OrGroup or_group; visitImpl(node, or_group); @@ -247,14 +250,14 @@ class CollectGroupsVisitor CNF::AndGroup and_group; private: - void visitImpl(QueryTreeNodePtr & node, CNF::OrGroup & or_group) + void visitImpl(const QueryTreeNodePtr & node, CNF::OrGroup & or_group) { checkStackSize(); auto * function_node = node->as(); if (!function_node || !isLogicalFunction(*function_node)) { - or_group.insert(CNFAtomicFormula{false, std::move(node)}); + or_group.insert(CNFAtomicFormula{false, node}); return; } @@ -262,8 +265,8 @@ class CollectGroupsVisitor if (name == "and") { - auto & arguments = function_node->getArguments().getNodes(); - for (auto & argument : arguments) + const auto & arguments = function_node->getArguments().getNodes(); + for (const auto & argument : arguments) { CNF::OrGroup argument_or_group; visitImpl(argument, argument_or_group); @@ -273,15 +276,15 @@ class CollectGroupsVisitor } else if (name == "or") { - auto & arguments = function_node->getArguments().getNodes(); - for (auto & argument : arguments) + const auto & arguments = function_node->getArguments().getNodes(); + for (const auto & argument : arguments) visitImpl(argument, or_group); } else { chassert(name == "not"); - auto & arguments = function_node->getArguments().getNodes(); - or_group.insert(CNFAtomicFormula{true, std::move(arguments[0])}); + const auto & arguments = function_node->getArguments().getNodes(); + or_group.insert(CNFAtomicFormula{true, arguments[0]}); } } }; @@ -295,9 +298,9 @@ std::optional tryInvertFunction( if (auto it = inverse_relations.find(function_node->getFunctionName()); it != inverse_relations.end()) { - auto inverse_function_resolver = FunctionFactory::instance().get(it->second, context); - function_node->resolveAsFunction(inverse_function_resolver); - return CNFAtomicFormula{!atom.negative, atom.node_with_hash.node}; + auto inverted_node = atom.node_with_hash.node->clone(); + inverted_node->as().resolveAsFunction(FunctionFactory::instance().get(it->second, context)); + return CNFAtomicFormula{!atom.negative, std::move(inverted_node)}; } return std::nullopt; diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 793cfb6e6497..520da50c2124 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -417,6 +417,8 @@ struct ExpressionFilters /// Excluded from the analysis: non-lossless conversions (they also veto the fold-to-false /// collapse), NaN constants, and everything when pruning is disabled. std::vector opaque_filters; + /// Index in `opaque_filters` of the first `equals` on this expression; set only when pruning is disabled. + std::optional first_equals_position; }; using ComparisonFilterMap = QueryTreeNodePtrWithHashMap; @@ -881,8 +883,8 @@ static void rebuildComparisonNode(ComparisonFilterInfo & filter, const ContextPt } /// Insert a new comparison filter for `expression` into `filter_map`. -/// When `enable_pruning` is true, performs type conversion, boundary folding, and -/// comparison against existing filters for the same expression. +/// Performs type conversion; with `enable_pruning` also boundary folding and comparison against +/// every existing filter for the expression, otherwise only against the first `equals` seen. /// Returns ALWAYS_FALSE if a contradiction is found, ALWAYS_TRUE if the condition holds /// for the column type or is implied by existing filters, or ADDED otherwise. static AddComparisonFilterResult addComparisonFilter( @@ -892,8 +894,10 @@ static AddComparisonFilterResult addComparisonFilter( bool enable_pruning, const ContextPtr & context) { - /// Pruning disabled — just store the filter without analysis. - if (!enable_pruning) + /// A comparison with a nullable result is ambiguous under NULL and must not be pruned or folded; + /// keep it as-is. Test the comparison node's result type, not the raw operand type, so nested and + /// carrier-hidden nullability (e.g. `LowCardinality(Nullable)`, `Dynamic`, `Variant`) is caught. + if (isNullableOrLowCardinalityNullable(new_filter.original_node->getResultType())) { filter_map[expression].opaque_filters.push_back(std::move(new_filter)); return AddComparisonFilterResult::ADDED; @@ -901,14 +905,16 @@ static AddComparisonFilterResult addComparisonFilter( /// Step 1: convert the constant to the column's type for uniform comparison. const auto & raw_type = expression->getResultType(); - chassert(!raw_type->isNullable()); auto expr_type = removeLowCardinality(raw_type); new_filter.converted_value = tryConvertToColumnType(new_filter.constant_node, expr_type); /// Step 2: for integer columns, try boundary folding / float-literal rewriting. - if (auto result = tryFoldBoundaryOrRewriteFloatForIntColumn(new_filter, expr_type)) - return *result; + if (enable_pruning) + { + if (auto result = tryFoldBoundaryOrRewriteFloatForIntColumn(new_filter, expr_type)) + return *result; + } auto & filters = filter_map[expression]; @@ -923,6 +929,24 @@ static AddComparisonFilterResult addComparisonFilter( return AddComparisonFilterResult::ADDED; } + if (!enable_pruning) + { + auto result = AddComparisonFilterResult::ADDED; + if (new_filter.function == ComparisonFunction::EQUALS) + { + if (filters.first_equals_position) + { + if (compareComparisonFilters(filters.opaque_filters[*filters.first_equals_position], new_filter) + == ValueComparisonResult::ALWAYS_FALSE) + result = AddComparisonFilterResult::ALWAYS_FALSE; + } + else + filters.first_equals_position = filters.opaque_filters.size(); + } + filters.opaque_filters.push_back(std::move(new_filter)); + return result; + } + /// Step 3: compare against the existing equals/range filters. auto & range_filters = filters.range_filters; for (size_t i = 0; i < range_filters.size(); ++i) @@ -1881,7 +1905,7 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont /** Optimize AND chains by analyzing comparison conditions on the same expression. * This method performs two things in a single pass: * - * (a) Comparison chain pruning (when `optimize_redundant_comparisons` is enabled): + * (a) Comparison chain pruning (when `optimize_redundant_comparisons` is enabled, except an always-false `equals` pair): * Given an AND expression where the same column appears in multiple comparisons * against constants (e.g. `a = 3 AND a < 5 AND a > 1`), we collect all conditions * on the same non-constant expression into a per-expression `ComparisonFilterMap`. @@ -1961,7 +1985,7 @@ class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithCont expression = lhs; } - /// Both sides are non-constant — keep as-is. + /// Both sides are non-constant (or the constant is NULL-valued) — keep as-is. if (!constant) { all_operands.emplace_back(argument_index, argument); diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h index 8ffb09f5e17b..253a3614ac30 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.h @@ -107,7 +107,7 @@ namespace DB * ------------------------------- * * 8. Prune redundant comparisons and detect conflicting comparison conditions on the same expression - * within AND chains. Controlled by setting `optimize_redundant_comparisons`. + * within AND chains. Controlled by `optimize_redundant_comparisons`, except an always-false `equals` pair. * Handles all six comparison operators (=, !=, <, <=, >, >=) and their combinations: * duplicate removal, contradiction detection, and range tightening. * ------------------------------- diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.cpp b/src/Analyzer/Resolve/IdentifierResolveScope.cpp index a169c74744eb..5fdb1eb7d443 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.cpp +++ b/src/Analyzer/Resolve/IdentifierResolveScope.cpp @@ -321,7 +321,13 @@ void dump_list(WriteBuffer & buffer, const String & list_name, const std::ranges dump_mapping(buffer, "Alias name to expression node", aliases.alias_name_to_expression_node); dump_mapping(buffer, "Alias name to function node", aliases.alias_name_to_lambda_node); dump_mapping(buffer, "Alias name to table expression node", aliases.alias_name_to_table_expression_node); - dump_mapping(buffer, "CTE name to query node", cte_name_to_query_node); + if (!cte_name_to_query_node.empty()) + { + buffer << "CTE name to query node table size: " << cte_name_to_query_node.size() << '\n'; + for (const auto & [cte_name, cte_nodes] : cte_name_to_query_node) + for (const auto & cte_node : cte_nodes) + buffer << " { '" << cte_name << "' : " << cte_node->formatASTForErrorMessage() << " }\n"; + } dump_mapping(buffer, "WINDOW name to window node", window_name_to_window_node); dump_list(buffer, "Nodes with duplicated aliases size ", aliases.nodes_with_duplicated_aliases); diff --git a/src/Analyzer/Resolve/IdentifierResolveScope.h b/src/Analyzer/Resolve/IdentifierResolveScope.h index 2864b0b31168..33609d26a3ae 100644 --- a/src/Analyzer/Resolve/IdentifierResolveScope.h +++ b/src/Analyzer/Resolve/IdentifierResolveScope.h @@ -154,8 +154,8 @@ struct IdentifierResolveScope std::list *> join_using_columns; - /// CTE name to query node - std::unordered_map cte_name_to_query_node; + /// CTE name to its definitions in declaration order (several only with `analyzer_compatibility_allow_cte_redefinition`) + std::unordered_map cte_name_to_query_node; /// Window name to window node std::unordered_map window_name_to_window_node; diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index c14879e0afb2..73cd1c3c40a9 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -305,6 +305,17 @@ std::shared_ptr IdentifierResolver::tryResolveTableIdentifier(const I StorageID storage_id(database_name, table_name); storage_id = context->resolveStorageID(storage_id); + + /// The view source carries the inserted block and its types. For a MV, return this source + /// directly as a table node instead of swapping it later for the storage from the catalog + /// which may have been changed by a concurrent ALTER (the MV types must match the + /// snapshot at the start of the INSERT, not the current types). + /// For an inner query of an ordinary view, keep the normal flow that resolves from the catalog. + if (auto view_source = context->getViewSource(); + view_source && !context->isViewInnerQuery() + && view_source->getStorageID().getFullNameNotQuoted() == storage_id.getFullNameNotQuoted()) + return std::make_shared(view_source, context); + bool is_temporary_table = storage_id.getDatabaseName() == DatabaseCatalog::TEMPORARY_DATABASE; StoragePtr storage; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 3c684a85673b..443e17706604 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -85,6 +85,7 @@ namespace Setting { extern const SettingsBool aggregate_functions_null_for_empty; extern const SettingsBool analyzer_compatibility_allow_non_aggregate_in_having; + extern const SettingsBool analyzer_compatibility_allow_cte_redefinition; extern const SettingsBool enable_streaming_queries; extern const SettingsBool analyzer_compatibility_join_using_top_level_identifier; extern const SettingsBool analyzer_compatibility_multiple_joins_qualify_column_names; @@ -149,6 +150,18 @@ namespace ErrorCodes namespace { +/// True for a `WITH` element declared `AS MATERIALIZED`, before or after its replacement by a `TableNode`. +bool isMaterializedCTEDefinition(const QueryTreeNodePtr & node) +{ + if (const auto * query_node = node->as()) + return query_node->isMaterialized(); + if (const auto * union_node = node->as()) + return union_node->isMaterialized(); + if (const auto * table_node = node->as()) + return table_node->isMaterializedCTE(); + return false; +} + /// Recursively clears aliases from `node` and all of its descendants, stopping at /// nested-scope boundaries (`QUERY`, `UNION`, `LAMBDA`). /// @@ -1181,6 +1194,10 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromAliases(const Ide { const auto & identifier_bind_part = identifier_lookup.identifier.front(); + /// Table aliases are single-part names, so a compound table identifier can only be `database.table`. + if (identifier_lookup.isTableExpressionLookup() && identifier_lookup.identifier.isCompound()) + return {}; + auto * it = scope.aliases.find(identifier_lookup, ScopeAliases::FindOption::FIRST_NAME); if (it == nullptr) return {}; @@ -1281,13 +1298,12 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromAliases(const Ide } return {}; } - if (identifier_lookup.isFunctionLookup() || identifier_lookup.isTableExpressionLookup()) + if (identifier_lookup.isFunctionLookup()) { throw Exception( ErrorCodes::UNKNOWN_IDENTIFIER, - "Compound identifier '{}' cannot be resolved as {}. In scope {}", + "Compound identifier '{}' cannot be resolved as function. In scope {}", identifier_lookup.identifier.getFullName(), - identifier_lookup.isFunctionLookup() ? "function" : "table expression", scope.scope_node->formatASTForErrorMessage()); } } @@ -1306,7 +1322,9 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromCTE( ) { auto full_name = identifier_lookup.identifier.getFullName(); - auto cte_query_node_it = scope.cte_name_to_query_node.find(full_name); + auto cte_nodes_it = scope.cte_name_to_query_node.find(full_name); + if (cte_nodes_it == scope.cte_name_to_query_node.end()) + return {}; /// CTE may reference table expressions with the same name, e.g.: /// @@ -1320,10 +1338,23 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierFromCTE( /// /// To accomplish this behaviour it's not allowed to resolve identifiers to /// CTE that is being resolved. - if (cte_query_node_it == scope.cte_name_to_query_node.end() || ctes_in_resolve_process.contains(cte_query_node_it->second)) + /// + /// With `analyzer_compatibility_allow_cte_redefinition` a name can have several definitions; the latest one + /// not being resolved wins, so a redefinition reads the previous definition and the query body the last one. + auto & cte_nodes = cte_nodes_it->second; + /// Every site that marks a scope-map CTE node as being resolved updates both sets. With one definition keep the + /// structural check: the materialized-CTE expression site inserts a clone, which only identity would miss. + const bool several_definitions = cte_nodes.size() > 1; + auto cte_node_it = std::find_if(cte_nodes.rbegin(), cte_nodes.rend(), [this, several_definitions](const QueryTreeNodePtr & node) + { + if (several_definitions) + return !cte_definitions_in_resolve_process.contains(node.get()); + return !ctes_in_resolve_process.contains(node); + }); + if (cte_node_it == cte_nodes.rend()) return {}; - auto & cte_node = cte_query_node_it->second; + auto & cte_node = *cte_node_it; auto * query_node = cte_node->as(); auto * union_node = cte_node->as(); @@ -3299,6 +3330,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( /// In this example argument of function `in` is being resolve here. If CTE `test1` is not forbidden, /// `test1` is resolved to CTE (not to the table) in `initializeQueryJoinTreeNode` function. ctes_in_resolve_process.insert(original_cte_node); + cte_definitions_in_resolve_process.insert(original_cte_node.get()); if (subquery_node) resolveQuery(resolved_identifier_node, subquery_scope); @@ -3306,6 +3338,7 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode( resolveUnion(resolved_identifier_node, subquery_scope); ctes_in_resolve_process.erase(original_cte_node); + cte_definitions_in_resolve_process.erase(original_cte_node.get()); } else if (table_node != nullptr && table_node->isMaterializedCTE()) { @@ -5986,12 +6019,18 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, QueryTreeNodePtr original_cte_node = try_get_original_cte_node(join_tree_node); if (original_cte_node) + { ctes_in_resolve_process.insert(original_cte_node); + cte_definitions_in_resolve_process.insert(original_cte_node.get()); + } resolveExpressionNode(join_tree_node, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/, true /*ignore_alias=*/); if (original_cte_node) + { ctes_in_resolve_process.erase(original_cte_node); + cte_definitions_in_resolve_process.erase(original_cte_node.get()); + } break; } case QueryTreeNodeType::TABLE_FUNCTION: @@ -6014,19 +6053,22 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, /// Prevent recursive CTE references during subquery resolution. const auto & cte_name = materialized_cte_ptr->cte_name; - QueryTreeNodePtr cte_map_node; + QueryTreeNodes cte_map_nodes; for (auto * s = &scope; s; s = s->parent_scope) { auto it = s->cte_name_to_query_node.find(cte_name); if (it != s->cte_name_to_query_node.end()) { - cte_map_node = it->second; + cte_map_nodes = it->second; break; } } - if (cte_map_node) + for (const auto & cte_map_node : cte_map_nodes) + { ctes_in_resolve_process.insert(cte_map_node); + cte_definitions_in_resolve_process.insert(cte_map_node.get()); + } IdentifierResolveScope & subquery_scope = createIdentifierResolveScope(subquery, &scope); subquery_scope.subquery_depth = scope.subquery_depth + 1; @@ -6036,8 +6078,11 @@ void QueryAnalyzer::resolveQueryJoinTreeNode(QueryTreeNodePtr & join_tree_node, else resolveUnion(subquery, subquery_scope); - if (cte_map_node) + for (const auto & cte_map_node : cte_map_nodes) + { ctes_in_resolve_process.erase(cte_map_node); + cte_definitions_in_resolve_process.erase(cte_map_node.get()); + } bool is_correlated = subquery->as() ? subquery->as()->isCorrelated() @@ -6490,12 +6535,32 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier continue; const auto & cte_name = subquery_node ? subquery_node->getCTEName() : union_node->getCTEName(); - auto [_, inserted] = scope.cte_name_to_query_node.emplace(cte_name, node); - if (!inserted) - throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, - "CTE with name {} already exists. In scope {}", - cte_name, - scope.scope_node->formatASTForErrorMessage()); + auto & cte_nodes = scope.cte_name_to_query_node[cte_name]; + if (!cte_nodes.empty()) + { + if (query_node_typed.isRecursiveWith()) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "CTE with name {} already exists and cannot be redefined in a recursive WITH clause. In scope {}", + cte_name, + scope.scope_node->formatASTForErrorMessage()); + + /// A redefinition is rejected on its second registration, so only the first node can be materialized. + if (isMaterializedCTEDefinition(node) || isMaterializedCTEDefinition(cte_nodes.front())) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "CTE with name {} already exists and cannot be redefined because it is declared as MATERIALIZED. In scope {}", + cte_name, + scope.scope_node->formatASTForErrorMessage()); + + /// Checked last, so the hint is given only when enabling the setting would help. + if (!scope.context->getSettingsRef()[Setting::analyzer_compatibility_allow_cte_redefinition]) + throw Exception(ErrorCodes::MULTIPLE_EXPRESSIONS_FOR_ALIAS, + "CTE with name {} already exists. Enable the setting analyzer_compatibility_allow_cte_redefinition " + "to let a later definition shadow the earlier one. In scope {}", + cte_name, + scope.scope_node->formatASTForErrorMessage()); + } + + cte_nodes.push_back(node); } /** WITH section can be safely removed, because WITH section only can provide aliases to query expressions diff --git a/src/Analyzer/Resolve/QueryAnalyzer.h b/src/Analyzer/Resolve/QueryAnalyzer.h index 129c38311cd7..b478854440f2 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.h +++ b/src/Analyzer/Resolve/QueryAnalyzer.h @@ -312,6 +312,9 @@ class QueryAnalyzer /// CTEs that are currently in resolve process QueryTreeNodePtrWithHashSet ctes_in_resolve_process; + /// Same as `ctes_in_resolve_process` but by identity: structural comparison cannot tell identical redefinitions apart. + std::unordered_set cte_definitions_in_resolve_process; + /// Window definitions that are currently in resolve process std::unordered_set windows_in_resolve_process; diff --git a/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp b/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp index 2b9c6355a330..93d4dfb653cb 100644 --- a/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp +++ b/src/Analyzer/Resolve/evaluateScalarSubqueryIfNeeded.cpp @@ -215,7 +215,9 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden addQueryTreePasses(query_tree_pass_manager, options.only_analyze); query_tree_pass_manager.run(query_tree); - if (auto storage = subquery_context->getViewSource()) + /// The inner query of an ordinary view referenced by the view query reads the table itself, + /// not the inserted block. + if (auto storage = subquery_context->getViewSource(); storage && !subquery_context->isViewInnerQuery()) replaceStorageInQueryTree(query_tree, subquery_context, storage); auto interpreter = std::make_unique(query_tree, subquery_context, options); diff --git a/src/Analyzer/Resolve/resolveFunction.cpp b/src/Analyzer/Resolve/resolveFunction.cpp index bb3e74a48993..750246027f42 100644 --- a/src/Analyzer/Resolve/resolveFunction.cpp +++ b/src/Analyzer/Resolve/resolveFunction.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include @@ -1310,7 +1311,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi scope.scope_node->formatASTForErrorMessage()); auto & table_node_typed = table_node->as(); - if (!std::dynamic_pointer_cast(table_node_typed.getStorage())) + if (!castStorage(table_node_typed.getStorage(), DeferredTable::Load)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} table '{}' should have engine StorageJoin. In scope {}", function_name, @@ -2141,8 +2142,9 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi } else { - /// Replace storage with values storage of insertion block - if (StoragePtr storage = scope.context->getViewSource()) + /// Replace storage with values storage of insertion block. + /// The inner query of an ordinary view referenced by the view query reads the table itself. + if (StoragePtr storage = scope.context->getViewSource(); storage && !scope.context->isViewInnerQuery()) { QueryTreeNodePtr table_expression = in_second_argument; diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 30446234dc67..5b6112e9f80f 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -13,8 +13,11 @@ #include #include +#include #include #include +#include +#include #include @@ -26,6 +29,7 @@ namespace DB namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; } @@ -61,7 +65,81 @@ namespace {"sdk_retry_max_backoff_ms", std::to_string(settings.sdk_retry_max_backoff_ms)}, }; } + +#if CLICKHOUSE_CLOUD + /// The recorded endpoint of a snapshot's source disk as a service URL: scheme, host, port and path, without + /// query parameters and trailing slashes. A source disk that authenticates with a SAS records it in the + /// query, and one configured with a connection string records the connection string itself; the snapshot + /// is read with the backup's credential in any case, so only the service URL is kept. + String snapshotServiceURL(const String & endpoint) + { + Azure::Core::Url parsed; + try + { + parsed = endpoint.starts_with("http") ? Azure::Core::Url(endpoint) + : Azure::Storage::_internal::ParseConnectionString(endpoint).BlobServiceUrl; + } + catch (const std::exception & e) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot parse the endpoint of a lightweight snapshot: {}", e.what()); + } + if (parsed.GetScheme().empty() || parsed.GetHost().empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The endpoint of a lightweight snapshot is not a service URL"); + + String url = parsed.GetScheme() + "://" + parsed.GetHost(); + if (parsed.GetPort() != 0) + url += ":" + std::to_string(parsed.GetPort()); + if (!parsed.GetPath().empty()) + url += "/" + parsed.GetPath(); + while (url.ends_with('/')) + url.pop_back(); + return url; + } + + /// The connection string with its blob endpoint replaced by `service_url`. `CreateFromConnectionString()` + /// takes the endpoint from the connection string alone, so this is how a connection string is pointed at + /// the recorded endpoint of a snapshot; the account key or SAS it carries then authorises the reads there, + /// or Azure refuses them if it belongs to another account. + String withBlobEndpoint(const String & connection_string, const String & service_url) + { + std::vector parts; + boost::split(parts, connection_string, boost::is_any_of(";")); + String result; + for (const auto & part : parts) + if (!part.empty() && !part.starts_with("BlobEndpoint=")) + result += part + ";"; + return result + "BlobEndpoint=" + service_url; + } +#endif +} + +#if CLICKHOUSE_CLOUD +AzureBlobStorage::ConnectionParams makeSnapshotSourceConnectionParams( + const AzureBlobStorage::ConnectionParams & backup_connection_params, const String & endpoint, const String & blob_namespace) +{ + auto connection_params = backup_connection_params; + + /// The objects are read from the recorded endpoint with the backup's credential, as the S3 reader does. + const String service_url = snapshotServiceURL(endpoint); + if (const auto * connection_string = std::get_if(&connection_params.auth_method)) + { + const String repointed = withBlobEndpoint(connection_string->toUnderType(), service_url); + connection_params.auth_method = AzureBlobStorage::ConnectionString{repointed}; + connection_params.endpoint.storage_account_url = repointed; + } + else + connection_params.endpoint.storage_account_url = service_url; + + const auto slash_pos = blob_namespace.find('/'); + connection_params.endpoint.container_name = blob_namespace.substr(0, slash_pos); + connection_params.endpoint.prefix = (slash_pos == String::npos) ? "" : blob_namespace.substr(slash_pos + 1); + + /// The snapshot was taken from this container, so it exists; the existence check is also a + /// container-level request. + connection_params.endpoint.container_already_exists = true; + return connection_params; } +#endif BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const AzureBlobStorage::ConnectionParams & connection_params_, diff --git a/src/Backups/BackupIO_AzureBlobStorage.h b/src/Backups/BackupIO_AzureBlobStorage.h index 3f262f83316b..4df6207442f1 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.h +++ b/src/Backups/BackupIO_AzureBlobStorage.h @@ -10,6 +10,22 @@ namespace DB { +#if CLICKHOUSE_CLOUD +/// Connection params for reading the objects a lightweight snapshot references, built from the endpoint +/// and object namespace its manifest records for the source disk. For Azure that namespace is `` +/// or `/` when the disk endpoint has a path below the container +/// (`AzureObjectStorage::getObjectsNamespace`), while the recorded object keys are relative to the prefix, so +/// it is split back into a container and a blob prefix: used as a container name, `/` +/// gives every container-level request extra path segments, which Azure rejects with `400 InvalidUri`. +/// The objects are read from the recorded endpoint with the backup's credential, as the S3 reader does. A +/// connection string is pointed there by replacing its `BlobEndpoint`, since `CreateFromConnectionString()` +/// takes the endpoint from the connection string alone; its key or SAS then authorises the reads, or Azure +/// refuses them if it belongs to another account. Query parameters of the recorded endpoint (a SAS of the +/// source disk) are dropped, and an endpoint recorded as a connection string is reduced to its service URL. +AzureBlobStorage::ConnectionParams makeSnapshotSourceConnectionParams( + const AzureBlobStorage::ConnectionParams & backup_connection_params, const String & endpoint, const String & blob_namespace); +#endif + /// Represents a backup stored to Azure class BackupReaderAzureBlobStorage : public BackupReaderDefault { diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 57e9c45cbfd3..2b7c176d0ecd 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -3,6 +3,7 @@ #if USE_AWS_S3 #include #include +#include #include #include #include @@ -131,6 +132,8 @@ class S3BackupClientCreator const S3Settings & settings, const ContextPtr & context) { + context->getGlobalContext()->getRemoteHostFilter().checkURL(s3_uri.uri); + Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); HTTPHeaderEntries headers; String session_token = settings.auth_settings[S3AuthSetting::session_token]; diff --git a/src/Backups/BackupInfo.cpp b/src/Backups/BackupInfo.cpp index 0357ab785981..5b68b55a10e4 100644 --- a/src/Backups/BackupInfo.cpp +++ b/src/Backups/BackupInfo.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include @@ -152,11 +151,22 @@ namespace return evaluated_literal->value.safeGet(); } - /// Rebuilds an `extra_credentials(...)` keeping only its non-secret keys, or `nullptr` if none are - /// left. `role_arn` and `role_session_name` only name the role to assume, which grants nothing - /// without the server's own identity and a matching trust policy, so `isNonSecretExtraCredentialsKey` - /// keeps them -- the same predicate that keeps them visible in a logged query. `external_id` is the - /// shared secret of the triple, and anything unclassifiable is dropped. + /// The `extra_credentials(...)` keys that the `` locator keeps in the `.backup` metadata: + /// the role identifiers `role_arn` and `role_session_name`. Every backup of a chain must be able to + /// reopen its base with the stored locator alone, and a trust policy may pin the session name through + /// `sts:RoleSessionName`, so dropping it would break such restores (see #116223). This deliberately + /// differs from the logged query text, where `role_session_name` is masked and only `role_arn` stays + /// visible (`FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey`): the metadata file lives + /// in the user's own backup bucket, next to the data it protects. `external_id` is the shared secret + /// of the triple and is dropped, as is anything unclassifiable. + bool isBaseBackupRoleIdentifierKey(std::string_view key) + { + return key == "role_arn" || key == "role_session_name"; + } + + /// Rebuilds an `extra_credentials(...)` keeping only its role identifiers, or `nullptr` if none are + /// left. They only name the role to assume and its session, which grants nothing without the server's + /// own identity and a matching trust policy. ASTPtr withoutSecretExtraCredentials(const ASTPtr & function_arg, const ContextPtr & context) { const auto * func = function_arg->as(); @@ -168,7 +178,7 @@ namespace for (const auto & child : func->arguments->children) { auto key = getEffectiveKeyValueArgName(child, context); - if (!key || !FunctionSecretArgumentsFinder::isNonSecretExtraCredentialsKey(*key)) + if (!key || !isBaseBackupRoleIdentifierKey(*key)) continue; /// The value is only classified, not rewritten: an expression the open path resolves stays as /// it was written, so a locator that loses nothing serializes byte for byte. @@ -286,7 +296,7 @@ BackupInfo BackupInfo::fromAST(const IAST & ast) { const auto * func = ast.as(); if (!func) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected function, got {}", ast.formatForErrorMessage()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a function as the backup locator"); BackupInfo res; res.backup_engine_name = func->name; @@ -295,7 +305,7 @@ BackupInfo BackupInfo::fromAST(const IAST & ast) { const auto * list = func->arguments->as(); if (!list) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected list, got {}", func->arguments->formatForErrorMessage()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected an argument list in the backup locator"); size_t index = 0; if (!list->children.empty()) @@ -329,7 +339,7 @@ BackupInfo BackupInfo::fromAST(const IAST & ast) res.function_arg = elem; break; } - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected literal, got {}", elem->formatForErrorMessage()); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a literal argument in the backup locator"); } res.args.push_back(lit->value); } @@ -418,9 +428,10 @@ BackupInfo BackupInfo::withoutS3Credentials(ContextPtr context) const /// S3(collection, secret_access_key = '...') -> S3(collection) /// The keys are the `S3` authentication arguments consumed by `registerBackupEngineS3` - /// and `S3StorageParsedArguments::collectCredentials`, minus the non-secret role identifiers, which - /// stay so that a role-authenticated base backup remains openable. The key is resolved with the - /// context, so that an expression key (e.g. concat('secret_', 'access_key')) is recognized as well. + /// and `S3StorageParsedArguments::collectCredentials`, minus the role identifiers (`role_arn`, + /// `role_session_name`), which stay so that a role-authenticated base backup remains openable (see + /// isBaseBackupRoleIdentifierKey). The key is resolved with the context, so that an expression key + /// (e.g. concat('secret_', 'access_key')) is recognized as well. res.kv_args.erase( std::remove_if( res.kv_args.begin(), diff --git a/src/Backups/BackupMetadataFinder.cpp b/src/Backups/BackupMetadataFinder.cpp index c9b6fee3f362..cfc3c257784d 100644 --- a/src/Backups/BackupMetadataFinder.cpp +++ b/src/Backups/BackupMetadataFinder.cpp @@ -304,8 +304,8 @@ void BackupMetadataFinder::findTableInBackupImpl( ErrorCodes::CANNOT_RESTORE_TABLE, "Extracted two different create queries for the same {}: {} and {}", tableNameWithTypeToString(table_name.database, table_name.table, false), - table_info.create_table_query_str, - create_table_query_str); + table_info.create_table_query->formatForErrorMessage(), + create_table_query->formatForErrorMessage()); } } @@ -395,8 +395,8 @@ void BackupMetadataFinder::findDatabaseInBackupImpl( ErrorCodes::CANNOT_RESTORE_DATABASE, "Extracted two different create queries for the same database {}: {} and {}", backQuoteIfNeed(database_name), - database_info.create_database_query_str, - create_database_query_str); + database_info.create_database_query->formatForErrorMessage(), + create_database_query->formatForErrorMessage()); } database_info.create_database_query = create_database_query; diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp index d4e63706d270..d3d4afa57d65 100644 --- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp +++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp @@ -301,12 +301,11 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory) params.write_settings, params.context); - auto snapshot_reader_creator = [&](const String & endpoint, const String & container_name) +#if CLICKHOUSE_CLOUD + auto snapshot_reader_creator = [&](const String & endpoint, const String & blob_namespace) { - connection_params.endpoint.storage_account_url = endpoint; - connection_params.endpoint.container_name = container_name; return std::make_shared( - connection_params, + makeSnapshotSourceConnectionParams(connection_params, endpoint, blob_namespace), "", params.allow_azure_native_copy, params.read_settings, @@ -315,6 +314,9 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory) }; return std::make_unique(params, archive_params, reader, snapshot_reader_creator); +#else + return std::make_unique(params, archive_params, reader); +#endif } auto writer = std::make_shared( diff --git a/src/Backups/registerBackupEngineS3.cpp b/src/Backups/registerBackupEngineS3.cpp index 4f3f9b075dee..feddeddd7887 100644 --- a/src/Backups/registerBackupEngineS3.cpp +++ b/src/Backups/registerBackupEngineS3.cpp @@ -304,8 +304,19 @@ void registerBackupEngineS3(BackupFactory & factory) /// copy: `BackupImpl::writeBackupMetadata` decides whether the base backup may take this /// backup's credentials by comparing the two locators as text, and a locator resolved on /// one side alone no longer matches the other written the same way. - if (!StorageS3Configuration::collectCredentials(params.backup_info.function_arg->clone(), auth_settings, params.context)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid argument: {}", params.backup_info.function_arg->formatForErrorMessage()); + /// The rejected argument is never echoed, so both failure modes report the same way: + /// a nested map like `headers(..)` is masked only by its enclosing formatter, so + /// formatting one alone prints its values, and evaluating one can throw quoting it. + try + { + if (!StorageS3Configuration::collectCredentials( + params.backup_info.function_arg->clone(), auth_settings, params.context)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 extra credentials"); + } + catch (const Exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 extra credentials"); + } role_arn = std::move(auth_settings[S3AuthSetting::role_arn]); role_session_name = std::move(auth_settings[S3AuthSetting::role_session_name]); diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp index 39209f2df658..2dc6f472ff86 100644 --- a/src/Client/HedgedConnections.cpp +++ b/src/Client/HedgedConnections.cpp @@ -136,6 +136,27 @@ void HedgedConnections::sendQueryPlan(const QueryPlan & query_plan) pipeline_for_new_replicas.add(send_query_plan); } +bool HedgedConnections::supportsQueryPlanSerializationVersion(UInt64 version) const +{ + /// The first replica is established before the query is sent, but a later hedge may + /// select any remaining replica: one whose version is not known yet, or an already + /// established usable but stale one that `setBestUsableReplica` keeps for later. + /// Use the SQL fallback rather than making that hedge unavailable after a timeout. + if (hedged_connections_factory.maySelectReplicaBelowQueryPlanSerializationVersion(version)) + return false; + + for (const OffsetState & offset_state : offset_states) + { + for (const ReplicaState & replica : offset_state.replicas) + { + if (replica.connection && replica.connection->getQueryPlanSerializationVersion() < version) + return false; + } + } + + return true; +} + void HedgedConnections::sendExternalTablesData(std::vector & data) { std::lock_guard lock(cancel_mutex); diff --git a/src/Client/HedgedConnections.h b/src/Client/HedgedConnections.h index 3fe98918f56d..5406fbdf816c 100644 --- a/src/Client/HedgedConnections.h +++ b/src/Client/HedgedConnections.h @@ -95,6 +95,8 @@ class HedgedConnections : public IConnections void sendQueryPlan(const QueryPlan & query_plan) override; + bool supportsQueryPlanSerializationVersion(UInt64 version) const override; + void sendClusterFunctionReadTaskResponse(const ClusterFunctionReadTaskResponse &) override { throw Exception(ErrorCodes::LOGICAL_ERROR, "sendReadTaskResponse in not supported with HedgedConnections"); diff --git a/src/Client/HedgedConnectionsFactory.cpp b/src/Client/HedgedConnectionsFactory.cpp index cfb938811d78..77fab23fcdf9 100644 --- a/src/Client/HedgedConnectionsFactory.cpp +++ b/src/Client/HedgedConnectionsFactory.cpp @@ -384,6 +384,29 @@ void HedgedConnectionsFactory::removeReplicaFromEpoll(int index, int fd) timeout_fd_to_replica_index.erase(replicas[index].change_replica_timeout.getDescriptor()); } +bool HedgedConnectionsFactory::maySelectReplicaBelowQueryPlanSerializationVersion(UInt64 version) const +{ + if (maySelectUnverifiedReplica()) + return true; + + if (!fallback_to_stale_replicas) + return false; + + /// Every pool has already been resolved, but the usable and not up-to-date entries are still + /// held by the factory and `setBestUsableReplica` may hand one of them out on a later hedge. + for (const ReplicaStatus & replica : replicas) + { + if (replica.is_ready) + continue; + + TryResult result = replica.connection_establisher->getResult(); + if (!result.entry.isNull() && result.is_usable && result.entry->getQueryPlanSerializationVersion() < version) + return true; + } + + return false; +} + size_t HedgedConnectionsFactory::numberOfProcessingReplicas() const { if (epoll.empty()) diff --git a/src/Client/HedgedConnectionsFactory.h b/src/Client/HedgedConnectionsFactory.h index ee5105d3f38d..bb50aad5c98c 100644 --- a/src/Client/HedgedConnectionsFactory.h +++ b/src/Client/HedgedConnectionsFactory.h @@ -82,6 +82,19 @@ class HedgedConnectionsFactory /// Tell Factory to not return connections with two level aggregation incompatibility. void skipReplicasWithTwoLevelAggregationIncompatibility() { skip_replicas_with_two_level_aggregation_incompatibility = true; } + /// Whether a future hedge could select a replica whose capabilities are not yet known. + bool maySelectUnverifiedReplica() const + { + return hasEventsInProcess() || entries_count + replicas_in_process_count + failed_pools_count < shuffled_pools.size(); + } + + /// Whether a future hedge could select a replica that does not support the given query-plan + /// serialization version. Besides the pools that were never dialled (whose version cannot be + /// known here), this also accounts for the already-established usable but not up-to-date + /// replicas: they are kept inside the factory, never handed out as READY, and `setBestUsableReplica` + /// can still pick one of them later when `fallback_to_stale_replicas_for_distributed_queries` is on. + bool maySelectReplicaBelowQueryPlanSerializationVersion(UInt64 version) const; + size_t getFailedPoolsCount() const { return failed_pools_count; } ~HedgedConnectionsFactory(); diff --git a/src/Client/IConnections.h b/src/Client/IConnections.h index 19c864306a16..e0f09ae8e2ee 100644 --- a/src/Client/IConnections.h +++ b/src/Client/IConnections.h @@ -29,6 +29,10 @@ class IConnections : boost::noncopyable virtual void sendQueryPlan(const QueryPlan & query_plan) = 0; + /// All currently connected replicas must support this version before the caller sends a + /// query-plan packet serialized at it. + virtual bool supportsQueryPlanSerializationVersion(UInt64 version) const = 0; + virtual void sendClusterFunctionReadTaskResponse(const ClusterFunctionReadTaskResponse &) = 0; virtual void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) = 0; virtual void sendMergeTreeAllRangesAnnouncementResponse(const InitialAllRangesAnnouncementResponse & response) = 0; diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 851a9e81c6cf..b23bdfb9b4cc 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -118,6 +118,17 @@ void MultiplexedConnections::sendQueryPlan(const QueryPlan & query_plan) } } +bool MultiplexedConnections::supportsQueryPlanSerializationVersion(UInt64 version) const +{ + for (const ReplicaState & state : replica_states) + { + if (state.connection && state.connection->getQueryPlanSerializationVersion() < version) + return false; + } + + return true; +} + void MultiplexedConnections::sendExternalTablesData(std::vector & data) { std::lock_guard lock(cancel_mutex); diff --git a/src/Client/MultiplexedConnections.h b/src/Client/MultiplexedConnections.h index 80164397d7da..a76cb9dd2bc4 100644 --- a/src/Client/MultiplexedConnections.h +++ b/src/Client/MultiplexedConnections.h @@ -41,6 +41,8 @@ class MultiplexedConnections final : public IConnections void sendQueryPlan(const QueryPlan & query_plan) override; + bool supportsQueryPlanSerializationVersion(UInt64 version) const override; + void sendClusterFunctionReadTaskResponse(const ClusterFunctionReadTaskResponse & response) override; void sendMergeTreeReadTaskResponse(const ParallelReadResponse & response) override; void sendMergeTreeAllRangesAnnouncementResponse(const InitialAllRangesAnnouncementResponse & response) override; diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 583883b5bd87..6a564e5e1965 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "config.h" // USE_SKIM @@ -437,6 +438,19 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_completions_context.clear(); hint_completions_context_size = 0; + /// A line that was just displayed programmatically (recalled from history, found by a + /// history search, pasted, brought back from the editor) must not pop hints by itself: + /// with hints visible, the next Up/Down press would navigate the hints instead of the + /// history. The display armed the one-shot and pinned the displayed text (the same + /// display can regenerate the hints once more when replxx replays a throttled + /// refresh); the first run for an edited text unpins and shows the hints again. + if (suppress_hints_once || (!suppress_hints_for_text.empty() && suppress_hints_for_text == rx.get_state().text())) + { + suppress_hints_once = false; + return replxx::Replxx::hints_t{}; + } + suppress_hints_for_text.clear(); + /// Mirror `set_complete_on_empty(false)` *before* matching: an empty last word matches /// every suggestion, and this callback runs on every zero-delay repaint, so we must not /// fold and stable-sort the whole dictionary only to drop the result here. @@ -487,12 +501,43 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) /// The modify callback runs on every dispatched action, so reset the mirror here to track /// replxx; the hint-navigation keys re-set it *after* invoking, so a real navigation stays. rx.set_modify_callback([this] (std::string &, int &) { hint_selection = -1; }); + + /// A pasted query is also a whole new line displayed at once, so it does not pop hints + /// either (replxx's default binding for the paste marker just invokes the same action; + /// the action reads the whole paste, so the buffer holds the pasted text afterwards). + rx.bind_key(Replxx::KEY::PASTE_START, [this](char32_t code) + { + suppress_hints_once = true; + auto result = rx.invoke(Replxx::ACTION::BRACKETED_PASTE, code); + /// The paste action fills the buffer directly, without invalidating replxx's hint + /// cache, which is keyed by the buffer text and lives across prompts. Pasting the + /// exact text that carried a visible hint on an earlier prompt would therefore + /// redisplay the cached hints without ever asking our hint callback, and the + /// suppression below would have nothing to suppress. Re-setting the state is what + /// invalidates that cache (see openEditor). + rx.set_state(rx.get_state()); + suppressHintsForDisplayedLine(); + return result; + }); } /// By default C-p/C-n bound to COMPLETE_NEXT/COMPLETE_PREV, /// bind C-p/C-n to history-previous/history-next like readline. - rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); }); - rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_NEXT, code); }); + rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + rx.bind_key(Replxx::KEY::meta(Replxx::KEY::DOWN), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_NEXT, code); }); + rx.bind_key(Replxx::KEY::meta(Replxx::KEY::UP), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + rx.bind_key(Replxx::KEY::meta('p'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH, code); }); + rx.bind_key(Replxx::KEY::meta('n'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_COMMON_PREFIX_SEARCH, code); }); + rx.bind_key(Replxx::KEY::meta('<'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_FIRST, code); }); + rx.bind_key(Replxx::KEY::PAGE_UP, [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_FIRST, code); }); + rx.bind_key(Replxx::KEY::meta('>'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_LAST, code); }); + rx.bind_key(Replxx::KEY::PAGE_DOWN, [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_LAST, code); }); + rx.bind_key(Replxx::KEY::control('G'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_RESTORE_CURRENT, code); }); + rx.bind_key(Replxx::KEY::meta('g'), [this](char32_t code) { return historyNavigate(Replxx::ACTION::HISTORY_RESTORE, code); }); + rx.bind_key(Replxx::KEY::control('R'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, code); }); + rx.bind_key(Replxx::KEY::control('S'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, code); }); + rx.bind_key(Replxx::KEY::meta('r'), [this](char32_t code) { return historySearch(Replxx::ACTION::HISTORY_SEEDED_INCREMENTAL_SEARCH, code); }); /// We don't want the default, "suspend" behavior, it confuses people. if (options.ignore_shell_suspend) @@ -561,7 +606,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_selection = next; return result; } - return rx.invoke(Replxx::ACTION::LINE_NEXT, code); + return historyNavigate(Replxx::ACTION::LINE_NEXT, code); }; /// Up navigates the hints only once a hint is selected; before that it keeps recalling /// command history, so the hints do not shadow it. @@ -574,7 +619,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_selection = next; return result; } - return rx.invoke(Replxx::ACTION::LINE_PREVIOUS, code); + return historyNavigate(Replxx::ACTION::LINE_PREVIOUS, code); }; rx.bind_key(Replxx::KEY::DOWN, hint_next); rx.bind_key(Replxx::KEY::UP, hint_previous); @@ -589,7 +634,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) hint_selection = next; return result; } - return rx.invoke(Replxx::ACTION::LINE_PREVIOUS, code); + return historyNavigate(Replxx::ACTION::LINE_PREVIOUS, code); }); /// Right accepts the chosen hint (the single one shown, or the one selected by navigating); @@ -669,14 +714,23 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) /// REPAINT before to avoid prompt overlap by the query rx.invoke(Replxx::ACTION::REPAINT, code); - if (!new_query.empty()) + const bool selected_query = !new_query.empty(); + if (selected_query) + { + /// The picked query is a whole new line displayed at once - do not pop hints on it + /// (see historyNavigate). + suppress_hints_once = true; rx.set_state(replxx::Replxx::State(new_query.c_str(), static_cast(new_query.size()))); + } if (bracketed_paste_enabled) enableBracketedPaste(); rx.invoke(Replxx::ACTION::CLEAR_SELF, code); - return rx.invoke(Replxx::ACTION::REPAINT, code); + auto result = rx.invoke(Replxx::ACTION::REPAINT, code); + if (selected_query) + suppressHintsForDisplayedLine(); + return result; }; rx.bind_key(Replxx::KEY::control(key_fuzzy), interactive_history_search); @@ -691,7 +745,7 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) { /// Reverse search is detected by C-R. uint32_t reverse_search = Replxx::KEY::control('R'); - return rx.invoke(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, reverse_search); + return historySearch(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, reverse_search); }); /// Change cursor style for overwrite mode to blinking (see console_codes(5)) @@ -735,14 +789,68 @@ bool ReplxxLineReader::hintChosen() return hintPopupActive() && (hint_selection >= 0 || hint_count == 1); } +replxx::Replxx::ACTION_RESULT ReplxxLineReader::historyNavigate(replxx::Replxx::ACTION action, char32_t code) +{ + /// The recalled entry is displayed (and its hints regenerated) inside the action, so the + /// suppression must be armed before it; the pin below keeps later regenerations of the + /// recalled text hintless (the refresh inside the action may be throttled and replayed after + /// this returns) and is cleared by the first edit. + suppress_hints_once = true; + auto result = rx.invoke(action, code); + if (rx.history_recalled()) + suppressHintsForDisplayedLine(); + else + suppress_hints_once = false; + return result; +} + +replxx::Replxx::ACTION_RESULT ReplxxLineReader::historySearch(replxx::Replxx::ACTION action, char32_t code) +{ + /// The selected entry is displayed (and its hints regenerated) inside the search action, so + /// the suppression must be armed before it. C-R, C-S, Meta-R, and the ClickHouse regular + /// history-search binding all use this wrapper. + suppress_hints_once = true; + auto result = rx.invoke(action, code); + if (rx.history_recalled()) + suppressHintsForDisplayedLine(); + else + suppress_hints_once = false; + return result; +} + +void ReplxxLineReader::suppressHintsForDisplayedLine() +{ + suppress_hints_once = false; + suppress_hints_for_text = rx.get_state().text(); +} + ReplxxLineReader::~ReplxxLineReader() { - if (history_file_fd >= 0 && close(history_file_fd)) - rx.print("Close of history file failed: %s\n", errnoToString().c_str()); + /// `Replxx::print` may fail with `std::runtime_error("write failed")` when e.g. the pty of the embedded + /// SSH client is gone already. A destructor is implicitly `noexcept`, so letting anything escape from + /// here would `std::terminate` the whole process. + try + { + if (history_file_fd >= 0 && close(history_file_fd)) + rx.print("Close of history file failed: %s\n", errnoToString().c_str()); - /// Reset cursor blinking - if (overwrite_mode) - rx.print("%s", "\033[0 q"); + /// Reset cursor blinking + if (overwrite_mode) + rx.print("%s", "\033[0 q"); + } + catch (...) + { + /// The reporting path must not be able to escape either: the `const char *` overload of + /// `tryLogCurrentException` builds a `String` for the logger name and calls `getLogger` + /// before it reaches its own `try`, so under memory pressure it can throw as well. + try + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + catch (...) // NOLINT(bugprone-empty-catch) Ok: reporting failed, nothing more to do + { + } + } } LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt) @@ -821,8 +929,19 @@ void ReplxxLineReader::openEditor(bool format_query) rx.print("\n"); } + /// The repaint below displays the whole buffer at once on every return path - the edited + /// query, or the original one brought back when the editor exited unsuccessfully or the + /// round trip threw. All of them are programmatic displays, so none of them may pop hints + /// (see historyNavigate); otherwise the hints left over from before the editor was opened + /// would stay live and the next Down would navigate them instead of the history. + /// replxx caches the hints by the buffer text, which is unchanged unless the edited query was + /// accepted, so re-setting the state is what makes it ask the hint callback again (and get an + /// empty list) instead of redisplaying the stale cached ones. + rx.set_state(rx.get_state()); + suppress_hints_once = true; rx.invoke(replxx::Replxx::ACTION::CLEAR_SELF, 0); rx.invoke(replxx::Replxx::ACTION::REPAINT, 0); + suppressHintsForDisplayedLine(); if (bracketed_paste_enabled) enableBracketedPaste(); @@ -846,6 +965,12 @@ void ReplxxLineReader::setInitialText(const String & text) if (!text.empty()) { rx.set_preload_buffer(text); + /// The preloaded query is displayed at once - do not pop hints on it (see + /// historyNavigate). The one-shot is consumed at the first render of the line inside + /// input(); the pin is set to the raw text (replxx may normalize whitespace in the + /// preload, in which case it just stays inert). + suppress_hints_once = true; + suppress_hints_for_text = text; } } diff --git a/src/Client/ReplxxLineReader.h b/src/Client/ReplxxLineReader.h index 81958ae8ca35..f7813514374d 100644 --- a/src/Client/ReplxxLineReader.h +++ b/src/Client/ReplxxLineReader.h @@ -52,6 +52,18 @@ class ReplxxLineReader : public LineReader int executeEditor(const std::string & path); void openEditor(bool format_query); + /// Run a history-navigation action with the hint suppression armed (see + /// `suppress_hints_once`): the entry it recalls must not pop hints by itself. + replxx::Replxx::ACTION_RESULT historyNavigate(replxx::Replxx::ACTION action, char32_t code); + + /// Run a history-search action with the hint suppression armed (see + /// `suppress_hints_once`): a selected entry must not pop hints by itself. + replxx::Replxx::ACTION_RESULT historySearch(replxx::Replxx::ACTION action, char32_t code); + + /// After a line was displayed programmatically, pin its text so that any hint regeneration + /// for it shows nothing (see `suppress_hints_for_text`). + void suppressHintsForDisplayedLine(); + /// Whether the text cursor is at the very end of the input (where as-you-type hints render). bool isCursorAtEndOfInput(); /// Whether the as-you-type hint "popup" is currently navigable here: hints are shown and the @@ -89,6 +101,20 @@ class ReplxxLineReader : public LineReader int hint_count = 0; int hint_selection = -1; + /// Suppression of the as-you-type hints for a line that is displayed programmatically - + /// recalled from history, found by a history search, pasted, brought back from the editor. + /// Such a display must not pop hints by itself: with hints visible, the next Up/Down press + /// would navigate the hints instead of the history. An edit shows the hints again. + /// `suppress_hints_once` is armed before the action that displays the line (the action + /// repaints, and regenerates the hints, inside itself) and consumed by the next run of the + /// hint callback. `suppress_hints_for_text` then pins the displayed text after the action, + /// because the same display can regenerate the hints again later - replxx replays a + /// throttled refresh after the key handler returns (its "rapid refresh" of e.g. a held-down + /// Up key) - so any later callback run for exactly this text shows no hints either; the + /// first run for an edited text clears the pin. + bool suppress_hints_once = false; + std::string suppress_hints_for_text; + /// Snapshot of the completion words computed when the hints were last displayed, plus the /// context (prefix and its length) they were computed for. The completion callback reuses it /// so that accepting a hint inserts exactly the word that was shown: replxx accepts a hint by diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index f4961ec20af5..7ac2afa63de3 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -718,6 +718,14 @@ void ColumnAggregateFunction::popBack(size_t n) data.resize_assume_reserved(new_size); } +void ColumnAggregateFunction::popBackWithoutDestroy(size_t n) +{ + if (n > size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot pop {} rows from {}: there are only {} rows", n, getName(), size()); + + data.resize_assume_reserved(data.size() - n); +} + ColumnPtr ColumnAggregateFunction::replicate(const IColumn::Offsets & offsets) const { size_t size = data.size(); diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h index 2689cb00816a..c9da4d7f5647 100644 --- a/src/Columns/ColumnAggregateFunction.h +++ b/src/Columns/ColumnAggregateFunction.h @@ -131,6 +131,18 @@ class ColumnAggregateFunction final : public COWHelperempty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Not empty data passed to ColumnArray, but no offsets passed"); @@ -638,13 +639,6 @@ bool ColumnArray::hasEqualOffsets(const ColumnArray & other) const } -ColumnPtr ColumnArray::convertToFullColumnIfConst() const -{ - /// It is possible to have an array with constant data and non-constant offsets. - /// Example is the result of expression: replicate('hello', [1]) - return ColumnArray::create(data->convertToFullColumnIfConst(), offsets); -} - void ColumnArray::getExtremes(Field & min, Field & max, size_t start, size_t end) const { min = Array(); @@ -1444,8 +1438,6 @@ ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const return replicateNumber(replicate_offsets); if (typeid_cast(data.get())) return replicateString(replicate_offsets); - if (typeid_cast(data.get())) - return replicateConst(replicate_offsets); if (typeid_cast(data.get())) return replicateNullable(replicate_offsets); if (typeid_cast(data.get())) @@ -1586,44 +1578,6 @@ ColumnPtr ColumnArray::replicateString(const Offsets & replicate_offsets) const } -ColumnPtr ColumnArray::replicateConst(const Offsets & replicate_offsets) const -{ - size_t col_size = size(); - if (col_size != replicate_offsets.size()) - throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of offsets doesn't match size of column."); - - if (0 == col_size) - return cloneEmpty(); - - const Offsets & src_offsets = getOffsets(); - - auto res_column_offsets = ColumnOffsets::create(); - Offsets & res_offsets = res_column_offsets->getData(); - res_offsets.reserve_exact(replicate_offsets.back()); - - Offset prev_replicate_offset = 0; - Offset prev_data_offset = 0; - Offset current_new_offset = 0; - - for (size_t i = 0; i < col_size; ++i) - { - size_t size_to_replicate = replicate_offsets[i] - prev_replicate_offset; - size_t value_size = src_offsets[i] - prev_data_offset; - - for (size_t j = 0; j < size_to_replicate; ++j) - { - current_new_offset += value_size; - res_offsets.push_back(current_new_offset); - } - - prev_replicate_offset = replicate_offsets[i]; - prev_data_offset = src_offsets[i]; - } - - return ColumnArray::create(getData().cloneResized(current_new_offset), std::move(res_column_offsets)); -} - - ColumnPtr ColumnArray::replicateGeneric(const Offsets & replicate_offsets) const { size_t col_size = size(); diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h index 1d4fade3dc32..50e68b18e551 100644 --- a/src/Columns/ColumnArray.h +++ b/src/Columns/ColumnArray.h @@ -133,7 +133,6 @@ class ColumnArray final : public COWHelper, ColumnArr size_t allocatedBytes() const override; void protect() override; ColumnPtr replicate(const Offsets & replicate_offsets) const override; - ColumnPtr convertToFullColumnIfConst() const override; void getExtremes(Field & min, Field & max, size_t start, size_t end) const override; bool hasEqualOffsets(const ColumnArray & other) const; @@ -244,13 +243,6 @@ class ColumnArray final : public COWHelper, ColumnArr /// Multiply the values if the nested column is ColumnString. The code is too complicated. ColumnPtr replicateString(const Offsets & replicate_offsets) const; - /** Non-constant arrays of constant values are quite rare. - * Most functions can not work with them, and does not create such columns as a result. - * An exception is the function `replicate` (see FunctionsMiscellaneous.h), which has service meaning for the implementation of lambda functions. - * Only for its sake is the implementation of the `replicate` method for ColumnArray(ColumnConst). - */ - ColumnPtr replicateConst(const Offsets & replicate_offsets) const; - /** The following is done by simply replicating of nested columns. */ ColumnPtr replicateTuple(const Offsets & replicate_offsets) const; diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index d927a82bad64..b40c289a53c6 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; extern const int LOGICAL_ERROR; extern const int INCORRECT_DATA; + extern const int PARAMETER_OUT_OF_BOUND; } void throwUnexpectedLowCardinalityIndexType(size_t size) @@ -30,6 +31,9 @@ void throwUnexpectedLowCardinalityIndexType(size_t size) namespace { + /// The range path starts to win around 64 rows when there are only a few distinct short keys. + constexpr size_t max_rows_to_translate_individually = 64; + void checkColumn(const IColumn & column) { if (!dynamic_cast(&column)) @@ -214,6 +218,22 @@ void ColumnLowCardinality::doInsertRangeFrom(const IColumn & src, size_t start, { compactIfSharedDictionary(); + if (length <= max_rows_to_translate_individually) + { + const IColumn & src_indexes = low_cardinality_src->getIndexes(); + if (start > src_indexes.size() || length > src_indexes.size() - start) + throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Parameters start = {}, length = {} are out of bound in " + "ColumnLowCardinality::insertRangeFrom method (size() = {}).", start, length, src_indexes.size()); + + std::array positions; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init) - only the first `length` entries are written before read + for (size_t i = 0; i < length; ++i) + positions[i] = src_indexes.getUInt(start + i); + getDictionary().uniqueInsertRowsFrom(*low_cardinality_src->getDictionary().getNestedColumn(), {positions.data(), length}); + for (size_t i = 0; i < length; ++i) + idx.insertIndex(positions[i]); + return; + } + /// TODO: Support native insertion from other unique column. It will help to avoid null map creation. auto sub_idx = IColumn::mutate(low_cardinality_src->getIndexes().cut(start, length)); diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index 7e178d368a11..ba512dc19290 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -21,7 +21,6 @@ namespace DB namespace ErrorCodes { - extern const int ILLEGAL_COLUMN; extern const int NOT_IMPLEMENTED; extern const int CANNOT_INSERT_VALUE_OF_DIFFERENT_SIZE_INTO_TUPLE; extern const int LOGICAL_ERROR; @@ -54,7 +53,7 @@ ColumnTuple::ColumnTuple(MutableColumns && mutable_columns) for (auto & column : mutable_columns) { if (isColumnConst(*column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnTuple cannot have ColumnConst as its element"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnTuple cannot have ColumnConst as its element"); columns.push_back(std::move(column)); } @@ -70,7 +69,7 @@ ColumnTuple::Ptr ColumnTuple::create(const Columns & columns) for (const auto & column : columns) if (isColumnConst(*column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnTuple cannot have ColumnConst as its element"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnTuple cannot have ColumnConst as its element"); auto column_tuple = ColumnTuple::create(columns[0]->size()); column_tuple->columns.assign(columns.begin(), columns.end()); @@ -85,7 +84,7 @@ ColumnTuple::Ptr ColumnTuple::create(const TupleColumns & columns) for (const auto & column : columns) if (isColumnConst(*column)) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "ColumnTuple cannot have ColumnConst as its element"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ColumnTuple cannot have ColumnConst as its element"); auto column_tuple = ColumnTuple::create(columns[0]->size()); column_tuple->columns = columns; diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h index c7234f3fb12c..447a52cdbe0c 100644 --- a/src/Columns/ColumnUnique.h +++ b/src/Columns/ColumnUnique.h @@ -63,6 +63,7 @@ class ColumnUnique final : public COWHelper rows) override; MutableColumnPtr uniqueInsertRangeFrom(const IColumn & src, size_t start, size_t length) override; IColumnUnique::IndexesWithOverflow uniqueInsertRangeWithOverflow(const IColumn & src, size_t start, size_t length, size_t max_dictionary_size) override; @@ -229,6 +230,12 @@ class ColumnUnique final : public COWHelper(column_holder.get()); } const ColumnType * getRawColumnPtr() const { return assert_cast(column_holder.get()); } + /// `src` must be `ColumnType` or `Nullable(ColumnType)`. + std::pair getValuesAndNullMap(const IColumn & src) const; + + /// The NULL and default value rules of `uniqueInsertRangeFrom`. Returns nothing for any other value. + ALWAYS_INLINE std::optional getSpecialValueIndex(const ColumnType & src, const NullMap * null_map, size_t row) const; + template MutableColumnPtr uniqueInsertRangeImpl( const IColumn & src, @@ -444,6 +451,57 @@ size_t ColumnUnique::uniqueInsertFrom(const IColumn & src, size_t n) return uniqueInsertData(ref.data(), ref.size()); } +template +void ColumnUnique::uniqueInsertRowsFrom(const IColumn & src, std::span rows) +{ + const auto [src_column, null_map] = getValuesAndNullMap(src); + UInt64 previous_row = std::numeric_limits::max(); + UInt64 previous_index = 0; + for (UInt64 & row : rows) + { + if (row != previous_row) + { + previous_row = row; + if (auto special_index = getSpecialValueIndex(*src_column, null_map, row)) + previous_index = *special_index; + else + previous_index = uniqueInsertFrom(*src_column, row); + } + row = previous_index; + } +} + +template +std::pair ColumnUnique::getValuesAndNullMap(const IColumn & src) const +{ + const ColumnType * src_column = nullptr; + const NullMap * null_map = nullptr; + + if (const auto * nullable_column = checkAndGetColumn(&src)) + { + src_column = typeid_cast(&nullable_column->getNestedColumn()); + null_map = &nullable_column->getNullMapData(); + } + else + src_column = typeid_cast(&src); + + if (src_column == nullptr) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Invalid column type for ColumnUnique::insertRangeFrom. " + "Expected {}, got {}", column_holder->getName(), src.getName()); + + return {src_column, null_map}; +} + +template +std::optional ColumnUnique::getSpecialValueIndex(const ColumnType & src, const NullMap * null_map, size_t row) const +{ + if (null_map && (*null_map)[row]) + return getNullValueIndex(); + if (getRawColumnPtr()->compareAt(getNestedTypeDefaultValueIndex(), row, src, 1) == 0) + return getNestedTypeDefaultValueIndex(); + return std::nullopt; +} + template size_t ColumnUnique::uniqueInsertData(const char * pos, size_t length) { @@ -657,8 +715,6 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeImpl( ReverseIndex * secondary_index, size_t max_dictionary_size) { - const ColumnType * src_column = nullptr; - const NullMap * null_map = nullptr; auto & positions = positions_column->getData(); auto update_position = [&](UInt64 & next_position) -> MutableColumnPtr @@ -692,17 +748,7 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeImpl( return nullptr; }; - if (const auto * nullable_column = checkAndGetColumn(&src)) - { - src_column = typeid_cast(&nullable_column->getNestedColumn()); - null_map = &nullable_column->getNullMapData(); - } - else - src_column = typeid_cast(&src); - - if (src_column == nullptr) - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Invalid column type for ColumnUnique::insertRangeFrom. " - "Expected {}, got {}", column_holder->getName(), src.getName()); + const auto [src_column, null_map] = getValuesAndNullMap(src); auto column = getRawColumnPtr(); @@ -724,10 +770,8 @@ MutableColumnPtr ColumnUnique::uniqueInsertRangeImpl( { auto row = start + num_added_rows; - if (null_map && (*null_map)[row]) - positions[num_added_rows] = static_cast(getNullValueIndex()); - else if (column->compareAt(getNestedTypeDefaultValueIndex(), row, *src_column, 1) == 0) - positions[num_added_rows] = static_cast(getNestedTypeDefaultValueIndex()); + if (auto special_index = getSpecialValueIndex(*src_column, null_map, row)) + positions[num_added_rows] = static_cast(*special_index); else { auto ref = src_column->getDataAt(row); diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h index 31baf6a282a4..42a35c50202f 100644 --- a/src/Columns/IColumnUnique.h +++ b/src/Columns/IColumnUnique.h @@ -44,6 +44,9 @@ class IColumnUnique : public IColumn virtual bool tryUniqueInsert(const Field & x, size_t & index) = 0; virtual size_t uniqueInsertFrom(const IColumn & src, size_t n) = 0; + /// Inserts rows `rows` of `src` with the rules of `uniqueInsertRangeFrom`, which differ from `uniqueInsertFrom` for NULL + /// and for values that compare equal to the default, such as `-0.0`, and replaces each row number with its index. + virtual void uniqueInsertRowsFrom(const IColumn & src, std::span rows) = 0; /// Appends range of elements from other column. /// Could be used to concatenate columns. virtual MutableColumnPtr uniqueInsertRangeFrom(const IColumn & src, size_t start, size_t length) = 0; diff --git a/src/Columns/LowCardinalityValueIndex.h b/src/Columns/LowCardinalityValueIndex.h new file mode 100644 index 000000000000..fb6fbecdad7d --- /dev/null +++ b/src/Columns/LowCardinalityValueIndex.h @@ -0,0 +1,70 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + +enum class LowCardinalityValueLookupResult +{ + /// The dictionary does not hold strings, so the caller has to compare the values itself. + Unsupported, + /// No row of the column can hold the value. + NotFound, + Found, +}; + +/// Resolves `value` to its position in the dictionary of `column` and calls +/// `callback(const IndexType * indexes, IndexType value_index)`, so that comparing a row of `column` +/// with `value` becomes an integer comparison of dictionary positions. +template +LowCardinalityValueLookupResult callWithLowCardinalityValueIndex( + const ColumnLowCardinality & column, std::string_view value, Callback && callback) +{ + const auto & dictionary = column.getDictionary(); + const auto & dictionary_values = *dictionary.getNestedNotNullableColumn(); + if (!typeid_cast(&dictionary_values) && !typeid_cast(&dictionary_values)) + return LowCardinalityValueLookupResult::Unsupported; + + auto value_index = dictionary.getOrFindValueIndex(value); + if (!value_index) + return LowCardinalityValueLookupResult::NotFound; + + const IColumn & indexes = column.getIndexes(); + + auto call_for_index_type = [&](IndexType) -> LowCardinalityValueLookupResult + { + /// A shared dictionary also holds values of other columns, so a dictionary position is not + /// necessarily representable in the index type of this column. No row here references it then. + if constexpr (!std::is_same_v) + { + if (*value_index > std::numeric_limits::max()) + return LowCardinalityValueLookupResult::NotFound; + } + + callback( + assert_cast &>(indexes).getData().data(), + static_cast(*value_index)); + + return LowCardinalityValueLookupResult::Found; + }; + + switch (column.getSizeOfIndexType()) + { + case sizeof(UInt8): return call_for_index_type(UInt8{}); + case sizeof(UInt16): return call_for_index_type(UInt16{}); + case sizeof(UInt32): return call_for_index_type(UInt32{}); + case sizeof(UInt64): return call_for_index_type(UInt64{}); + default: throwUnexpectedLowCardinalityIndexType(column.getSizeOfIndexType()); + } +} + +} diff --git a/src/Columns/tests/gtest_column_array.cpp b/src/Columns/tests/gtest_column_array.cpp index a9b55a582735..c2b95118b018 100644 --- a/src/Columns/tests/gtest_column_array.cpp +++ b/src/Columns/tests/gtest_column_array.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -23,6 +24,14 @@ ColumnArray::MutablePtr createArray(std::vector data_values, std::vector return ColumnArray::create(std::move(data), std::move(offsets)); } +/// One array of two rows of a constant: the offsets match the nested column, so only the constant is wrong. +ColumnArray::MutablePtr createArrayOverConst() +{ + auto offsets = ColumnArray::ColumnOffsets::create(); + offsets->getData().push_back(2); + return ColumnArray::create(ColumnConst::create(ColumnUInt64::create(1, 42), 2), std::move(offsets)); +} + } /// A ColumnArray is created from an already populated nested column and offsets, so its offsets @@ -64,6 +73,12 @@ TEST(ColumnArray, InconsistentOffsetsAreRejected) EXPECT_THROW(createArray({10}, {}), Exception); } +TEST(ColumnArray, ConstNestedColumnIsRejected) +{ + EXPECT_THROW(createArrayOverConst(), Exception); + EXPECT_THROW(ColumnArray::create(ColumnConst::create(ColumnUInt64::create(1, 42), 0)), Exception); +} + #endif /// A decreasing offset makes `sizeAt` underflow to a huge value even when the last offset matches @@ -86,4 +101,14 @@ TEST(ColumnArrayDeathTest, NonMonotonicOffsetsAreRejected) EXPECT_DEATH((createArray({10, 20, 30}, {3, 0, 3})), "not monotonically increasing"); } +TEST(ColumnArrayDeathTest, ConstNestedColumnIsRejected) +{ + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + EXPECT_DEATH(createArrayOverConst(), "ColumnArray cannot have ColumnConst as its nested column"); + EXPECT_DEATH( + ColumnArray::create(ColumnConst::create(ColumnUInt64::create(1, 42), 0)), + "ColumnArray cannot have ColumnConst as its nested column"); +} + #endif diff --git a/src/Columns/tests/gtest_column_tuple.cpp b/src/Columns/tests/gtest_column_tuple.cpp index cc784d9ff53e..bf073267ae49 100644 --- a/src/Columns/tests/gtest_column_tuple.cpp +++ b/src/Columns/tests/gtest_column_tuple.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -18,6 +19,7 @@ namespace DB namespace ErrorCodes { extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; +extern const int LOGICAL_ERROR; } } @@ -142,3 +144,71 @@ TEST(ColumnTuple, EmptyTuplePermute) ASSERT_EQ(e.code(), ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); } } + +namespace +{ + +/// Two rows of a constant next to a full element of two rows: the sizes match, so only the constant is wrong. +ColumnPtr createTupleFromMutableColumnsWithConst() +{ + MutableColumns elements; + elements.push_back(ColumnUInt64::create(2, 1)); + elements.push_back(ColumnConst::create(ColumnUInt64::create(1, 42), 2)); + return ColumnTuple::create(std::move(elements)); +} + +ColumnPtr createTupleFromColumnsWithConst() +{ + return ColumnTuple::create(Columns{ColumnUInt64::create(2, 1), ColumnConst::create(ColumnUInt64::create(1, 42), 2)}); +} + +ColumnPtr createTupleFromTupleColumnsWithConst() +{ + VectorWithMemoryTracking elements; + elements.emplace_back(ColumnUInt64::create(2, 1)); + elements.emplace_back(ColumnConst::create(ColumnUInt64::create(1, 42), 2)); + return ColumnTuple::create(elements); +} + +} + +/// Skipped under debug/sanitizers: LOGICAL_ERROR aborts there, so the exception can't be caught. +#ifndef DEBUG_OR_SANITIZER_BUILD + +namespace +{ + +int errorCodeOf(ColumnPtr (*create)()) +{ + try + { + (void)create(); + } + catch (const Exception & e) + { + return e.code(); + } + return 0; +} + +} + +TEST(ColumnTuple, ConstElementIsRejected) +{ + EXPECT_EQ(errorCodeOf(createTupleFromMutableColumnsWithConst), ErrorCodes::LOGICAL_ERROR); + EXPECT_EQ(errorCodeOf(createTupleFromColumnsWithConst), ErrorCodes::LOGICAL_ERROR); + EXPECT_EQ(errorCodeOf(createTupleFromTupleColumnsWithConst), ErrorCodes::LOGICAL_ERROR); +} + +#else + +TEST(ColumnTupleDeathTest, ConstElementIsRejected) +{ + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + + EXPECT_DEATH(createTupleFromMutableColumnsWithConst(), "ColumnTuple cannot have ColumnConst as its element"); + EXPECT_DEATH(createTupleFromColumnsWithConst(), "ColumnTuple cannot have ColumnConst as its element"); + EXPECT_DEATH(createTupleFromTupleColumnsWithConst(), "ColumnTuple cannot have ColumnConst as its element"); +} + +#endif diff --git a/src/Columns/tests/gtest_low_cardinality.cpp b/src/Columns/tests/gtest_low_cardinality.cpp index 7656d4fc9676..5ece835fbc1d 100644 --- a/src/Columns/tests/gtest_low_cardinality.cpp +++ b/src/Columns/tests/gtest_low_cardinality.cpp @@ -3,9 +3,16 @@ #include #include +#include +#include #include +#include +#include +#include +#include + using namespace DB; template @@ -111,3 +118,155 @@ TEST(ColumnLowCardinality, EmptyDictionaryEmptyIndexes) ASSERT_EQ(column->size(), 0); } + +namespace +{ + +MutableColumnPtr makeLowCardinality(const DataTypePtr & type, const std::vector & values) +{ + auto column = std::make_shared(type)->createColumn(); + for (const auto & value : values) + column->insert(value); + return column; +} + +/// 1000 rows drawn from `keys`, with runs of repeated values. +std::vector makeSourceValues(const std::vector & keys, UInt64 seed) +{ + pcg64 rng(seed); + std::vector values; + while (values.size() < 1000) + { + const Field & key = keys[rng() % keys.size()]; + for (size_t repeat = 1 + rng() % 3; repeat > 0 && values.size() < 1000; --repeat) + values.push_back(key); + } + return values; +} + +/// A range short enough for the per-row path must give the same rows and the same dictionary prefix +/// as a range of the same rows that is long enough for the range path. +void checkShortRangeMatchesRangePath(const IColumn & source, const std::function & make_destination) +{ + const auto & source_lc = assert_cast(source); + const auto & source_keys = *source_lc.getDictionary().getNestedNotNullableColumn(); + constexpr size_t long_length = 600; + for (size_t start : {0, 1, 7, 300}) + { + for (size_t length = 1; length <= 256; ++length) + { + auto short_range = make_destination(); + auto long_range = make_destination(); + const size_t offset = short_range->size(); + short_range->insertRangeFrom(source, start, length); + long_range->insertRangeFrom(source, start, long_length); + + const auto & short_lc = assert_cast(*short_range); + const auto & long_lc = assert_cast(*long_range); + ASSERT_EQ(short_range->size(), offset + length); + for (size_t row = offset; row < offset + length; ++row) + { + SCOPED_TRACE(fmt::format("start {}, length {}, row {}", start, length, row)); + ASSERT_EQ(short_lc.getIndexes().getUInt(row), long_lc.getIndexes().getUInt(row)); + ASSERT_EQ(short_lc.isNullAt(row), long_lc.isNullAt(row)); + if (!short_lc.isNullAt(row)) + ASSERT_EQ(short_lc.getDataAt(row), long_lc.getDataAt(row)); + + /// Both paths apply the same NULL and default rules, so check those against the source too. + const size_t source_row = start + row - offset; + const auto & dictionary = short_lc.getDictionary(); + const size_t default_index = dictionary.getNestedTypeDefaultValueIndex(); + if (source.isNullAt(source_row)) + ASSERT_EQ(short_lc.getIndexes().getUInt(row), dictionary.getNullValueIndex()); + else if (dictionary.getNestedNotNullableColumn()->compareAt( + default_index, source_lc.getIndexes().getUInt(source_row), source_keys, 1) == 0) + ASSERT_EQ(short_lc.getIndexes().getUInt(row), default_index); + } + + const auto & short_keys = *short_lc.getDictionary().getNestedNotNullableColumn(); + const auto & long_keys = *long_lc.getDictionary().getNestedNotNullableColumn(); + ASSERT_LE(short_keys.size(), long_keys.size()); + for (size_t key = 0; key < short_keys.size(); ++key) + ASSERT_EQ(short_keys.getDataAt(key), long_keys.getDataAt(key)) << "start " << start << ", length " << length; + } + } +} + +} + +TEST(ColumnLowCardinality, ShortRangeFromDifferentDictionaryMatchesRangePath) +{ + const auto string_type = std::make_shared(); + const auto nullable_string_type = std::make_shared(string_type); + + std::vector string_keys{Field(""), Field("a"), Field("b"), Field("c"), Field("d"), Field("e"), Field(std::string(40, 'f'))}; + auto string_source = makeLowCardinality(string_type, makeSourceValues(string_keys, 1)); + auto make_string_destination = [&] { return makeLowCardinality(string_type, {Field("b"), Field("x"), Field(""), Field("a"), Field("y")}); }; + + { + SCOPED_TRACE("LowCardinality(String)"); + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*string_source, make_string_destination)); + } + { + SCOPED_TRACE("LowCardinality(Nullable(String))"); + std::vector nullable_keys = string_keys; + nullable_keys.push_back(Field()); + auto source = makeLowCardinality(nullable_string_type, makeSourceValues(nullable_keys, 2)); + auto make_destination = [&] { return makeLowCardinality(nullable_string_type, {Field("b"), Field(), Field("x")}); }; + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_destination)); + } + { + SCOPED_TRACE("LowCardinality(String) into LowCardinality(Nullable(String))"); + auto make_destination = [&] { return makeLowCardinality(nullable_string_type, {Field("b"), Field(), Field("x")}); }; + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*string_source, make_destination)); + } + { + SCOPED_TRACE("Shared source dictionary"); + const auto & string_source_lc = assert_cast(*string_source); + auto source = ColumnLowCardinality::create(string_source_lc.getDictionaryPtr(), string_source_lc.getIndexesPtr(), /* is_shared = */ true); + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_string_destination)); + } + { + /// The keys are set directly, so that the dictionary holds `-0.0` next to `0.0` and two NaN payloads. + SCOPED_TRACE("LowCardinality(Float64)"); + auto keys = ColumnFloat64::create(); + for (Float64 key : {0.0, -0.0, 1.5, std::bit_cast(0x7ff8000000000001ULL), std::bit_cast(0xfff8000000000002ULL)}) + keys->insertValue(key); + ColumnPtr dictionary = DataTypeLowCardinality::createColumnUnique(DataTypeFloat64(), std::move(keys)); + auto indexes = ColumnUInt8::create(); + pcg64 rng(3); + for (size_t row = 0; row < 1000; ++row) + indexes->insertValue(static_cast(rng() % 5)); + auto source = ColumnLowCardinality::create(dictionary, std::move(indexes), /* is_shared = */ false); + auto make_destination = [&] { return makeLowCardinality(std::make_shared(), {Field(2.5), Field(1.5)}); }; + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_destination)); + } + { + SCOPED_TRACE("LowCardinality(UInt64), index type grows"); + const auto uint64_type = std::make_shared(); + std::vector keys; + for (UInt64 key = 0; key < 300; ++key) + keys.push_back(Field(key * 10)); + auto source = makeLowCardinality(uint64_type, makeSourceValues(keys, 4)); + std::vector destination_values; + for (UInt64 value = 1; value <= 250; ++value) + destination_values.push_back(Field(value * 7)); + auto make_destination = [&] { return makeLowCardinality(uint64_type, destination_values); }; + ASSERT_EQ(assert_cast(*make_destination()).getSizeOfIndexType(), sizeof(UInt8)); + ASSERT_NO_FATAL_FAILURE(checkShortRangeMatchesRangePath(*source, make_destination)); + + auto destination = make_destination(); + destination->insertRangeFrom(*source, 0, 32); + ASSERT_EQ(assert_cast(*destination).getSizeOfIndexType(), sizeof(UInt16)); + } + { + SCOPED_TRACE("Out of bound range"); + auto destination = make_string_destination(); + const size_t size_before = destination->size(); + const size_t dictionary_size_before = assert_cast(*destination).getDictionary().size(); + EXPECT_THROW(destination->insertRangeFrom(*string_source, string_source->size() - 1, 2), Exception); + EXPECT_THROW(destination->insertRangeFrom(*string_source, std::numeric_limits::max(), 1), Exception); + EXPECT_EQ(destination->size(), size_before); + EXPECT_EQ(assert_cast(*destination).getDictionary().size(), dictionary_size_before); + } +} diff --git a/src/Common/Config/ConfigReloader.cpp b/src/Common/Config/ConfigReloader.cpp index dd9531c6493d..1dc2fd0ab747 100644 --- a/src/Common/Config/ConfigReloader.cpp +++ b/src/Common/Config/ConfigReloader.cpp @@ -222,8 +222,17 @@ struct ConfigReloader::FileWithTimestamp void ConfigReloader::FilesChangesTracker::addIfExists(const std::string & path_to_add) { - if (!path_to_add.empty() && fs::exists(path_to_add)) - files.emplace(path_to_add); + if (path_to_add.empty() || !fs::exists(path_to_add)) + return; + + files.emplace(path_to_add); + + /// E.g. a directory with CA certificates, a change of a file in it should be noticed too. + std::error_code ec; + if (fs::is_directory(path_to_add, ec)) + for (const auto & entry : fs::directory_iterator(path_to_add, ec)) + if (entry.is_regular_file(ec)) + files.emplace(entry.path().string()); } bool ConfigReloader::FilesChangesTracker::isDifferOrNewerThan(const FilesChangesTracker & rhs) diff --git a/src/Common/Crypto/X509Certificate.cpp b/src/Common/Crypto/X509Certificate.cpp index a2009ddf3f4f..8efa49f8e527 100644 --- a/src/Common/Crypto/X509Certificate.cpp +++ b/src/Common/Crypto/X509Certificate.cpp @@ -173,26 +173,45 @@ std::string X509Certificate::subjectName() const return buffer; } +/// Extract the value of the first entry with the given NID from an X509 name as a length-delimited +/// string. We read the ASN1_STRING bytes directly instead of X509_NAME_get_text_by_NID because that +/// function copies into a fixed C buffer and NUL-terminates: an embedded NUL byte (e.g. a CN of +/// "admin\0.evil.com") would be silently truncated to "admin", letting a certificate impersonate a +/// different subject during authentication. Preserving the exact bytes makes such a value compare +/// unequal to any NUL-free configured subject, and also avoids silent truncation of long names. +static std::string extractNameEntry(X509_NAME * name, uint nid) +{ + if (!name) + return {}; + + const int index = X509_NAME_get_index_by_NID(name, static_cast(nid), -1); + if (index < 0) + return {}; + + const X509_NAME_ENTRY * entry = X509_NAME_get_entry(name, index); + if (!entry) + return {}; + + const ASN1_STRING * data = X509_NAME_ENTRY_get_data(entry); + if (!data) + return {}; + + const unsigned char * bytes = ASN1_STRING_get0_data(data); + const int length = ASN1_STRING_length(data); + if (!bytes || length < 0) + return {}; + + return std::string(reinterpret_cast(bytes), static_cast(length)); +} + std::string X509Certificate::issuerName(uint nid) const { - if (X509_NAME * issuer = X509_get_issuer_name(certificate)) - { - char buffer[X509Certificate::NAME_BUFFER_SIZE]; - if (X509_NAME_get_text_by_NID(issuer, nid, buffer, sizeof(buffer)) >= 0) - return std::string(buffer); - } - return {}; + return extractNameEntry(X509_get_issuer_name(certificate), nid); } std::string X509Certificate::subjectName(uint nid) const { - if (X509_NAME * subj = X509_get_subject_name(certificate)) - { - char buffer[X509Certificate::NAME_BUFFER_SIZE]; - if (X509_NAME_get_text_by_NID(subj, nid, buffer, sizeof(buffer)) >= 0) - return std::string(buffer); - } - return {}; + return extractNameEntry(X509_get_subject_name(certificate), nid); } std::string X509Certificate::commonName() const diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index da6d80d86a4a..d356f79975b9 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -687,6 +687,7 @@ M(1013, AMBIGUOUS_HANDLER) \ M(1014, TRANSACTION_ROLLBACK_PARTIAL_FAILURE) \ M(1015, FILE_CHANGED_DURING_READ) \ + M(1021, AZURE_OBJECT_CHANGED_DURING_READ) \ /* See END */ #ifdef APPLY_FOR_EXTERNAL_ERROR_CODES @@ -703,7 +704,7 @@ namespace ErrorCodes APPLY_FOR_ERROR_CODES(M) #undef M - constexpr ErrorCode END = 1015; + constexpr ErrorCode END = 1021; #if !defined(CLICKHOUSE_PARSER_MINIMAL_BUILD) /** One `ErrorPairHolder` per error code, each holding two `Error` structs - the last message, diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 5df9d5d8f3b1..a372a97e8b9c 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -75,6 +75,7 @@ static struct InitFiu ONCE(rmt_lightweight_update_sleep_after_block_allocation) \ ONCE(rmt_merge_task_sleep_in_prepare) \ ONCE(merge_tree_refresh_parts_throw_once) \ + REGULAR(patch_part_index_write_empty) \ ONCE(s3_read_buffer_throw_expired_token) \ ONCE(s3_send_request_throw_expired_token) \ REGULAR(s3_read_inject_etag_mismatch) \ @@ -161,6 +162,7 @@ static struct InitFiu PAUSEABLE_ONCE(kafka2_remove_zk_before_get_children) \ PAUSEABLE_ONCE(kafka2_remove_zk_before_final_multi) \ PAUSEABLE_ONCE(keeper_map_delete_pause_before_multi) \ + PAUSEABLE_ONCE(paimon_incremental_read_pause_before_is_active_remove) \ PAUSEABLE(dummy_pausable_failpoint) \ PAUSEABLE(paimon_incremental_read_pause_after_watermark_commit) \ ONCE(execute_query_calling_empty_set_result_func_on_exception) \ @@ -203,6 +205,7 @@ static struct InitFiu REGULAR(claim_inject_stale_part_dir) \ PAUSEABLE(infinite_sleep) \ PAUSEABLE(async_insert_flush_pause_in_executor) \ + PAUSEABLE(async_insert_pause_before_schedule) \ PAUSEABLE(system_replicas_schedule_requests_pause) \ PAUSEABLE(stop_moving_part_before_swap_with_active) \ REGULAR(replicated_merge_tree_all_replicas_stale) \ @@ -347,6 +350,7 @@ static struct InitFiu PAUSEABLE(keeper_changelog_readahead_pre_drain) \ PAUSEABLE(object_storage_source_pause_before_virtual_columns) \ REGULAR(keeper_changelog_readahead_fill_exception) \ + ONCE(keeper_changelog_preallocate_no_space) \ REGULAR(distributed_plan_record_failure_while_starting_tasks) \ ONCE(zk_send_thread_request_window_throw) \ ONCE(zk_send_thread_operations_insert_throw) \ @@ -360,7 +364,9 @@ static struct InitFiu PAUSEABLE_ONCE(limit_by_transform_mid_loop_pause) \ PAUSEABLE_ONCE(aggregating_in_order_transform_mid_loop_pause) \ REGULAR(smt_force_takeover_predicate_true) \ - REGULAR(smt_takeover_fake_hardware_error_after_set) + REGULAR(smt_takeover_fake_hardware_error_after_set) \ + REGULAR(aggregate_function_state_transfer_throw) \ + REGULAR(aggregate_function_state_transfer_throw_after_child) namespace FailPoints { diff --git a/src/Common/FieldBinaryEncoding.cpp b/src/Common/FieldBinaryEncoding.cpp index d93976e82321..410f6abf2a9c 100644 --- a/src/Common/FieldBinaryEncoding.cpp +++ b/src/Common/FieldBinaryEncoding.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -185,6 +186,9 @@ void FieldVisitorEncodeBinary::operator() (const AggregateFunctionStateData & x, void FieldVisitorEncodeBinary::operator() (const Array & x, WriteBuffer & buf) const { + /// Nothing but the value itself bounds the nesting depth of a Field. + checkStackSize(); + writeBinary(UInt8(FieldBinaryTypeIndex::Array), buf); size_t size = x.size(); writeVarUInt(size, buf); @@ -194,6 +198,8 @@ void FieldVisitorEncodeBinary::operator() (const Array & x, WriteBuffer & buf) c void FieldVisitorEncodeBinary::operator() (const Tuple & x, WriteBuffer & buf) const { + checkStackSize(); + writeBinary(UInt8(FieldBinaryTypeIndex::Tuple), buf); size_t size = x.size(); writeVarUInt(size, buf); @@ -203,6 +209,8 @@ void FieldVisitorEncodeBinary::operator() (const Tuple & x, WriteBuffer & buf) c void FieldVisitorEncodeBinary::operator() (const Map & x, WriteBuffer & buf) const { + checkStackSize(); + writeBinary(UInt8(FieldBinaryTypeIndex::Map), buf); size_t size = x.size(); writeVarUInt(size, buf); @@ -216,6 +224,8 @@ void FieldVisitorEncodeBinary::operator() (const Map & x, WriteBuffer & buf) con void FieldVisitorEncodeBinary::operator() (const Object & x, WriteBuffer & buf) const { + checkStackSize(); + writeBinary(UInt8(FieldBinaryTypeIndex::Object), buf); size_t size = x.size(); @@ -265,25 +275,28 @@ T decodeValueLittleEndian(ReadBuffer & buf) return value; } +Field decodeFieldImpl(ReadBuffer & buf, size_t & complexity, size_t max_complexity); + template -T decodeArrayLikeField(ReadBuffer & buf) +T decodeArrayLikeField(ReadBuffer & buf, size_t & complexity, size_t max_complexity) { + /// The nesting depth comes from the data, so nothing bounds the recursion on its own. + checkStackSize(); + size_t size = 0; readVarUInt(size, buf); T value; for (size_t i = 0; i != size; ++i) - value.push_back(decodeField(buf)); + value.push_back(decodeFieldImpl(buf, complexity, max_complexity)); return value; } -} -void encodeField(const Field & x, WriteBuffer & buf) +Field decodeFieldImpl(ReadBuffer & buf, size_t & complexity, size_t max_complexity) { - Field::dispatch([&buf] (const auto & val) { FieldVisitorEncodeBinary()(val, buf); }, x); -} + ++complexity; + if (max_complexity > 0 && complexity > max_complexity) + throw Exception(ErrorCodes::INCORRECT_DATA, "Binary type decoding complexity limit exceeded: {} > {} (adjust input_format_binary_max_type_complexity)", complexity, max_complexity); -Field decodeField(ReadBuffer & buf) -{ UInt8 type = 0; readBinary(type, buf); switch (FieldBinaryTypeIndex(type)) @@ -343,25 +356,29 @@ Field decodeField(ReadBuffer & buf) return value; } case FieldBinaryTypeIndex::Array: - return decodeArrayLikeField(buf); + return decodeArrayLikeField(buf, complexity, max_complexity); case FieldBinaryTypeIndex::Tuple: - return decodeArrayLikeField(buf); + return decodeArrayLikeField(buf, complexity, max_complexity); case FieldBinaryTypeIndex::Map: { + checkStackSize(); + size_t size = 0; readVarUInt(size, buf); Map map; for (size_t i = 0; i != size; ++i) { Tuple key_and_value; - key_and_value.push_back(decodeField(buf)); - key_and_value.push_back(decodeField(buf)); + key_and_value.push_back(decodeFieldImpl(buf, complexity, max_complexity)); + key_and_value.push_back(decodeFieldImpl(buf, complexity, max_complexity)); map.push_back(key_and_value); } return map; } case FieldBinaryTypeIndex::Object: { + checkStackSize(); + size_t size = 0; readVarUInt(size, buf); Object value; @@ -369,7 +386,7 @@ Field decodeField(ReadBuffer & buf) { String name; readStringBinary(name, buf); - value[name] = decodeField(buf); + value[name] = decodeFieldImpl(buf, complexity, max_complexity); } return value; } @@ -387,3 +404,21 @@ Field decodeField(ReadBuffer & buf) } } + +void encodeField(const Field & x, WriteBuffer & buf) +{ + Field::dispatch([&buf] (const auto & val) { FieldVisitorEncodeBinary()(val, buf); }, x); +} + +Field decodeField(ReadBuffer & buf) +{ + size_t complexity = 0; + return decodeFieldImpl(buf, complexity, 0); +} + +Field decodeField(ReadBuffer & buf, size_t & complexity, size_t max_complexity) +{ + return decodeFieldImpl(buf, complexity, max_complexity); +} + +} diff --git a/src/Common/FieldBinaryEncoding.h b/src/Common/FieldBinaryEncoding.h index a8f94f08c7ea..7d216184feea 100644 --- a/src/Common/FieldBinaryEncoding.h +++ b/src/Common/FieldBinaryEncoding.h @@ -40,4 +40,9 @@ Binary encoding for Fields: void encodeField(const Field &, WriteBuffer & buf); Field decodeField(ReadBuffer & buf); +/// Same, but charges every decoded Field node to the complexity budget of the surrounding type decoding +/// (see decodeDataType), so that a small type header cannot expand into an unbounded amount of work +/// through the parameters of an AggregateFunction type. max_complexity == 0 means unlimited. +Field decodeField(ReadBuffer & buf, size_t & complexity, size_t max_complexity); + } diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index c100f5999c0e..3a17b07f8f6a 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -903,7 +903,8 @@ The server successfully detected this situation and will download merged part fr M(FilesystemCacheStateLockMicroseconds, "Lock filesystem cache time for state lock", ValueType::Microseconds) \ M(FilesystemCacheReserveMicroseconds, "Filesystem cache space reservation time", ValueType::Microseconds) \ M(FilesystemCacheReserveAttempts, "Filesystem cache space reservation attempt", ValueType::Number) \ - M(FilesystemCacheFailedReserveAttempts, "Number of failed file cache space reservation attempts.", ValueType::Number) \ + M(FilesystemCacheReserveAheadRetries, "Number of times a filesystem cache reservation with reserve-ahead did not fit and was retried with exactly the requested size", ValueType::Number) \ + M(FilesystemCacheFailedReserveAttempts, "Number of failed file cache space reservation attempts, including reserve-ahead attempts that were retried with the exact size (see FilesystemCacheReserveAheadRetries).", ValueType::Number) \ M(FilesystemCacheEvictMicroseconds, "Filesystem cache eviction time", ValueType::Microseconds) \ M(FilesystemCacheGetOrSetMicroseconds, "Filesystem cache getOrSet() time", ValueType::Microseconds) \ M(FilesystemCacheGetMicroseconds, "Filesystem cache get() time", ValueType::Microseconds) \ diff --git a/src/Common/RemoteHostFilter.cpp b/src/Common/RemoteHostFilter.cpp index fe7bf878596b..f3e8462f1026 100644 --- a/src/Common/RemoteHostFilter.cpp +++ b/src/Common/RemoteHostFilter.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -10,6 +11,7 @@ namespace DB { namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int UNACCEPTABLE_URL; } @@ -29,6 +31,29 @@ void RemoteHostFilter::checkHostAndPort(const std::string & host, const std::str "see ", host, port); } +std::string RemoteHostFilter::checkAndGetCanonicalHostAndPort( + const std::string & host_and_port, UInt16 default_port, const std::string & description) const +{ + for (const char c : host_and_port) + { + const bool is_visible_ascii = isPrintableASCII(c) && c != ' '; + const bool is_url_separator = c == '/' || c == '@' || c == '\\'; + if (!is_visible_ascii || is_url_separator) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected character '{}' in {} '{}': expected host[:port]", + c, description, host_and_port); + } + + const auto [host, port] = parseAddress(host_and_port, default_port); + if (host.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Empty host in {} '{}'", description, host_and_port); + + checkHostAndPort(host, toString(port)); + + return host + ':' + toString(port); +} + void RemoteHostFilter::setValuesFromConfig(const Poco::Util::AbstractConfiguration & config) { if (config.has("remote_url_allow_hosts")) diff --git a/src/Common/RemoteHostFilter.h b/src/Common/RemoteHostFilter.h index 4c8983205fa3..ae0a166b8119 100644 --- a/src/Common/RemoteHostFilter.h +++ b/src/Common/RemoteHostFilter.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace Poco { class URI; } @@ -26,6 +27,14 @@ class RemoteHostFilter void checkHostAndPort(const std::string & host, const std::string & port) const; /// Does the same as checkURL, but for host and port. + /// Parses a `host[:port]` string, checks it as checkHostAndPort does, and returns it rebuilt as + /// `host:port` with an explicit port - the string to hand to a client library in place of the + /// original value, so the library dials exactly what the filter saw. A string whose re-parse by + /// a client library could disagree with this parse is rejected with `BAD_ARGUMENTS`: only + /// visible ASCII is accepted, and `/`, `@`, `\` and an empty host are rejected. + /// `description` names the checked value in error messages, e.g. "Kafka broker". + std::string checkAndGetCanonicalHostAndPort(const std::string & host_and_port, UInt16 default_port, const std::string & description) const; + private: std::atomic_bool is_initialized = false; diff --git a/src/Common/Scheduler/CPULeaseAllocation.cpp b/src/Common/Scheduler/CPULeaseAllocation.cpp index bc29236f5702..19549f5bc595 100644 --- a/src/Common/Scheduler/CPULeaseAllocation.cpp +++ b/src/Common/Scheduler/CPULeaseAllocation.cpp @@ -528,6 +528,13 @@ bool CPULeaseAllocation::renew(Lease & lease) // It is better to run less threads, but utilize CPU better to avoid frequent context switches. This is how down-scaling works. setPreempted(thread_num); + // No thread is running now, so nothing reports consumption until a grant resumes one. + // With no running thread, preemption implies `consumed_ns >= requested_ns`: every request + // in consumption is fully consumed, but `consume` finishes only one per report. The rest + // would hold their slots while we wait, and the grant we wait for may need one of them. + if (threads.running_count == 0) + finishConsumedRequests(lock); + std::optional preemption_span; if (settings.trace_cpu_scheduling) { @@ -628,6 +635,30 @@ void CPULeaseAllocation::consume(std::unique_lock & lock, ResourceCo } } +void CPULeaseAllocation::finishConsumedRequests(std::unique_lock & lock) +{ + if (allocated == 0) + return; + + while (allocated > 0) + { + chassert(consumed_ns >= requests.getMaxConsumed()); + --allocated; + --granted; + requests.finish(); + LOG_EVENT(C); + } + if (granted <= 0 && !exception) + acquirable.store(false, std::memory_order_relaxed); + + // Ask for a slot to resume a preempted thread + if (!requests.hasEnqueued()) + { + if (!schedule(lock)) + grantImpl(lock); + } +} + bool CPULeaseAllocation::schedule(std::unique_lock &) { if (allocated >= current_max_slots || shutdown) diff --git a/src/Common/Scheduler/CPULeaseAllocation.h b/src/Common/Scheduler/CPULeaseAllocation.h index d774d576e3d8..60481f2f6a86 100644 --- a/src/Common/Scheduler/CPULeaseAllocation.h +++ b/src/Common/Scheduler/CPULeaseAllocation.h @@ -220,6 +220,10 @@ class CPULeaseAllocation final : public ISlotAllocation /// Accounts consumed resource void consume(std::unique_lock & lock, ResourceCost delta_ns); + /// Finishes every request in consumption state. Used when the last running thread is preempted: + /// the requests are fully consumed, and no thread would report consumption to finish them. + void finishConsumedRequests(std::unique_lock & lock); + /// Enqueue a resource request to the scheduler if necessary. /// Returns true if request is enqueued, false if it is noncompeting and should be granted immediately. bool schedule(std::unique_lock & lock); diff --git a/src/Common/Scheduler/Nodes/tests/gtest_cpu_lease_allocation.cpp b/src/Common/Scheduler/Nodes/tests/gtest_cpu_lease_allocation.cpp new file mode 100644 index 000000000000..b0326ba56b4d --- /dev/null +++ b/src/Common/Scheduler/Nodes/tests/gtest_cpu_lease_allocation.cpp @@ -0,0 +1,72 @@ +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +void burnCPU(ResourceCost cpu_ns) +{ + UInt64 start = clock_gettime_ns(CLOCK_THREAD_CPUTIME_ID); + while (clock_gettime_ns(CLOCK_THREAD_CPUTIME_ID) - start < static_cast(cpu_ns)) + { + } +} + +} + +/// Only worker threads compete for slots (like `CREATE RESOURCE cpu (WORKER THREAD)`), so the master request +/// is free and finishing it returns no slot. A single report that covers several quanta finishes only the master +/// request, and the master thread is preempted while its worker request is fully consumed and holds the only slot. +/// The next worker request needs that slot, so the query must give it back on preemption, or it waits for itself. +TEST(SchedulerCPULeaseAllocation, FullPreemptionReturnsConsumedSlots) +{ + TimeSharedScheduler scheduler; + auto semaphore = std::make_shared(scheduler.event_queue, SchedulerNodeInfo{}, /*max_requests=*/ 1); + auto queue = std::make_shared(scheduler.event_queue, SchedulerNodeInfo{}); + queue->basename = "queue"; + semaphore->attachChild(queue); + scheduler.attachChild(semaphore); + scheduler.start(ThreadName::TEST_SCHEDULER); + + constexpr ResourceCost quantum_ns = 10'000'000; + CPULeaseSettings settings; + settings.quantum_ns = quantum_ns; + settings.report_ns = quantum_ns / 10; + ResourceLink worker_link; + worker_link.queue = queue.get(); + auto allocation = std::make_shared(/*max_threads=*/ 3, ResourceLink{}, worker_link, settings); + + // The free master request is granted immediately, the first worker request takes the only slot, + // and the second one waits in the queue. + while (semaphore->getInflights().first != 1 || queue->getQueueLengthAndCost().first != 1) + std::this_thread::yield(); + + auto master = std::async(std::launch::async, [&] + { + auto slot = allocation->acquire(); + auto * lease = dynamic_cast(slot.get()); + lease->startConsumption(); + burnCPU(4 * quantum_ns); // One processor step that consumes more than all three requests + return lease->renew(); + }); + + bool resumed = master.wait_for(std::chrono::seconds(30)) == std::future_status::ready; + if (!resumed) + allocation->free(); // Wakes the master thread, so the test fails instead of hanging + EXPECT_TRUE(resumed) << "the preempted master thread waits for a slot that its own query holds"; + EXPECT_EQ(master.get(), resumed); + + allocation->free(); + scheduler.stop(); +} diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp index 0e61bf81933b..039ff73ba69c 100644 --- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp +++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp @@ -1785,14 +1785,17 @@ void ZooKeeper::pushRequest(RequestInfo && info) info.request->spans.maybeInitialize(KeeperSpan::ClientRequestsQueue, info.request->tracing_context.get()); - if (!requests_queue.tryPush(std::move(info), args.operation_timeout_ms)) + /// A failed push kills the session (the `catch` below calls `finalize`), so be patient here. + const UInt64 push_timeout_ms = 3 * static_cast(args.session_timeout_ms); + + if (!requests_queue.tryPush(std::move(info), push_timeout_ms)) { if (requests_queue.isFinished()) throw Exception::fromMessage(Error::ZSESSIONEXPIRED, "Session expired"); throw Exception(Error::ZOPERATIONTIMEOUT, - "Cannot push request to queue within operation timeout of {} ms", - args.operation_timeout_ms); + "Cannot push request to queue within {} ms", + push_timeout_ms); } } catch (...) diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index afc505b185e8..29038c229fb7 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -73,15 +73,9 @@ inline bool maskURIPassword(std::string * uri) return false; } -/** Mask the userinfo part of a URL: `scheme://anything@rest` becomes `scheme://[HIDDEN]@rest`. - * Returns whether anything was masked. - * - * This used to be the regular expression `^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/?#]+@` rewritten to - * `\1[HIDDEN]@`. Only a match at the start of the string counts, and the userinfo is taken - * greedily up to the last '@' before the path, so a password that itself contains an at-sign is - * masked whole. `src/Common/tests/gtest_mask_uri_password.cpp` checks this against re2. +/** The offset just past the `://` of a value that starts with an RFC 3986 scheme, `npos` otherwise. */ -inline bool maskURIUserinfo(std::string & url) +inline size_t findURIAuthority(std::string_view uri) { static constexpr std::string_view SEPARATOR = "://"; @@ -90,19 +84,35 @@ inline bool maskURIUserinfo(std::string & url) auto is_letter = [](char c) { return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); }; auto is_letter_or_digit = [&](char c) { return is_letter(c) || ('0' <= c && c <= '9'); }; - if (url.empty() || !is_letter(url[0])) - return false; + if (uri.empty() || !is_letter(uri[0])) + return std::string_view::npos; size_t scheme_end = 1; - while (scheme_end < url.length() - && (is_letter_or_digit(url[scheme_end]) || url[scheme_end] == '+' || url[scheme_end] == '.' || url[scheme_end] == '-')) + while (scheme_end < uri.length() + && (is_letter_or_digit(uri[scheme_end]) || uri[scheme_end] == '+' || uri[scheme_end] == '.' || uri[scheme_end] == '-')) ++scheme_end; - if (url.compare(scheme_end, SEPARATOR.length(), SEPARATOR) != 0) + if (uri.compare(scheme_end, SEPARATOR.length(), SEPARATOR) != 0) + return std::string_view::npos; + + return scheme_end + SEPARATOR.length(); +} + +/** Mask the userinfo part of a URL: `scheme://anything@rest` becomes `scheme://[HIDDEN]@rest`. + * Returns whether anything was masked. + * + * This used to be the regular expression `^([a-zA-Z][a-zA-Z0-9+.-]*://)[^/?#]+@` rewritten to + * `\1[HIDDEN]@`. Only a match at the start of the string counts, and the userinfo is taken + * greedily up to the last '@' before the path, so a password that itself contains an at-sign is + * masked whole. `src/Common/tests/gtest_mask_uri_password.cpp` checks this against re2. + */ +inline bool maskURIUserinfo(std::string & url) +{ + size_t authority_begin = findURIAuthority(url); + if (authority_begin == std::string::npos) return false; /// `[^/?#]+@` - the userinfo, greedy, so it ends at the last '@' before the path. - size_t authority_begin = scheme_end + SEPARATOR.length(); size_t authority_end = url.find_first_of("/?#", authority_begin); if (authority_end == std::string::npos) authority_end = url.length(); diff --git a/src/Common/quoteString.cpp b/src/Common/quoteString.cpp index 0559e85d8a50..cc3dbaac0e62 100644 --- a/src/Common/quoteString.cpp +++ b/src/Common/quoteString.cpp @@ -39,6 +39,17 @@ String doubleQuoteString(std::string_view x) } +String doubleQuoteStringSQLite(std::string_view x) +{ + String res(2 + x.size(), '\0'); + { + WriteBufferFromString wb(res); + writeDoubleQuotedStringSQLite(x, wb); + } + return res; +} + + String backQuote(std::string_view x) { String res(2 + x.size(), '\0'); diff --git a/src/Common/quoteString.h b/src/Common/quoteString.h index b6ee14c0b878..15733c7c2d6a 100644 --- a/src/Common/quoteString.h +++ b/src/Common/quoteString.h @@ -30,6 +30,8 @@ namespace DB /// Double quote the string. String doubleQuoteString(std::string_view x); +String doubleQuoteStringSQLite(std::string_view x); + /// Quote the identifier with backquotes. String backQuote(std::string_view x); diff --git a/src/Common/tests/gtest_ssl_error_queue.cpp b/src/Common/tests/gtest_ssl_error_queue.cpp new file mode 100644 index 000000000000..53e2a560ecf5 --- /dev/null +++ b/src/Common/tests/gtest_ssl_error_queue.cpp @@ -0,0 +1,392 @@ +#include "config.h" + +#if USE_SSL + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace +{ + +void leaveShutdownWhileInInit(Poco::Net::Context::Ptr context) +{ + ERR_clear_error(); + SSL * ssl = SSL_new(context->sslContext()); + ASSERT_NE(ssl, nullptr); + SSL_set_connect_state(ssl); + ASSERT_LT(SSL_shutdown(ssl), 0); + SSL_free(ssl); + + const auto error = ERR_peek_last_error(); + /// Unwrap the error manually because OpenSSL declares `ERR_GET_LIB` and + /// `ERR_GET_REASON` with `ossl_unused`, which causes compiler warnings when used. + ASSERT_EQ((error >> ERR_LIB_OFFSET) & ERR_LIB_MASK, ERR_LIB_SSL); + ASSERT_EQ(error & ERR_REASON_MASK, SSL_R_SHUTDOWN_WHILE_IN_INIT); +} + +int failReadWithoutErrno(BIO *, char *, int) +{ + /// Return a fatal `BIO` error without setting either `errno` or the OpenSSL error queue. + return -1; +} + +long failReadCtrl(BIO *, int command, long, void *) // NOLINT(google-runtime-int) +{ + if (command == BIO_CTRL_FLUSH) + return 1; + return 0; +} + +int failReadCreate(BIO * bio) +{ + BIO_set_init(bio, 1); + BIO_set_data(bio, nullptr); + return 1; +} + +int failReadDestroy(BIO *) +{ + return 1; +} + +const BIO_METHOD * failReadBioMethod() +{ + static const BIO_METHOD * method = [] + { + BIO_METHOD * result = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "fail-read-without-errno"); + BIO_meth_set_read(result, failReadWithoutErrno); + BIO_meth_set_ctrl(result, failReadCtrl); + BIO_meth_set_create(result, failReadCreate); + BIO_meth_set_destroy(result, failReadDestroy); + return result; + }(); + return method; +} + +int retryRead(BIO * bio, char *, int) +{ + BIO_clear_retry_flags(bio); + BIO_set_retry_read(bio); + return -1; +} + +int retryWrite(BIO * bio, const char *, int) +{ + BIO_clear_retry_flags(bio); + BIO_set_retry_write(bio); + return -1; +} + +long retryCtrl(BIO *, int command, long, void *) // NOLINT(google-runtime-int) +{ + if (command == BIO_CTRL_FLUSH) + return 1; + return 0; +} + +int retryCreate(BIO * bio) +{ + BIO_set_init(bio, 1); + BIO_set_data(bio, nullptr); + return 1; +} + +int retryDestroy(BIO *) +{ + return 1; +} + +const BIO_METHOD * retryBioMethod() +{ + static const BIO_METHOD * method = [] + { + BIO_METHOD * result = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "always-retry"); + BIO_meth_set_read(result, retryRead); + BIO_meth_set_write(result, retryWrite); + BIO_meth_set_ctrl(result, retryCtrl); + BIO_meth_set_create(result, retryCreate); + BIO_meth_set_destroy(result, retryDestroy); + return result; + }(); + return method; +} + +class ExhaustTimeoutStreamSocketImpl final : public Poco::Net::StreamSocketImpl +{ +public: + bool pollImpl(Poco::Timespan & timeout, int) override + { + timeout = 0; + return true; + } +}; + +class LiveTLSPair +{ +public: + LiveTLSPair() + : server_context(cert.makeContext(Poco::Net::Context::SERVER_USE)) + , client_context(cert.makeContext(Poco::Net::Context::CLIENT_USE)) + , listener(Poco::Net::SocketAddress("127.0.0.1", 0), 1, server_context) + , server_thread([this] + { + runServer(); + }) + { + try + { + client = std::make_unique( + Poco::Net::SocketAddress("127.0.0.1", listener.address().port()), client_context); + if (client->sendBytes("x", 1) != 1) + throw std::runtime_error("TLS client could not send the handshake byte"); + server_ready.wait(); + if (server_exception) + std::rethrow_exception(server_exception); + } + catch (...) + { + stop(); + throw; + } + } + + ~LiveTLSPair() + { + stop(); + } + + void sendMalformedRecord() + { + send_malformed_record.store(true, std::memory_order_release); + action_requested.count_down(); + action_was_requested = true; + action_done.wait(); + if (server_exception) + std::rethrow_exception(server_exception); + } + + Poco::Net::SecureStreamSocket & getClient() + { + return *client; + } + + Poco::Net::Context::Ptr getClientContext() const + { + return client_context; + } + +private: + void runServer() + { + try + { + Poco::Net::SecureStreamSocket peer(listener.acceptConnection()); + char byte = 0; + if (peer.receiveBytes(&byte, 1) != 1) + throw std::runtime_error("TLS server did not receive the handshake byte"); + + server_ready.count_down(); + action_requested.wait(); + + if (send_malformed_record.load(std::memory_order_acquire)) + { + /// A syntactically TLS-looking application-data record with an invalid payload. + /// It bypasses the server's `SSL` object and causes a fatal record-layer error on + /// the client while leaving the underlying TCP connection open. + constexpr unsigned char malformed_record[] = {0x17, 0x03, 0x03, 0x00, 0x01, 0xff}; + const ssize_t sent = ::send( + peer.impl()->sockfd(), malformed_record, sizeof(malformed_record), MSG_NOSIGNAL); + if (sent != static_cast(sizeof(malformed_record))) + throw std::runtime_error("TLS server could not send the malformed record"); + } + + action_done.count_down(); + finish.wait(); + peer.abort(); + } + catch (...) + { + server_exception = std::current_exception(); + server_ready.count_down(); + action_done.count_down(); + } + } + + void stop() noexcept + { + if (stopped) + return; + stopped = true; + + if (client) + { + try + { + client->abort(); + } + catch (...) + { + /// Ok: report the failure but continue cleanup so that the server thread is always joined. + ADD_FAILURE() << "Failed to abort the TLS client during cleanup"; + } + } + + if (!action_was_requested) + action_requested.count_down(); + finish.count_down(); + try + { + listener.close(); + } + catch (...) + { + /// Ok: report the failure but continue cleanup so that the server thread is always joined. + ADD_FAILURE() << "Failed to close the TLS listener during cleanup"; + } + if (server_thread.joinable()) + server_thread.join(); + } + + EphemeralCert cert; + Poco::Net::Context::Ptr server_context; + Poco::Net::Context::Ptr client_context; + Poco::Net::SecureServerSocket listener; + std::latch server_ready{1}; + std::latch action_requested{1}; + std::latch action_done{1}; + std::latch finish{1}; + std::atomic send_malformed_record{false}; + std::exception_ptr server_exception; + std::thread server_thread; + std::unique_ptr client; + bool action_was_requested = false; + bool stopped = false; +}; + +} + + +TEST(SSLErrorQueue, StaleErrorDoesNotChangeNonBlockingReadRetry) +{ + LiveTLSPair pair; + auto & client = pair.getClient(); + client.setBlocking(false); + + char byte = 0; + leaveShutdownWhileInInit(pair.getClientContext()); + EXPECT_EQ(client.receiveBytes(&byte, 1), Poco::Net::SecureStreamSocket::ERR_SSL_WANT_READ); + EXPECT_EQ(ERR_peek_error(), 0UL); + + /// The external async socket path retries `receiveBytes` after polling. The queue must be + /// cleared for every attempt, not only for the first operation on the connection. + leaveShutdownWhileInInit(pair.getClientContext()); + EXPECT_EQ(client.receiveBytes(&byte, 1), Poco::Net::SecureStreamSocket::ERR_SSL_WANT_READ); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + + +TEST(SSLErrorQueue, StaleErrnoDoesNotRetryFatalBioError) +{ + LiveTLSPair pair; + auto & client = pair.getClient(); + client.setSendTimeout(Poco::Timespan(0, 10'000)); + client.setReceiveTimeout(Poco::Timespan(0, 10'000)); + + auto * client_impl = static_cast(client.impl()); + SSL * ssl = client_impl->ssl(); + ASSERT_NE(ssl, nullptr); + + /// Replace only the read `BIO`; `SSL_set0_rbio` transfers its ownership to `SSL`. + BIO * failing_read_bio = BIO_new(failReadBioMethod()); + ASSERT_NE(failing_read_bio, nullptr); + SSL_set0_rbio(ssl, failing_read_bio); + + /// The failing `BIO` deliberately leaves `errno` unchanged. A stale retriable value must not + /// make `SecureSocketImpl::mustRetry` repeat the operation or report an unrelated socket error. + errno = EINTR; + char byte = 0; + EXPECT_THROW(client.receiveBytes(&byte, 1), Poco::Net::SSLConnectionUnexpectedlyClosedException); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + + +TEST(SSLErrorQueue, BlockingConnectDoesNotAcceptTimedOutHandshake) +{ + Poco::Net::ServerSocket listener(Poco::Net::SocketAddress("127.0.0.1", 0)); + EphemeralCert cert; + auto client_context = cert.makeContext(Poco::Net::Context::CLIENT_USE); + + Poco::AutoPtr client_impl + = new Poco::Net::SecureStreamSocketImpl(new ExhaustTimeoutStreamSocketImpl, client_context); + client_impl->setBioMethod(retryBioMethod()); + + /// The custom `BIO` keeps returning `WANT_READ` or `WANT_WRITE`. The first poll reports + /// readiness but consumes the entire timeout, so a blocking handshake must not be accepted. + EXPECT_THROW(client_impl->connect(listener.address()), Poco::TimeoutException); +} + + +TEST(SSLErrorQueue, FatalReadSkipsTLSShutdown) +{ + LiveTLSPair pair; + pair.sendMalformedRecord(); + + char byte = 0; + EXPECT_THROW(pair.getClient().receiveBytes(&byte, 1), Poco::Net::SSLException); + + /// `shutdown` must close the transport directly. Calling `SSL_shutdown` after the fatal read + /// would raise another SSL exception and could contaminate the thread's error queue. + EXPECT_NO_THROW(pair.getClient().shutdown()); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + + +TEST(SSLErrorQueue, FatalPeekSkipsTLSShutdown) +{ + LiveTLSPair pair; + pair.sendMalformedRecord(); + + auto state = DB::SocketState::Idle; + /// `poll` can first wake for a TLS 1.3 post-handshake message. Repeat while `SSL_peek` + /// consumes only such control records, with a total timeout of five seconds. + for (int attempt = 0; attempt < 50 && state == DB::SocketState::Idle; ++attempt) + { + if (pair.getClient().poll(Poco::Timespan(0, 100'000), Poco::Net::Socket::SELECT_READ)) + state = DB::getSocketState(pair.getClient()); + } + EXPECT_EQ(state, DB::SocketState::Closed); + EXPECT_EQ(ERR_peek_error(), 0UL); + + /// The diagnostic `SSL_peek` saw a fatal TLS error, so `shutdown` must close the transport + /// directly instead of attempting the forbidden `SSL_shutdown` operation. + EXPECT_NO_THROW(pair.getClient().shutdown()); + EXPECT_EQ(ERR_peek_error(), 0UL); +} + +#endif diff --git a/src/Common/tests/gtest_ssl_send_pending_data.cpp b/src/Common/tests/gtest_ssl_send_pending_data.cpp new file mode 100644 index 000000000000..20a26adc9613 --- /dev/null +++ b/src/Common/tests/gtest_ssl_send_pending_data.cpp @@ -0,0 +1,177 @@ +#include "config.h" + +#if USE_SSL + +#include + +#include + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace +{ + +/// This custom `BIO` makes every TLS write fail as if the underlying socket was reset. +/// It is installed as the write `BIO` only, so the real read `BIO` and buffered application data remain intact. +int resetWrite(BIO * bio, const char *, int) +{ + BIO_clear_retry_flags(bio); + errno = ECONNRESET; + return -1; +} + +long resetCtrl(BIO *, int command, long, void *) // NOLINT(google-runtime-int) +{ + if (command == BIO_CTRL_FLUSH) + return 1; + return 0; +} + +int resetCreate(BIO * bio) +{ + BIO_set_init(bio, 1); + BIO_set_data(bio, nullptr); + return 1; +} + +int resetDestroy(BIO *) +{ + return 1; +} + +const BIO_METHOD * resetWriteBioMethod() +{ + /// `SSL_set0_wbio` transfers ownership of the `BIO` to `SSL`, so the method must support its full lifetime. + /// A successful flush also lets OpenSSL perform routine `BIO` housekeeping without obscuring the write failure. + static const BIO_METHOD * method = [] + { + BIO_METHOD * result = BIO_meth_new(BIO_get_new_index() | BIO_TYPE_SOURCE_SINK, "reset-write"); + BIO_meth_set_write(result, resetWrite); + BIO_meth_set_ctrl(result, resetCtrl); + BIO_meth_set_create(result, resetCreate); + BIO_meth_set_destroy(result, resetDestroy); + return result; + }(); + return method; +} + +} + + +TEST(SSLSocketError, SendFailureDoesNotConsumePendingApplicationData) +{ + EphemeralCert cert; + auto server_context = cert.makeContext(Poco::Net::Context::SERVER_USE); + auto client_context = cert.makeContext(Poco::Net::Context::CLIENT_USE); + + Poco::Net::SecureServerSocket server_socket(Poco::Net::SocketAddress("127.0.0.1", 0), 1, server_context); + const Poco::Net::SocketAddress server_address("127.0.0.1", server_socket.address().port()); + const Poco::Timespan timeout(5, 0); + + /// Establish TCP before starting the server thread. The lazy TLS handshake is completed below, + /// concurrently with the first server-side write. + Poco::Net::SecureStreamSocket client(client_context); + client.setLazyHandshake(true); + client.connect(server_address); + client.setSendTimeout(timeout); + client.setReceiveTimeout(timeout); + + constexpr std::array payload{'p', 'e', 'n', 'd', 'i', 'n', 'g', '!'}; + std::exception_ptr server_exception; + std::jthread server_thread([&] + { + try + { + auto accepted = server_socket.acceptConnection(); + accepted.setSendTimeout(timeout); + accepted.setReceiveTimeout(timeout); + if (accepted.sendBytes(payload.data(), static_cast(payload.size())) != static_cast(payload.size())) + throw Poco::Net::NetException("Could not send the complete TLS test payload"); + } + catch (...) + { + server_exception = std::current_exception(); + } + }); + + auto * client_impl = static_cast(client.impl()); + ASSERT_EQ(client_impl->completeHandshake(), 1); + + SSL * ssl = client_impl->ssl(); + ASSERT_NE(ssl, nullptr); + + /// The failing write `BIO` installed below would also reject a TLS shutdown write. Mark shutdown as complete + /// during scope cleanup, while still letting `SSL` own and destroy the failing `BIO`. + SCOPE_EXIT(SSL_set_shutdown(ssl, SSL_SENT_SHUTDOWN | SSL_RECEIVED_SHUTDOWN)); + + /// `SSL_peek` processes the incoming TLS record without consuming its first application-data byte. + /// Because the server sends the payload in one call, the complete payload must then be visible in `SSL_pending`. + char first_byte = 0; + const int peek_result = SSL_peek(ssl, &first_byte, 1); + + /// Wait until the server has completed its single write and propagate any server-side exception. + server_thread.join(); + if (server_exception) + std::rethrow_exception(server_exception); + + ASSERT_EQ(peek_result, 1); + EXPECT_EQ(first_byte, payload.front()); + const int pending_before_send = SSL_pending(ssl); + ASSERT_EQ(pending_before_send, static_cast(payload.size())); + + /// Replace only the write `BIO`. The read `BIO` and application data already buffered by `SSL` are preserved, + /// and `SSL_set0_wbio` transfers ownership of the failing `BIO` to `SSL`. + BIO * reset_write_bio = BIO_new(resetWriteBioMethod()); + ASSERT_NE(reset_write_bio, nullptr); + SSL_set0_wbio(ssl, reset_write_bio); + + /// OpenSSL requires the current thread's error queue to be empty before TLS I/O so that `SSL_get_error` + /// classifies this `SSL_write` result rather than an earlier error. Check the exact exception type because + /// the removed workaround also changed a connection reset into an `SSLException`. + ERR_clear_error(); + errno = 0; + const char byte_to_send = 'x'; + try + { + client.sendBytes(&byte_to_send, 1); + ADD_FAILURE() << "Expected Poco::Net::ConnectionResetException"; + } + catch (const Poco::Net::ConnectionResetException & exception) + { + EXPECT_EQ(typeid(exception), typeid(Poco::Net::ConnectionResetException)); + } + catch (const Poco::Exception & exception) + { + ADD_FAILURE() << "Expected Poco::Net::ConnectionResetException, got " << exception.className(); + } + catch (...) + { + /// Ok: this catch turns a non-Poco exception into a test failure. + ADD_FAILURE() << "Expected Poco::Net::ConnectionResetException, got a non-Poco exception"; + } + + /// Previously `SecureSocketImpl::sendBytes` called `SSL_read` after this write failure, silently consuming + /// one pending application-data byte. Handling the write error must leave all incoming data untouched. + EXPECT_EQ(SSL_pending(ssl), pending_before_send); +} + + +#endif diff --git a/src/Common/tests/gtest_ssl_send_timeout.cpp b/src/Common/tests/gtest_ssl_send_timeout.cpp index 7f56fd605945..e17f43b4add7 100644 --- a/src/Common/tests/gtest_ssl_send_timeout.cpp +++ b/src/Common/tests/gtest_ssl_send_timeout.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -15,6 +16,12 @@ #include +#include + +#include + +#include +#include #include #include #include @@ -100,6 +107,125 @@ TEST(SSLSocketTimeout, SendBytesThrowsTimeoutOnBlockingSocket) } +namespace +{ + +/// Checks that shutting a blocking SSL socket down after its write timed out returns +/// promptly, instead of waiting for the peer for another full I/O timeout and throwing. +void checkShutdownAfterSendTimeout(bool receive_timeout_before_shutdown) +{ + EphemeralCert cert; + auto server_ctx = cert.makeContext(Poco::Net::Context::SERVER_USE); + auto client_ctx = cert.makeContext(Poco::Net::Context::CLIENT_USE); + + Poco::Net::SecureServerSocket server_socket( + Poco::Net::SocketAddress("127.0.0.1", 0), 1, server_ctx); + auto port = server_socket.address().port(); + + std::atomic server_done{false}; + + /// Server thread: accept and handshake, then sit idle (never read). + std::jthread server_thread([&] + { + try + { + auto accepted = server_socket.acceptConnection(); + /// Handshake happens on first I/O. Do a small read to trigger it. + char buf[1]; + try { accepted.receiveBytes(buf, 1); } catch (...) {} /// Ok: handshake may fail. NOLINT(bugprone-empty-catch) + /// Keep the connection open, and unread, until the test completes. + while (!server_done.load()) + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + catch (...) {} /// Ok: server thread cleanup, test checks client-side behavior. NOLINT(bugprone-empty-catch) + }); + + /// Declared after the thread so that it runs before the thread is joined, on every exit path. + /// Closing the listening socket also unblocks acceptConnection if the client never connected. + SCOPE_EXIT(server_done.store(true); server_socket.close()); + + std::optional client; + try + { + client.emplace(Poco::Net::SocketAddress("127.0.0.1", port), client_ctx); + } + catch (const Poco::Exception & e) + { + /// Connection setup can fail on some systems; skip gracefully. + GTEST_SKIP() << "SSL setup failed: " << e.displayText(); + } + + /// Very short send timeout so the test doesn't wait long. + client->setSendTimeout(Poco::Timespan(0, 200'000)); /// 200ms + + /// Write enough data to fill the TCP send buffer and SSL buffer. + /// Typical TCP buffer is 128KB-256KB. Write 4MB to be sure. + std::vector data(4 * 1024 * 1024, 'X'); + + bool got_timeout = false; + try + { + size_t offset = 0; + while (offset < data.size()) + { + int sent = client->sendBytes(data.data() + offset, static_cast(data.size() - offset)); + if (sent > 0) + offset += sent; + else + break; + } + } + catch (const Poco::TimeoutException &) + { + got_timeout = true; + } + + /// Without a timed out write there is no unsent data left behind and nothing to test. + ASSERT_TRUE(got_timeout) << "Expected Poco::TimeoutException when writing to a non-reading SSL peer"; + + auto * client_impl = static_cast(client->impl()); + SSL * ssl = client_impl->ssl(); + ASSERT_NE(ssl, nullptr); + + if (receive_timeout_before_shutdown) + { + /// The peer never writes either, so the read times out too. + client->setReceiveTimeout(Poco::Timespan(0, 200'000)); /// 200ms + char buf[1]; + EXPECT_THROW(client->receiveBytes(buf, 1), Poco::TimeoutException); + /// The read replaces the state that `SSL_want_write` reports, while the write stays pending. + ASSERT_FALSE(SSL_want_write(ssl)); + } + + /// The shutdown budget is max(send timeout, receive timeout), so raising the receive + /// timeout now separates a shutdown that waits for the peer from one that does not. + client->setReceiveTimeout(Poco::Timespan(10, 0)); /// 10s + + const auto started = std::chrono::steady_clock::now(); + EXPECT_NO_THROW(client->shutdown()); + const auto elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - started).count(); + + EXPECT_LT(elapsed_ms, 2000) << "shutdown() blocked for " << elapsed_ms + << "ms waiting for a peer that is not reading"; + + /// The orderly TLS shutdown must still be attempted once, so skipping it entirely does not pass. + EXPECT_TRUE(SSL_get_shutdown(ssl) & SSL_SENT_SHUTDOWN); +} + +} + +TEST(SSLSocketTimeout, ShutdownAfterSendTimeoutDoesNotWaitForPeer) +{ + checkShutdownAfterSendTimeout(/* receive_timeout_before_shutdown= */ false); +} + +TEST(SSLSocketTimeout, ShutdownAfterSendAndReceiveTimeoutsDoesNotWaitForPeer) +{ + checkShutdownAfterSendTimeout(/* receive_timeout_before_shutdown= */ true); +} + + /// Test that SSL handshake throws TimeoutException when the peer /// is a plain TCP listener that never speaks SSL. /// No server thread needed -- the kernel's listen backlog completes the diff --git a/src/Compression/CompressedReadBufferBase.cpp b/src/Compression/CompressedReadBufferBase.cpp index b32d51ef27e4..d116c7baa1d0 100644 --- a/src/Compression/CompressedReadBufferBase.cpp +++ b/src/Compression/CompressedReadBufferBase.cpp @@ -310,6 +310,12 @@ void CompressedReadBufferBase::decompress(BufferBase::Buffer & to, size_t size_d "Can't decompress data: the compressed data size ({}, this should include header size) is less than the header size ({})", size_compressed_without_checksum, static_cast(header_size)); + if (size_compressed_without_checksum - header_size != size_decompressed) + throw Exception(external_data ? ErrorCodes::CANNOT_DECOMPRESS : ErrorCodes::CORRUPTED_DATA, + "Can't decompress data: the compressed data size without header ({}) does not match size_decompressed ({}) " + "for a codec that stores data uncompressed", + size_compressed_without_checksum - header_size, size_decompressed); + to = BufferBase::Buffer(compressed_buffer + header_size, compressed_buffer + size_compressed_without_checksum); } else diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 4b358776b2c1..a5d204ea5bce 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -92,6 +92,7 @@ namespace FailPoints extern const char keeper_changelog_readahead_park_armed[]; extern const char keeper_changelog_readahead_pre_drain[]; extern const char keeper_changelog_readahead_fill_exception[]; + extern const char keeper_changelog_preallocate_no_space[]; } namespace @@ -347,7 +348,8 @@ class ChangelogWriter bool appendRecord(ChangelogRecord && record) { const auto * file_buffer = tryGetFileBaseBuffer(); - chassert(file_buffer && current_file_description); + if (!file_buffer || !current_file_description) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Log writer wasn't initialized for any file"); chassert(record.header.index - getStartIndex() <= current_file_description->expectedEntriesCountInLog()); // check if log file reached the limit for amount of records it can contain @@ -590,6 +592,12 @@ class ChangelogWriter file_buffer->getFD(), FALLOC_FL_KEEP_SIZE, 0, log_file_settings.max_size + log_file_settings.overallocate_size); } while (res < 0 && errno == EINTR); + fiu_do_on(FailPoints::keeper_changelog_preallocate_no_space, + { + res = -1; + errno = ENOSPC; + }); + if (res != 0) { if (errno == ENOSPC) @@ -4154,9 +4162,6 @@ void Changelog::appendCompletionThread() bool append_ok = false; while (append_completion_queue.pop(append_ok)) { - if (!append_ok) - current_writer->finalize(); - // we shouldn't start the raft_server before sending it here if (auto raft_server_locked = raft_server.lock()) raft_server_locked->notify_log_append_completion(append_ok); diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index c1d19bbc92bd..b453b94dea7c 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -148,7 +148,9 @@ auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std if (config.has(root_ca_file_property)) params.caLocation = config.getString(root_ca_file_property); - params.loadDefaultCAs = config.getBool(load_default_ca_file_property, false); + /// Unlike `Poco::Net::SSLManager`, the default CA certificates are not trusted unless `loadDefaultCAFile` is set. + constexpr bool load_default_cas_default = false; + params.loadDefaultCAs = config.getBool(load_default_ca_file_property, load_default_cas_default); params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString(verification_mode_property, "none")); const String cipher_list_property = config_prefix + "cipherList"; @@ -194,7 +196,7 @@ auto getSslContextProvider(const Poco::Util::AbstractConfiguration & config, std /// Try to register with CertificateReloader for hot-reload support. /// If registration fails, fall back to static certificate loading. - if (!CertificateReloader::instance().registerAdditionalContext(ssl_ctx, config_prefix)) + if (!CertificateReloader::instance().registerAdditionalContext(ssl_ctx, config_prefix, load_default_cas_default)) { /// For passphrase-protected keys, load certificates manually if (certificate_data) diff --git a/src/Coordination/tests/gtest_coordination_changelog.cpp b/src/Coordination/tests/gtest_coordination_changelog.cpp index 74e6c2b1a2ff..ef2a02920b17 100644 --- a/src/Coordination/tests/gtest_coordination_changelog.cpp +++ b/src/Coordination/tests/gtest_coordination_changelog.cpp @@ -30,6 +30,7 @@ namespace FailPoints { extern const char keeper_changelog_read_plan_resolved[]; extern const char keeper_changelog_removed_from_disk_set[]; + extern const char keeper_changelog_preallocate_no_space[]; } namespace ErrorCodes @@ -346,6 +347,36 @@ TEST_P(CoordinationTestWithCompression, ChangelogTestFlushThrottling) EXPECT_GE(watch.elapsedMilliseconds(), 100); } +/// A failed preallocation (e.g. `ENOSPC`) fails the batch, but must leave the writer usable: +/// the next append retries the preallocation. Previously the append completion thread +/// finalized the writer without holding the writer lock, and the next append dereferenced +/// the destroyed file buffer. +TEST_P(CoordinationTestWithCompression, ChangelogTestAppendAfterPreallocationFailure) +{ + ChangelogDirTest test("./logs"); + this->setLogDirectory("./logs"); + + DB::KeeperLogStore changelog( + DB::LogFileSettings{ + .force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1000, .max_size = 1024 * 1024}, + DB::FlushSettings(), + DB::ReadAheadSettings{}, + this->keeper_context); + changelog.init(0, 0); + + DB::FailPointInjection::enableFailPoint(DB::FailPoints::keeper_changelog_preallocate_no_space); + + auto entry = getLogEntry("hello world", 77); + changelog.append(entry); + EXPECT_FALSE(changelog.flush()); + + for (size_t i = 0; i < 10; ++i) + { + changelog.append(entry); + EXPECT_TRUE(changelog.flush()); + } +} + TEST_P(CoordinationTestWithCompression, ChangelogTestFile) { ChangelogDirTest test("./logs"); diff --git a/src/Core/BaseSettings.cpp b/src/Core/BaseSettings.cpp index cbedfdb85d0f..da6718854c74 100644 --- a/src/Core/BaseSettings.cpp +++ b/src/Core/BaseSettings.cpp @@ -28,7 +28,7 @@ void BaseSettingsHelpers::writeString(std::string_view str, WriteBuffer & out) String BaseSettingsHelpers::readString(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); return str; } diff --git a/src/Core/ExternalTable.cpp b/src/Core/ExternalTable.cpp index 584b2b50f1fb..a03f0109a3ae 100644 --- a/src/Core/ExternalTable.cpp +++ b/src/Core/ExternalTable.cpp @@ -216,11 +216,12 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, const Settings & settings = getContext()->getSettingsRef(); - if (settings[Setting::http_max_multipart_form_data_size]) + const size_t form_data_size_limit = settings[Setting::http_max_multipart_form_data_size]; + if (form_data_size_limit) read_buffer = std::make_unique( stream, LimitReadBuffer::Settings{ - .read_no_more = settings[Setting::http_max_multipart_form_data_size], + .read_no_more = form_data_size_limit > form_data_bytes_read ? form_data_size_limit - form_data_bytes_read : 0, .expect_eof = true, .excetion_hint = "the maximum size of multipart/form-data. This limit can be tuned by 'http_max_multipart_form_data_size' setting", }); @@ -293,6 +294,14 @@ void ExternalTablesHandler::handlePart(const Poco::Net::MessageHeader & header, CompletedPipelineExecutor executor(pipeline); executor.execute(); + + /// Whatever the format left unread still belongs to the part, and `HTMLForm` would skip it + /// outside the limiter. Read it out through the limiter so it counts against the budget, and so + /// that a part running past the budget trips `expect_eof` rather than slipping by. + if (form_data_size_limit) + read_buffer->ignoreAll(); + + form_data_bytes_read += read_buffer->count(); } } diff --git a/src/Core/ExternalTable.h b/src/Core/ExternalTable.h index e7c5918a209d..c120fb22e2cd 100644 --- a/src/Core/ExternalTable.h +++ b/src/Core/ExternalTable.h @@ -91,6 +91,9 @@ class ExternalTablesHandler : public HTMLForm::PartHandler, BaseExternalTable, W private: const Poco::Net::NameValueCollection & params; + /// `http_max_multipart_form_data_size` is one budget across the form's external-table parts, not a + /// fresh limit per part. Other fields have their own bound, `http_max_field_value_size`. + size_t form_data_bytes_read = 0; }; diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 12ab3ae32e44..5e9998838ce8 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -439,7 +439,7 @@ static Field getBinaryValue(UInt8 type, ReadBuffer & buf) case Field::Types::String: { std::string value; - readStringBinary(value, buf); + readStringBinaryGrowing(value, buf); return value; } case Field::Types::Array: @@ -469,8 +469,8 @@ static Field getBinaryValue(UInt8 type, ReadBuffer & buf) case Field::Types::AggregateFunctionState: { AggregateFunctionStateData value; - readStringBinary(value.name, buf); - readStringBinary(value.data, buf); + readStringBinaryGrowing(value.name, buf); + readStringBinaryGrowing(value.data, buf); return value; } case Field::Types::Bool: @@ -597,7 +597,7 @@ void readBinary(Object & x, ReadBuffer & buf) UInt8 type = 0; String key; readBinary(type, buf); - readBinary(key, buf); + readStringBinaryGrowing(key, buf); x[key] = getBinaryValue(type, buf); } } diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index 5bc1e8766ec4..be23cba6625f 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -852,7 +852,7 @@ The maximum allowed size for String in RowBinary format. It prevents allocating The maximum allowed size for Array in RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit )", 0) \ DECLARE(UInt64, input_format_binary_max_type_complexity, 1000, R"( -Max type nodes when decoding binary types (not depth, but total count). `Map(String, UInt32)` = 3 nodes. Protects against malicious inputs. 0 = unlimited. +Max type nodes when decoding binary types (not depth, but total count). `Map(String, UInt32)` = 3 nodes. Parameters of `AggregateFunction` types count as one node per value, including nested ones. Protects against malicious inputs. 0 = unlimited. )", 0) \ DECLARE(UInt64, format_binary_max_object_size, 100000, R"( The maximum allowed number of paths in a single Object for JSON type RowBinary format. It prevents allocating large amount of memory in case of corrupted data. 0 means there is no limit @@ -1613,7 +1613,7 @@ Possible values: Use the precise float parsing algorithm, which always returns the closest representable value to the input. When disabled, a faster but less accurate algorithm is used that may differ from the precise result by the least significant bits. )", 0) \ DECLARE(DateTimeOverflowBehavior, date_time_overflow_behavior, "ignore", R"( -Defines the behavior when [Date](/reference/data-types/date), [Date32](/reference/data-types/date32), [DateTime](/reference/data-types/datetime), [DateTime64](/reference/data-types/datetime64) or integers are converted into Date, Date32, DateTime or DateTime64 but the value cannot be represented in the result type. +Defines the behavior when [Date](/reference/data-types/date), [Date32](/reference/data-types/date32), [DateTime](/reference/data-types/datetime), [DateTime64](/reference/data-types/datetime64) or integers are converted into Date, Date32, DateTime or DateTime64 but the value cannot be represented in the result type. It also applies when a `Date` or `DateTime` is parsed from text, including by an input format. Possible values: diff --git a/src/Core/MySQL/IMySQLReadPacket.cpp b/src/Core/MySQL/IMySQLReadPacket.cpp index e29fb0385a56..d40af18658ba 100644 --- a/src/Core/MySQL/IMySQLReadPacket.cpp +++ b/src/Core/MySQL/IMySQLReadPacket.cpp @@ -18,6 +18,11 @@ namespace MySQLProtocol void IMySQLReadPacket::readPayload(ReadBuffer & in, uint8_t & sequence_id) { MySQLPacketPayloadReadBuffer payload(in, sequence_id); + readPayloadFrom(payload); +} + +void IMySQLReadPacket::readPayloadFrom(ReadBuffer & payload) +{ payload.next(); readPayloadImpl(payload); if (!payload.eof()) @@ -35,8 +40,10 @@ void IMySQLReadPacket::readPayloadWithUnpacked(ReadBuffer & in) void LimitedReadPacket::readPayload(ReadBuffer &in, uint8_t &sequence_id) { - LimitReadBuffer limited(in, {.read_no_more = 10000, .expect_eof = true, .excetion_hint = "too long MySQL packet."}); - IMySQLReadPacket::readPayload(limited, sequence_id); + /// On the payload, not on `in`: `in` is the connection, which continues with the next packet. + MySQLPacketPayloadReadBuffer payload(in, sequence_id); + LimitReadBuffer limited(payload, {.read_no_more = 10000, .expect_eof = true, .excetion_hint = "too long MySQL packet."}); + readPayloadFrom(limited); } void LimitedReadPacket::readPayloadWithUnpacked(ReadBuffer & in) diff --git a/src/Core/MySQL/IMySQLReadPacket.h b/src/Core/MySQL/IMySQLReadPacket.h index b6c3d59f5eef..1e7fb1b501b4 100644 --- a/src/Core/MySQL/IMySQLReadPacket.h +++ b/src/Core/MySQL/IMySQLReadPacket.h @@ -23,6 +23,9 @@ class IMySQLReadPacket protected: virtual void readPayloadImpl(ReadBuffer & buf) = 0; + + /// `payload` has to end where the packet ends. + void readPayloadFrom(ReadBuffer & payload); }; class LimitedReadPacket : public IMySQLReadPacket diff --git a/src/Core/ProtocolDefines.h b/src/Core/ProtocolDefines.h index 4941b6179c76..ec172045ffa6 100644 --- a/src/Core/ProtocolDefines.h +++ b/src/Core/ProtocolDefines.h @@ -95,7 +95,10 @@ static constexpr auto DBMS_MERGE_TREE_PART_INFO_VERSION = 1; /// per-field version gate; the rest rely on the whole stream being rejected by its leading version. /// Version 9 registers the `Rollup` and `Cube` steps, so a plan with `GROUP BY ... WITH ROLLUP` /// or `WITH CUBE` can be shipped under `make_distributed_plan`. -static constexpr auto DBMS_QUERY_PLAN_SERIALIZATION_VERSION = 9; +/// Version 10 serializes the plan-level `max_threads` and `concurrency_control` fields. They are not +/// properties of individual steps, so a remote plan fragment would otherwise execute with its default +/// execution limits after deserialization. +static constexpr auto DBMS_QUERY_PLAN_SERIALIZATION_VERSION = 10; /// The parallel-replicas remote plan is serialized once (at DBMS_QUERY_PLAN_SERIALIZATION_VERSION) and /// that one blob is reused for every replica, so a replica below this version must be excluded up front /// rather than sent a blob it cannot parse. Tied to DBMS_QUERY_PLAN_SERIALIZATION_VERSION itself so a @@ -112,6 +115,8 @@ static constexpr auto DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_PACKED_STRI /// `adaptive_aggregator_freeze_threshold` plan setting names. Gates writing them in /// `AggregatingStep::serializeSettings`. static constexpr auto DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_ADAPTIVE_AGGREGATOR = 7; +/// First query-plan serialization version that preserves plan-level `max_threads` and `concurrency_control`. +static constexpr auto DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS = 10; /// Version 1 added the initiator's settings changes to the task. /// Version 2 added per-stream streaming-exchange ports to exchange_stream_sources. static constexpr auto DBMS_DISTRIBUTED_TASK_SERIALIZATION_VERSION = 2; diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 0f94a06ba3bc..209c0521f77b 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -1886,7 +1886,7 @@ Configured as `named_collections_storage.type` (`` contains the certificate.)", 0, "openSSL.server.certificateFile") \ - DECLARE(String, openssl_server_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/).)", 0, "openSSL.server.caConfig") \ + DECLARE(String, openssl_server_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/). The CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`; new connections are verified against the reloaded certificates.)", 0, "openSSL.server.caConfig") \ DECLARE(String, openssl_server_verification_mode, "relaxed", R"(The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``, ``, ``, ``.)", 0, "openSSL.server.verificationMode") \ DECLARE(UInt64, openssl_server_verification_depth, 9, R"(The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.)", 0, "openSSL.server.verificationDepth") \ DECLARE(Bool, openssl_server_load_default_ca_file, true, R"(Determines whether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.server.loadDefaultCAFile") \ @@ -1906,7 +1906,7 @@ Configured as `named_collections_storage.type` (`` contains the certificate.)", 0, "openSSL.client.certificateFile") \ - DECLARE(String, openssl_client_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/).)", 0, "openSSL.client.caConfig") \ + DECLARE(String, openssl_client_ca_config, "", R"(Path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://docs.openssl.org/3.0/man3/SSL_CTX_load_verify_locations/). The CA certificates are reloaded without a restart when the file changes or on `SYSTEM RELOAD CONFIG`; new connections are verified against the reloaded certificates.)", 0, "openSSL.client.caConfig") \ DECLARE(String, openssl_client_verification_mode, "relaxed", R"(The method for checking the node's certificates. Details are in the description of the [Context](https://github.com/ClickHouse/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: ``, ``, ``, ``.)", 0, "openSSL.client.verificationMode") \ DECLARE(UInt64, openssl_client_verification_depth, 9, R"(The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.)", 0, "openSSL.client.verificationDepth") \ DECLARE(Bool, openssl_client_load_default_ca_file, true, R"(Determines whether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `` (resp. the directory ``) or in file (resp. directory) specified by the environment variable `` (resp. ``).)", 0, "openSSL.client.loadDefaultCAFile") \ diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 2311dd217c00..eaaa1018cc33 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -679,6 +679,9 @@ Check each uploaded object to s3 with head request to be sure that upload was su )", 0) \ DECLARE(Bool, s3_validate_etag_on_read, true, R"( When reading an object from S3 (or an S3-compatible store such as GCS), check that every GET request returns the same ETag that was observed when the object was listed. A single file read issues many ranged GET requests; if the object is overwritten in place between them (for example by an external writer rewriting a fixed key), the reads can otherwise be stitched together from two different object generations and surface as a corrupted checksum or parse error. When a mismatch is detected the read fails with `S3_OBJECT_CHANGED_DURING_READ` instead of returning inconsistent data. Disable only for workloads that intentionally read objects that are being overwritten and can tolerate inconsistent reads. +)", 0) \ + DECLARE(Bool, azure_validate_etag_on_read, true, R"( +When reading a blob from Azure Blob Storage through the `azureBlobStorage` / `azureBlobStorageCluster` table functions or the `AzureBlobStorage` table engine, pin every `GET` request to the generation of the blob that was observed when it was listed by sending its `ETag` in `If-Match`, and check the `ETag` of the response. A single file read issues many ranged `GET` requests; if the blob is overwritten in place between them (for example by an external writer rewriting a fixed key), the reads can otherwise be stitched together from two different generations of the blob and surface as a corrupted checksum or parse error. The size recorded at listing time is also used as the right bound of the read, so it is only correct for the generation it was recorded for. When a mismatch is detected the read fails with `AZURE_OBJECT_CHANGED_DURING_READ` instead of returning inconsistent data. Disable only for workloads that intentionally read blobs that are being overwritten and can tolerate inconsistent reads. )", 0) \ DECLARE(Bool, azure_check_objects_after_upload, false, R"( Check each uploaded object in azure blob storage to be sure that upload was successful @@ -4564,6 +4567,11 @@ Possible values: \ DECLARE(Bool, allow_execute_multiif_columnar, true, R"( Allow execute multiIf function columnar +)", 0) \ + DECLARE(Bool, allow_executable_tables, true, R"( +Allow reading through the `executable` table function and from `Executable` and `ExecutablePool` tables. + +Disabling this refuses reads only: creating, attaching, dropping and describing such tables still works. `ExecutablePool` processes that have already started are left running rather than terminated, but each read is still refused until the setting is enabled again. )", 0) \ DECLARE(Bool, formatdatetime_f_prints_single_zero, false, R"( Formatter '%f' in function 'formatDateTime' prints a single zero instead of six zeros if the formatted value has no fractional seconds. @@ -6711,6 +6719,20 @@ This is an expert-level setting which should only be used for debugging by devel Possible values: +- 0 - Disable +- 1 - Enable +)", 0) \ + DECLARE(Bool, query_plan_filter_push_down_below_limit_by, true, R"( +Toggles pushing filters on `LIMIT BY` key columns below the `LIMIT BY` step. +Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1. +It is read independently of [query_plan_filter_push_down](#query_plan_filter_push_down): the push-down pass also runs, with that setting off, once a `JOIN` runtime filter has been added. + +:::note +This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed. +::: + +Possible values: + - 0 - Disable - 1 - Enable )", 0) \ @@ -7065,6 +7087,17 @@ If enabled, any URL parameter not recognized as a known parameter, setting, or ` - A plain `name=value` becomes the equality `` `name` = 'value' `` (the identifier is back-quoted, the value is quoted as a string literal). - A comparison operator (`!=`, `>`, `<`, `>=`, `<=`, `<>`) makes it a comparison: either split across the parameter (`?a!=2`, `?a>=2`) or written inline when the URL has no `=` to split on (`?a<>2`, `?f(x)>3`), in which case the reassembled `name[=value]` is parsed as a full SQL expression. +)", 0) \ + DECLARE(Bool, http_x_clickhouse_format_overrides_output_format, true, R"( +Controls which setting the `X-ClickHouse-Format` HTTP header maps to. + +If enabled (the default), the header is an alias for the `output_format` setting: it is an explicit override of the response format that wins over the `FORMAT` clause in the query and over the file extension in the URL path. + +If disabled, the header is an alias for the `default_format` setting, as it was before version 26.8: it only selects the format used when the query has no `FORMAT` clause and no other format override is applied. + +In both cases the header overrides the URL parameter of the same name (`output_format` or `default_format`, respectively) and never changes how the request body of an `INSERT` is parsed. + +This is a compatibility setting for the HTTP interface: the header is consumed before the query is parsed, so it must be supplied via a URL parameter or a user profile, not via an in-query `SETTINGS` clause. Like `output_format` and `default_format` themselves, it can always be changed in read-only mode (`readonly = 1`), so a read-only user can pass it as a URL parameter. )", 0) \ \ DECLARE(UInt64, function_range_max_elements_in_block, 500000000, R"( @@ -8100,6 +8133,11 @@ Default partition strategy for file like engines. Applied only to `CREATE` queri )", 0) \ DECLARE(Bool, use_iceberg_partition_pruning, true, R"( Use Iceberg partition pruning for Iceberg tables +)", 0) \ + DECLARE(Bool, iceberg_tolerate_conflicting_manifest_schemas, true, R"( +If enabled and the schema-id of an Iceberg manifest file is already registered from metadata.json, the metadata.json schema is used and the copy in the `schema` key of the manifest file header is ignored without being compared. If disabled, the manifest header copy is compared with the metadata.json schema, and a conflict fails the query with an ICEBERG_SPECIFICATION_VIOLATION error. + +The manifest header schema is only a copy of the table schema at the time the manifest was written, and some writers (e.g. AWS S3 Tables maintenance jobs) have been observed storing degraded copies there. Other query engines resolve schemas from metadata.json and ignore divergent header copies, so the default follows them. A conflict between two metadata.json schema definitions still always fails the query. )", 0) \ DECLARE(Bool, optimize_distinct_in_order, true, R"( Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement @@ -8321,6 +8359,14 @@ SELECT ll.Date FROM (SELECT * FROM t AS ll LEFT JOIN t1 ON ll.k = t1.k LEFT JOIN ``` Takes effect only when the analyzer is enabled (`enable_analyzer = 1`). +)", 0) \ + DECLARE(Bool, analyzer_compatibility_allow_cte_redefinition, false, R"( +Allow a Common Table Expression name to be defined more than once in a single `WITH` clause. A reference to such a name binds to the latest definition that is not being resolved at that moment: a redefinition can read the previous definition of the same name, and the query body reads the last one. This matches the query analysis that ClickHouse used before v24.3, where a later definition silently shadowed the earlier ones. One shape differs from that analysis: a CTE declared between two definitions of a name also binds to the last definition, where the old analysis bound it to the definition visible at its declaration point. By default a redefinition is rejected with `MULTIPLE_EXPRESSIONS_FOR_ALIAS`. A CTE declared as `MATERIALIZED` and a CTE in a `WITH RECURSIVE` clause cannot be redefined even when the setting is enabled. + +Possible values: + +- 0 - A CTE name can be defined only once in a `WITH` clause. +- 1 - A later definition of a CTE name shadows the earlier ones. )", 0) \ DECLARE(Bool, enable_identifier_resolve_cache, true, R"( Enable the identifier resolution cache in the query analyzer. The cache shares resolved alias nodes to prevent AST explosion when the same alias is referenced multiple times. Set to false to disable caching if incorrect results are suspected. @@ -8439,7 +8485,7 @@ Allow extracting common expressions from disjunctions in WHERE, PREWHERE, ON, HA Populate constant comparison in AND chains to enhance filtering ability. Support operators `<`, `<=`, `>`, `>=`, `=` and mix of them. For example, `(a < b) AND (b < c) AND (c < 5)` would be `(a < b) AND (b < c) AND (c < 5) AND indexHint(b < 5) AND indexHint(a < 5)`. The derived comparisons are wrapped in `indexHint`: they participate in index analysis (primary key, partition key, skipping indexes) and prune the read set, but cost nothing per row and do not affect PREWHERE. A comparison derived through expressions of different tables stays executable (`(t1.a < t2.b) AND (t2.b < 5)` derives plain `t1.a < 5`): it is the only condition that can be pushed below the join, where it filters a join input the original chain cannot reach. Derived comparisons that contradict an existing condition are also added as plain conditions, so the `AND` folds to `false`. )", 0) \ DECLARE(Bool, optimize_redundant_comparisons, true, R"( -Detect conflicting and redundant comparison conditions on the same expression within AND chains. For example, `a < 1 AND a > 5` would be rewritten to `false`. +Detect conflicting and redundant comparison conditions on the same expression within AND chains. For example, `a < 1 AND a > 5` would be rewritten to `false`. A contradiction between two `equals` on the same expression (for example, `a = 1 AND a = 2`) is detected independently of this setting. )", 0) \ DECLARE(UInt64, optimize_and_compare_chain_max_hash_work, 5'000'000, R"( Work budget for the `optimize_and_compare_chain` optimization during query analysis, measured in the number of query-tree nodes hashed by `getTreeHash` (the dominant cost of this optimization). Once a query has hashed more than this many nodes while applying the optimization, it stops applying it for the rest of the query. This bounds analysis time for queries with very many or very large `AND`-chains of comparisons, where the optimization can otherwise dominate analysis while folding nothing. Stopping early is always safe: it only forgoes an optimization and never changes results. Set to `0` to disable the budget (unlimited). @@ -8735,21 +8781,21 @@ Initial delay in milliseconds before the first retry of a failed AI function API DECLARE(Bool, ai_function_throw_on_error, true, R"( If true (default), an AI function call that fails permanently after exhausting all retries aborts the query with an exception. If false, the failed row receives the default value for the column type (empty string for String) and processing continues. )", BETA) \ - DECLARE(UInt64, ai_function_max_input_tokens_per_query, 1000000, R"( -Maximum total input (prompt) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of input tokens per in-flight request, since a call's input tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. + DECLARE(UInt64, ai_function_max_input_tokens_per_query, 0, R"( +Maximum total input (prompt) tokens across all AI function API calls in a single query. 0 (default) disables the limit. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of input tokens per in-flight request, since a call's input tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. This limit is only enforced for providers that report a `usage` object in their response (OpenAI, Anthropic, vLLM). Providers that omit token usage (notably HuggingFace TEI) cause the counter to stay at 0 — use `ai_function_max_api_calls_per_query` instead to bound such calls. )", BETA) \ - DECLARE(UInt64, ai_function_max_output_tokens_per_query, 500000, R"( -Maximum total output (completion) tokens across all AI function API calls in a single query. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of output tokens per in-flight request, since a call's output tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. + DECLARE(UInt64, ai_function_max_output_tokens_per_query, 0, R"( +Maximum total output (completion) tokens across all AI function API calls in a single query. 0 (default) disables the limit. Tracked cumulatively from provider responses. Note that this limit may be exceeded by up to one call's worth of output tokens per in-flight request, since a call's output tokens are not known until its response arrives. Like the other AI quotas, it is enforced per server / query fragment, not summed across a distributed query, and must be set in the top-level query - a sub-query `SETTINGS` override is ignored. This limit is only enforced for providers that report a `usage` object in their response (OpenAI, Anthropic, vLLM). It does not apply to the embedding functions (`aiEmbed`, `aiSimilarity`), which never produce output tokens. )", BETA) \ - DECLARE(UInt64, ai_function_max_api_calls_per_query, 1000, R"( -Maximum number of HTTP requests that AI functions may dispatch per query. Enforced independently by each server and query fragment: within one execution context it is an exact cap shared by every AI function, block, and thread there, but a distributed query (across shards or parallel-replica fragments) may dispatch up to this many requests per shard/fragment. It must be set in the top-level query - a sub-query `SETTINGS` override is ignored. Set to 0 to disable. + DECLARE(UInt64, ai_function_max_api_calls_per_query, 0, R"( +Maximum number of HTTP requests that AI functions may dispatch per query. 0 (default) disables the limit. Enforced independently by each server and query fragment: within one execution context it is an exact cap shared by every AI function, block, and thread there, but a distributed query (across shards or parallel-replica fragments) may dispatch up to this many requests per shard/fragment. It must be set in the top-level query - a sub-query `SETTINGS` override is ignored. )", BETA) \ DECLARE(Bool, ai_function_throw_on_quota_exceeded, true, R"( -If true (default), exceeding an AI function quota limit (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`, or `ai_function_max_api_calls_per_query`) aborts the query with an exception. If false, remaining rows receive the default value for the column type (empty string for String). Like the quota limits, this must be set in the top-level query - a sub-query `SETTINGS` override is ignored. +If true (default), exceeding an AI function quota limit (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`, or `ai_function_max_api_calls_per_query`) aborts the query with an exception. All three limits are disabled by default, so this has no effect until one of them is set. If false, remaining rows receive the default value for the column type (empty string for String). Like the quota limits, this must be set in the top-level query - a sub-query `SETTINGS` override is ignored. )", BETA) \ DECLARE(NonZeroUInt64, ai_function_embedding_max_batch_size, 100, R"( Maximum number of texts to include in a single HTTP request made by the embedding functions (`aiEmbed`, `aiSimilarity`). Texts are grouped into batches of this size to reduce API call overhead. For example, 500 unique texts with a batch size of 100 result in 5 HTTP requests. @@ -8885,9 +8931,12 @@ Maximal selectivity of the filter to use the hint built from the inverted text i )", 0) \ DECLARE(Bool, use_text_index_like_evaluation_by_dictionary_scan, true, R"( Enable evaluation of LIKE/ILIKE queries by scanning the inverted text index dictionary. + +The accelerated patterns are `%value%`, `value%` and `%value`, as well as the `startsWith` and `endsWith` calls that `optimize_rewrite_like_perfect_affix` rewrites into `value%` and `%value`. )", 0) \ DECLARE(UInt64, text_index_like_min_pattern_length, 4, R"( -Minimum length of the alphanumeric needle in a LIKE/ILIKE pattern required to use the text index LIKE evaluation by the dictionary scan. +Minimum length of the alphanumeric needle in a LIKE/ILIKE pattern, or of a `startsWith`/`endsWith` needle, +required to use the text index LIKE evaluation by the dictionary scan. Patterns shorter than this threshold match too many dictionary tokens and are skipped to avoid expensive scans. Requires `use_text_index_like_evaluation_by_dictionary_scan` to be enabled. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index a13257bba353..a33162430ed9 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -41,9 +41,16 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() /// controls new feature and it's 'true' by default, use 'false' as previous_value). /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) /// Note: please check if the key already exists to prevent duplicate entries. + addSettingsChanges(settings_changes_history, "26.10", + { + {"azure_validate_etag_on_read", false, true, "New setting to detect concurrent in-place overwrites of Azure blobs during a read by pinning every `GET` to the listed `ETag` with `If-Match` and validating the `ETag` of the response, like `s3_validate_etag_on_read` does for S3. `compatibility` with versions before 26.10 restores the previous behavior (no validation)."}, + }); addSettingsChanges(settings_changes_history, "26.8", { + {"iceberg_tolerate_conflicting_manifest_schemas", false, true, "New setting: when an Iceberg manifest file header carries a schema that conflicts with the schema registered for the same schema-id from metadata.json, prefer the metadata.json schema instead of failing the query, matching the behavior of other query engines. `compatibility` below 26.8 restores the previous strict behavior."}, + {"analyzer_compatibility_allow_cte_redefinition", false, false, "New compatibility setting. When enabled, the analyzer accepts a CTE name defined more than once in a single `WITH` clause and lets a later definition shadow the earlier ones, as the query analysis before v24.3 did."}, {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, + {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, @@ -102,7 +109,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"shrink_over_allocated_columns_min_waste_ratio", 1.0, 1.0, "New setting to shrink over-allocated columns to fit on INSERT to reduce peak memory usage. Disabled by default (1.0)."}, {"shrink_over_allocated_columns_min_waste_bytes", 16 * 1024 * 1024, 16 * 1024 * 1024, "New setting: minimum absolute wasted memory in a column for it to be shrunk to fit on INSERT."}, {"ai_function_allow_insecure_endpoint", true, false, "AI functions now reject insecure (http) endpoints to remote hosts by default."}, - {"ai_function_max_api_calls_per_query", 0, 1000, "Bound outbound AI function HTTP calls per query by default (previously 0 - unlimited)."}, + {"ai_function_max_input_tokens_per_query", 1000000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, + {"ai_function_max_output_tokens_per_query", 500000, 0, "The AI function per-query quotas are disabled by default: 0 means no limit."}, {"join_runtime_filter_min_probe_rows", 0, 1000, "New setting to control minimum probe side size for installing JOIN runtime filters. It wasn't limited before, so previous value is 0 meaning always install."}, {"read_in_order_use_virtual_row", false, true, "Enable the virtual row optimization by default. When reading in order of the primary key over many parts, it lets `MergingSortedTransform` reprioritize sources using primary key values from the sparse index, so parts that are not relevant for the query are not read, plus a bounded read-ahead window of at most `max_threads` parts that keeps reads parallel. This significantly reduces peak memory consumption (see https://github.com/ClickHouse/ClickHouse/issues/52624)."}, {"page", 0., 0., "New setting for paginated HTTP responses, equivalent to offset = limit * (page - 1). Float so it can hold negative or fractional values (passed through to SQL `LIMIT`/`OFFSET`)."}, @@ -122,6 +130,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"http_allow_table_as_file", false, false, "New setting to recognize a table name in the URL path of HTTP requests, with optional format/compression extensions."}, {"http_allow_filters_as_path", false, false, "New setting to recognize hive-style `name=value` filters in the URL path of HTTP requests."}, {"http_allow_filters_as_unrecognized_url_parameters", false, false, "New setting to treat unrecognized URL parameters as filter expressions in HTTP requests."}, + {"http_x_clickhouse_format_overrides_output_format", false, true, "Controls whether the `X-ClickHouse-Format` HTTP header is an alias for `output_format` (overriding the query's `FORMAT` clause), as since 26.8, or for `default_format`, as before 26.8. The entry sits in the 26.8 block because that is the release whose behavior it restores, so `compatibility` with a version before 26.8 brings back the old header behavior."}, {"ignore_on_cluster_for_replicated_handler_queries", false, false, "New setting to ignore the ON CLUSTER clause for handler management queries when handlers are backed by replicated (Keeper) storage."}, {"materialized_views_populate_atomically", false, true, "New setting that makes plain `CREATE MATERIALIZED VIEW ... POPULATE` locally atomic: existing data is snapshotted and the view is subscribed to new inserts together, under a brief exclusive lock on the source, so rows inserted through the same server are neither missed nor duplicated. The guarantee covers the local insert path only - inserts arriving on another replica or through a distributed write are outside the cut - and it requires a source that can provide a pinned snapshot (the `MergeTree` family and `Memory`); other sources, as well as `CREATE OR REPLACE` / `REPLACE`, keep the legacy non-atomic population. Set to `false` for the legacy non-atomic behavior everywhere."}, {"input_format_json_max_object_size", 512 * 1024 * 1024, 512 * 1024 * 1024, "New setting to limit the maximum size of a single JSON object in bytes"}, @@ -157,6 +166,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"iceberg_compaction_commit_batch_size", 100, 100, "New setting"}, {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, + {"query_plan_filter_push_down_below_limit_by", false, true, "New setting to control pushing a filter on the `LIMIT BY` key columns below the `LIMIT BY` step. Set it to false to keep the filter above the `LIMIT BY`."}, }); addSettingsChanges(settings_changes_history, "26.7", { diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 0849ead59791..001edbfeb629 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -247,7 +247,7 @@ void SettingFieldNumber::readBinary(ReadBuffer & in) { static_assert(std::is_floating_point_v); String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = ::DB::parseFromString(str); } } @@ -461,7 +461,7 @@ void SettingFieldString::writeBinary(WriteBuffer & out) const void SettingFieldString::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = std::move(str); } @@ -544,7 +544,7 @@ void SettingFieldChar::writeBinary(WriteBuffer & out) const void SettingFieldChar::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = stringToChar(str); } @@ -557,7 +557,7 @@ void SettingFieldURI::writeBinary(WriteBuffer & out) const void SettingFieldURI::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = Poco::URI{str}; } @@ -570,7 +570,7 @@ void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & ou String SettingFieldEnumHelpers::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); return str; } @@ -582,7 +582,7 @@ void SettingFieldTimezone::writeBinary(WriteBuffer & out) const void SettingFieldTimezone::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); *this = std::move(str); } @@ -615,7 +615,7 @@ void SettingFieldCustom::writeBinary(WriteBuffer & out) const void SettingFieldCustom::readBinary(ReadBuffer & in) { String str; - readStringBinary(str, in); + readStringBinaryGrowing(str, in); parseFromString(str); } diff --git a/src/Core/SettingsSecrets.h b/src/Core/SettingsSecrets.h index dd6ad89674a7..41dcce0c253b 100644 --- a/src/Core/SettingsSecrets.h +++ b/src/Core/SettingsSecrets.h @@ -27,14 +27,22 @@ using ValueMaskingFunc = std::function; /// precondition holds by construction and needs no check. inline bool maskURLCredentials(String & value) { - bool masked = maskURIPassword(&value); + /// A statement is masked for logging before its settings are validated, so a value that no URI + /// parser can read still reaches a log. + if (findURIAuthority(value) == String::npos && value.contains('@')) + { + value = "[HIDDEN]"; + return true; + } + + bool masked = maskURIUserinfo(value); masked |= maskPresignedURLParameters(value); return masked; } /// The settings of the query-level `Settings` collection whose value can carry a credential, and how /// each one is masked. `system.query_log.query` shows -/// `format_avro_schema_registry_url = 'http://user:[HIDDEN]@registry:8080'`, so every other place that +/// `format_avro_schema_registry_url = 'http://[HIDDEN]@registry:8080'`, so every other place that /// prints the same value hides the same secret through this map. /// /// Mirrors the per-engine `SETTINGS_TO_HIDE` maps (`Kafka_fwd.h`, `NATS_fwd.h`, ...), which do this diff --git a/src/Core/tests/gtest_mysql_limited_read_packet.cpp b/src/Core/tests/gtest_mysql_limited_read_packet.cpp new file mode 100644 index 000000000000..62fbdb893a44 --- /dev/null +++ b/src/Core/tests/gtest_mysql_limited_read_packet.cpp @@ -0,0 +1,121 @@ +#include + +#include +#include +#include +#include + +#include + +using namespace DB; +using namespace DB::MySQLProtocol; + +namespace DB::ErrorCodes +{ +extern const int LIMIT_EXCEEDED; +} + +namespace +{ + +/// The limit `LimitedReadPacket` applies, from IMySQLReadPacket.cpp. +constexpr size_t PACKET_PAYLOAD_LIMIT = 10000; + +/// Serves its data a few bytes at a time. The limiter only reaches its `expect_eof` check when it +/// has to refill, which a fully buffered source never makes it do. +class ChunkedSource : public ReadBuffer +{ +public: + ChunkedSource(String data_, size_t chunk_) : ReadBuffer(nullptr, 0), data(std::move(data_)), chunk(chunk_) {} + +private: + bool nextImpl() override + { + if (consumed >= data.size()) + return false; + + const size_t count = std::min(chunk, data.size() - consumed); + BufferBase::set(data.data() + consumed, count, 0); + consumed += count; + return true; + } + + String data; + size_t chunk; + size_t consumed = 0; +}; + +struct PayloadCollector : public LimitedReadPacket +{ + String payload; + + void readPayloadImpl(ReadBuffer & buf) override { readStringUntilEOF(payload, buf); } +}; + +String packet(const String & payload, uint8_t sequence_id) +{ + String result; + const size_t length = payload.size(); + result += static_cast(length & 0xFF); + result += static_cast((length >> 8) & 0xFF); + result += static_cast((length >> 16) & 0xFF); + result += static_cast(sequence_id); + result += payload; + return result; +} + +} + +/// The limit counts one packet's payload, not the connection. A payload exactly at the limit is +/// allowed even though the 4-byte header pushes the connection past it, and the next packet is not +/// mistaken for overflow. +TEST(MySQLLimitedReadPacket, PayloadAtTheLimitFollowedByAnotherPacket) +{ + const String first(PACKET_PAYLOAD_LIMIT, 'a'); + ChunkedSource in(packet(first, 0) + packet("second", 1), 64); + + uint8_t sequence_id = 0; + PayloadCollector collector; + collector.readPayload(in, sequence_id); + + EXPECT_EQ(collector.payload, first); + EXPECT_EQ(sequence_id, 1); + + PayloadCollector next; + next.readPayload(in, sequence_id); + EXPECT_EQ(next.payload, "second"); +} + +TEST(MySQLLimitedReadPacket, PayloadOverTheLimitIsRejected) +{ + ChunkedSource in(packet(String(PACKET_PAYLOAD_LIMIT + 1, 'a'), 0), 64); + + uint8_t sequence_id = 0; + PayloadCollector collector; + try + { + collector.readPayload(in, sequence_id); + FAIL() << "An oversized payload was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::LIMIT_EXCEEDED); + } +} + +/// The unpacked variant is handed a payload that already ends where the packet ends. +TEST(MySQLLimitedReadPacket, UnpackedPayloadOverTheLimitIsRejected) +{ + ChunkedSource in(String(PACKET_PAYLOAD_LIMIT + 1, 'a'), 64); + + PayloadCollector collector; + try + { + collector.readPayloadWithUnpacked(in); + FAIL() << "An oversized unpacked payload was accepted"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::LIMIT_EXCEEDED); + } +} diff --git a/src/Core/tests/gtest_settings.cpp b/src/Core/tests/gtest_settings.cpp index fb18ce76337b..7a75f2e11052 100644 --- a/src/Core/tests/gtest_settings.cpp +++ b/src/Core/tests/gtest_settings.cpp @@ -1,18 +1,26 @@ #include +#include +#include #include +#include +#include #include #include #include #include #include #include +#include #include +#include #include +#include namespace DB::ErrorCodes { + extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int INCORRECT_DATA; } @@ -287,3 +295,148 @@ GTEST_TEST(SettingsTier, GetTierDecodesEveryEncoding) EXPECT_EQ(BaseSettingsHelpers::getTier(private_preview | Flags::CUSTOM), SettingsTierType::PRIVATE_PREVIEW); EXPECT_EQ(BaseSettingsHelpers::getTier(private_preview | Flags::HOT_RELOAD), SettingsTierType::PRIVATE_PREVIEW); } + +namespace +{ + +/// Restores the global hard limit and the operator-new throw threshold on scope exit. +struct MemoryLimitGuard +{ + Int64 prev_hard_limit; + MemoryLimitGuard() : prev_hard_limit(total_memory_tracker.getHardLimit()) {} + ~MemoryLimitGuard() + { + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(0); + total_memory_tracker.setHardLimit(prev_hard_limit); + } +}; + +/// Makes the next allocation of a megabyte or more overshoot the global limit, while smaller +/// allocations (the exception being built, the settings object's own bookkeeping) still succeed. +void armMemoryLimitOnAllocationsOfAtLeastAMegabyte() +{ + MainThreadStatus::getInstance(); + CurrentThread::flushUntrackedMemory(); + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(1ULL << 20); + total_memory_tracker.setHardLimit(total_memory_tracker.get() + 1024); +} + +} + +GTEST_TEST(Settings, ADeclaredLengthOnTheSettingsWireIsNotAnAllocation) +{ + /// Each entry declares a large string length on the settings wire and then sends no payload, so + /// a read that grows with the bytes that actually arrive runs out of buffer immediately, while + /// one that sizes its destination from the declared length allocates before it can fail. + constexpr size_t declared_size = 64 * 1024 * 1024; + + /// `writeBinary(const Map &)` writes a fixed width element count, then one tagged field per + /// element, and puts no type constraint on an element. + const auto write_map_element_tag = [](WriteBuffer & out, Field::Types::Which field_type) + { + BaseSettingsHelpers::writeString("additional_table_filters", out); + writeBinary(static_cast(1), out); + writeBinary(static_cast(field_type), out); + }; + + struct Case + { + std::string_view covers; + SettingsWriteFormat format; + std::function write_wire; + }; + + const std::vector cases = { + {"the setting name, i.e. every BaseSettingsHelpers::readString carrier", + SettingsWriteFormat::STRINGS_WITH_FLAGS, + [&](WriteBuffer & out) { writeVarUInt(declared_size, out); }}, + {"SettingFieldString::readBinary", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + BaseSettingsHelpers::writeString("log_comment", out); + writeVarUInt(declared_size, out); + }}, + {"SettingFieldNumber::readBinary", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + BaseSettingsHelpers::writeString("totals_auto_threshold", out); + writeVarUInt(declared_size, out); + }}, + {"a String inside a Map setting", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + write_map_element_tag(out, Field::Types::String); + writeVarUInt(declared_size, out); + }}, + {"an AggregateFunctionState name inside a Map setting", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + write_map_element_tag(out, Field::Types::AggregateFunctionState); + writeVarUInt(declared_size, out); + }}, + {"an Object key inside a Map setting", + SettingsWriteFormat::BINARY, + [&](WriteBuffer & out) + { + write_map_element_tag(out, Field::Types::Object); + writeBinary(static_cast(1), out); + writeBinary(static_cast(Field::Types::String), out); + writeVarUInt(declared_size, out); + }}, + }; + + for (const auto & test_case : cases) + { + SCOPED_TRACE(test_case.covers); + + /// Build the wire and the settings object before clamping, so only the read runs clamped. + WriteBufferFromOwnString out; + test_case.write_wire(out); + const std::string wire = out.str(); + Settings settings; + ReadBufferFromString in(wire); + + try + { + MemoryLimitGuard guard; + armMemoryLimitOnAllocationsOfAtLeastAMegabyte(); + settings.read(in, test_case.format); + ADD_FAILURE() << "reading a declared length of " << declared_size << " did not throw"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::ATTEMPT_TO_READ_AFTER_EOF) << e.displayText(); + } + } +} + +GTEST_TEST(Settings, WellFormedSettingsStillReadUnderAClampedMemoryLimit) +{ + /// The in-range control: the clamp itself does not make a read fail, and the empty string end + /// marker still terminates the loop. + Settings sent; + sent.set("log_comment", "hello"); + sent.set("max_block_size", UInt64(4096)); + sent.set("totals_auto_threshold", 0.25); + + WriteBufferFromOwnString out; + sent.write(out, SettingsWriteFormat::STRINGS_WITH_FLAGS); + const std::string wire = out.str(); + + Settings settings; + ReadBufferFromString in(wire); + { + MemoryLimitGuard guard; + armMemoryLimitOnAllocationsOfAtLeastAMegabyte(); + ASSERT_NO_THROW(settings.read(in, SettingsWriteFormat::STRINGS_WITH_FLAGS)); + } + + ASSERT_EQ(settings.get("log_comment"), Field(String("hello"))); + ASSERT_EQ(settings.get("max_block_size"), Field(UInt64(4096))); + ASSERT_EQ(settings.get("totals_auto_threshold"), Field(0.25)); + ASSERT_TRUE(in.eof()); +} diff --git a/src/DataTypes/DataTypeMapHelpers.cpp b/src/DataTypes/DataTypeMapHelpers.cpp index 63aa451627f6..5443542a3be6 100644 --- a/src/DataTypes/DataTypeMapHelpers.cpp +++ b/src/DataTypes/DataTypeMapHelpers.cpp @@ -1,11 +1,15 @@ #include #include +#include #include +#include #include #include #include #include +#include +#include #include #include @@ -35,21 +39,30 @@ struct KeyMatcherGeneric bool match(size_t keys_row) const { - return keys_column.compareAt(keys_row, 0, key, 0) == 0; + /// The direction hint must not be zero. When exactly one side is a `NaN`, or a NULL of a + /// `Nullable` nested in the key, `compareAt` answers with the hint itself, so a zero hint + /// reports them as equal to every other value: a row holding such a key would match any + /// requested key, and requesting such a key would match any key. + return keys_column.compareAt(keys_row, 0, key, 1) == 0; } }; -/// Specialized key matcher for ColumnVector. Compares values directly -/// without virtual dispatch. -template +/// Specialized key matcher for columns holding a flat array of values (ColumnVector, +/// ColumnDecimal). Compares values directly without virtual dispatch. +template struct KeyMatcherVector { - const typename ColumnVector::Container & data; + using T = typename ColumnType::ValueType; + + const typename ColumnType::Container & data; T key_value; bool match(size_t keys_row) const { - return data[keys_row] == key_value; + /// `CompareHelper` is a plain `==` for every type but the floating point ones, where it + /// keeps the matcher in agreement with `KeyMatcherGeneric`: a `NaN` key is found by a + /// requested `NaN` and by nothing else. + return CompareHelper::equals(data[keys_row], key_value, 1); } }; @@ -86,6 +99,20 @@ struct KeyMatcherFixedString } }; +/// Specialized key matcher for ColumnLowCardinality. The requested key is resolved to its dictionary +/// position once, so a key is matched without comparing the key values. +template +struct KeyMatcherLowCardinality +{ + const IndexType * indexes; + IndexType key_index; + + bool match(size_t keys_row) const + { + return indexes[keys_row] == key_index; + } +}; + /// The core position-finding loop, parametrized by Matcher type. /// For each row in [start, end), finds the flat index of the matching key. /// @@ -126,6 +153,59 @@ void findKeyPositions( } } +/// The value of the single-row key column, if the key is a String or a FixedString, +/// possibly wrapped in LowCardinality. +std::optional tryGetStringKey(const IColumn & key) +{ + const IColumn * key_values = &key; + if (const auto * key_low_cardinality = typeid_cast(&key)) + key_values = key_low_cardinality->getDictionary().getNestedNotNullableColumn().get(); + + if (!typeid_cast(key_values) && !typeid_cast(key_values)) + return {}; + + return key.getDataAt(0); +} + +/// Finds the key positions by comparing dictionary positions when the keys are LowCardinality. +/// Returns false if the key column types are not supported, leaving it to the generic matcher. +bool tryFindKeyPositionsLowCardinality( + const IColumn & keys_column, + const ColumnArray::Offsets & offsets, + const IColumn & key, + size_t start, + size_t end, + PaddedPODArray & matched_positions) +{ + const auto * keys_low_cardinality = typeid_cast(&keys_column); + if (!keys_low_cardinality) + return false; + + auto key_value = tryGetStringKey(key); + if (!key_value) + return false; + + auto lookup_result = callWithLowCardinalityValueIndex( + *keys_low_cardinality, + *key_value, + [&](const auto * indexes, auto key_index) + { + KeyMatcherLowCardinality matcher{indexes, key_index}; + findKeyPositions(offsets, matcher, start, end, matched_positions); + }); + + if (lookup_result == LowCardinalityValueLookupResult::Unsupported) + return false; + + if (lookup_result == LowCardinalityValueLookupResult::NotFound) + { + matched_positions.clear(); + matched_positions.resize_fill(end - start, KEY_NOT_FOUND); + } + + return true; +} + /// Dispatches to the appropriate specialized matcher based on the key column type, /// then calls findKeyPositions with that matcher. void findKeyPositionsDispatch( @@ -138,31 +218,47 @@ void findKeyPositionsDispatch( { TypeIndex type_id = keys_column.getDataType(); - /// Try ColumnVector specializations. switch (type_id) { -#define DISPATCH_VECTOR(T) \ +#define DISPATCH_COLUMN(T, ColType) \ case TypeIndex::T: \ { \ - using ColType = ColumnVector; \ const auto & typed_col = assert_cast(keys_column); \ const auto & key_col = assert_cast(key); \ - KeyMatcherVector matcher{typed_col.getData(), key_col.getData()[0]}; \ + KeyMatcherVector matcher{typed_col.getData(), key_col.getData()[0]}; \ findKeyPositions(offsets, matcher, start, end, matched_positions); \ return; \ } +#define DISPATCH_VECTOR(T) DISPATCH_COLUMN(T, ColumnVector) +#define DISPATCH_DECIMAL(T) DISPATCH_COLUMN(T, ColumnDecimal) DISPATCH_VECTOR(UInt8) DISPATCH_VECTOR(UInt16) DISPATCH_VECTOR(UInt32) DISPATCH_VECTOR(UInt64) + DISPATCH_VECTOR(UInt128) + DISPATCH_VECTOR(UInt256) DISPATCH_VECTOR(Int8) DISPATCH_VECTOR(Int16) DISPATCH_VECTOR(Int32) DISPATCH_VECTOR(Int64) + DISPATCH_VECTOR(Int128) + DISPATCH_VECTOR(Int256) + DISPATCH_VECTOR(BFloat16) DISPATCH_VECTOR(Float32) DISPATCH_VECTOR(Float64) + DISPATCH_VECTOR(UUID) + DISPATCH_VECTOR(IPv4) + DISPATCH_VECTOR(IPv6) + DISPATCH_DECIMAL(Decimal32) + DISPATCH_DECIMAL(Decimal64) + DISPATCH_DECIMAL(Decimal128) + DISPATCH_DECIMAL(Decimal256) + DISPATCH_DECIMAL(DateTime64) + DISPATCH_DECIMAL(Time64) +#undef DISPATCH_DECIMAL #undef DISPATCH_VECTOR +#undef DISPATCH_COLUMN case TypeIndex::String: { @@ -181,13 +277,19 @@ void findKeyPositionsDispatch( findKeyPositions(offsets, matcher, start, end, matched_positions); return; } - default: + case TypeIndex::LowCardinality: { - /// Fallback: generic matcher using virtual compareAt. - KeyMatcherGeneric matcher{keys_column, key}; - findKeyPositions(offsets, matcher, start, end, matched_positions); + if (tryFindKeyPositionsLowCardinality(keys_column, offsets, key, start, end, matched_positions)) + return; + break; } + default: + break; } + + /// Fallback: generic matcher using virtual compareAt. + KeyMatcherGeneric matcher{keys_column, key}; + findKeyPositions(offsets, matcher, start, end, matched_positions); } /// --------------------------------------------------------------------------- @@ -213,17 +315,20 @@ void extractValuesGeneric( } } -/// Specialized value extractor for ColumnVector, with optional Nullable support. +/// Specialized value extractor for columns holding a flat array of values (ColumnVector, +/// ColumnDecimal), with optional Nullable support. /// If src_null_map / dst_null_map are non-null, propagates null flags. /// For missing keys, inserts a default value and sets the null flag to 1. -template +template void extractValuesVector( - const ColumnVector & values_column, - ColumnVector & result, + const ColumnType & values_column, + ColumnType & result, const PaddedPODArray & matched_positions, const NullMap * src_null_map = nullptr, NullMap * dst_null_map = nullptr) { + using T = typename ColumnType::ValueType; + const auto & src_data = values_column.getData(); auto & dst_data = result.getData(); size_t old_size = dst_data.size(); @@ -374,28 +479,45 @@ void extractValuesDispatch( switch (type_id) { -#define DISPATCH_VECTOR(T) \ +#define DISPATCH_COLUMN(T, ColType) \ case TypeIndex::T: \ { \ - using ColType = ColumnVector; \ - extractValuesVector( \ + extractValuesVector( \ assert_cast(*data_column), \ assert_cast(*result_data_column), \ matched_positions, src_null_map, dst_null_map); \ return; \ } +#define DISPATCH_VECTOR(T) DISPATCH_COLUMN(T, ColumnVector) +#define DISPATCH_DECIMAL(T) DISPATCH_COLUMN(T, ColumnDecimal) DISPATCH_VECTOR(UInt8) DISPATCH_VECTOR(UInt16) DISPATCH_VECTOR(UInt32) DISPATCH_VECTOR(UInt64) + DISPATCH_VECTOR(UInt128) + DISPATCH_VECTOR(UInt256) DISPATCH_VECTOR(Int8) DISPATCH_VECTOR(Int16) DISPATCH_VECTOR(Int32) DISPATCH_VECTOR(Int64) + DISPATCH_VECTOR(Int128) + DISPATCH_VECTOR(Int256) + DISPATCH_VECTOR(BFloat16) DISPATCH_VECTOR(Float32) DISPATCH_VECTOR(Float64) + DISPATCH_VECTOR(UUID) + DISPATCH_VECTOR(IPv4) + DISPATCH_VECTOR(IPv6) + DISPATCH_DECIMAL(Decimal32) + DISPATCH_DECIMAL(Decimal64) + DISPATCH_DECIMAL(Decimal128) + DISPATCH_DECIMAL(Decimal256) + DISPATCH_DECIMAL(DateTime64) + DISPATCH_DECIMAL(Time64) +#undef DISPATCH_DECIMAL #undef DISPATCH_VECTOR +#undef DISPATCH_COLUMN case TypeIndex::String: { diff --git a/src/DataTypes/DataTypesBinaryEncoding.cpp b/src/DataTypes/DataTypesBinaryEncoding.cpp index 0fbfc72ed104..ad24dc392291 100644 --- a/src/DataTypes/DataTypesBinaryEncoding.cpp +++ b/src/DataTypes/DataTypesBinaryEncoding.cpp @@ -292,7 +292,7 @@ std::tuple decodeAggregateFunction(ReadB Array parameters; parameters.reserve(num_parameters); for (size_t i = 0; i != num_parameters; ++i) - parameters.push_back(decodeField(buf)); + parameters.push_back(decodeField(buf, complexity, max_complexity)); size_t num_arguments = 0; readVarUInt(num_arguments, buf); if (num_arguments > MAX_ARRAY_SIZE) diff --git a/src/DataTypes/Native.cpp b/src/DataTypes/Native.cpp index 3784b161efef..6c670eafe12b 100644 --- a/src/DataTypes/Native.cpp +++ b/src/DataTypes/Native.cpp @@ -311,6 +311,8 @@ llvm::Value * nativeCastWithDecimalScale( } if (from_w.isFloat32() || from_w.isFloat64()) { + /// A float source must not reach here: `fptosi` has no defined result outside the destination range. + chassert(false, "Float to Decimal must not be JIT-compiled"); /// Float → `Decimal`: multiply by `10^to_scale` in floating point first, /// then truncate to the target integer storage type. if (to_scale == 0) diff --git a/src/DataTypes/Serializations/SerializationDate.cpp b/src/DataTypes/Serializations/SerializationDate.cpp index 0552baf4fd04..8b33f626b671 100644 --- a/src/DataTypes/Serializations/SerializationDate.cpp +++ b/src/DataTypes/Serializations/SerializationDate.cpp @@ -38,26 +38,26 @@ void SerializationDate::deserializeWholeText(IColumn & column, ReadBuffer & istr throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date"); } -bool SerializationDate::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - if (!tryReadDateText(x, istr, time_zone) || !istr.eof()) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !istr.eof()) return false; assert_cast(column).getData().push_back(x); return true; } -void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assert_cast(column).getData().push_back(x); } -bool SerializationDate::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - if (!tryReadDateText(x, istr, time_zone)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow())) return false; assert_cast(column).getData().push_back(x); return true; @@ -75,19 +75,19 @@ void SerializationDate::serializeTextQuoted(const IColumn & column, size_t row_n writeChar('\'', ostr); } -void SerializationDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; assertChar('\'', istr); - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assertChar('\'', istr); assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } -bool SerializationDate::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum x; - if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone) || !checkChar('\'', istr)) + if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !checkChar('\'', istr)) return false; assert_cast(column).getData().push_back(x); @@ -109,7 +109,7 @@ void SerializationDate::deserializeTextJSON(IColumn & column, ReadBuffer & istr, return; } DayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !format_settings.throwOnDateTimeOverflow()); assertChar('"', istr); assert_cast(column).getData().push_back(x); } @@ -120,7 +120,7 @@ bool SerializationDate::tryDeserializeTextJSON(IColumn & column, ReadBuffer & is return SerializationNumber::tryDeserializeTextJSON(column, istr, format_settings); DayNum x; - if (!tryReadDateText(x, istr, time_zone) || !checkChar('"', istr)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !format_settings.throwOnDateTimeOverflow()) || !checkChar('"', istr)) return false; assert_cast(column).getData().push_back(x); return true; @@ -133,17 +133,17 @@ void SerializationDate::serializeTextCSV(const IColumn & column, size_t row_num, writeChar('"', ostr); } -void SerializationDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum value; - readCSV(value, istr, time_zone); + readCSV(value, istr, time_zone, !settings.throwOnDateTimeOverflow()); assert_cast(column).getData().push_back(value); } -bool SerializationDate::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { DayNum value; - if (!tryReadCSV(value, istr, time_zone)) + if (!tryReadCSV(value, istr, time_zone, !settings.throwOnDateTimeOverflow())) return false; assert_cast(column).getData().push_back(value); return true; diff --git a/src/DataTypes/Serializations/SerializationDate32.cpp b/src/DataTypes/Serializations/SerializationDate32.cpp index 0f17f1de8779..085bbf971765 100644 --- a/src/DataTypes/Serializations/SerializationDate32.cpp +++ b/src/DataTypes/Serializations/SerializationDate32.cpp @@ -10,6 +10,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_PARSE_DATE; +} + UInt128 SerializationDate32::getHash(const DateLUTImpl & time_zone_) { SipHash hash; @@ -37,26 +42,26 @@ void SerializationDate32::deserializeWholeText(IColumn & column, ReadBuffer & is throwUnexpectedDataAfterParsedValue(column, istr, settings, "Date32"); } -bool SerializationDate32::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeWholeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - if (!tryReadDateText(x, istr, time_zone) || !istr.eof()) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !istr.eof()) return false; assert_cast(column).getData().push_back(x); return true; } -void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assert_cast(column).getData().push_back(x); } -bool SerializationDate32::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextEscaped(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - if (!tryReadDateText(x, istr, time_zone)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow())) return false; assert_cast(column).getData().push_back(x); return true; @@ -74,19 +79,19 @@ void SerializationDate32::serializeTextQuoted(const IColumn & column, size_t row writeChar('\'', ostr); } -void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; assertChar('\'', istr); - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !settings.throwOnDateTimeOverflow()); assertChar('\'', istr); assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. } -bool SerializationDate32::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextQuoted(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { ExtendedDayNum x; - if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone) || !checkChar('\'', istr)) + if (!checkChar('\'', istr) || !tryReadDateText(x, istr, time_zone, nullptr, !settings.throwOnDateTimeOverflow()) || !checkChar('\'', istr)) return false; assert_cast(column).getData().push_back(x); /// It's important to do this at the end - for exception safety. return true; @@ -107,7 +112,7 @@ void SerializationDate32::deserializeTextJSON(IColumn & column, ReadBuffer & ist return; } ExtendedDayNum x; - readDateText(x, istr, time_zone); + readDateText(x, istr, time_zone, !format_settings.throwOnDateTimeOverflow()); assertChar('"', istr); assert_cast(column).getData().push_back(x); } @@ -118,7 +123,7 @@ bool SerializationDate32::tryDeserializeTextJSON(IColumn & column, ReadBuffer & return SerializationNumber::tryDeserializeTextJSON(column, istr, format_settings); ExtendedDayNum x; - if (!tryReadDateText(x, istr, time_zone) || !checkChar('"', istr)) + if (!tryReadDateText(x, istr, time_zone, nullptr, !format_settings.throwOnDateTimeOverflow()) || !checkChar('"', istr)) return false; assert_cast(column).getData().push_back(x); return true; @@ -131,18 +136,25 @@ void SerializationDate32::serializeTextCSV(const IColumn & column, size_t row_nu writeChar('"', ostr); } -void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +void SerializationDate32::deserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { LocalDate value; readCSV(value, istr); + /// This one goes through `LocalDate`, which accepts a calendar-invalid date and resolves it to a default + if (settings.throwOnDateTimeOverflow() + && !tryToMakeDayNum(time_zone, value.year(), value.month(), value.day())) + throw Exception(ErrorCodes::CANNOT_PARSE_DATE, "Cannot parse date"); assert_cast(column).getData().push_back(value.getExtenedDayNum()); } -bool SerializationDate32::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings &) const +bool SerializationDate32::tryDeserializeTextCSV(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const { LocalDate value; if (!tryReadCSV(value, istr)) return false; + if (settings.throwOnDateTimeOverflow() + && !tryToMakeDayNum(time_zone, value.year(), value.month(), value.day())) + return false; assert_cast(column).getData().push_back(value.getExtenedDayNum()); return true; } diff --git a/src/DataTypes/Serializations/SerializationDateTime.cpp b/src/DataTypes/Serializations/SerializationDateTime.cpp index 042807705667..16df898076f8 100644 --- a/src/DataTypes/Serializations/SerializationDateTime.cpp +++ b/src/DataTypes/Serializations/SerializationDateTime.cpp @@ -43,16 +43,19 @@ namespace inline void readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { + const auto overflow = settings.throwOnDateTimeOverflow() + ? DateTimeOverflow::Report + : DateTimeOverflow::Saturate; switch (settings.date_time_input_format) { case FormatSettings::DateTimeInputFormat::Basic: - readDateTimeTextImpl<>(x, istr, time_zone); + readDateTimeTextImpl<>(x, istr, time_zone, nullptr, nullptr, overflow == DateTimeOverflow::Saturate); break; case FormatSettings::DateTimeInputFormat::BestEffort: - parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone); + parseDateTimeBestEffort(x, istr, time_zone, utc_time_zone, overflow); break; case FormatSettings::DateTimeInputFormat::BestEffortUS: - parseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone); + parseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone, overflow); break; } @@ -62,17 +65,20 @@ readText(time_t & x, ReadBuffer & istr, const FormatSettings & settings, const D inline bool tryReadText( time_t & x, ReadBuffer & istr, const FormatSettings & settings, const DateLUTImpl & time_zone, const DateLUTImpl & utc_time_zone) { + const auto overflow = settings.throwOnDateTimeOverflow() + ? DateTimeOverflow::Report + : DateTimeOverflow::Saturate; bool res = false; switch (settings.date_time_input_format) { case FormatSettings::DateTimeInputFormat::Basic: - res = tryReadDateTimeText(x, istr, time_zone); + res = tryReadDateTimeText(x, istr, time_zone, nullptr, nullptr, overflow == DateTimeOverflow::Saturate); break; case FormatSettings::DateTimeInputFormat::BestEffort: - res = tryParseDateTimeBestEffort(x, istr, time_zone, utc_time_zone); + res = tryParseDateTimeBestEffort(x, istr, time_zone, utc_time_zone, overflow); break; case FormatSettings::DateTimeInputFormat::BestEffortUS: - res = tryParseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone); + res = tryParseDateTimeBestEffortUS(x, istr, time_zone, utc_time_zone, overflow); break; } @@ -181,11 +187,11 @@ void SerializationDateTime::deserializeTextQuoted(IColumn & column, ReadBuffer & } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - readDateTimeAsRawValue(x, istr); + readDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow()); } else /// Just 1504193808 or 1703363853.5 (a Unix timestamp, possibly with a sub-second part) { - readDateTimeAsNumber(x, istr); + readDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow()); } /// It's important to do this at the end - for exception safety. @@ -202,12 +208,12 @@ bool SerializationDateTime::tryDeserializeTextQuoted(IColumn & column, ReadBuffe } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - if (!tryReadDateTimeAsRawValue(x, istr)) + if (!tryReadDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow())) return false; } else /// Just 1504193808 or 1703363853.5 (a Unix timestamp, possibly with a sub-second part) { - if (!tryReadDateTimeAsNumber(x, istr)) + if (!tryReadDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow())) return false; } @@ -234,11 +240,11 @@ void SerializationDateTime::deserializeTextJSON(IColumn & column, ReadBuffer & i } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - readDateTimeAsRawValue(x, istr); + readDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow()); } else { - readDateTimeAsNumber(x, istr); + readDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow()); } assert_cast(column).getData().push_back(static_cast(x)); @@ -254,12 +260,12 @@ bool SerializationDateTime::tryDeserializeTextJSON(IColumn & column, ReadBuffer } else if (settings.read_datetime_number_as_raw_value) /// Legacy: the raw value (seconds). { - if (!tryReadDateTimeAsRawValue(x, istr)) + if (!tryReadDateTimeAsRawValue(x, istr, !settings.throwOnDateTimeOverflow())) return false; } else { - if (!tryReadDateTimeAsNumber(x, istr)) + if (!tryReadDateTimeAsNumber(x, istr, !settings.throwOnDateTimeOverflow())) return false; } diff --git a/src/DataTypes/Serializations/SerializationDynamic.cpp b/src/DataTypes/Serializations/SerializationDynamic.cpp index 210627550921..c7ab75fecf62 100644 --- a/src/DataTypes/Serializations/SerializationDynamic.cpp +++ b/src/DataTypes/Serializations/SerializationDynamic.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -21,6 +22,7 @@ #include #include +#include namespace DB { @@ -34,15 +36,7 @@ namespace ErrorCodes namespace { -/// `num_types` is the length of a list of Dynamic's nested types read from a (possibly untrusted, -/// e.g. Native format) stream. It must not be handed to `reserve` directly: a count the container -/// cannot hold escapes as an uncaught `std::length_error` instead of a `DB::Exception`, and a -/// large-but-representable count (e.g. `100000000`, far below `max_size()`) would drive a huge -/// up-front allocation and fail as `std::bad_alloc` / OOM before a single type is read. Reject the -/// first as corruption, and cap the `reserve` hint for the second: `reserve` is only a sizing hint, -/// so the caller's read loop still appends each type as it is decoded (growing the container on -/// demand for a legitimately large count), while a corrupted over-count trips a normal read error -/// at end of stream instead of a huge allocation. +/// The count is untrusted, so use it only as a capped hint; the caller appends types as it reads them. template void reserveOrThrowTooManyTypes(Container & container, size_t num_types) { @@ -78,7 +72,7 @@ struct SerializeBinaryBulkStateDynamic : public ISerialization::SerializeBinaryB /// For flattened serialization only. std::optional flattened_column; - std::vector flattened_states; + VectorWithMemoryTracking flattened_states; ISerialization::SerializeBinaryBulkStatePtr flattened_indexes_state; explicit SerializeBinaryBulkStateDynamic(SerializationDynamic::SerializationVersion structure_version_) @@ -94,7 +88,7 @@ struct DeserializeBinaryBulkStateDynamic : public ISerialization::DeserializeBin ISerialization::DeserializeBinaryBulkStatePtr structure_state; /// For flattened serialization only. - std::vector flattened_states; + VectorWithMemoryTracking flattened_states; ISerialization::DeserializeBinaryBulkStatePtr flattened_indexes_state; ISerialization::DeserializeBinaryBulkStatePtr clone() const override @@ -152,6 +146,23 @@ void SerializationDynamic::SerializationVersion::checkVersion(UInt64 version) throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Dynamic structure serialization: {}", version); } +void SerializationDynamic::SerializationVersion::checkVersion(UInt64 version, bool native_format) +{ + checkVersion(version); + + if (native_format && version == V3) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Version {} of Dynamic structure serialization is written only into MergeTree data parts and is not allowed in Native format", + version); + + if (!native_format && version == FLATTENED) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Version {} of Dynamic structure serialization is written only in Native format and is not allowed in MergeTree data part", + version); +} + SerializationDynamic::SerializationVersion::SerializationVersion(MergeTreeDynamicSerializationVersion version) { switch (version) @@ -378,6 +389,7 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD /// Read structure serialization version. UInt64 structure_version = 0; readBinaryLittleEndian(structure_version, *structure_stream); + SerializationVersion::checkVersion(structure_version, settings.native_format); auto structure_state = std::make_shared(structure_version); if (structure_state->structure_version.value == SerializationVersion::FLATTENED) { @@ -386,17 +398,30 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD readVarUInt(num_types, *structure_stream); reserveOrThrowTooManyTypes(structure_state->flattened_data_types, num_types); String data_type_name; + std::unordered_set type_names; for (size_t i = 0; i != num_types; ++i) { + DataTypePtr data_type; if (settings.native_format && settings.format_settings && settings.format_settings->native.decode_types_in_binary_format) { - structure_state->flattened_data_types.push_back(decodeDataType(*structure_stream, settings.format_settings->binary.max_binary_type_complexity)); + data_type = decodeDataType(*structure_stream, settings.format_settings->binary.max_binary_type_complexity); } else { readStringBinary(data_type_name, *structure_stream); - structure_state->flattened_data_types.push_back(getDataTypesCache().getType(data_type_name)); + data_type = getDataTypesCache().getType(data_type_name); } + + /// Nothing is not stored as a variant, so such a type would have no discriminator to unflatten into. + if (isNothing(data_type)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Type Nothing is not allowed in the list of types of a flattened Dynamic column"); + + /// Duplicates would map two different indexes onto the same variant discriminator, + /// which makes the offsets of that variant inconsistent with its size. + if (!type_names.insert(data_type->getName()).second) + throw Exception(ErrorCodes::INCORRECT_DATA, "Duplicate type {} in the list of types of a flattened Dynamic column", data_type->getName()); + + structure_state->flattened_data_types.push_back(std::move(data_type)); } structure_state->flattened_indexes_type = getSmallestIndexesType(num_types + 1); /// +1 for NULL index. @@ -412,9 +437,7 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationDynamic::deserializeD /// Read information about variants. DataTypes variants; readVarUInt(structure_state->num_dynamic_types, *structure_stream); - /// A `Dynamic` column can have at most `ColumnDynamic::MAX_DYNAMIC_TYPES_LIMIT` regular variants. - /// Check this before doing the `+ 1` below: for a corrupted count equal to `SIZE_MAX`, - /// `num_dynamic_types + 1` would wrap around to `0` and defeat the check entirely. + /// Check before the `+ 1` below, which would wrap a corrupted `SIZE_MAX` count to `0`. if (structure_state->num_dynamic_types > ColumnDynamic::MAX_DYNAMIC_TYPES_LIMIT) throw Exception(ErrorCodes::INCORRECT_DATA, "Dynamic column has too many types: {}", structure_state->num_dynamic_types); /// +1 for shared variant. @@ -645,6 +668,13 @@ void SerializationDynamic::deserializeBinaryBulkWithMultipleStreams( /// First, read indexes. auto indexes_serialization = flattened_column.indexes_type->getDefaultSerialization(); indexes_serialization->deserializeBinaryBulkWithMultipleStreams(*mutable_indexes_column, limit, settings, dynamic_state->flattened_indexes_state, cache); + if (mutable_indexes_column->size() != limit) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Mismatch in flattened Dynamic column: {} rows are expected, but the indexes stream contains only {} rows", + limit, + mutable_indexes_column->size()); + flattened_column.indexes_column = std::move(mutable_indexes_column); /// Second, read data of all flattened types in corresponding order. auto flattened_limits = getLimitsForFlattenedDynamicColumn(*flattened_column.indexes_column, flattened_column.types.size()); diff --git a/src/DataTypes/Serializations/SerializationDynamic.h b/src/DataTypes/Serializations/SerializationDynamic.h index 5f6875cb2a07..953b84c278e3 100644 --- a/src/DataTypes/Serializations/SerializationDynamic.h +++ b/src/DataTypes/Serializations/SerializationDynamic.h @@ -56,6 +56,8 @@ class SerializationDynamic final : public ISerialization Value value; static void checkVersion(UInt64 version); + /// Each version is written into one channel only, so reading data of the other channel means the data is corrupted. + static void checkVersion(UInt64 version, bool native_format); explicit SerializationVersion(UInt64 version); explicit SerializationVersion(MergeTreeDynamicSerializationVersion version); diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 46eabe5e14c1..a2304073915e 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -8,6 +8,7 @@ #include +#include #include #include @@ -35,19 +36,7 @@ namespace ErrorCodes namespace { -/// The number of paths in a `JSON` / `Object` column is read from a possibly-untrusted stream -/// (e.g. `Native` input, or the statistics of a corrupted on-disk part) and used only as a sizing -/// hint before the actual paths are read one by one. It must not be handed to a container's -/// `resize` / `reserve` directly: -/// * A count the container cannot hold (`> max_size()`, e.g. close to `SIZE_MAX`) escapes as an -/// uncaught non-`DB::Exception` (`std::length_error`, `std::bad_array_new_length` or, for a -/// hash table, `std::bad_alloc`), so reject it as corruption up front. -/// * A large-but-representable count (e.g. `100000000`) is far below `max_size()` for a -/// `std::vector`, yet handing it to `resize` / `reserve` would allocate gigabytes -/// before a single path byte is read and fail as `std::bad_alloc` / OOM. -/// So cap the hint at `DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS`: the caller's read loop appends each -/// path as it is decoded (growing the container on demand for a legitimately large count), while a -/// corrupted over-count trips a normal read error at end of stream instead of a huge allocation. +/// The count is untrusted, so use it only as a capped hint; the caller appends paths as it reads them. template void reserveOrThrowTooManyPaths(Container & container, size_t num_paths) { @@ -56,14 +45,6 @@ void reserveOrThrowTooManyPaths(Container & container, size_t num_paths) container.reserve(std::min(num_paths, DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS)); } -/// `shared_data_buckets` in a V3 `Object` prefix is another raw count from a possibly-untrusted -/// stream, later used to size per-bucket state vectors and `Columns`. Unlike the path / type counts, -/// this one has a tight writer-side invariant: the number of buckets is chosen from the small -/// MergeTree settings `object_shared_data_buckets_for_{compact,wide}_part`, which are non-zero and -/// capped at `MAX_OBJECT_SHARED_DATA_BUCKETS`. So the only legitimate on-wire range is -/// `1 .. MAX_OBJECT_SHARED_DATA_BUCKETS`; any value outside it (including a large-but-representable -/// count such as `100000`, which the generic `Native` column-count cap would let through) can only -/// be corruption and must be rejected up front, before it is used to size the per-bucket state. void throwIfInvalidNumberOfBuckets(size_t num_buckets) { if (num_buckets == 0 || num_buckets > MAX_OBJECT_SHARED_DATA_BUCKETS) @@ -145,6 +126,23 @@ void SerializationObject::SerializationVersion::checkVersion(UInt64 version) throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid version for Object structure serialization: {}", version); } +void SerializationObject::SerializationVersion::checkVersion(UInt64 version, bool native_format) +{ + checkVersion(version); + + if (native_format && version == V3) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Version {} of Object structure serialization is written only into MergeTree data parts and is not allowed in Native format", + version); + + if (!native_format && (version == STRING || version == FLATTENED)) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Version {} of Object structure serialization is written only in Native format and is not allowed in MergeTree data part", + version); +} + struct SerializeBinaryBulkStateObject: public ISerialization::SerializeBinaryBulkState { SerializationObject::SerializationVersion serialization_version; @@ -481,6 +479,16 @@ void SerializationObject::serializeBinaryBulkStatePrefix( state = std::move(object_state); } +void SerializationObject::checkPathIsNotTyped(const String & path, bool native_format) const +{ + /// Such a path would be stored in the column twice, giving the object two values under one key. + if (typed_paths_serializations.contains(path)) + throw Exception( + native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, + "Path {} of an Object column is stored as a dynamic path, but it is a typed path in the type of the column", + path); +} + void SerializationObject::deserializeBinaryBulkStatePrefix( DeserializeBinaryBulkSettings & settings, DeserializeBinaryBulkStatePtr & state, @@ -500,6 +508,18 @@ void SerializationObject::deserializeBinaryBulkStatePrefix( return; } + /// Safety check for a corrupted data part or a malformed Native block. + if (structure_state_concrete->serialization_version.value == SerializationVersion::FLATTENED) + { + for (const auto & path : structure_state_concrete->flattened_paths) + checkPathIsNotTyped(path, settings.native_format); + } + else + { + for (const auto & path : *structure_state_concrete->sorted_dynamic_paths) + checkPathIsNotTyped(path, settings.native_format); + } + settings.path.push_back(Substream::ObjectData); /// Call callback for newly discovered dynamic subcolumns if needed. @@ -734,12 +754,11 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb /// Read structure serialization version. UInt64 serialization_version = 0; readBinaryLittleEndian(serialization_version, *structure_stream); + SerializationVersion::checkVersion(serialization_version, settings.native_format); auto structure_state = std::make_shared(serialization_version); if (structure_state->serialization_version.value == SerializationVersion::FLATTENED) { - /// Read the list of flattened paths. Append one path at a time (with a capped `reserve` - /// hint) rather than pre-sizing to the untrusted `paths_size`, so a corrupted count - /// cannot drive a huge allocation before any path is read (see `reserveOrThrowTooManyPaths`). + /// Read the list of flattened paths. size_t paths_size = 0; readVarUInt(paths_size, *structure_stream); reserveOrThrowTooManyPaths(structure_state->flattened_paths, paths_size); @@ -749,6 +768,17 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb readStringBinary(path, *structure_stream); structure_state->flattened_paths.push_back(std::move(path)); } + + /// The same path twice would be added to the column twice, as a dynamic path and into shared + /// data. The views are taken after the list is complete: the capped `reserve` above means the + /// vector can still grow while it is read, which would invalidate views of short (SSO) paths. + std::unordered_set unique_paths; + unique_paths.reserve(structure_state->flattened_paths.size()); + for (const auto & path : structure_state->flattened_paths) + { + if (!unique_paths.insert(path).second) + throw Exception(ErrorCodes::INCORRECT_DATA, "Duplicate path {} in the list of paths of a flattened Object column", path); + } } else if (structure_state->serialization_version.value == SerializationVersion::STRING) { @@ -763,7 +793,7 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb readVarUInt(max_dynamic_paths, *structure_stream); } - /// Read the sorted list of dynamic paths (same append-on-demand handling as flattened paths). + /// Read the sorted list of dynamic paths. size_t dynamic_paths_size = 0; readVarUInt(dynamic_paths_size, *structure_stream); structure_state->sorted_dynamic_paths = std::make_shared>(); @@ -775,6 +805,11 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObject::deserializeOb structure_state->sorted_dynamic_paths->push_back(std::move(path)); } structure_state->dynamic_paths.insert(structure_state->sorted_dynamic_paths->begin(), structure_state->sorted_dynamic_paths->end()); + /// A duplicate would leave the column with fewer dynamic paths than the data has. + if (structure_state->dynamic_paths.size() != structure_state->sorted_dynamic_paths->size()) + throw Exception( + settings.native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, + "Duplicate path in the list of dynamic paths of an Object column"); /// If we have V3 Object serialization, read shared data serialization version. if (structure_state->serialization_version.value == SerializationVersion::V3) @@ -1065,6 +1100,8 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams( if (structure_state->serialization_version.value == SerializationVersion::FLATTENED) { + const size_t prev_num_rows = column_object.size(); + settings.path.push_back(Substream::ObjectData); for (const auto & path : sorted_typed_paths) { @@ -1072,6 +1109,14 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams( settings.path.back().object_path_name = path; typed_paths_serializations.at(path)->deserializeBinaryBulkWithMultipleStreams(*typed_paths[path], limit, settings, object_state->typed_path_states[path], cache); settings.path.pop_back(); + + if (typed_paths[path]->size() != prev_num_rows + limit) + throw Exception( + settings.native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, + "Unexpected size of typed path {} in flattened Object column: {}. Expected size {}", + path, + typed_paths[path]->size() - prev_num_rows, + limit); } MutableColumns flattened_paths_columns; @@ -1083,6 +1128,14 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams( flattened_paths_columns.emplace_back(dynamic_type->createColumn()); dynamic_serialization->deserializeBinaryBulkWithMultipleStreams(*flattened_paths_columns.back(), limit, settings, object_state->dynamic_path_states[path], cache); settings.path.pop_back(); + + if (flattened_paths_columns.back()->size() != limit) + throw Exception( + settings.native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, + "Unexpected size of flattened path {} in flattened Object column: {}. Expected size {}", + path, + flattened_paths_columns.back()->size(), + limit); } settings.path.pop_back(); diff --git a/src/DataTypes/Serializations/SerializationObject.h b/src/DataTypes/Serializations/SerializationObject.h index 110140bd0c45..f8d911b09f28 100644 --- a/src/DataTypes/Serializations/SerializationObject.h +++ b/src/DataTypes/Serializations/SerializationObject.h @@ -60,6 +60,8 @@ class SerializationObject : public ISerialization Value value; static void checkVersion(UInt64 version); + /// Each version is written into one channel only, so reading data of the other channel means the data is corrupted. + static void checkVersion(UInt64 version, bool native_format); explicit SerializationVersion(UInt64 version); explicit SerializationVersion(MergeTreeObjectSerializationVersion version); @@ -140,7 +142,7 @@ class SerializationObject : public ISerialization ColumnObject::StatisticsPtr statistics; /// For flattened serialization only. - std::vector flattened_paths; + VectorWithMemoryTracking flattened_paths; explicit DeserializeBinaryBulkStateObjectStructure(UInt64 serialization_version_) : serialization_version(serialization_version_) @@ -181,6 +183,8 @@ class SerializationObject : public ISerialization SerializationPtr dynamic_serialization; private: + void checkPathIsNotTyped(const String & path, bool native_format) const; + std::vector sorted_typed_paths; }; diff --git a/src/DataTypes/Serializations/SerializationObjectDistinctPaths.cpp b/src/DataTypes/Serializations/SerializationObjectDistinctPaths.cpp index 6db21419b282..21883c742112 100644 --- a/src/DataTypes/Serializations/SerializationObjectDistinctPaths.cpp +++ b/src/DataTypes/Serializations/SerializationObjectDistinctPaths.cpp @@ -9,6 +9,7 @@ namespace DB namespace ErrorCodes { + extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; } @@ -279,11 +280,21 @@ void SerializationObjectDistinctPaths::deserializeBinaryBulkWithMultipleStreams( if (bucket == 0) num_new_rows = bucket_shared_data_paths_column->size(); + /// All buckets store the same rows, and the number of rows of the first one is used as + /// the number of rows of the result. + else if (bucket_shared_data_paths_column->size() != num_new_rows) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Bucket {} of Object shared data has {} rows, but bucket 0 has {} rows", + bucket, + bucket_shared_data_paths_column->size(), + num_new_rows); } break; } case SerializationObjectSharedData::SerializationVersion::ADVANCED: { + std::shared_ptr first_bucket_structure_granules; for (size_t bucket = 0; bucket < object_structure_state->shared_data_buckets; ++bucket) { settings.path.push_back(Substream::Bucket); @@ -291,6 +302,11 @@ void SerializationObjectDistinctPaths::deserializeBinaryBulkWithMultipleStreams( auto * shared_data_structure_state = checkAndGetState(object_distinct_paths_state->bucket_shared_data_structure_states[bucket]); auto structure_granules = SerializationObjectSharedData::deserializeStructure(limit, settings, *shared_data_structure_state, cache); + if (bucket == 0) + first_bucket_structure_granules = structure_granules; + else + SerializationObjectSharedData::checkGranulesMatchFirstBucket(*structure_granules, *first_bucket_structure_granules, bucket); + for (const auto & structure_granule : *structure_granules) { for (const auto & path : structure_granule.all_paths) @@ -305,8 +321,12 @@ void SerializationObjectDistinctPaths::deserializeBinaryBulkWithMultipleStreams( } } - array_column.getOffsets().push_back(paths_column.size()); - array_column.insertManyDefaults(num_new_rows - 1); + /// The streams may return no rows at all, and then there is no first row to hold the paths. + if (num_new_rows != 0) + { + array_column.getOffsets().push_back(paths_column.size()); + array_column.insertManyDefaults(num_new_rows - 1); + } settings.path.pop_back(); settings.path.pop_back(); diff --git a/src/DataTypes/Serializations/SerializationObjectHelpers.cpp b/src/DataTypes/Serializations/SerializationObjectHelpers.cpp index d544170f0abd..570ef54337e6 100644 --- a/src/DataTypes/Serializations/SerializationObjectHelpers.cpp +++ b/src/DataTypes/Serializations/SerializationObjectHelpers.cpp @@ -25,7 +25,8 @@ std::vector> flattenPaths(const ColumnObj return all_paths; } -void unflattenAndInsertPaths(const std::vector & flattened_paths, MutableColumns && flattened_columns, ColumnObject & object_column, size_t num_rows) +template +void unflattenAndInsertPaths(const PathsContainer & flattened_paths, MutableColumns && flattened_columns, ColumnObject & object_column, size_t num_rows) { /// Iterate over paths and try to add them to dynamic paths until the limit is reached. /// All remaining paths will be inserted into shared data. @@ -53,6 +54,12 @@ void unflattenAndInsertPaths(const std::vector & flattened_paths, Mutabl } } +template void unflattenAndInsertPaths>( + const std::vector &, MutableColumns &&, ColumnObject &, size_t); + +template void unflattenAndInsertPaths>( + const VectorWithMemoryTracking &, MutableColumns &&, ColumnObject &, size_t); + size_t getSharedDataPathBucket(std::string_view path, size_t num_buckets) { /// Do not change the hash function here, it will break paths @@ -193,6 +200,18 @@ void collectSharedDataFromBuckets(const Columns & shared_data_buckets, IColumn & std::tie(shared_data_paths_buckets[i], shared_data_values_buckets[i], shared_data_offsets_buckets[i]) = ColumnObject::getSharedDataPathsValuesAndOffsets(*shared_data_buckets[i]); size_t num_rows = shared_data_buckets[0]->size(); + /// Every row is collected from all buckets at once, so a shorter bucket would be indexed out of bounds. + for (size_t bucket = 1; bucket != shared_data_buckets.size(); ++bucket) + { + if (shared_data_buckets[bucket]->size() != num_rows) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Bucket {} of Object shared data has {} rows, but bucket 0 has {} rows", + bucket, + shared_data_buckets[bucket]->size(), + num_rows); + } + for (size_t i = 0; i != num_rows; ++i) { /// Shared data contains paths in sorted order in each row. @@ -254,7 +273,7 @@ ColumnPtr createPathsIndexesImpl(const std::unordered_map -void deserializeIndexesAndCollectPathsImpl(ColumnString & paths_column, ReadBuffer & istr, std::vector && paths, size_t limit) +void deserializeIndexesAndCollectPathsImpl(ColumnString & paths_column, ReadBuffer & istr, VectorWithMemoryTracking && paths, size_t limit) { auto & data = paths_column.getChars(); auto & offsets = paths_column.getOffsets(); @@ -263,11 +282,10 @@ void deserializeIndexesAndCollectPathsImpl(ColumnString & paths_column, ReadBuff /// Avoiding calling resize in a loop improves the performance. data.resize(std::max(data.capacity(), static_cast(4096))); + /// The number of indexes comes from the offsets, so stopping at the end of the stream would leave the + /// paths column shorter than the offsets say and it would be read out of bounds. for (size_t i = 0; i != limit; ++i) { - if (istr.eof()) - break; - T index; readBinaryLittleEndian(index, istr); @@ -309,7 +327,7 @@ std::pair createPathsIndexes(const std::unordered_map && paths, size_t limit) +void deserializeIndexesAndCollectPaths(IColumn & paths_column, ReadBuffer & istr, VectorWithMemoryTracking && paths, size_t limit) { auto & paths_string_column = assert_cast(paths_column); auto indexes_type = getSmallestIndexesType(paths.size()); diff --git a/src/DataTypes/Serializations/SerializationObjectHelpers.h b/src/DataTypes/Serializations/SerializationObjectHelpers.h index 6e40fa117a4c..3779530cf761 100644 --- a/src/DataTypes/Serializations/SerializationObjectHelpers.h +++ b/src/DataTypes/Serializations/SerializationObjectHelpers.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -18,7 +19,9 @@ namespace DB std::vector> flattenPaths(const ColumnObject & object_column); /// Insert data from flattened representation of an Object column to a usual Object column. -void unflattenAndInsertPaths(const std::vector & flattened_paths, MutableColumns && flattened_columns, ColumnObject & object_column, size_t num_rows); +/// Callers hold the paths in different containers, so it is explicitly instantiated for each of them. +template +void unflattenAndInsertPaths(const PathsContainer & flattened_paths, MutableColumns && flattened_columns, ColumnObject & object_column, size_t num_rows); /// Get the bucket number for a specific path. size_t getSharedDataPathBucket(std::string_view path, size_t num_buckets); @@ -65,6 +68,6 @@ void collectSharedDataFromBuckets(const Columns & shared_data_buckets, IColumn & /// Create a column that will contain indexes of paths from paths_column column based on provided mapping path_to_index. std::pair createPathsIndexes(const std::unordered_map & path_to_index, const IColumn & paths_column, size_t start, size_t end); /// Deserialize up to limit indexes from the read buffer and collect corresponding paths to the paths_column. -void deserializeIndexesAndCollectPaths(IColumn & paths_column, ReadBuffer & istr, std::vector && paths, size_t limit); +void deserializeIndexesAndCollectPaths(IColumn & paths_column, ReadBuffer & istr, VectorWithMemoryTracking && paths, size_t limit); } diff --git a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp index 9a85415ff5c7..f7799e836374 100644 --- a/src/DataTypes/Serializations/SerializationObjectSharedData.cpp +++ b/src/DataTypes/Serializations/SerializationObjectSharedData.cpp @@ -13,12 +13,14 @@ #include #include #include +#include namespace DB { namespace ErrorCodes { + extern const int CANNOT_READ_ALL_DATA; extern const int LOGICAL_ERROR; extern const int INCORRECT_DATA; extern const int NOT_IMPLEMENTED; @@ -27,19 +29,7 @@ namespace ErrorCodes namespace { -/// A per-granule count (the number of paths, or the number of substreams of a path) is read from a -/// possibly-untrusted stream (e.g. a corrupted on-disk `Object` part) and used only as a sizing hint -/// before the corresponding items are read one by one. It must not be handed to a container's -/// `reserve` directly, for the same reasons as the outer path lists (see `reserveOrThrowTooManyPaths` -/// in `SerializationObject.cpp`): -/// * A count the container cannot hold (`> max_size()`, close to `SIZE_MAX`) would escape as an -/// uncaught non-`DB::Exception` (`std::length_error`), so reject it as corruption up front. -/// * A large-but-representable count (e.g. `100000000`) is far below `max_size()` for a -/// `std::vector`, yet handing it to `reserve` would allocate gigabytes before a single -/// byte of payload is read and fail as `std::bad_alloc` / OOM. -/// So cap the hint at `DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS`: the caller's read loop appends each -/// item as it is decoded (growing the container on demand for a legitimately large count), while a -/// corrupted over-count trips a normal read error at end of stream instead of a huge allocation. +/// The count is untrusted, so use it only as a capped hint; the caller appends items as it reads them. template void reserveOrThrowTooMany(Container & container, size_t count, const char * what) { @@ -644,6 +634,32 @@ ISerialization::DeserializeBinaryBulkStatePtr SerializationObjectSharedData::des return state; } +void SerializationObjectSharedData::checkGranulesMatchFirstBucket( + const StructureGranules & granules, const StructureGranules & first_bucket_granules, size_t bucket) +{ + if (granules.size() != first_bucket_granules.size()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Bucket {} of Object shared data has {} granules, but bucket 0 has {} granules", + bucket, + granules.size(), + first_bucket_granules.size()); + + for (size_t granule = 0; granule != granules.size(); ++granule) + { + if (granules[granule].limit != first_bucket_granules[granule].limit || granules[granule].offset != first_bucket_granules[granule].offset) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Granule {} of bucket {} of Object shared data has {} rows at offset {}, but the same granule of bucket 0 has {} rows at offset {}", + granule, + bucket, + granules[granule].limit, + granules[granule].offset, + first_bucket_granules[granule].limit, + first_bucket_granules[granule].offset); + } +} + void SerializationObjectSharedData::deserializeStructureGranulePrefix( ReadBuffer & buf, SerializationObjectSharedData::StructureGranule & structure_granule, @@ -1063,6 +1079,17 @@ std::shared_ptr SerializationO { auto subcolumn = subcolumns_infos[pos].type->createColumn(); subcolumns_substream_data[pos].serialization->deserializeBinaryBulkWithMultipleStreams(*subcolumn, structure_granule.num_rows, deserialization_settings, subcolumns_substream_data[pos].deserialize_state, &cache_for_subcolumns); + /// The callers read the rows of the granule out of this column, so a shorter one + /// would be read out of bounds. + if (subcolumn->size() != structure_granule.num_rows) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Unexpected size of subcolumn {} of path {} in Object shared data: {}. Expected size {}", + subcolumns_infos[pos].name, + requested_path, + subcolumn->size(), + structure_granule.num_rows); + paths_data_granule.paths_subcolumns_data[requested_path][subcolumns_infos[pos].name] = std::move(subcolumn); } } @@ -1075,6 +1102,14 @@ std::shared_ptr SerializationO auto dynamic_column = dynamic_type->createColumn(); dynamic_serialization->deserializeBinaryBulkStatePrefix(deserialization_settings, path_state, nullptr); dynamic_serialization->deserializeBinaryBulkWithMultipleStreams(*dynamic_column, structure_granule.num_rows, deserialization_settings, path_state, nullptr); + if (dynamic_column->size() != structure_granule.num_rows) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Unexpected size of path {} in Object shared data: {}. Expected size {}", + requested_path, + dynamic_column->size(), + structure_granule.num_rows); + paths_data_granule.paths_data[requested_path] = std::move(dynamic_column); } } @@ -1134,7 +1169,7 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( /// In Compact part we always read one whole granule, so we don't need to worry about reading data from multiple granules. if (settings.data_part_type == MergeTreeDataPartType::Compact) { - std::vector paths; + VectorWithMemoryTracking paths; /// Collect all paths stored in this granule in all buckets. for (size_t bucket = 0; bucket != buckets; ++bucket) @@ -1283,7 +1318,16 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( if (!values_stream) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty stream for shared data copy values"); + size_t values_size_before = values_column.size(); SerializationString::create()->deserializeBinaryBulk(values_column, *values_stream, nested_limit, 0); + /// The number of values comes from the offsets, so a shorter column would be read out of bounds. + if (values_column.size() != values_size_before + nested_limit) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Unexpected number of values in Object shared data: {}. Expected {}", + values_column.size() - values_size_before, + nested_limit); + settings.path.pop_back(); settings.path.pop_back(); @@ -1292,9 +1336,10 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( else { /// Collect list of paths from all buckets for each granule. - std::vector> granules_paths; + std::vector> granules_paths; /// Collect the number of rows to read for each granule. std::vector granules_limits; + std::shared_ptr first_bucket_structure_granules; for (size_t bucket = 0; bucket != buckets; ++bucket) { @@ -1310,11 +1355,16 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( /// Initialize granules_paths/granules_limits on first bucket. if (bucket == 0) { + first_bucket_structure_granules = structure_granules; granules_paths.resize(structure_granules->size()); granules_limits.reserve(structure_granules->size()); for (size_t granule = 0; granule != structure_granules->size(); ++granule) granules_limits.push_back((*structure_granules)[granule].limit); } + else + { + checkGranulesMatchFirstBucket(*structure_granules, *first_bucket_structure_granules, bucket); + } for (size_t granule = 0; granule != structure_granules->size(); ++granule) granules_paths[granule].insert(granules_paths[granule].end(), (*structure_granules)[granule].all_paths.begin(), (*structure_granules)[granule].all_paths.end()); @@ -1351,6 +1401,19 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( /// Each granule has its own set of indexes, we should deserialize them granule by granule. size_t offsets_current_granule_start = prev_offset_size; auto & offsets = shared_data_array_column.getOffsets(); + + /// The granules and the sizes stream describe the same rows: covering more would index the + /// offsets out of bounds, fewer would leave the paths short. Reported as a short read, like + /// a truncated elements stream in SerializationArray. + size_t num_granules_rows = std::accumulate(granules_limits.begin(), granules_limits.end(), size_t(0)); + size_t num_offsets_rows = offsets.size() - prev_offset_size; + if (num_granules_rows != num_offsets_rows) + throw Exception( + ErrorCodes::CANNOT_READ_ALL_DATA, + "Granules of Object shared data contain {} rows, but {} rows were read from the sizes stream", + num_granules_rows, + num_offsets_rows); + for (size_t granule = 0; granule != granules_paths.size(); ++granule) { /// Calculate how many index entries should be read for this granule. @@ -1370,7 +1433,19 @@ void SerializationObjectSharedData::deserializeBinaryBulkWithMultipleStreams( /// Read values. settings.path.push_back(Substream::ObjectSharedDataCopyValues); auto * values_stream = settings.getter(settings.path); + if (!values_stream) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty stream for object shared data copy values"); + + size_t values_size_before = values_column.size(); SerializationString::create()->deserializeBinaryBulk(values_column, *values_stream, nested_limit, 0); + /// The number of values comes from the offsets, so a shorter column would be read out of bounds. + if (values_column.size() != values_size_before + nested_limit) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Unexpected number of values in Object shared data: {}. Expected {}", + values_column.size() - values_size_before, + nested_limit); + settings.path.pop_back(); settings.path.pop_back(); diff --git a/src/DataTypes/Serializations/SerializationObjectSharedData.h b/src/DataTypes/Serializations/SerializationObjectSharedData.h index 70bc1921fcc7..e926a8ffe47c 100644 --- a/src/DataTypes/Serializations/SerializationObjectSharedData.h +++ b/src/DataTypes/Serializations/SerializationObjectSharedData.h @@ -3,6 +3,8 @@ #pragma clang diagnostic ignored "-Wdocumentation-html" #include +#include +#include #include #include @@ -134,7 +136,7 @@ class SerializationObjectSharedData final : public SimpleTextSerialization std::map position_to_requested_path; /// List of all paths stored in the granule. It is filled only if whole /// shared data is deserialized when we need the list of all paths. - std::vector all_paths; + VectorWithMemoryTracking all_paths; /// Number of rows in this granule. size_t num_rows = 0; /// How much rows should be read from this granule. Can be less than num_rows if we read only a part of the granule. @@ -217,6 +219,9 @@ class SerializationObjectSharedData final : public SimpleTextSerialization DeserializeBinaryBulkStateObjectSharedDataStructure & structure_state, SubstreamsCache * cache); + /// The readers apply the granule layout of the first bucket to all of them. + static void checkGranulesMatchFirstBucket(const StructureGranules & granules, const StructureGranules & first_bucket_granules, size_t bucket); + /// Deserialize prefix of the granule in ObjectSharedDataStructure(Prefix) stream that contains: /// - number of rows in the granule /// - list of all paths stored in the granule @@ -247,9 +252,9 @@ class SerializationObjectSharedData final : public SimpleTextSerialization /// Mark of the substreams marks in ObjectSharedDataSubstreamsMarks stream for this path. MarkInCompressedFile substreams_marks_mark{}; /// List of substreams for this path. - std::vector substreams; + VectorWithMemoryTracking substreams; /// Map Substream -> its mark in ObjectSharedDataData stream. - std::unordered_map substream_to_mark; + UnorderedMapWithMemoryTracking substream_to_mark; }; struct PathsInfos diff --git a/src/DataTypes/Serializations/SerializationSubObject.cpp b/src/DataTypes/Serializations/SerializationSubObject.cpp index e7312a6be33a..6fc89c3689d0 100644 --- a/src/DataTypes/Serializations/SerializationSubObject.cpp +++ b/src/DataTypes/Serializations/SerializationSubObject.cpp @@ -11,6 +11,8 @@ namespace DB namespace ErrorCodes { + extern const int INCORRECT_DATA; + extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; } @@ -267,6 +269,20 @@ void SerializationSubObject::deserializeBinaryBulkWithMultipleStreams( settings.path.pop_back(); settings.path.pop_back(); + + /// Verify that all typed paths, dynamic paths and shared data has consistent sizes + size_t expected_size = column_object.getSharedDataPtr()->size(); + for (const auto & [path, path_column] : typed_paths) + { + if (path_column->size() != expected_size) + throw Exception(settings.native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, "Unexpected size of typed path {}: {}. Expected size {}", path, path_column->size(), expected_size); + } + + for (const auto & [path, path_column] : dynamic_paths) + { + if (path_column->size() != expected_size) + throw Exception(settings.native_format ? ErrorCodes::INCORRECT_DATA : ErrorCodes::LOGICAL_ERROR, "Unexpected size of dynamic path {}: {}. Expected size {}", path, path_column->size(), expected_size); + } } size_t SerializationSubObject::allocatedBytes() const diff --git a/src/DataTypes/Serializations/SerializationSubObjectSharedData.cpp b/src/DataTypes/Serializations/SerializationSubObjectSharedData.cpp index 9c513b413e16..5d9cfcdb542b 100644 --- a/src/DataTypes/Serializations/SerializationSubObjectSharedData.cpp +++ b/src/DataTypes/Serializations/SerializationSubObjectSharedData.cpp @@ -310,6 +310,7 @@ void SerializationSubObjectSharedData::deserializeBinaryBulkWithMultipleStreams( std::vector> bucket_paths_data_granules(buckets); /// We need to remember offset and limit from each granule to know which rows to insert in the result. std::vector> granules_offset_and_limit; + std::shared_ptr first_bucket_structure_granules; for (size_t bucket = 0; bucket != buckets; ++bucket) { settings.path.push_back(Substream::Bucket); @@ -323,10 +324,16 @@ void SerializationSubObjectSharedData::deserializeBinaryBulkWithMultipleStreams( /// Init offset and limit for each granule if (bucket == 0) { + first_bucket_structure_granules = structure_granules; granules_offset_and_limit.reserve(structure_granules->size()); for (size_t granule = 0; granule != structure_granules->size(); ++granule) granules_offset_and_limit.emplace_back((*structure_granules)[granule].offset, (*structure_granules)[granule].limit); } + else + { + SerializationObjectSharedData::checkGranulesMatchFirstBucket(*structure_granules, *first_bucket_structure_granules, bucket); + } + settings.path.pop_back(); } diff --git a/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.cpp b/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.cpp index 949f5323e982..43a7eff68321 100644 --- a/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.cpp +++ b/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include namespace DB { @@ -9,15 +11,16 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +template std::vector getSubcolumnsDeserializationOrder( const String & column_name, const std::vector & subcolumns_data, - const std::vector & substreams_in_serialization_order, + const SubstreamsContainer & substreams_in_serialization_order, ISerialization::EnumerateStreamsSettings & enumerate_settings, const ISerialization::StreamFileNameSettings & stream_file_name_settings) { /// Create map (substream) -> (pos in serialization order). - std::unordered_map substream_to_pos; + UnorderedMapWithMemoryTracking substream_to_pos; substream_to_pos.reserve(substreams_in_serialization_order.size()); for (size_t i = 0; i != substreams_in_serialization_order.size(); ++i) substream_to_pos[substreams_in_serialization_order[i]] = i; @@ -68,5 +71,19 @@ std::vector getSubcolumnsDeserializationOrder( return subcolumns_positions; } +template std::vector getSubcolumnsDeserializationOrder>( + const String &, + const std::vector &, + const std::vector &, + ISerialization::EnumerateStreamsSettings &, + const ISerialization::StreamFileNameSettings &); + +template std::vector getSubcolumnsDeserializationOrder>( + const String &, + const std::vector &, + const VectorWithMemoryTracking &, + ISerialization::EnumerateStreamsSettings &, + const ISerialization::StreamFileNameSettings &); + } diff --git a/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.h b/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.h index 88eaa9cdd999..19a04bf89803 100644 --- a/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.h +++ b/src/DataTypes/Serializations/getSubcolumnsDeserializationOrder.h @@ -7,10 +7,13 @@ namespace DB /// When we deserialize several subcolumns of the same column it's always better to deserialize /// subcolumns in order of their serialization, so we can avoid seeks back in the data files. /// This function determines this order. +/// Explicitly instantiated in the .cpp for the two containers callers hold the substreams in: +/// `std::vector` and `VectorWithMemoryTracking`. +template std::vector getSubcolumnsDeserializationOrder( const String & column_name, const std::vector & subcolumns_data, - const std::vector & substreams_in_serialization_order, + const SubstreamsContainer & substreams_in_serialization_order, ISerialization::EnumerateStreamsSettings & enumerate_settings, const ISerialization::StreamFileNameSettings & stream_file_name_settings); diff --git a/src/Databases/DataLake/DataLakeConstants.h b/src/Databases/DataLake/DataLakeConstants.h index bc5c96ac093f..93156b46fd0e 100644 --- a/src/Databases/DataLake/DataLakeConstants.h +++ b/src/Databases/DataLake/DataLakeConstants.h @@ -29,11 +29,14 @@ static inline std::unordered_map SETTINGS_TO_HIDE = {"aws_access_key_id", DEFAULT_MASKING_RULE}, {"aws_secret_access_key", DEFAULT_MASKING_RULE}, {"aws_external_id", DEFAULT_MASKING_RULE}, + /// A trust policy can require a specific session name (`sts:RoleSessionName`), so it is a secret too. + {"aws_role_session_name", DEFAULT_MASKING_RULE}, /// Legacy storage_* aliases (declared in DataLakeStorageSettings.h, originally for the Glue catalog) {"storage_catalog_credential", DEFAULT_MASKING_RULE}, {"storage_auth_header", DEFAULT_MASKING_RULE}, {"storage_aws_access_key_id", DEFAULT_MASKING_RULE}, {"storage_aws_secret_access_key", DEFAULT_MASKING_RULE}, + {"storage_aws_role_session_name", DEFAULT_MASKING_RULE}, /// OneLake credentials {"onelake_client_secret", DEFAULT_MASKING_RULE}, {"onelake_bearer_token", DEFAULT_MASKING_RULE}, diff --git a/src/Databases/DataLake/DatabaseDataLake.cpp b/src/Databases/DataLake/DatabaseDataLake.cpp index 38c41f1e182e..6a276373d7ef 100644 --- a/src/Databases/DataLake/DatabaseDataLake.cpp +++ b/src/Databases/DataLake/DatabaseDataLake.cpp @@ -920,7 +920,7 @@ StoragePtr DatabaseDataLake::tryGetTableImpl(const String & name, ContextPtr con const auto is_secondary_query = context_->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY; /// When we applied static credentials from database settings, they are authoritative: - /// do not let a catalog-vended refresh callback (e.g. Unity/REST `requestReadCredentials`) + /// do not let a catalog-vended refresh callback (e.g. Unity/REST `requestCredentials`) /// silently re-fetch credentials and override them. The same holds when the user disabled /// `vended_credentials` and no static credentials were applied (e.g. relying on default or /// environment S3 auth): the object storage layer invokes the refresh callback after an diff --git a/src/Databases/DataLake/ICatalog.h b/src/Databases/DataLake/ICatalog.h index e79ebf686f68..6077a30586cc 100644 --- a/src/Databases/DataLake/ICatalog.h +++ b/src/Databases/DataLake/ICatalog.h @@ -286,6 +286,11 @@ class ICatalog return std::nullopt; } + virtual CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & /*storage_id*/) + { + return std::nullopt; + } + /// Result of `prepareSettingsChanges`: the new catalog state built off to the side, /// ready to be published by `commitSettingsChanges`. struct PreparedSettingsChanges diff --git a/src/Databases/DataLake/UnityCatalog.cpp b/src/Databases/DataLake/UnityCatalog.cpp index 2d78df3ec7d8..5f8ff8578902 100644 --- a/src/Databases/DataLake/UnityCatalog.cpp +++ b/src/Databases/DataLake/UnityCatalog.cpp @@ -1,5 +1,6 @@ #include #include +#include #if USE_PARQUET @@ -132,11 +133,11 @@ void UnityCatalog::getTableMetadata( throw DB::Exception(DB::ErrorCodes::DATALAKE_DATABASE_ERROR, "No response from unity catalog"); } -Poco::JSON::Object::Ptr UnityCatalog::requestReadCredentials(const String & table_id) const +Poco::JSON::Object::Ptr UnityCatalog::requestCredentials(const String & table_id, const String & operation) const { Poco::JSON::Object request_body; request_body.set("table_id", table_id); - request_body.set("operation", "READ"); + request_body.set("operation", operation); auto callback = [&request_body] (std::ostream & os) { request_body.stringify(os); }; auto [json, _] = postJSONRequest(TEMPORARY_CREDENTIALS_ENDPOINT, callback); @@ -172,7 +173,7 @@ void UnityCatalog::getCredentials(const String & table_id, TableMetadata & metad if (storage_type != StorageType::S3 && storage_type != StorageType::Azure) return; - auto response = requestReadCredentials(table_id); + auto response = requestCredentials(table_id, "READ"); std::shared_ptr creds; switch (storage_type) @@ -489,12 +490,28 @@ ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsConfigurationCa "Cannot build a Unity credentials refresh callback for `{}`: StorageID has no UUID", table_id.getNameForLogs()); - const String unity_table_id = toString(table_id.uuid); + return getCredentialsCallbackForOperation(DB::toString(table_id.uuid), "READ"); +} + +/// `StorageID::uuid` of a `DataLakeCatalog` table is the `table_id` returned by Unity. +ICatalog::CredentialsRefreshCallback UnityCatalog::getWriteCredentialsConfigurationCallback(const DB::StorageID & table_id) +{ + if (table_id.uuid == DB::UUIDHelpers::Nil) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "Cannot build a Unity credentials refresh callback for `{}`: the table has no UUID", + table_id.getNameForLogs()); + + return getCredentialsCallbackForOperation(DB::toString(table_id.uuid), "READ_WRITE"); +} - return [this, unity_table_id] () -> std::shared_ptr { - LOG_DEBUG(log, "Update credentials in the catalog"); +ICatalog::CredentialsRefreshCallback UnityCatalog::getCredentialsCallbackForOperation(const String & unity_table_id, const String & operation) +{ + return [this, unity_table_id, operation] () -> std::shared_ptr + { + LOG_DEBUG(log, "Update {} credentials in the catalog", operation); - return parseS3Credentials(requestReadCredentials(unity_table_id)); + return parseS3Credentials(requestCredentials(unity_table_id, operation)); }; } diff --git a/src/Databases/DataLake/UnityCatalog.h b/src/Databases/DataLake/UnityCatalog.h index 840822b0e0da..783d0fb2b018 100644 --- a/src/Databases/DataLake/UnityCatalog.h +++ b/src/Databases/DataLake/UnityCatalog.h @@ -66,7 +66,7 @@ class UnityCatalog final : public ICatalog, private DB::WithContext CatalogTables listTablesInNamespaceDirect(const std::string & namespace_name) const override; void getCredentials(const String & table_id, TableMetadata & metadata) const; - Poco::JSON::Object::Ptr requestReadCredentials(const String & table_id) const; + Poco::JSON::Object::Ptr requestCredentials(const String & table_id, const String & operation) const; std::shared_ptr parseS3Credentials(const Poco::JSON::Object::Ptr & response) const; std::shared_ptr parseAzureCredentials(const Poco::JSON::Object::Ptr & response) const; @@ -77,6 +77,10 @@ class UnityCatalog final : public ICatalog, private DB::WithContext TableMetadata & result) const; ICatalog::CredentialsRefreshCallback getCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + + ICatalog::CredentialsRefreshCallback getWriteCredentialsConfigurationCallback(const DB::StorageID & table_id) override; + + ICatalog::CredentialsRefreshCallback getCredentialsCallbackForOperation(const String & unity_table_id, const String & operation); }; } diff --git a/src/Databases/DatabaseBackup.cpp b/src/Databases/DatabaseBackup.cpp index 0fa7ae4fe9c7..97d063e5fa30 100644 --- a/src/Databases/DatabaseBackup.cpp +++ b/src/Databases/DatabaseBackup.cpp @@ -43,6 +43,7 @@ #include #include #include +#include namespace CurrentMetrics @@ -71,6 +72,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int INCORRECT_FILE_NAME; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; extern const int CANNOT_GET_CREATE_TABLE_QUERY; } @@ -464,8 +466,11 @@ ASTPtr DatabaseBackup::getCreateDatabaseQueryImpl() const { const auto & settings = getContext()->getSettingsRef(); + /// The locator is emitted as the function it is, not as a string literal holding its text: this + /// definition is what `ALTER DATABASE ... MODIFY COMMENT` writes back into the metadata file, and + /// the load path parses the second argument with `BackupInfo::fromAST`, which takes a function. const String query = fmt::format("CREATE DATABASE {} ENGINE = Backup({}, {})", - backQuoteIfNeed(database_name), quoteString(config.database_name), quoteString(config.backup_info.toString())); + backQuoteIfNeed(database_name), quoteString(config.database_name), config.backup_info.toString()); ParserCreateQuery parser; ASTPtr ast = parseQuery(parser, @@ -493,7 +498,7 @@ std::vector> DatabaseBackup::getTablesForBackup(co namespace { -DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr) +DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr, bool allow_locator_in_string_literal) { if (engine_args.size() != 2) throw Exception::createRuntime(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -501,7 +506,43 @@ DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr) DatabaseBackup::Configuration result; - result.database_name = checkAndGetLiteralArgument(engine_args[0], "database_name"); + /// `checkAndGetLiteralArgument` formats the argument it rejects, and a locator written in this + /// position would format its credentials in plaintext. + try + { + result.database_name = checkAndGetLiteralArgument(engine_args[0], "database_name"); + } + catch (const Exception &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument 'database_name' must be a string literal"); + } + + /** A locator held in a string literal (`Backup('db', 'File(\'backup.zip\')')`) is the form that + * metadata rewritten by an older server carries, so it has to keep loading - a server that cannot + * parse its own metadata does not start at all. + * + * Only there: in a statement a user writes the locator must be the function it is, because that is + * the form `FunctionSecretArgumentsFinder` knows how to redact, and a quoted one would carry its + * credentials verbatim into `query_log`, `SHOW PROCESSLIST` and the distributed DDL payload. + */ + if (allow_locator_in_string_literal) + { + if (const auto * locator = engine_args[1]->as(); locator && locator->value.getType() == Field::Types::String) + { + result.backup_info = BackupInfo::fromString(locator->value.safeGet()); + return result; + } + } + + /// A locator held in a string literal is exactly the text that can carry credentials, and a rejection + /// message reaches the error log and the `exception` column of `query_log`, so name the accepted form + /// here rather than the text given. + if (!engine_args[1]->as()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Expected function as the backup destination of a `Backup` database. It must be spelled as the " + "function it is, such as `File('backup')` or `S3(...)`. The text given is not shown, because a " + "destination held in a string literal cannot be redacted and may carry credentials"); + result.backup_info = BackupInfo::fromAST(*engine_args[1]); return result; @@ -511,11 +552,20 @@ DatabaseBackup::Configuration parseArguments(ASTs engine_args, ContextPtr) void DatabaseBackup::parseAndAuthorizeLocator(const ASTs & engine_args, ContextPtr query_context) { - /// A locator we cannot parse opens nothing: creation rejects it, so there is nothing to authorize. - if (engine_args.size() == 2 && !engine_args[1]->as()) - return; - - auto config = parseArguments(engine_args, query_context); + /** A locator that is not a function is refused right here rather than left to creation time. + * + * This preflight is the last point that still runs as the real user, and the creation it would + * otherwise rely on does not always run: `RESTORE DATABASE` issues `CREATE DATABASE IF NOT EXISTS`, + * which returns from `InterpreterCreateQuery::createDatabase` before `DatabaseFactory::get` when the + * target database already exists. With `allow_different_database_def = 1` the definition mismatch is + * waived as well, so a manifest an older server left holding a quoted locator would pass through the + * whole restore with its embedded source never authorized, while the function form of the same + * manifest is checked. + * + * Only `parseArguments` may formulate the refusal: the offending text can carry credentials and must + * not be echoed. + */ + auto config = parseArguments(engine_args, query_context, /*allow_locator_in_string_literal=*/ false); BackupFactory::instance().checkSourceAccess(config.backup_info, query_context, IBackup::OpenMode::READ); } @@ -531,13 +581,23 @@ void registerDatabaseBackup(DatabaseFactory & factory) if (engine->arguments) engine_args = engine->arguments->children; - auto config = parseArguments(engine_args, args.context); - /// Authorize only a newly introduced definition: one read back from this server's metadata was /// already validated, and a context with no user cannot be checked per user. + /// + /// Metadata is read back on three paths: the short `ATTACH DATABASE db`, a load under `force_restore_data`, + /// and the replay of the stored full `ATTACH DATABASE db ENGINE = Backup(...)` statement at server start. + /// The last one runs in plain `ATTACH` mode, so neither of the first two conditions covers it - and it is + /// exactly the path that has to load metadata an older server rewrote. + /// + /// The loader flag, not `internal`, is the discriminator: wrappers such as `PARALLEL WITH` run user + /// statements as internal ones, and a user's `ATTACH DATABASE ... ENGINE = Backup(...)` must neither + /// skip the source authorization nor get its locator accepted in the form the secret masker cannot redact. const bool has_real_user = args.context->getAccess()->getUserID().has_value(); + const bool is_internal_metadata_replay = args.is_metadata_replay && args.mode >= LoadingStrictnessLevel::ATTACH; const bool from_existing_metadata - = isLoadingFromExistingMetadata(args.mode) || args.create_query.attach_short_syntax; + = isLoadingFromExistingMetadata(args.mode) || args.create_query.attach_short_syntax || is_internal_metadata_replay; + + auto config = parseArguments(engine_args, args.context, /*allow_locator_in_string_literal=*/ from_existing_metadata); if (has_real_user && !from_existing_metadata) BackupFactory::instance().checkSourceAccess(config.backup_info, args.context, IBackup::OpenMode::READ); diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 7bb6c4ac0100..0f9ee0107929 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -33,12 +33,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include namespace fs = std::filesystem; @@ -438,6 +440,15 @@ void DatabaseOrdinary::loadTableFromMetadata( } } +/// Background work that deferring would cancel, or nothing to load, so these are never deferred. +static bool isEagerEngine(const String & engine_name) +{ + static const std::unordered_set eager_engines + = {"Distributed", "Buffer", "MaterializedPostgreSQL", "Merge", "Memory"}; + + return eager_engines.contains(engine_name); +} + bool DatabaseOrdinary::shouldLazyLoad(const ASTCreateQuery & query, LoadingStrictnessLevel mode) const { if (!database_metadata_disk_settings[DatabaseMetadataDiskSetting::lazy_load_tables]) @@ -452,6 +463,9 @@ bool DatabaseOrdinary::shouldLazyLoad(const ASTCreateQuery & query, LoadingStric if (query.is_time_series_table) return false; + if (query.storage && query.storage->engine && isEagerEngine(query.storage->engine->name)) + return false; + /// Already handled by `StorageTableFunctionProxy`. if (query.as_table_function) return false; @@ -534,7 +548,7 @@ LoadTaskPtr DatabaseOrdinary::loadTableFromMetadataAsync( void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr table, const QualifiedTableName & name) { - auto * rmt = table->as(); + auto rmt = castStorage(table, DeferredTable::Skip); if (!rmt) return; diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 19906928b8c8..d83f38975cca 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -2903,7 +2904,7 @@ bool DatabaseReplicated::shouldReplicateQuery(const ContextPtr & query_context, auto table_id = query_context->resolveStorageID(ast, Context::ResolveOrdinary); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, query_context); - return table->as() != nullptr; + return castStorage(table, DeferredTable::Load) != nullptr; }; const auto is_replicated_table = [&](const ASTPtr & ast) diff --git a/src/Databases/DatabaseURL.cpp b/src/Databases/DatabaseURL.cpp index b08e1332830f..36874c55942e 100644 --- a/src/Databases/DatabaseURL.cpp +++ b/src/Databases/DatabaseURL.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -42,26 +43,6 @@ namespace ErrorCodes namespace { -/// Check that the string starts with a valid RFC 3986 scheme followed by "://". -bool hasURLScheme(const String & url) -{ - auto scheme_end = url.find("://"); - if (scheme_end == String::npos || scheme_end == 0) - return false; - - if (!std::isalpha(static_cast(url[0]))) - return false; - - for (size_t i = 1; i < scheme_end; ++i) - { - char c = url[i]; - if (!std::isalnum(static_cast(c)) && c != '+' && c != '-' && c != '.') - return false; - } - - return true; -} - /// A table of a `URL` database is a thin wrapper over the `url` table function, which dispatches /// the URL to the matching backend (`file`, `s3`, `azureBlobStorage`, ...). struct URLTableDelegate @@ -151,6 +132,7 @@ class StorageURLDatabaseTable final : public StorageProxy } StoragePtr getNested() const override { return nested; } + StoragePtr tryGetNested() const override { return nested; } String getName() const override { return nested->getName(); } /// Engine classification is used by policy checks (e.g. the `disable_insertion_and_mutation` @@ -265,7 +247,7 @@ DatabaseURL::DatabaseURL(const String & name_, const String & base_url_, Context : IDatabase(name_), WithContext(context_->getGlobalContext()), base_url(base_url_) { /// Not echoed back: password masking anchors on the `://` this value lacks, so it would log the password. - if (!base_url.empty() && !hasURLScheme(base_url)) + if (!base_url.empty() && findURIAuthority(base_url) == String::npos) throw Exception(ErrorCodes::BAD_ARGUMENTS, "The base URL of a URL database must contain a scheme (e.g. https://)"); } @@ -274,7 +256,7 @@ String DatabaseURL::getTableURL(const String & name) const { String resolved = StorageURL::resolveURLBase(name, base_url, "base URL of the URL database"); - if (!hasURLScheme(resolved)) + if (findURIAuthority(resolved) == String::npos) return {}; return resolved; } diff --git a/src/Dictionaries/CassandraSource.cpp b/src/Dictionaries/CassandraSource.cpp index 83e7869ddeb6..8030b120b82c 100644 --- a/src/Dictionaries/CassandraSource.cpp +++ b/src/Dictionaries/CassandraSource.cpp @@ -25,7 +25,7 @@ CassandraSource::CassandraSource( const String & query_str, SharedHeader & sample_block, size_t max_block_size_) - : ISource(sample_block) + : ISource(std::make_shared(sample_block->cloneEmpty())) , session(session_) , statement(query_str.c_str(), /*parameters count*/ 0) , max_block_size(max_block_size_) diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index 248efbe0e5ac..c96a071d945d 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -20,6 +20,8 @@ namespace ErrorCodes namespace { + /// Holds one row per column, an attribute's row being its `null_value` default. A port header + /// must have no rows, so a source publishing this block as its header has to strip them first. Block createSampleBlock(const DictionaryStructure & dict_struct) { Block block; diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 684bd77f4c83..05d98fc6c70a 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -346,14 +346,14 @@ class SourceFromQueryPipeline : public ISource { public: explicit SourceFromQueryPipeline(QueryPipeline & pipeline_) - : ISource(pipeline_.getSharedHeader()) + : ISource(std::make_shared(pipeline_.getSharedHeader()->cloneEmpty())) , executor(pipeline_) { pipeline_.setConcurrencyControl(false); } explicit SourceFromQueryPipeline(BlockIO io) - : ISource(io.pipeline.getSharedHeader()) + : ISource(std::make_shared(io.pipeline.getSharedHeader()->cloneEmpty())) , io_holder(std::move(io)) , executor(io_holder->pipeline) { diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 3e70047a8906..6d7970523bef 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -413,6 +413,8 @@ Setting fields: | `value` | Value set for a specific identifier name. | When creating a dictionary using the DDL command (`CREATE DICTIONARY ...`) remote hosts for HTTP dictionaries are checked against the contents of `remote_url_allow_hosts` section from config to prevent database users to access arbitrary HTTP server. + +The headers, including their names, are shown as `HEADERS ('[HIDDEN]')` in the output of `SHOW CREATE DICTIONARY`, in `system.tables` and in the query logs, the same way as the password. As with the password, a query that cannot be parsed is logged as is, with only [`query_masking_rules`](/reference/settings/server-settings/settings/query#query_masking_rules) applied. To display the headers in `SHOW CREATE DICTIONARY` and `system.tables`, enable the server setting [`display_secrets_in_show_and_select`](/reference/settings/server-settings/settings/other#display_secrets_in_show_and_select) and the format setting [`format_display_secrets_in_show_and_select`](/reference/settings/formats/format#format_display_secrets_in_show_and_select); the user also needs the `displaySecretsInShowAndSelect` privilege. These settings do not affect the query logs. )DOCS_MD", .syntax = "SOURCE(HTTP(url 'https://host/path' format 'CSV'))", .related = {"file"}}); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 62965ab14b05..e18fdb935dee 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -88,6 +88,26 @@ static void checkNoSSLPaths(const Poco::Util::AbstractConfiguration & config, co key, contents_key); } } + +/// `enable_local_infile` sets `MYSQL_OPT_LOCAL_INFILE`, which lets the MySQL endpoint ask the client +/// for the contents of a file of its choosing, read with the server's own privileges: the option is +/// off by default because it is insecure (`mysqlxx/Connection.h`). Same reasoning as above. +/// `fallback_prefix` is the parent prefix a `` inherits the value from, resolved in the same +/// order as `Pool::Pool`, so what is checked is the value the connection will actually use. +static void checkNoLocalInfile( + const Poco::Util::AbstractConfiguration & config, + const std::string & prefix, + const std::string & fallback_prefix = {}) +{ + const bool inherited + = !fallback_prefix.empty() && config.getBool(fallback_prefix + ".enable_local_infile", false); + + if (config.getBool(prefix + ".enable_local_infile", inherited)) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`enable_local_infile` cannot be enabled in a dictionary created with a DDL query. " + "It is only accepted in a dictionary defined in a server configuration file"); +} #endif void registerDictionarySourceMysql(DictionarySourceFactory & factory); @@ -197,6 +217,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) { const auto replica_prefix = settings_config_prefix + "." + replica_key; checkNoSSLPaths(config, replica_prefix); + checkNoLocalInfile(config, replica_prefix, settings_config_prefix); global_context->getRemoteHostFilter().checkHostAndPort( config.getString(replica_prefix + ".host"), toString(config.getInt(replica_prefix + ".port", 3306))); @@ -205,6 +226,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) } else { + checkNoLocalInfile(config, settings_config_prefix); global_context->getRemoteHostFilter().checkHostAndPort( config.getString(settings_config_prefix + ".host"), toString(config.getInt(settings_config_prefix + ".port", 3306))); @@ -300,6 +322,7 @@ Setting fields: | `fail_on_connection_loss` | Controls behavior of the server on connection loss. If `true`, an exception is thrown immediately if the connection between client and server was lost. If `false`, the server retries to fetch data at least three times before reporting an error. Note that retrying leads to increased response times. Default value: `false`. | | `query` | The custom query. Optional. | | `enable_compression` | Enables zlib compression for the MySQL protocol connection. When set to `1`, ClickHouse requests protocol-level compression from the MySQL server. Can also be set per-replica inside ``. Default value: `0`. | +| `enable_local_infile` | Allows the MySQL server to ask ClickHouse for the contents of a local file (`LOAD DATA LOCAL INFILE`). Only a dictionary defined in a server configuration file may enable it; a `CREATE DICTIONARY` query may leave it unset or set it to `0`. Can also be set per-replica inside ``, and a `` that does not set it inherits the value above. Default value: `0`. | | `ssl_ca_pem` | Contents of the CA certificate that the MySQL server certificate is verified against. Optional. | | `ssl_cert_pem` | Contents of the client certificate, for certificate-based authentication. Optional. | | `ssl_key_pem` | Contents of the private key belonging to `ssl_cert_pem`. Optional. | @@ -313,6 +336,10 @@ The `table` or `where` fields cannot be used together with the `query` field. An `ssl_ca`, `ssl_cert` and `ssl_key` name files that the server opens with its own privileges, so they are only accepted for a dictionary defined in a server configuration file, or through a named collection defined there. A `CREATE DICTIONARY` query that specifies the TLS credentials directly must pass their contents instead, in `ssl_ca_pem`, `ssl_cert_pem` and `ssl_key_pem`. Those values are masked in logs and in `SHOW` queries, the same way passwords are. + +`enable_local_infile` lets the MySQL server ask ClickHouse for the contents of a file of its choosing, which the server reads with its own privileges, so it is only accepted for a dictionary defined in a server configuration file. A `CREATE DICTIONARY` query that enables it, at the source or at one of its `` entries, is rejected with `BAD_ARGUMENTS`. + + There is no explicit parameter `secure`. When establishing an SSL-connection security is mandatory. diff --git a/src/Dictionaries/NullDictionarySource.cpp b/src/Dictionaries/NullDictionarySource.cpp index c940c28e5ef7..beb30e8b8f8d 100644 --- a/src/Dictionaries/NullDictionarySource.cpp +++ b/src/Dictionaries/NullDictionarySource.cpp @@ -22,7 +22,7 @@ BlockIO NullDictionarySource::loadAll() { LOG_TRACE(getLogger("NullDictionarySource"), "loadAll {}", toString()); BlockIO io; - io.pipeline = QueryPipeline(std::make_shared(sample_block)); + io.pipeline = QueryPipeline(std::make_shared(std::make_shared(sample_block->cloneEmpty()))); return io; } diff --git a/src/Dictionaries/RedisSource.cpp b/src/Dictionaries/RedisSource.cpp index a606d9cc69cb..c2c38623dce3 100644 --- a/src/Dictionaries/RedisSource.cpp +++ b/src/Dictionaries/RedisSource.cpp @@ -26,7 +26,7 @@ namespace DB const RedisStorageType & storage_type_, SharedHeader sample_block, size_t max_block_size_) - : ISource(sample_block) + : ISource(std::make_shared(sample_block->cloneEmpty())) , connection(std::move(connection_)) , keys(keys_) , storage_type(storage_type_) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 21641d98d0b8..4e37f48d5039 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -54,6 +54,7 @@ namespace DB namespace ErrorCodes { extern const int AZURE_BLOB_STORAGE_ERROR; + extern const int AZURE_OBJECT_CHANGED_DURING_READ; extern const int UNSUPPORTED_METHOD; } @@ -249,6 +250,26 @@ std::unique_ptr AzureObjectStorage::readObject( /// NOLI blob_storage_log->local_path = object.local_path; } + /// An object whose size is known to be zero is read as empty without a single `Download` + /// request, so no `If-Match` condition pins that read to the generation the caller has seen. + /// A blob listed as empty and replaced with a non-empty one since must still be rejected rather + /// than returned as a clean empty file, so its generation is checked on the properties instead. + if (object.bytes_size == 0 && !object.etag.empty()) + getObjectMetadataOfListedGeneration(object); + + /// The size of the object recorded in the metadata is a locally known bound, so it is used + /// as the right bound of the read rather than trusting the length of whatever the endpoint + /// answers with. `UnknownSize` is the only sentinel for "the size was never determined": + /// it is the default-constructed value, so a caller that does not know the size leaves it + /// there, and `bytes_size == 0` means a genuinely empty object, which must read as empty + /// rather than unbounded. + /// `S3ObjectStorage::readObject` maps `0` to "no bound" as well, because + /// `ReadBufferFromS3::read_until_position` is a plain `size_t` whose `0` already means + /// "unbounded" and therefore cannot express the empty range at all. + const std::optional known_size = object.bytes_size != StoredObject::UnknownSize + ? std::optional(object.bytes_size) + : std::nullopt; + return std::make_unique( client.get(), object.remote_path, @@ -257,9 +278,18 @@ std::unique_ptr AzureObjectStorage::readObject( /// NOLI settings_ptr->max_single_download_retries, use_external_buffer, restrict_seek, - /* read_until_position */0, + /* read_until_position */ known_size, std::move(blob_storage_log), - connection_params.getContainer()); + connection_params.getContainer(), + /// The size is only a correct bound for the generation of the object it was recorded for, + /// so the read is pinned to that generation when the caller has recorded its `ETag`, and + /// a replaced object is rejected instead of being truncated to the stale size. + object.etag, + /// The same size is what the buffer reports as the size of the file, so that a wrapper + /// that sizes itself by `getFileSize` (`CachedInMemoryReadBufferFromFile` does, and treats + /// an earlier end of the file as corruption) sees the bound the read has, and not the size + /// of whatever generation a live `GetProperties` request would meet. + /* file_size */ known_size); } SmallObjectDataWithMetadata AzureObjectStorage::readSmallObjectAndGetObjectMetadata( /// NOLINT @@ -268,6 +298,17 @@ SmallObjectDataWithMetadata AzureObjectStorage::readSmallObjectAndGetObjectMetad size_t max_size_bytes, std::optional read_hint) const { + /// An object whose size is known to be zero is read as empty without issuing a single + /// `Download` request, because `readObject` bounds the read by that size. There is then no + /// response to take the metadata from, so it is requested explicitly, and the check that + /// `readObject` performs on the `ETag` of the response is made on the properties instead. + if (object.bytes_size == 0) + { + SmallObjectDataWithMetadata result; + result.metadata = getObjectMetadataOfListedGeneration(object); + return result; + } + auto buffer = readObject(object, read_settings, read_hint); SmallObjectDataWithMetadata result; WriteBufferFromString out(result.data); @@ -582,6 +623,22 @@ void AzureObjectStorage::tagObjects(const StoredObjects & objects, const std::st setAzureBlobTag(client_ptr, blob_names, tag_key, tag_value); } +ObjectMetadata AzureObjectStorage::getObjectMetadataOfListedGeneration(const StoredObject & object) const +{ + ObjectMetadata metadata = getObjectMetadata(object.remote_path, /* with_tags */ false); + + /// A listing spells the `ETag` without the quotes that the `ETag` header of the properties has. + const String listed_etag = ReadBufferFromAzureBlobStorage::quotedETag(object.etag); + const String current_etag = ReadBufferFromAzureBlobStorage::quotedETag(metadata.etag); + if (!listed_etag.empty() && current_etag != listed_etag) + throw Exception( + ErrorCodes::AZURE_OBJECT_CHANGED_DURING_READ, + "Azure blob {} was replaced during read (ETag changed from {} to {}); retry the query, or set azure_validate_etag_on_read=0 to disable this check for table reads", + object.remote_path, listed_etag, current_etag); + + return metadata; +} + ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path, bool) const { auto client_ptr = client.get(); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h index 88adfe903284..e741b238d7a6 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h @@ -130,6 +130,12 @@ class AzureObjectStorage : public IObjectStorage } private: + /// The properties of `object` as they are now, checked against the generation the caller has + /// seen: throws `AZURE_OBJECT_CHANGED_DURING_READ` when `object.etag` is set and names another + /// generation. Used for an object whose size is known to be zero, which is read without a + /// single `Download` request, so that no `If-Match` condition pins it. + ObjectMetadata getObjectMetadataOfListedGeneration(const StoredObject & object) const; + void removeObjectImpl( const StoredObject & object, const std::shared_ptr & client_ptr, diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h index 8c0a5c2858fb..d176c3b94f0f 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/IObjectStorage.h @@ -451,6 +451,8 @@ class IObjectStorage /// `ReadBufferFromS3` / `getObjectMetadata` error handlers that normally invoke this. virtual bool tryRefreshCredentialsViaCallback() { return false; } + virtual bool hasCredentialsRefreshCallback() const { return false; } + #if USE_AZURE_BLOB_STORAGE || USE_AWS_S3 /// Assign tag on objects virtual void tagObjects(const StoredObjects &, const std::string &, const std::string &) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h index ab26f5e998d0..5339d5b3908e 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/S3/S3ObjectStorage.h @@ -40,7 +40,7 @@ class S3ObjectStorage : public IObjectStorage ObjectStorageKeyGeneratorPtr key_generator_, const String & disk_name_, bool for_disk_s3_ = true, - const S3CredentialsRefreshCallback & credentials_refresh_callback_ = [] -> std::unique_ptr{ return nullptr; }, + const S3CredentialsRefreshCallback & credentials_refresh_callback_ = {}, bool client_restricts_server_credentials_ = true) : uri(uri_) , disk_name(disk_name_) @@ -155,6 +155,7 @@ class S3ObjectStorage : public IObjectStorage std::shared_ptr tryGetS3StorageClient() override; bool tryRefreshCredentialsViaCallback() override; + bool hasCredentialsRefreshCallback() const override { return static_cast(credentials_refresh_callback); } S3::URI getURI() const { return uri; } S3Settings getS3Settings() const { return *s3_settings.get(); } diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index 884a1243aea2..5a4c13c7c676 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -1131,7 +1131,8 @@ bool CachedOnDiskReadBufferFromFile::predownloadForFileSegment( info.cache_settings.reserve_space_wait_lock_timeout_milliseconds, failure_reason, /* reserve_stat */nullptr, - reserve_hint); + reserve_hint, + &info.reserve_ahead); if (continue_predownload) { @@ -1634,7 +1635,8 @@ size_t CachedOnDiskReadBufferFromFile::readFromFileSegment( info.cache_settings.reserve_space_wait_lock_timeout_milliseconds, failure_reason, /* reserve_stat */nullptr, - reserve_hint); + reserve_hint, + &info.reserve_ahead); if (success) { @@ -2064,6 +2066,7 @@ off_t CachedOnDiskReadBufferFromFile::seek(off_t offset, int whence) first_offset = file_offset_of_buffer_end = new_pos; info.reset(); + info.reserve_ahead.reset(); state.reset(); initialized = false; diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h index ee80a207acea..7aff6baf5228 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.h @@ -129,6 +129,8 @@ class CachedOnDiskReadBufferFromFile : public ReadBufferFromFileBase, public IRe /// List of file segments which we need to read /// given initial [start_offset, read_until_position). FileSegmentsHolderPtr file_segments; + /// Kept across file segments and setReadUntilPosition(), reset on seek. + FileCacheReserveAhead reserve_ahead; void reset(); }; diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp index 1c036c9ef357..538ebe5025b0 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.cpp @@ -197,7 +197,9 @@ bool FileSegmentRangeWriter::write(char * data, size_t size, size_t offset, File } size_t size_to_write = std::min(available_size, size); - bool reserved = file_segment->reserve(size_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason); + bool reserved = file_segment->reserve( + size_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason, + /* reserve_stat */nullptr, /* reserve_hint */std::nullopt, &reserve_ahead); if (!reserved) { appendFilesystemCacheLog(*file_segment); @@ -423,6 +425,7 @@ void FileSegmentRangeWriter::jumpToPosition(size_t position) expected_write_offset = position; ignore_bytes = 0; + reserve_ahead.reset(); } CachedOnDiskWriteBufferFromFile::CachedOnDiskWriteBufferFromFile( diff --git a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h index e07609a9df39..b9c7f342eae7 100644 --- a/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h +++ b/src/Disks/IO/CachedOnDiskWriteBufferFromFile.h @@ -82,6 +82,8 @@ class FileSegmentRangeWriter size_t expected_write_offset = 0; + FileCacheReserveAhead reserve_ahead; + bool finalized = false; }; diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 9cbed6ada133..e8bd82ffe962 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -15,6 +15,8 @@ #include #include +#include + namespace ProfileEvents { @@ -37,6 +39,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int CANNOT_ALLOCATE_MEMORY; extern const int NOT_INITIALIZED; + extern const int AZURE_OBJECT_CHANGED_DURING_READ; } ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage( @@ -47,9 +50,11 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage( size_t max_single_download_retries_, bool use_external_buffer_, bool restricted_seek_, - size_t read_until_position_, + std::optional read_until_position_, BlobStorageLogWriterPtr blob_storage_log_, - String container_for_logging_) + String container_for_logging_, + String expected_etag_, + std::optional file_size_) : ReadBufferFromFileBase() , blob_container_client(blob_container_client_) , path(path_) @@ -60,10 +65,13 @@ ReadBufferFromAzureBlobStorage::ReadBufferFromAzureBlobStorage( , use_external_buffer(use_external_buffer_) , restricted_seek(restricted_seek_) , read_until_position(read_until_position_) + , expected_etag(quotedETag(std::move(expected_etag_))) , last_object_metadata(std::make_unique>()) , blob_storage_log(std::move(blob_storage_log_)) , container_for_logging(std::move(container_for_logging_)) { + file_size = file_size_; + if (!use_external_buffer) { tmp_buffer.resize(tmp_buffer_size); @@ -76,7 +84,7 @@ void ReadBufferFromAzureBlobStorage::setReadUntilEnd() { if (read_until_position) { - read_until_position = 0; + read_until_position.reset(); if (initialized) { offset = getPosition(); @@ -88,7 +96,22 @@ void ReadBufferFromAzureBlobStorage::setReadUntilEnd() void ReadBufferFromAzureBlobStorage::setReadUntilPosition(size_t position) { + if (read_until_position == position) + return; + read_until_position = position; + + /// The bytes that the previous bound allowed to buffer are already in `working_buffer`, and + /// some of them can be past the new bound. Give them up and let the next `nextImpl` download + /// the range again, so that the new bound takes effect immediately, as required by + /// `supportsRightBoundedReads`. `offset` is rewound to the position the caller has read up to, + /// because the buffered bytes after it are no longer handed out. + if (!working_buffer.empty()) + { + offset = getPosition(); + resetWorkingBuffer(); + } + initialized = false; } @@ -96,11 +119,15 @@ bool ReadBufferFromAzureBlobStorage::nextImpl() { if (read_until_position) { - if (read_until_position == offset) + if (*read_until_position == static_cast(offset)) return false; - if (read_until_position < offset) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to read beyond right offset ({} > {})", offset, read_until_position - 1); + if (*read_until_position < static_cast(offset)) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Attempt to read beyond right offset ({} > {})", + offset, + static_cast(*read_until_position) - 1); } if (!initialized) @@ -183,7 +210,7 @@ off_t ReadBufferFromAzureBlobStorage::seek(off_t offset_, int whence) ErrorCodes::CANNOT_SEEK_THROUGH_FILE, "Seek is allowed only before first read attempt from the buffer (current offset: " "{}, new offset: {}, reading until position: {}, available: {})", - getPosition(), offset_, read_until_position, available()); + getPosition(), offset_, read_until_position ? std::to_string(*read_until_position) : "none", available()); } if (whence != SEEK_SET) @@ -238,10 +265,11 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) Azure::Storage::Blobs::DownloadBlobOptions download_options; Azure::Nullable length {}; - if (read_until_position != 0) - length = {static_cast(read_until_position - offset)}; + if (read_until_position) + length = {static_cast(*read_until_position - offset)}; download_options.Range = {static_cast(offset), length}; + setAccessConditions(download_options); Azure::Core::Context azure_context = Azure::Core::Context().WithValue(PocoAzureHTTPClient::getSDKContextKeyForBufferRetry(), attempt); @@ -263,6 +291,7 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) ProfileEvents::increment(ProfileEvents::DiskAzureGetObject); auto download_response = blob_client->Download(download_options, azure_context); + checkReturnedGeneration(download_response.Value.Details); setMetadataFromResponse(download_response.Value.Details, download_response.Value.BlobSize); data_stream = std::move(download_response.Value.BodyStream); @@ -293,6 +322,7 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) ProfileEvents::increment(ProfileEvents::ReadBufferFromAzureRequestsErrors); LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); + rethrowIfGenerationChanged(e); if (i + 1 == max_single_download_retries || !isRetryableAzureException(e)) throw; @@ -313,8 +343,10 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) ProfileEvents::increment(ProfileEvents::ReadBufferFromAzureRequestsErrors); LOG_DEBUG(log, "Exception caught during Azure Download for file {} at attempt {}/{}: {}", path, i + 1, max_single_download_retries, getCurrentExceptionMessage(false)); - /// It doesn't make sense to retry allocator errors - if (getCurrentExceptionCode() == ErrorCodes::CANNOT_ALLOCATE_MEMORY) + /// It doesn't make sense to retry allocator errors, and a blob that has been replaced + /// stays replaced. + if (getCurrentExceptionCode() == ErrorCodes::CANNOT_ALLOCATE_MEMORY + || getCurrentExceptionCode() == ErrorCodes::AZURE_OBJECT_CHANGED_DURING_READ) throw; if (i + 1 == max_single_download_retries) @@ -328,11 +360,71 @@ void ReadBufferFromAzureBlobStorage::initialize(size_t attempt) if (data_stream == nullptr) throw Exception(ErrorCodes::RECEIVED_EMPTY_DATA, "Null data stream obtained while downloading file {} from Blob Storage", path); - total_size = data_stream->Length() + offset; + total_size = getTotalSizeOfCurrentDownload(data_stream->Length(), offset, read_until_position); initialized = true; } +size_t ReadBufferFromAzureBlobStorage::getTotalSizeOfCurrentDownload(int64_t reported_length, off_t offset_, std::optional read_until_position_) +{ + /// `reported_length` is the `Content-Length` of the response, which is chosen by the remote + /// endpoint: an endpoint that answers a ranged request with more data than was requested must + /// not be able to push bytes past the right bound into the caller. A negative value means that + /// the length of the response is unknown. + /// + /// The size of the blob from the same response is not used to bound it, because it comes from + /// the same untrusted place; only `read_until_position`, which is set locally by the caller, + /// is a trustworthy bound. + size_t total = reported_length >= 0 + ? static_cast(offset_) + static_cast(reported_length) + : std::numeric_limits::max(); + + if (read_until_position_) + total = std::min(total, *read_until_position_); + + return total; +} + +String ReadBufferFromAzureBlobStorage::quotedETag(String etag) +{ + if (etag.empty()) + return etag; + + if (etag.size() >= 2 && etag.front() == '"' && etag.back() == '"') + return etag; + + return "\"" + etag + "\""; +} + +void ReadBufferFromAzureBlobStorage::setAccessConditions(Azure::Storage::Blobs::DownloadBlobOptions & download_options) const +{ + if (!expected_etag.empty()) + download_options.AccessConditions.IfMatch = Azure::ETag(expected_etag); +} + +void ReadBufferFromAzureBlobStorage::checkReturnedGeneration(const Azure::Storage::Blobs::Models::DownloadBlobDetails & details) const +{ + /// A response without an `ETag` cannot be checked, the same as in `ReadBufferFromS3`. + if (expected_etag.empty() || !details.ETag.HasValue()) + return; + + const String returned_etag = quotedETag(details.ETag.ToString()); + if (returned_etag != expected_etag) + throw Exception( + ErrorCodes::AZURE_OBJECT_CHANGED_DURING_READ, + "Azure blob {} was replaced during read (ETag changed from {} to {}); retry the query, or set azure_validate_etag_on_read=0 to disable this check for table reads", + path, expected_etag, returned_etag); +} + +void ReadBufferFromAzureBlobStorage::rethrowIfGenerationChanged(const Azure::Core::RequestFailedException & e) const +{ + if (!expected_etag.empty() && e.StatusCode == Azure::Core::Http::HttpStatusCode::PreconditionFailed) + throw Exception( + ErrorCodes::AZURE_OBJECT_CHANGED_DURING_READ, + "Azure blob {} was replaced during read (If-Match on ETag {} failed); retry the query, or set azure_validate_etag_on_read=0 to disable this check for table reads", + path, expected_etag); +} + std::optional ReadBufferFromAzureBlobStorage::tryGetFileSize() { if (!blob_client) @@ -375,9 +467,12 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran Azure::Storage::Blobs::DownloadBlobOptions download_options; download_options.Range = {static_cast(range_begin), n}; + setAccessConditions(download_options); Azure::Core::Context azure_context = Azure::Core::Context().WithValue(PocoAzureHTTPClient::getSDKContextKeyForBufferRetry(), size_t{0}); auto download_response = blob_client->Download(download_options, azure_context); + checkReturnedGeneration(download_response.Value.Details); + if (blob_storage_log) { blob_storage_log->addEvent( @@ -413,6 +508,7 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran ProfileEvents::increment(ProfileEvents::ReadBufferFromAzureRequestsErrors); LOG_DEBUG(log, "Exception caught during Azure Download for file {} at offset {} at attempt {}/{}: {}", path, offset, i + 1, max_single_download_retries, e.Message); + rethrowIfGenerationChanged(e); if (i + 1 == max_single_download_retries || !isRetryableAzureException(e)) throw; @@ -433,8 +529,10 @@ size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t ran ProfileEvents::increment(ProfileEvents::ReadBufferFromAzureRequestsErrors); LOG_DEBUG(log, "Exception caught during Azure Download for file {} at attempt {}/{}: {}", path, i + 1, max_single_download_retries, getCurrentExceptionMessage(false)); - /// It doesn't make sense to retry allocator errors - if (getCurrentExceptionCode() == ErrorCodes::CANNOT_ALLOCATE_MEMORY) + /// It doesn't make sense to retry allocator errors, and a blob that has been replaced + /// stays replaced. + if (getCurrentExceptionCode() == ErrorCodes::CANNOT_ALLOCATE_MEMORY + || getCurrentExceptionCode() == ErrorCodes::AZURE_OBJECT_CHANGED_DURING_READ) throw; if (i + 1 == max_single_download_retries) diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h index cc0a75e02c58..a7da6b4f5712 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h @@ -32,9 +32,22 @@ class ReadBufferFromAzureBlobStorage : public ReadBufferFromFileBase size_t max_single_download_retries_, bool use_external_buffer_ = false, bool restricted_seek_ = false, - size_t read_until_position_ = 0, + std::optional read_until_position_ = {}, BlobStorageLogWriterPtr blob_storage_log_ = {}, - String container_for_logging_ = {}); + String container_for_logging_ = {}, + String expected_etag_ = {}, + /// The size of the blob as the caller knows it, when the caller knows it. It is reported by + /// `tryGetFileSize` instead of the size a live `GetProperties` request would return, so that + /// a wrapper that sizes itself by `getFileSize` (the page cache, for one) sees the very size + /// the read is bounded by, rather than that of a generation the caller has never seen. + std::optional file_size_ = {}); + + /// The `ETag` of one generation of a blob arrives in two spellings: quoted in HTTP headers + /// (`ETag: "0x8DA..."`), as RFC 9110 requires, and unquoted in the XML body of a blob listing + /// (`0x8DA...`). Returns the quoted spelling, so that the two compare equal and + /// so that `If-Match` carries the spelling the HTTP specification prescribes. An empty `ETag` + /// stays empty. + static String quotedETag(String etag); off_t seek(off_t off, int whence) override; @@ -70,6 +83,24 @@ class ReadBufferFromAzureBlobStorage : public ReadBufferFromFileBase void initialize(size_t attempt); void setMetadataFromResponse(const Azure::Storage::Blobs::Models::DownloadBlobDetails & details, size_t blob_size) const; + /// The offset just past the last byte that the current download is allowed to deliver. + /// `reported_length` is the length of the response body as reported by the remote endpoint, + /// and is not trusted: it is bounded by `read_until_position_`, which is set locally. + static size_t getTotalSizeOfCurrentDownload(int64_t reported_length, off_t offset_, std::optional read_until_position_); + + /// Pins the download to the generation of the blob named by `expected_etag`, if there is one, + /// so that the endpoint rejects the request with `412 Precondition Failed` once the blob has + /// been replaced. + void setAccessConditions(Azure::Storage::Blobs::DownloadBlobOptions & download_options) const; + + /// Rejects a response whose `ETag` differs from `expected_etag`: an endpoint that ignored the + /// `If-Match` condition must not be able to hand out the bytes of another generation. + void checkReturnedGeneration(const Azure::Storage::Blobs::Models::DownloadBlobDetails & details) const; + + /// Turns the `412 Precondition Failed` that the `If-Match` condition produces into + /// `AZURE_OBJECT_CHANGED_DURING_READ`, which is never retried. + void rethrowIfGenerationChanged(const Azure::Core::RequestFailedException & e) const; + std::unique_ptr data_stream; ContainerClientPtr blob_container_client; BlobClientPtr blob_client; @@ -86,7 +117,19 @@ class ReadBufferFromAzureBlobStorage : public ReadBufferFromFileBase /// (non-disk seek is applied for seekable input formats: orc, arrow, parquet). bool restricted_seek; - off_t read_until_position = 0; + /// The offset just past the last byte the caller is allowed to read, when the caller has set + /// a bound. An empty optional means "no bound", so that a bound of zero - the empty range + /// `[0, 0)` - is honoured as a bound and reports EOF right away, as `supportsRightBoundedReads` + /// promises, instead of being taken for an unbounded read. + std::optional read_until_position; + + /// The `ETag` of the generation of the blob that the caller has seen (from a listing or from + /// the properties), or empty when the caller has not seen one. A locally known size is only a + /// correct bound for that generation: a blob that was replaced with a longer one after the + /// listing would otherwise be silently truncated to the stale size instead of rejected. + /// So, as `ReadBufferFromS3` does, each download is pinned to it with `If-Match` and the `ETag` + /// of the response is checked against it. Kept in the spelling of `quotedETag`. + const String expected_etag; off_t offset = 0; size_t total_size{}; diff --git a/src/Disks/tests/gtest_azure_read_buffer.cpp b/src/Disks/tests/gtest_azure_read_buffer.cpp new file mode 100644 index 000000000000..141924dc2f02 --- /dev/null +++ b/src/Disks/tests/gtest_azure_read_buffer.cpp @@ -0,0 +1,869 @@ +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int AZURE_OBJECT_CHANGED_DURING_READ; +} + +namespace +{ + +class FixedBodyStream : public Azure::Core::IO::BodyStream +{ +public: + FixedBodyStream(std::vector data_, int64_t reported_length_) + : data(std::move(data_)), reported_length(reported_length_) + { + } + + int64_t Length() const override { return reported_length; } + + void Rewind() override { position = 0; } + +private: + size_t OnRead(uint8_t * buffer, size_t count, const Azure::Core::Context &) override + { + const size_t available = data.size() - position; + const size_t to_read = std::min(count, available); + if (to_read != 0) + memcpy(buffer, data.data() + position, to_read); + position += to_read; + return to_read; + } + + std::vector data; + int64_t reported_length; + size_t position = 0; +}; + +/// Serves a range response that advertises `claimed_size` bytes but whose body stream only +/// yields `served_size` bytes. claimed_size > requested exercises the overlong branch (the +/// body must not be copied past the caller's buffer); served_size < requested exercises the +/// truncated branch (the body stream hits EOF before the requested amount). +class RangeResponseTransport : public Azure::Core::Http::HttpTransport +{ +public: + RangeResponseTransport(size_t claimed_size_, size_t served_size_) + : claimed_size(claimed_size_), served_size(served_size_) + { + } + + std::unique_ptr Send( + Azure::Core::Http::Request & request, const Azure::Core::Context &) override + { + const bool is_download = request.GetMethod() == Azure::Core::Http::HttpMethod::Get; + + auto response = std::make_unique( + 1, + 1, + is_download ? Azure::Core::Http::HttpStatusCode::PartialContent : Azure::Core::Http::HttpStatusCode::Ok, + is_download ? "Partial Content" : "OK"); + response->SetHeader("Content-Length", std::to_string(claimed_size)); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("ETag", "\"0x8DA000000000000\""); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + + if (!is_download) + { + response->SetBodyStream(std::make_unique(std::vector{}, 0)); + return response; + } + + response->SetHeader("Content-Range", "bytes 0-" + std::to_string(claimed_size - 1) + "/" + std::to_string(claimed_size)); + + std::vector data(served_size); + for (size_t i = 0; i < served_size; ++i) + data[i] = static_cast(i); + /// Report the claimed (advertised) length while only serving served_size bytes, so a + /// reader that trusted the reported length would over-read or over-report. + response->SetBodyStream(std::make_unique(std::move(data), static_cast(claimed_size))); + return response; + } + +private: + size_t claimed_size; + size_t served_size; +}; + +/// Reads a blob sequentially from an endpoint that answers every ranged request with `response_size` +/// bytes counting up from zero, with the right bound set to `read_until_position` and a +/// `buffer_size`-byte reading buffer. +std::string readWithRightBound(size_t response_size, size_t read_until_position, size_t buffer_size) +{ + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = std::make_shared(response_size, response_size); + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + DB::ReadSettings read_settings; + read_settings.remote_fs_settings.buffer_size = buffer_size; + + DB::ReadBufferFromAzureBlobStorage buffer( + container_client, + "blob", + read_settings, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 1); + + buffer.setReadUntilPosition(read_until_position); + + std::string result; + DB::readStringUntilEOF(result, buffer); + return result; +} + +void assertCountsUpFromZero(const std::string & data) +{ + for (size_t i = 0; i < data.size(); ++i) + ASSERT_EQ(static_cast(data[i]), static_cast(i)) << "at position " << i; +} + +} + +/// The endpoint answers a 100-byte ranged request with 128 bytes. The reader must stop at the right +/// bound instead of handing the extra 28 bytes to the caller: it reads through a 64-byte buffer, so +/// with the bound derived from the `Content-Length` of the response the second `nextImpl` call +/// already delivers bytes 100..127, and only the third one trips the right-bound check. +TEST(AzureReadUntilPosition, OverlongRangeResponse) +{ + std::string data; + ASSERT_NO_THROW(data = readWithRightBound(/* response_size */ 128, /* read_until_position */ 100, /* buffer_size */ 64)); + + ASSERT_EQ(data.size(), static_cast(100)); + assertCountsUpFromZero(data); +} + +/// The same, with a reading buffer larger than the requested range: a single response must not +/// overrun the right bound either. +TEST(AzureReadUntilPosition, OverlongRangeResponseWithLargeBuffer) +{ + std::string data; + ASSERT_NO_THROW(data = readWithRightBound(/* response_size */ 128, /* read_until_position */ 100, /* buffer_size */ 1024)); + + ASSERT_EQ(data.size(), static_cast(100)); + assertCountsUpFromZero(data); +} + +/// A well-behaved endpoint returns exactly the requested range. +TEST(AzureReadUntilPosition, ExactRangeResponse) +{ + std::string data; + ASSERT_NO_THROW(data = readWithRightBound(/* response_size */ 100, /* read_until_position */ 100, /* buffer_size */ 64)); + + ASSERT_EQ(data.size(), static_cast(100)); + assertCountsUpFromZero(data); +} + +namespace +{ + +/// Serves every ranged request with the bytes of a blob that counts up from zero, and optionally +/// with `extra_bytes` more than were requested, so that a reader that trusts the length of the +/// response hands out bytes from outside the requested range. +class CountingRangeTransport : public Azure::Core::Http::HttpTransport +{ +public: + static constexpr auto default_etag = "\"0x8DA000000000000\""; + + /// The blob is served with `etag_` as its current generation. An honest endpoint rejects a + /// download whose `If-Match` names another generation with `412 Precondition Failed`; one with + /// `honours_if_match_ == false` ignores the condition and serves the current generation anyway. + explicit CountingRangeTransport(size_t extra_bytes_, std::string etag_ = default_etag, bool honours_if_match_ = true) + : extra_bytes(extra_bytes_), etag(std::move(etag_)), honours_if_match(honours_if_match_) + { + } + + /// How many blob downloads the reader has issued, so that a test can assert that a bound + /// that allows no bytes at all does not go to the endpoint in the first place. + size_t getDownloadCount() const { return downloads; } + + /// How many `GetProperties` (`HEAD`) requests the reader has issued. + size_t getPropertiesCount() const { return properties_requests; } + + /// The `If-Match` condition of the last download, or empty if it had none. + const std::string & getLastIfMatch() const { return last_if_match; } + + std::unique_ptr Send( + Azure::Core::Http::Request & request, const Azure::Core::Context &) override + { + const bool is_download = request.GetMethod() == Azure::Core::Http::HttpMethod::Get; + if (request.GetMethod() == Azure::Core::Http::HttpMethod::Head) + ++properties_requests; + if (is_download) + { + ++downloads; + + /// The SDK stores the names of the headers of a request in lower case. + const auto headers = request.GetHeaders(); + auto it = headers.find("if-match"); + last_if_match = it == headers.end() ? "" : it->second; + + if (honours_if_match && !last_if_match.empty() && last_if_match != etag) + { + auto rejection = std::make_unique( + 1, 1, Azure::Core::Http::HttpStatusCode::PreconditionFailed, "Precondition Failed"); + rejection->SetHeader("x-ms-error-code", "ConditionNotMet"); + rejection->SetHeader("Content-Length", "0"); + rejection->SetBodyStream(std::make_unique(std::vector{}, 0)); + return rejection; + } + } + + auto response = std::make_unique( + 1, + 1, + is_download ? Azure::Core::Http::HttpStatusCode::PartialContent : Azure::Core::Http::HttpStatusCode::Ok, + is_download ? "Partial Content" : "OK"); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("ETag", etag); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + + if (!is_download) + { + response->SetHeader("Content-Length", "0"); + response->SetBodyStream(std::make_unique(std::vector{}, 0)); + return response; + } + + const auto [range_begin, range_end] = parseRange(request); + const size_t length = range_end - range_begin + extra_bytes; + + std::vector data(length); + for (size_t i = 0; i < length; ++i) + data[i] = static_cast(range_begin + i); + + response->SetHeader("Content-Length", std::to_string(length)); + response->SetHeader( + "Content-Range", + "bytes " + std::to_string(range_begin) + "-" + std::to_string(range_begin + length - 1) + "/" + std::to_string(blob_size)); + response->SetBodyStream(std::make_unique(std::move(data), static_cast(length))); + return response; + } + +private: + /// `bytes=-`, where `` is inclusive and can be absent. Returns the + /// half-open range. + std::pair parseRange(const Azure::Core::Http::Request & request) const + { + const auto headers = request.GetHeaders(); + auto it = headers.find("x-ms-range"); + if (it == headers.end()) + it = headers.find("range"); + if (it == headers.end()) + return {0, blob_size}; + + const std::string & value = it->second; + const size_t equals_pos = value.find('='); + const size_t dash_pos = value.find('-', equals_pos + 1); + + const size_t range_begin = std::stoul(value.substr(equals_pos + 1, dash_pos - equals_pos - 1)); + if (dash_pos + 1 == value.size()) + return {range_begin, blob_size}; + + return {range_begin, std::stoul(value.substr(dash_pos + 1)) + 1}; + } + + static constexpr size_t blob_size = 1024; + size_t extra_bytes; + std::string etag; + bool honours_if_match; + size_t downloads = 0; + size_t properties_requests = 0; + std::string last_if_match; +}; + +std::unique_ptr makeCountingBuffer(size_t extra_bytes, size_t buffer_size) +{ + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = std::make_shared(extra_bytes); + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + DB::ReadSettings read_settings; + read_settings.remote_fs_settings.buffer_size = buffer_size; + + return std::make_unique( + container_client, + "blob", + read_settings, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 1); +} + +} + +/// `supportsRightBoundedReads` promises that a bound set by `setReadUntilPosition` takes effect +/// immediately. Tightening it after a part of a wider download has already been buffered must not +/// hand out the bytes past the new bound that are still sitting in the working buffer. +TEST(AzureReadUntilPosition, TightenedAfterPartialRead) +{ + auto buffer = makeCountingBuffer(/* extra_bytes */ 0, /* buffer_size */ 64); + buffer->setReadUntilPosition(100); + + std::array head{}; + ASSERT_EQ(buffer->read(head.data(), head.size()), head.size()); + ASSERT_EQ(buffer->getPosition(), static_cast(10)); + + buffer->setReadUntilPosition(20); + + std::string rest; + ASSERT_NO_THROW(DB::readStringUntilEOF(rest, *buffer)); + + ASSERT_EQ(rest.size(), static_cast(10)); + for (size_t i = 0; i < rest.size(); ++i) + ASSERT_EQ(static_cast(rest[i]), static_cast(10 + i)) << "at position " << i; + + ASSERT_TRUE(buffer->eof()); + ASSERT_EQ(buffer->getPosition(), static_cast(20)); +} + +/// The same, with an endpoint that answers every ranged request with more bytes than were +/// requested: neither the stale buffer nor the overlong response may cross the new bound. +TEST(AzureReadUntilPosition, TightenedAfterPartialReadWithOverlongResponse) +{ + auto buffer = makeCountingBuffer(/* extra_bytes */ 28, /* buffer_size */ 64); + buffer->setReadUntilPosition(100); + + std::array head{}; + ASSERT_EQ(buffer->read(head.data(), head.size()), head.size()); + + buffer->setReadUntilPosition(20); + + std::string rest; + ASSERT_NO_THROW(DB::readStringUntilEOF(rest, *buffer)); + + ASSERT_EQ(rest.size(), static_cast(10)); + for (size_t i = 0; i < rest.size(); ++i) + ASSERT_EQ(static_cast(rest[i]), static_cast(10 + i)) << "at position " << i; + + ASSERT_TRUE(buffer->eof()); +} + +/// Widening the bound after a partial read must keep the already-read prefix and continue from the +/// position the caller has read up to. +TEST(AzureReadUntilPosition, WidenedAfterPartialRead) +{ + auto buffer = makeCountingBuffer(/* extra_bytes */ 0, /* buffer_size */ 64); + buffer->setReadUntilPosition(20); + + std::array head{}; + ASSERT_EQ(buffer->read(head.data(), head.size()), head.size()); + + buffer->setReadUntilPosition(200); + + std::string rest; + ASSERT_NO_THROW(DB::readStringUntilEOF(rest, *buffer)); + + ASSERT_EQ(rest.size(), static_cast(190)); + for (size_t i = 0; i < rest.size(); ++i) + ASSERT_EQ(static_cast(rest[i]), static_cast(10 + i)) << "at position " << i; +} + +namespace +{ + +/// The same as `makeCountingBuffer`, but also hands the transport back, so that a test can look at +/// the requests the reader has issued. +std::unique_ptr makeCountingBuffer( + size_t extra_bytes, size_t buffer_size, std::shared_ptr & transport) +{ + transport = std::make_shared(extra_bytes); + + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = transport; + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + DB::ReadSettings read_settings; + read_settings.remote_fs_settings.buffer_size = buffer_size; + + return std::make_unique( + container_client, + "blob", + read_settings, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 1); +} + +} + +/// The empty range `[0, 0)` is a bound like any other: `supportsRightBoundedReads` promises that +/// the reader stops at it, so it must report EOF right away instead of taking a bound of zero for +/// an unbounded read. It must not go to the endpoint either, because no byte of the response could +/// be handed out. +TEST(AzureReadUntilPosition, EmptyRange) +{ + std::shared_ptr transport; + auto buffer = makeCountingBuffer(/* extra_bytes */ 0, /* buffer_size */ 64, transport); + buffer->setReadUntilPosition(0); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_TRUE(data.empty()); + ASSERT_TRUE(buffer->eof()); + ASSERT_EQ(transport->getDownloadCount(), static_cast(0)); +} + +/// `setReadUntilEnd` is the way to say "no bound", and it must still work after the bound has been +/// set to the empty range. +TEST(AzureReadUntilPosition, EmptyRangeThenReadUntilEnd) +{ + auto buffer = makeCountingBuffer(/* extra_bytes */ 0, /* buffer_size */ 64); + buffer->setReadUntilPosition(0); + ASSERT_TRUE(buffer->eof()); + + buffer->setReadUntilEnd(); + + std::array head{}; + ASSERT_EQ(buffer->read(head.data(), head.size()), head.size()); + for (size_t i = 0; i < head.size(); ++i) + ASSERT_EQ(static_cast(head[i]), static_cast(i)) << "at position " << i; +} + +/// A bound below the current position is allowed as long as the caller seeks back before reading, +/// which `ReadBuffer::setReadUntilPosition` explicitly recommends supporting. Tightening it all the +/// way down to the empty range is no different. +TEST(AzureReadUntilPosition, TightenedToEmptyRangeAfterPartialRead) +{ + std::shared_ptr transport; + auto buffer = makeCountingBuffer(/* extra_bytes */ 0, /* buffer_size */ 64, transport); + buffer->setReadUntilPosition(100); + + std::array head{}; + ASSERT_EQ(buffer->read(head.data(), head.size()), head.size()); + + const size_t downloads_after_head = transport->getDownloadCount(); + + buffer->setReadUntilPosition(0); + buffer->seek(0, SEEK_SET); + + std::string rest; + ASSERT_NO_THROW(DB::readStringUntilEOF(rest, *buffer)); + + ASSERT_TRUE(rest.empty()); + ASSERT_EQ(transport->getDownloadCount(), downloads_after_head); +} + +/// An endpoint that honours the requested range must keep working: reading a bounded range from a +/// nonzero offset returns exactly the bytes of the blob at that offset. +TEST(AzureReadUntilPosition, SeekToNonZeroOffsetWithHonestEndpoint) +{ + auto buffer = makeCountingBuffer(/* extra_bytes */ 0, /* buffer_size */ 64); + + buffer->seek(100, SEEK_SET); + buffer->setReadUntilPosition(200); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_EQ(data.size(), static_cast(100)); + for (size_t i = 0; i < data.size(); ++i) + ASSERT_EQ(static_cast(data[i]), static_cast(100 + i)) << "at position " << i; +} + +namespace +{ + +/// An `AzureObjectStorage` whose every request is answered by `RangeResponseTransport`, which +/// serves the blob from its beginning regardless of the requested range and reports +/// `served_size` as the length of the body. +std::unique_ptr makeObjectStorage(size_t claimed_size, size_t served_size) +{ + DB::AzureBlobStorage::ConnectionParams connection_params; + connection_params.endpoint.container_name = "container"; + connection_params.client_options.Retry.MaxRetries = 0; + connection_params.client_options.Transport.Transport + = std::make_shared(claimed_size, served_size); + + auto container_client = std::make_unique( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", connection_params.client_options), + /* blob_prefix */ ""); + + return std::make_unique( + "azure", + connection_params.auth_method, + std::move(container_client), + std::make_unique(), + connection_params, + /* object_namespace */ "container", + /* description */ "azure", + /* common_key_prefix */ ""); +} + +} + +/// A whole-object read through `AzureObjectStorage::readObject` sets no right bound of its own, so +/// before `StoredObject::bytes_size` was threaded through as the bound, the size of the read came +/// from the length of the response. An endpoint answering a request for a 100-byte object with 128 +/// bytes then handed the caller 28 bytes past the end of the object. The size from the metadata is +/// known locally, so it, and not the response, decides where the object ends. +TEST(AzureReadObject, BoundedByTheObjectSizeFromTheMetadata) +{ + auto object_storage = makeObjectStorage(/* claimed_size */ 128, /* served_size */ 128); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_EQ(data.size(), static_cast(100)); + assertCountsUpFromZero(data); +} + +/// An object whose size was never determined carries the `UnknownSize` sentinel, which is not a +/// bound: such a read must still run to the end of whatever the endpoint returns rather than +/// stopping immediately or reading a sentinel-sized range. +TEST(AzureReadObject, UnknownObjectSizeReadsToTheEnd) +{ + auto object_storage = makeObjectStorage(/* claimed_size */ 100, /* served_size */ 100); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ DB::StoredObject::UnknownSize); + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_EQ(data.size(), static_cast(100)); + assertCountsUpFromZero(data); +} + +/// `UnknownSize` is the only sentinel for an undetermined size, so `bytes_size == 0` describes a +/// genuinely empty object and is a bound like any other. Treating it as "no bound" made an empty +/// object read unbounded, so an endpoint answering with a non-empty body handed the caller bytes +/// of an object that the metadata says has none. +TEST(AzureReadObject, EmptyObjectReadsAsEmpty) +{ + auto object_storage = makeObjectStorage(/* claimed_size */ 128, /* served_size */ 128); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 0); + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_TRUE(data.empty()); + ASSERT_TRUE(buffer->eof()); +} + +/// Bounding the read by `bytes_size` means an empty object is read without issuing a single +/// request, so the metadata of the last request does not exist. `readSmallObjectAndGetObjectMetadata` +/// must still return the metadata of the object instead of throwing `NOT_INITIALIZED`. +TEST(AzureReadObject, EmptyObjectStillReportsMetadata) +{ + auto object_storage = makeObjectStorage(/* claimed_size */ 0, /* served_size */ 0); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 0); + + DB::SmallObjectDataWithMetadata result; + ASSERT_NO_THROW(result = object_storage->readSmallObjectAndGetObjectMetadata(object, DB::ReadSettings{}, /* max_size_bytes */ 4096)); + + ASSERT_TRUE(result.data.empty()); + ASSERT_EQ(result.metadata.size_bytes, static_cast(0)); + ASSERT_EQ(result.metadata.etag, "\"0x8DA000000000000\""); +} + +namespace +{ + +/// An `AzureObjectStorage` whose every request is answered by `CountingRangeTransport`, so that the +/// blob has a generation (`etag`) and the endpoint either honours `If-Match` or ignores it. +std::unique_ptr makeCountingObjectStorage(std::shared_ptr transport) +{ + DB::AzureBlobStorage::ConnectionParams connection_params; + connection_params.endpoint.container_name = "container"; + connection_params.client_options.Retry.MaxRetries = 0; + connection_params.client_options.Transport.Transport = std::move(transport); + + auto container_client = std::make_unique( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", connection_params.client_options), + /* blob_prefix */ ""); + + return std::make_unique( + "azure", + connection_params.auth_method, + std::move(container_client), + std::make_unique(), + connection_params, + /* object_namespace */ "container", + /* description */ "azure", + /* common_key_prefix */ ""); +} + +constexpr auto listed_etag = "\"0x8DA000000000000\""; +constexpr auto replaced_etag = "\"0x8DA000000000001\""; + +/// Runs `action` and asserts that it fails with `AZURE_OBJECT_CHANGED_DURING_READ`. +template +void assertRejectsReplacedBlob(Action && action) +{ + try + { + action(); + FAIL() << "the read of a replaced blob succeeded"; + } + catch (const DB::Exception & e) + { + ASSERT_EQ(e.code(), DB::ErrorCodes::AZURE_OBJECT_CHANGED_DURING_READ) << e.message(); + } +} + +} + +/// `StoredObject::bytes_size` and `StoredObject::etag` describe one generation of the object. When +/// the blob has been replaced with a longer one after the listing, a read bounded by the stale size +/// alone would return a clean EOF after the first `bytes_size` bytes of the new generation. The +/// download is pinned to the listed generation with `If-Match`, so an honest endpoint rejects it +/// with `412 Precondition Failed`, which is reported as `AZURE_OBJECT_CHANGED_DURING_READ` and is +/// not retried as if it were transient. +TEST(AzureReadObject, RejectsReplacedBlobThroughIfMatch) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, replaced_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + object.etag = listed_etag; + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + assertRejectsReplacedBlob([&] { DB::readStringUntilEOF(data, *buffer); }); + + ASSERT_TRUE(data.empty()); + ASSERT_EQ(transport->getLastIfMatch(), listed_etag); + ASSERT_EQ(transport->getDownloadCount(), static_cast(1)); +} + +/// An endpoint that ignores `If-Match` and answers `206` with the current generation must not get +/// its bytes through either: the `ETag` of the response is checked against the listed one. +TEST(AzureReadObject, RejectsReplacedBlobWhenIfMatchIsIgnored) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, replaced_etag, /* honours_if_match */ false); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + object.etag = listed_etag; + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + assertRejectsReplacedBlob([&] { DB::readStringUntilEOF(data, *buffer); }); + + ASSERT_TRUE(data.empty()); +} + +/// The blob that was listed is the one that is read: the download carries the listed `ETag` as its +/// `If-Match` condition and the bytes come through, bounded by the listed size. +TEST(AzureReadObject, ReadsTheListedGeneration) +{ + auto transport = std::make_shared(/* extra_bytes */ 28, listed_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + object.etag = listed_etag; + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_EQ(data.size(), static_cast(100)); + assertCountsUpFromZero(data); + ASSERT_EQ(transport->getLastIfMatch(), listed_etag); +} + +/// A caller that has not seen a generation of the blob sets no condition and gets whatever the +/// endpoint currently holds. +TEST(AzureReadObject, UnknownGenerationIsNotPinned) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, replaced_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_EQ(data.size(), static_cast(100)); + ASSERT_TRUE(transport->getLastIfMatch().empty()); +} + +/// An object listed as empty is read without a download, so no `If-Match` pins it; the `ETag` of +/// the properties that stand in for the response is checked against the listed one instead. +TEST(AzureReadObject, EmptyObjectMetadataRejectsReplacedBlob) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, replaced_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 0); + object.etag = listed_etag; + + assertRejectsReplacedBlob([&] { object_storage->readSmallObjectAndGetObjectMetadata(object, DB::ReadSettings{}, /* max_size_bytes */ 4096); }); + ASSERT_EQ(transport->getDownloadCount(), static_cast(0)); +} + +/// The plain `readObject` path of an object listed as empty issues no download either, so +/// `If-Match` never reaches the endpoint: the generation is checked on the properties before the +/// buffer is handed out, instead of returning the replaced blob as a clean empty file. +TEST(AzureReadObject, EmptyObjectRejectsReplacedBlob) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, replaced_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 0); + object.etag = listed_etag; + + assertRejectsReplacedBlob([&] { object_storage->readObject(object, DB::ReadSettings{}); }); + ASSERT_EQ(transport->getDownloadCount(), static_cast(0)); +} + +/// The XML body of a blob listing spells the `ETag` without the quotes that the `ETag` header of a +/// download has (`0x8DA...` against `"0x8DA..."`), so the two spellings name one generation and +/// must compare equal, and `If-Match` must carry the quoted spelling that HTTP prescribes. +constexpr auto unquoted_listed_etag = "0x8DA000000000000"; + +TEST(AzureReadObject, AcceptsTheUnquotedETagOfAListing) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, listed_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + object.etag = unquoted_listed_etag; + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + + ASSERT_EQ(data.size(), static_cast(100)); + ASSERT_EQ(transport->getLastIfMatch(), listed_etag); +} + +TEST(AzureReadObject, EmptyObjectAcceptsTheUnquotedETagOfAListing) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, listed_etag, /* honours_if_match */ true); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 0); + object.etag = unquoted_listed_etag; + + ASSERT_NO_THROW(object_storage->readObject(object, DB::ReadSettings{})); + + DB::SmallObjectDataWithMetadata result; + ASSERT_NO_THROW(result = object_storage->readSmallObjectAndGetObjectMetadata(object, DB::ReadSettings{}, /* max_size_bytes */ 4096)); + ASSERT_TRUE(result.data.empty()); + ASSERT_EQ(result.metadata.etag, listed_etag); + ASSERT_EQ(transport->getDownloadCount(), static_cast(0)); +} + +/// `readObject` bounds the read by the size the caller recorded, so the buffer must report that +/// very size as the size of the file. Before the recorded size was handed to the buffer as its +/// `file_size`, `getFileSize` asked the endpoint with a live `GetProperties` request, whose answer +/// describes whatever generation the blob has by now: `CachedInMemoryReadBufferFromFile` sizes +/// itself by that answer in its constructor and throws `UNEXPECTED_END_OF_FILE` when the inner +/// buffer ends earlier, so a blob grown since it was listed turned a bounded read into an exception. +/// Here the endpoint reports the blob as empty on `HEAD`, while the caller has listed it as 100 +/// bytes: the size the caller knows must win, and no request is needed to learn it. +TEST(AzureReadObject, FileSizeIsTheListedSize) +{ + auto transport = std::make_shared(/* extra_bytes */ 0); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ 100); + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + ASSERT_EQ(buffer->getFileSize(), static_cast(100)); + ASSERT_EQ(transport->getPropertiesCount(), static_cast(0)); + + std::string data; + ASSERT_NO_THROW(DB::readStringUntilEOF(data, *buffer)); + ASSERT_EQ(data.size(), static_cast(100)); +} + +/// Without a recorded size there is nothing local to report, so the size is still learned from +/// the endpoint, as before. +TEST(AzureReadObject, UnknownObjectSizeIsAskedFromTheEndpoint) +{ + auto transport = std::make_shared(/* extra_bytes */ 0); + auto object_storage = makeCountingObjectStorage(transport); + + DB::StoredObject object("blob", /* local_path */ "", /* bytes_size */ DB::StoredObject::UnknownSize); + auto buffer = object_storage->readObject(object, DB::ReadSettings{}); + + ASSERT_EQ(buffer->getFileSize(), static_cast(0)); + ASSERT_EQ(transport->getPropertiesCount(), static_cast(1)); +} + +TEST(AzureQuotedETag, Spellings) +{ + ASSERT_EQ(DB::ReadBufferFromAzureBlobStorage::quotedETag(""), ""); + ASSERT_EQ(DB::ReadBufferFromAzureBlobStorage::quotedETag("0x8DA000000000000"), "\"0x8DA000000000000\""); + ASSERT_EQ(DB::ReadBufferFromAzureBlobStorage::quotedETag("\"0x8DA000000000000\""), "\"0x8DA000000000000\""); + ASSERT_EQ(DB::ReadBufferFromAzureBlobStorage::quotedETag("\""), "\"\"\""); +} + +/// `readBigAt` issues its own downloads, so it carries the same condition and the same check. +TEST(AzureReadBigAt, RejectsReplacedBlobThroughIfMatch) +{ + auto transport = std::make_shared(/* extra_bytes */ 0, replaced_etag, /* honours_if_match */ true); + + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = transport; + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + DB::ReadBufferFromAzureBlobStorage buffer( + container_client, + "blob", + DB::ReadSettings{}, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 3, + /* use_external_buffer */ false, + /* restricted_seek */ false, + /* read_until_position */ std::nullopt, + /* blob_storage_log */ nullptr, + /* container_for_logging */ "", + listed_etag, + /* file_size */ 1000); + + /// In this version `readBigAt` uses the client that `tryGetFileSize` creates, so it is called first, + /// as the callers of `readBigAt` do. The size is known, so it issues no request of its own. + ASSERT_EQ(buffer.getFileSize(), static_cast(1000)); + + std::array out{}; + assertRejectsReplacedBlob([&] { buffer.readBigAt(out.data(), out.size(), /* range_begin */ 100, nullptr); }); + + /// A replaced blob stays replaced, so the rejection is not retried. + ASSERT_EQ(transport->getDownloadCount(), static_cast(1)); +} + +#endif diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 42f62e34fb22..99385f454d62 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -133,6 +133,8 @@ struct FormatSettings DateTimeOverflowBehavior date_time_overflow_behavior = DateTimeOverflowBehavior::Ignore; + bool throwOnDateTimeOverflow() const { return date_time_overflow_behavior == DateTimeOverflowBehavior::Throw; } + bool input_format_ipv4_default_on_conversion_error = false; bool input_format_ipv6_default_on_conversion_error = false; bool check_conversion_from_numbers_to_enum = true; diff --git a/src/Formats/JSONExtractTree.cpp b/src/Formats/JSONExtractTree.cpp index c8e2d6c27bc0..c00dac73ec2b 100644 --- a/src/Formats/JSONExtractTree.cpp +++ b/src/Formats/JSONExtractTree.cpp @@ -701,7 +701,7 @@ class DateNode : public JSONExtractTreeNode auto data = element.getString(); ReadBufferFromMemory buf(data); DateType date; - if (!tryReadDateText(date, buf) || !buf.eof()) + if (!tryReadDateText(date, buf, DateLUT::instance(), nullptr, !format_settings.throwOnDateTimeOverflow()) || !buf.eof()) { error = fmt::format("cannot parse Date value here: {}", data); return false; @@ -737,7 +737,7 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi time_t value = 0; if (element.isString()) { - if (!tryParse(value, element.getString(), format_settings.date_time_input_format)) + if (!tryParse(value, element.getString(), format_settings.date_time_input_format, !format_settings.throwOnDateTimeOverflow())) { error = fmt::format("cannot parse DateTime value here: {}", element.getString()); return false; @@ -756,12 +756,22 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi return false; } value = element.getInt64(); + if (format_settings.throwOnDateTimeOverflow() && (value < 0 || value > 0xFFFFFFFF)) + { + error = fmt::format("value {} is out of bounds of type DateTime", value); + return false; + } } else { /// Clamp in the unsigned domain before narrowing to time_t, /// because values above INT64_MAX would wrap to negative on cast. UInt64 raw = element.getUInt64(); + if (format_settings.throwOnDateTimeOverflow() && raw > 0xFFFFFFFF) + { + error = fmt::format("value {} is out of bounds of type DateTime", raw); + return false; + } value = static_cast(std::min(raw, UInt64(0xFFFFFFFF))); } } @@ -774,7 +784,7 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi /// exactly can cross the second boundary (`1703363853.9999999` arrives here as `1703363854.0`). String str_value = jsonElementToString(element, format_settings); ReadBufferFromMemory buf(str_value); - if (!tryReadDateTimeAsNumber(value, buf) || !buf.eof()) + if (!tryReadDateTimeAsNumber(value, buf, !format_settings.throwOnDateTimeOverflow()) || !buf.eof()) { error = fmt::format("cannot read DateTime value from JSON element: {}", str_value); return false; @@ -790,21 +800,22 @@ class DateTimeNode : public JSONExtractTreeNode, public TimezoneMixi return true; } - bool tryParse(time_t & value, std::string_view data, FormatSettings::DateTimeInputFormat date_time_input_format) const + bool tryParse(time_t & value, std::string_view data, FormatSettings::DateTimeInputFormat date_time_input_format, bool saturate_on_overflow) const { + const auto overflow = saturate_on_overflow ? DateTimeOverflow::Saturate : DateTimeOverflow::Report; ReadBufferFromMemory buf(data); switch (date_time_input_format) { case FormatSettings::DateTimeInputFormat::Basic: - if (tryReadDateTimeText(value, buf, time_zone) && buf.eof()) + if (tryReadDateTimeText(value, buf, time_zone, nullptr, nullptr, saturate_on_overflow) && buf.eof()) return true; break; case FormatSettings::DateTimeInputFormat::BestEffort: - if (tryParseDateTimeBestEffort(value, buf, time_zone, utc_time_zone) && buf.eof()) + if (tryParseDateTimeBestEffort(value, buf, time_zone, utc_time_zone, overflow) && buf.eof()) return true; break; case FormatSettings::DateTimeInputFormat::BestEffortUS: - if (tryParseDateTimeBestEffortUS(value, buf, time_zone, utc_time_zone) && buf.eof()) + if (tryParseDateTimeBestEffortUS(value, buf, time_zone, utc_time_zone, overflow) && buf.eof()) return true; break; } diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 96adfe2a3e82..206717bab661 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -1867,7 +1867,8 @@ class FunctionBinaryArithmetic : public IFunction, WithContext ? array_element_function->executeImpl(new_arguments, result_array_type, rows_count) : executeImpl(new_arguments, result_array_type, rows_count); - return ColumnArray::create(res, typeid_cast(arguments[0].column.get())->getOffsetsPtr()); + /// The element-wise result can be a constant (for example a NULL), the data of an array cannot. + return ColumnArray::create(res->convertToFullColumnIfConst(), typeid_cast(arguments[0].column.get())->getOffsetsPtr()); } ColumnPtr executeArrayWithNumericImpl(const ColumnsWithTypeAndName & args, const DataTypePtr & result_type, size_t input_rows_count) const diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index f5e46079539c..5eaa0bf567a1 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -498,6 +498,29 @@ bool allArgumentColumnsAreConstant(const ColumnsWithTypeAndName & args) return true; } +bool plusMinusWithConstantsIsInjective( + const ColumnWithTypeAndName & left, const ColumnWithTypeAndName & right, const DataTypePtr & return_type) +{ + /// Two varying operands are not injective (`x + y` maps many pairs to one sum), and with both + /// fixed there is no varying argument to be injective in. + const bool left_is_const = left.column && isColumnConst(*left.column); + const bool right_is_const = right.column && isColumnConst(*right.column); + if (left_is_const == right_is_const) + return false; + + auto is_integer_type = [](const DataTypePtr & type) + { return type && isInteger(*removeNullable(recursiveRemoveLowCardinality(type))); }; + + if (!is_integer_type(left.type) || !is_integer_type(right.type) || !is_integer_type(return_type)) + return false; + + /// A NULL among the varying argument's values maps to NULL one-to-one, so only the fixed operand + /// matters. Its `ColumnConst` nests a column of size 1, so the value is readable even when the + /// constant itself was materialized with size 0, as query-plan constants are. + const ColumnWithTypeAndName & constant = left_is_const ? left : right; + return !constant.column->onlyNull(); +} + bool convertLowCardinalityColumnsToFull(ColumnsWithTypeAndName & args) { bool converted = false; diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 8a77b9a3eda2..9260ae9861d0 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -217,6 +217,12 @@ bool isLowCardinalityType(const IDataType & type); bool hasLowCardinalityTypes(const ColumnsWithTypeAndName & args); /// Returns true if all of the arguments have constant columns. bool allArgumentColumnsAreConstant(const ColumnsWithTypeAndName & args); +/// Whether `plus`/`minus` is injective in its varying argument, given the other one fixed. Only +/// integer arithmetic qualifies: integer wrap-around is a bijection, while every other operand class +/// collapses distinct arguments somewhere - an `Interval` at end-of-month days and DST transitions, a +/// float or `Decimal` by rounding or rescaling, a narrower date constant, a NULL constant. +bool plusMinusWithConstantsIsInjective( + const ColumnWithTypeAndName & left, const ColumnWithTypeAndName & right, const DataTypePtr & return_type); bool convertLowCardinalityColumnsToFull(ColumnsWithTypeAndName & args); void checkFunctionArgumentSizes(const ColumnsWithTypeAndName & arguments, size_t input_rows_count); diff --git a/src/Functions/FunctionIfBase.h b/src/Functions/FunctionIfBase.h index e80cd3b75694..6ec99c209760 100644 --- a/src/Functions/FunctionIfBase.h +++ b/src/Functions/FunctionIfBase.h @@ -64,7 +64,7 @@ class FunctionIfBase : public IFunction b.CreateCondBr(nativeBoolCast(b, cond), then, next); b.SetInsertPoint(then); - /// Use `nativeCastWithDecimalScale` to correctly lift integer/float branches to a + /// Use `nativeCastWithDecimalScale` to correctly lift integer branches to a /// `Decimal` `result_type` (and to convert between `Decimal` types of different scales). /// Plain `nativeCast` reinterprets the integer bits without applying the `10^scale` /// factor, which silently produces wrong values when the analyzer leaves a non-`Decimal` diff --git a/src/Functions/FunctionJoinGet.cpp b/src/Functions/FunctionJoinGet.cpp index e3d7b87f97d8..613b1b1aaf6b 100644 --- a/src/Functions/FunctionJoinGet.cpp +++ b/src/Functions/FunctionJoinGet.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -168,7 +169,7 @@ getJoin(const ColumnsWithTypeAndName & arguments, ContextPtr context) const auto storage_id = context->resolveStorageID({qualified_name.database, qualified_name.table}); auto table = DatabaseCatalog::instance().getTable(storage_id, std::const_pointer_cast(context)); - auto storage_join = std::dynamic_pointer_cast(table); + auto storage_join = castStorage(table, DeferredTable::Load); if (!storage_join) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Table {} should have engine StorageJoin", join_name); diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp index 726781d7ea8c..d20712d3f738 100644 --- a/src/Functions/FunctionsConversion.cpp +++ b/src/Functions/FunctionsConversion.cpp @@ -3351,6 +3351,34 @@ bool castBothTypes(const IDataType * left, const IDataType * right, F && f) return castType(left, [&](const auto & left_) { return castType(right, [&](const auto & right_) { return f(left_, right_); }); }); } +/// Whether a numeric conversion `from` -> `to` can be JIT-compiled. A float source is refused for an +/// integer or `Decimal` destination, because `fptosi` / `fptoui` have no defined result outside the +/// destination range. A `Bool` destination stays allowed, it is compiled through `nativeBoolCast`. +static bool isCompilableNumericConversion(const IDataType * from, const IDataType * to) +{ + return castBothTypes(from, to, [](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + + if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) + { + if constexpr (IsDataTypeNumber && IsDataTypeNumber) + { + if constexpr (is_floating_point + && !is_floating_point) + return isBool(right.getPtr()); + return true; + } + else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) + return !is_floating_point; + else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) + return true; + } + return false; + }); +} + bool convertIsCompilableImpl(const DataTypes & types, const DataTypePtr & result_type) { if (types.empty()) @@ -3359,25 +3387,7 @@ bool convertIsCompilableImpl(const DataTypes & types, const DataTypePtr & result if (!canBeNativeType(types[0]) || !canBeNativeType(result_type)) return false; - return castBothTypes( - types[0].get(), - result_type.get(), - [](const auto & left, const auto & right) - { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - - if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) - { - if constexpr (IsDataTypeNumber && IsDataTypeNumber) - return true; - else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) - return true; - else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) - return true; - } - return false; - }); + return isCompilableNumericConversion(types[0].get(), result_type.get()); } llvm::Value * convertCompileImpl(llvm::IRBuilderBase & builder, const ValuesWithType & arguments, const DataTypePtr & result_type) @@ -3479,21 +3489,7 @@ bool FunctionCast::isCompilable() const if (!canBeNativeType(denull_input_type) || !canBeNativeType(denull_result_type)) return false; - return castBothTypes(denull_input_type.get(), denull_result_type.get(), [](const auto & left, const auto & right) - { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - if constexpr (IsDataTypeDecimalOrNumber && IsDataTypeDecimalOrNumber) - { - if constexpr (IsDataTypeNumber && IsDataTypeNumber) - return true; - else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) - return true; - else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) - return true; - } - return false; - }); + return isCompilableNumericConversion(denull_input_type.get(), denull_result_type.get()); } llvm::Value * FunctionCast::compile(llvm::IRBuilderBase & builder, const ValuesWithType & arguments) const diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e3158387728c..913b78ffaa5d 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -1084,7 +1084,7 @@ inline void convertFromTime(DataTypeTime::FieldType & x, time_t & /** Conversion of strings to numbers, dates, datetimes: through parsing. */ template -void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing) +void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing, bool) { if constexpr (is_floating_point) { @@ -1098,33 +1098,33 @@ void parseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTI } template <> -inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { DayNum tmp(0); - readDateText(tmp, rb, *time_zone); + readDateText(tmp, rb, *time_zone, saturate_on_overflow); x = tmp; } template <> -inline void parseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { ExtendedDayNum tmp(0); - readDateText(tmp, rb, *time_zone); + readDateText(tmp, rb, *time_zone, saturate_on_overflow); x = tmp; } // NOTE: no need of extra overload of DateTime64, since readDateTimeText64 has different signature and that case is explicitly handled in the calling code. template <> -inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { time_t time = 0; - readDateTimeText(time, rb, *time_zone); + readDateTimeText(time, rb, *time_zone, saturate_on_overflow); convertFromTime(x, time); } template <> -inline void parseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline void parseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool) { time_t time = 0; readTimeText(time, rb, *time_zone); @@ -1132,7 +1132,7 @@ inline void parseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb } template <> -inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { UUID tmp; readUUIDText(tmp, rb); @@ -1140,7 +1140,7 @@ inline void parseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb } template <> -inline void parseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline void parseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv4 tmp; readIPv4Text(tmp, rb); @@ -1148,7 +1148,7 @@ inline void parseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb } template <> -inline void parseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline void parseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv6 tmp; readIPv6Text(tmp, rb); @@ -1156,7 +1156,7 @@ inline void parseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb } template -bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing) +bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool precise_float_parsing, bool) { if constexpr (is_floating_point) { @@ -1170,37 +1170,37 @@ bool tryParseImpl(typename DataType::FieldType & x, ReadBuffer & rb, const DateL } template <> -inline bool tryParseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline bool tryParseImpl(DataTypeDate::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { DayNum tmp(0); - if (!tryReadDateText(tmp, rb, *time_zone)) + if (!tryReadDateText(tmp, rb, *time_zone, nullptr, saturate_on_overflow)) return false; x = tmp; return true; } template <> -inline bool tryParseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline bool tryParseImpl(DataTypeDate32::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { ExtendedDayNum tmp(0); - if (!tryReadDateText(tmp, rb, *time_zone)) + if (!tryReadDateText(tmp, rb, *time_zone, nullptr, saturate_on_overflow)) return false; x = tmp; return true; } template <> -inline bool tryParseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +inline bool tryParseImpl(DataTypeDateTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool saturate_on_overflow) { time_t time = 0; - if (!tryReadDateTimeText(time, rb, *time_zone)) + if (!tryReadDateTimeText(time, rb, *time_zone, nullptr, nullptr, saturate_on_overflow)) return false; convertFromTime(x, time); return true; } template <> -[[maybe_unused]]inline bool tryParseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool) +[[maybe_unused]]inline bool tryParseImpl(DataTypeTime::FieldType & x, ReadBuffer & rb, const DateLUTImpl * time_zone, bool, bool) { time_t time = 0; if (!tryReadTimeText(time, rb, *time_zone)) @@ -1210,7 +1210,7 @@ template <> } template <> -inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { UUID tmp; if (!tryReadUUIDText(tmp, rb)) @@ -1221,7 +1221,7 @@ inline bool tryParseImpl(DataTypeUUID::FieldType & x, ReadBuffer & } template <> -inline bool tryParseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline bool tryParseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv4 tmp; if (!tryReadIPv4Text(tmp, rb)) @@ -1232,7 +1232,7 @@ inline bool tryParseImpl(DataTypeIPv4::FieldType & x, ReadBuffer & } template <> -inline bool tryParseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool) +inline bool tryParseImpl(DataTypeIPv6::FieldType & x, ReadBuffer & rb, const DateLUTImpl *, bool, bool) { IPv6 tmp; if (!tryReadIPv6Text(tmp, rb)) @@ -1365,6 +1365,11 @@ struct ConvertThroughParsing utc_time_zone = &DateLUT::instance("UTC"); } + /// Nothing to wrap around when parsing text, so `ignore` behaves like `saturate` + const bool saturate_on_overflow [[maybe_unused]] + = settings.date_time_overflow_behavior != FormatSettings::DateTimeOverflowBehavior::Throw; + const auto overflow [[maybe_unused]] = saturate_on_overflow ? DateTimeOverflow::Saturate : DateTimeOverflow::Report; + const IColumn * col_from = arguments[0].column.get(); const ColumnString * col_from_string = checkAndGetColumn(col_from); const ColumnFixedString * col_from_fixed_string = checkAndGetColumn(col_from); @@ -1459,7 +1464,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone); + parseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i], res); } } @@ -1486,7 +1491,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + parseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i], res); } } @@ -1532,11 +1537,11 @@ struct ConvertThroughParsing } if constexpr (std::is_same_v) { - if (!tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing)) + if (!tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing, saturate_on_overflow)) throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "Cannot parse string to type {}", TypeName); } else - parseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing); + parseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing, saturate_on_overflow); } while (false); } } @@ -1571,7 +1576,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone); + parsed = tryParseDateTimeBestEffort(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i],res); } } @@ -1598,7 +1603,7 @@ struct ConvertThroughParsing else { time_t res = 0; - parsed = tryParseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone); + parsed = tryParseDateTimeBestEffortUS(res, read_buffer, *local_time_zone, *utc_time_zone, overflow); convertFromTime(vec_to[i],res); } } @@ -1631,7 +1636,7 @@ struct ConvertThroughParsing } else { - parsed = tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing); + parsed = tryParseImpl(vec_to[i], read_buffer, local_time_zone, settings.precise_float_parsing, saturate_on_overflow); } } @@ -4365,8 +4370,14 @@ struct ToDateMonotonicity { static bool has() { return true; } - static IFunction::Monotonicity get(const IDataType & type, const Field & left, const Field & right) + static IFunction::Monotonicity get(const IDataType & type_with_wrappers, const Field & left, const Field & right) { + const IDataType * type_without_wrappers = &type_with_wrappers; + if (const auto * low_cardinality_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = low_cardinality_type->getDictionaryType().get(); + if (const auto * nullable_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = nullable_type->getNestedType().get(); + const IDataType & type = *type_without_wrappers; auto which = WhichDataType(type); if (which.isDateOrDate32() || which.isTime() || which.isTime64() || which.isDateTime() || which.isDateTime64() || which.isInt8() || which.isInt16() || which.isUInt8() || which.isUInt16()) @@ -4404,8 +4415,14 @@ struct ToDateTimeMonotonicity { static bool has() { return true; } - static IFunction::Monotonicity get(const IDataType & type, const Field &, const Field &) + static IFunction::Monotonicity get(const IDataType & type_with_wrappers, const Field &, const Field &) { + const IDataType * type_without_wrappers = &type_with_wrappers; + if (const auto * low_cardinality_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = low_cardinality_type->getDictionaryType().get(); + if (const auto * nullable_type = typeid_cast(type_without_wrappers)) + type_without_wrappers = nullable_type->getNestedType().get(); + const IDataType & type = *type_without_wrappers; if (type.isValueRepresentedByNumber()) { auto which = WhichDataType(type); diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index 663b58c53340..a041de61cb25 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -1855,6 +1856,17 @@ struct MatcherNumberConst bool match(size_t row_data, size_t /* row_index */) const { return data[row_data] == index; } }; +/// Matcher for keys of a ColumnLowCardinality. The requested key is resolved to its dictionary +/// position once, so a key is matched without comparing the key values. +template +struct MatcherLowCardinalityConst +{ + const IndexType * indexes; + IndexType key_index; + + bool match(size_t row_data, size_t /* row_index */) const { return indexes[row_data] == key_index; } +}; + } template @@ -1921,11 +1933,6 @@ bool castColumnString(const IColumn * column, F && f) return castTypeToEither(column, std::forward(f)); } -bool isStringOrFixedStringColumn(const IColumn & column) -{ - return typeid_cast(&column) || typeid_cast(&column); -} - template bool FunctionArrayElement::matchKeyToIndexStringConst( const IColumn & data, const Offsets & offsets, const Field & index, PaddedPODArray & matched_idxs) @@ -1933,37 +1940,27 @@ bool FunctionArrayElement::matchKeyToIndexStringConst( if (index.getType() != Field::Types::String) return false; - /// The dictionary lookup below is defined only for String and FixedString keys. For other - /// LowCardinality key types, fall through so that the regular dispatch reports the type error - /// instead of silently finding no match. - const auto * low_cardinality_data = typeid_cast(&data); - if (low_cardinality_data - && isStringOrFixedStringColumn(*low_cardinality_data->getDictionary().getNestedNotNullableColumn())) + if (const auto * low_cardinality_data = typeid_cast(&data)) { - const auto & requested_key = index.safeGet(); - auto dictionary_index = low_cardinality_data->getDictionary().getOrFindValueIndex(requested_key); matched_idxs.reserve(offsets.size()); - if (!dictionary_index) + auto lookup_result = callWithLowCardinalityValueIndex( + *low_cardinality_data, + index.safeGet(), + [&](const auto * indexes, auto key_index) + { + MatcherLowCardinalityConst matcher{indexes, key_index}; + executeMatchKeyToIndex(offsets, matched_idxs, matcher); + }); + + /// For LowCardinality key types without a dictionary lookup, fall through so that the regular + /// dispatch reports the type error instead of silently finding no match. + if (lookup_result != LowCardinalityValueLookupResult::Unsupported) { - matched_idxs.resize_fill(offsets.size()); + if (lookup_result == LowCardinalityValueLookupResult::NotFound) + matched_idxs.resize_fill(offsets.size()); return true; } - - struct MatcherLowCardinalityStringConst - { - const ColumnLowCardinality & data; - UInt64 dictionary_index; - - bool match(size_t row_data, size_t /* row_index */) const - { - return data.getIndexAt(row_data) == dictionary_index; - } - }; - - MatcherLowCardinalityStringConst matcher{*low_cardinality_data, *dictionary_index}; - executeMatchKeyToIndex(offsets, matched_idxs, matcher); - return true; } return castColumnString( diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp index e732014562fb..c8784dfdebfe 100644 --- a/src/Functions/stem.cpp +++ b/src/Functions/stem.cpp @@ -99,9 +99,10 @@ class Stemmer /// Rows where null_map[i] != 0 are skipped and emitted as empty strings. /// For FixedString, getDataAt returns the value with null-byte padding; trimRight removes it. /// For String, trailing zero bytes are valid data and must not be trimmed. - /// Snowball stemming never lengthens a word, so upper_bound bytes is a safe pre-allocation. MutableColumnPtr stemColumn(const IColumn & col, size_t input_rows_count, const NullMap * null_map = nullptr) { + /// upper_bound is only an initial estimate: some stemmers lengthen a word (e.g. Turkish maps + /// the ASCII 'i' to the 2-byte 'ı'), so the loop grows res_data when the output overflows it. size_t upper_bound = 0; const bool is_fixed_string = checkAndGetColumn(&col) != nullptr; if (const auto * col_str = checkAndGetColumn(&col)) @@ -131,7 +132,10 @@ class Stemmer if (is_fixed_string) trimRight(word, '\0'); std::string_view stemmed = stem(word); - chassert(data_size + stemmed.size() <= res_data.size()); + + /// Stemming can lengthen the word, grow the output buffer. + if (data_size + stemmed.size() > res_data.size()) + res_data.resize(data_size + stemmed.size()); memcpy(res_data.data() + data_size, stemmed.data(), stemmed.size()); data_size += stemmed.size(); diff --git a/src/Functions/toLowCardinality.cpp b/src/Functions/toLowCardinality.cpp index 04dc60289fb6..b0cfb9c4f751 100644 --- a/src/Functions/toLowCardinality.cpp +++ b/src/Functions/toLowCardinality.cpp @@ -54,6 +54,13 @@ class FunctionToLowCardinality final : public IFunction typeid_cast(*column).insertRangeFromFullColumn(*arg.column, 0, arg.column->size()); return column; } + + bool hasInformationAboutMonotonicity() const override { return true; } + + Monotonicity getMonotonicityForRange(const IDataType &, const Field &, const Field &) const override + { + return { .is_monotonic = true, .is_positive = true, .is_always_monotonic = true, .is_strict = true }; + } }; } diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp index 1df1d5757abc..ff44f7aea860 100644 --- a/src/Functions/vectorFunctions.cpp +++ b/src/Functions/vectorFunctions.cpp @@ -630,7 +630,7 @@ struct FunctionTupleOperationInterval final : public ITupleFunction else tuple_columns.resize(2); - tuple_columns[0] = arguments[0].column->convertToFullColumnIfConst(); + tuple_columns[0] = arguments[0].column; } else if (first_tuple) { @@ -675,15 +675,13 @@ struct FunctionTupleOperationInterval final : public ITupleFunction { auto minus = FunctionFactory::instance().get("minus", context); auto elem_minus = minus->build({left, arguments[1]}); - last_column = elem_minus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false) - ->convertToFullColumnIfConst(); + last_column = elem_minus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false); } else { auto plus = FunctionFactory::instance().get("plus", context); auto elem_plus = plus->build({left, arguments[1]}); - last_column = elem_plus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false) - ->convertToFullColumnIfConst(); + last_column = elem_plus->execute({left, arguments[1]}, arguments[1].type, input_rows_count, /* dry_run = */ false); } } else @@ -700,6 +698,10 @@ struct FunctionTupleOperationInterval final : public ITupleFunction } } + /// Either operand can be a constant, which a tuple cannot hold. + for (auto & column : tuple_columns) + column = column->convertToFullColumnIfConst(); + return ColumnTuple::create(tuple_columns); } }; diff --git a/src/IO/DiskCacheProvider.cpp b/src/IO/DiskCacheProvider.cpp index ebb06055ee9b..1e8a5633f8ce 100644 --- a/src/IO/DiskCacheProvider.cpp +++ b/src/IO/DiskCacheProvider.cpp @@ -280,10 +280,15 @@ size_t DiskCacheWriter::write(ChainedBuffers data, const Claim & claim) data.copyTo(flat_buf.data(), write_range); std::string failure_reason; + /// Don't reserve ahead past the end of the miss range. + const size_t reserve_hint = write_end_max - write_offset; const bool reserved = seg.reserve( contiguous, cache_settings.reserve_space_wait_lock_timeout_milliseconds, - failure_reason); + failure_reason, + /* reserve_stat */nullptr, + reserve_hint, + &reserve_ahead); if (!reserved) { LOG_TRACE(log, "DiskCacheWriter::write: reserve failed for [{}, {}]: {}", diff --git a/src/IO/DiskCacheProvider.h b/src/IO/DiskCacheProvider.h index 92eb337b68e2..f24ac534b3e7 100644 --- a/src/IO/DiskCacheProvider.h +++ b/src/IO/DiskCacheProvider.h @@ -92,6 +92,8 @@ class DiskCacheWriter : public CacheWriter /// same time, so both update this set. mutable std::mutex committed_mutex; ByteRange aligned_range; + /// One state per writer: it owns a single append-only segment. + FileCacheReserveAhead reserve_ahead; LoggerPtr log = getLogger("DiskCacheWriter"); }; diff --git a/src/IO/LimitReadBuffer.cpp b/src/IO/LimitReadBuffer.cpp index 6f580b64f9aa..aa727c5b57cd 100644 --- a/src/IO/LimitReadBuffer.cpp +++ b/src/IO/LimitReadBuffer.cpp @@ -20,15 +20,14 @@ bool LimitReadBuffer::nextImpl() /// Let underlying buffer calculate read bytes in `next()` call. in->position() = position(); - if (bytes >= settings.read_no_less) + if (bytes >= settings.read_no_more) { - if (settings.expect_eof && bytes > settings.read_no_more) + /// A stream ending exactly at the limit is not an error, so the check waits until the limit is + /// reached. `eof` refills the nested buffer, it consumes nothing. + if (settings.expect_eof && !in->eof()) throw Exception(ErrorCodes::LIMIT_EXCEEDED, "Limit for LimitReadBuffer exceeded: {}", settings.excetion_hint); - if (bytes >= settings.read_no_more) - return false; - - //throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Unexpected data, got {} bytes, expected {}", bytes, settings.read_atmost); + return false; } if (!in->next()) diff --git a/src/IO/LimitReadBuffer.h b/src/IO/LimitReadBuffer.h index fbcb4570e83d..af4626a11cc2 100644 --- a/src/IO/LimitReadBuffer.h +++ b/src/IO/LimitReadBuffer.h @@ -19,6 +19,9 @@ class LimitReadBuffer : public ReadBuffer { size_t read_no_less = 0; size_t read_no_more = std::numeric_limits::max(); + /// Throw instead of reporting EOF when the nested buffer still has data at the limit. Do not set + /// it when the nested buffer carries an unrelated message after this one, such as the next + /// keep-alive request: those bytes are not part of what the limit counts. bool expect_eof = false; std::string excetion_hint = {}; }; diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 7d0c89e43caa..f347a259cc88 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -47,6 +47,7 @@ namespace ErrorCodes extern const int TOO_DEEP_RECURSION; extern const int TOO_LARGE_STRING_SIZE; extern const int SYNTAX_ERROR; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } /// Converts num_bytes hex-encoded bytes from src to dst in a single pass, folding validity into @@ -776,8 +777,7 @@ void readEscapedStringIntoImpl(Vector & s, ReadBuffer & buf) } } } - - if constexpr (support_crlf) + else if constexpr (support_crlf) { if (*buf.position() == '\r') { @@ -1695,8 +1695,9 @@ ReturnType readDateTimeTextFallback( second = (s[6] - '0') * 10 + (s[7] - '0'); } - if constexpr (throw_exception) + if (saturate_on_overflow) { + /// Use saturating version - makeDateTime saturates out-of-range years if (unlikely(year == 0)) datetime = 0; else @@ -1704,29 +1705,28 @@ ReturnType readDateTimeTextFallback( } else { - if (saturate_on_overflow) + /// Use non-saturating version - report out-of-range values instead of clamping them + auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); + if (!datetime_maybe) { - /// Use saturating version - makeDateTime saturates out-of-range years - if (unlikely(year == 0)) - datetime = 0; + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse DateTime"); else - datetime = makeDateTime(date_lut, year, month, day, hour, minute, second); - } - else - { - /// Use non-saturating version - return false for out-of-range values - auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); - if (!datetime_maybe) return false; + } - if constexpr (!dt64_mode) + if constexpr (!dt64_mode) + { + if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) { - if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", *datetime_maybe); + else return false; } - - datetime = *datetime_maybe; } + + datetime = *datetime_maybe; } } else @@ -1758,6 +1758,8 @@ ReturnType readDateTimeTextFallback( return false; } + return checkParsedDateTimeRange(datetime, saturate_on_overflow); + } return ReturnType(true); @@ -2069,10 +2071,7 @@ bool trySkipJSONField(ReadBuffer & buf, std::string_view name_of_field, const Fo } -/// The same as `readStringBinary`, but the string grows as the bytes arrive instead of being resized -/// to the declared size first, so that a size declared by the peer cannot become an allocation on -/// its own when the payload never follows. -static void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size = DEFAULT_MAX_STRING_SIZE) +void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size) { size_t size = 0; readVarUInt(size, buf); @@ -2595,6 +2594,15 @@ namespace /// Scale `value` to whole seconds and clamp it to the `DateTime` range. The multiplication is bound-checked /// with truncating division rather than `common::mulOverflow`, a no-op stub for big-int types. +/// Whether `value` scaled to whole seconds fits the DateTime range, using the same truncating bound +bool datetimeSecondsInRange(Int128 value, UInt32 unread_scale) +{ + static constexpr Int128 max_seconds = 0xFFFFFFFF; + if (value < 0) + return false; + return value <= max_seconds / DecimalUtils::scaleMultiplier(unread_scale); +} + time_t datetimeSecondsFromNumber(Int128 value, UInt32 unread_scale) { static constexpr Int128 max_seconds = 0xFFFFFFFF; @@ -2619,7 +2627,7 @@ bool datetime64TicksFromNumber(DateTime64 & x, Int128 value, UInt32 unread_scale } template -ReturnType readDateTimeAsNumberImpl(time_t & x, ReadBuffer & buf) +ReturnType readDateTimeAsNumberImpl(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { static constexpr bool throw_exception = std::is_same_v; Decimal128 tmp; @@ -2639,12 +2647,20 @@ ReturnType readDateTimeAsNumberImpl(time_t & x, ReadBuffer & buf) else return ReturnType(false); } + if (!saturate_on_overflow && !datetimeSecondsInRange(tmp.value, unread_scale)) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value is out of bounds of type DateTime"); + else + return ReturnType(false); + } + x = datetimeSecondsFromNumber(tmp.value, unread_scale); return ReturnType(true); } template -ReturnType readDateTimeAsRawValueImpl(time_t & x, ReadBuffer & buf) +ReturnType readDateTimeAsRawValueImpl(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { static constexpr bool throw_exception = std::is_same_v; /// Saturating 128-bit read: a plain `readIntText` does not check overflow, so an out-of-range value would @@ -2655,6 +2671,14 @@ ReturnType readDateTimeAsRawValueImpl(time_t & x, ReadBuffer & buf) else if (!readIntText128Saturating(tmp, buf)) return ReturnType(false); + if (!saturate_on_overflow && !datetimeSecondsInRange(tmp, 0)) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value is out of bounds of type DateTime"); + else + return ReturnType(false); + } + x = datetimeSecondsFromNumber(tmp, 0); return ReturnType(true); } @@ -2685,6 +2709,7 @@ ReturnType readDateTime64AsNumberImpl(DateTime64 & x, UInt32 scale, ReadBuffer & else return ReturnType(false); } + return ReturnType(true); } @@ -2705,15 +2730,16 @@ ReturnType readDateTime64AsRawValueImpl(DateTime64 & x, ReadBuffer & buf) else return ReturnType(false); } + return ReturnType(true); } } -void readDateTimeAsNumber(time_t & x, ReadBuffer & buf) { readDateTimeAsNumberImpl(x, buf); } -bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf) { return readDateTimeAsNumberImpl(x, buf); } -void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf) { readDateTimeAsRawValueImpl(x, buf); } -bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf) { return readDateTimeAsRawValueImpl(x, buf); } +void readDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { readDateTimeAsNumberImpl(x, buf, saturate_on_overflow); } +bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { return readDateTimeAsNumberImpl(x, buf, saturate_on_overflow); } +void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { readDateTimeAsRawValueImpl(x, buf, saturate_on_overflow); } +bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow) { return readDateTimeAsRawValueImpl(x, buf, saturate_on_overflow); } void readDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf) { readDateTime64AsNumberImpl(x, scale, buf); } bool tryReadDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf) { return readDateTime64AsNumberImpl(x, scale, buf); } diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 2d216684a75b..755861fa88c3 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -57,6 +57,7 @@ namespace ErrorCodes extern const int TOO_LARGE_STRING_SIZE; extern const int TOO_LARGE_ARRAY_SIZE; extern const int SIZE_OF_FIXED_STRING_DOESNT_MATCH; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } /// Helper functions for formatted input. @@ -170,6 +171,11 @@ inline void skipStringBinary(ReadBuffer & buf) buf.ignore(size); } +/// The same as `readStringBinary`, but the string grows as the bytes arrive instead of being resized +/// to the declared size first, so that a size declared by the peer cannot become an allocation on +/// its own when the payload never follows. +void readStringBinaryGrowing(String & s, ReadBuffer & buf, size_t max_string_size = DEFAULT_MAX_STRING_SIZE); + /// For historical reasons we store IPv6 as a String inline void readIPv6Binary(IPv6 & ip, ReadBuffer & buf) { @@ -645,39 +651,40 @@ inline ReturnType readDateTextImpl(DayNum & date, ReadBuffer & buf, const DateLU LocalDate local_date; if constexpr (throw_exception) - { readDateTextImpl(local_date, buf, allowed_delimiters); + else if (!readDateTextImpl(local_date, buf, allowed_delimiters)) + return false; + + if (saturate_on_overflow) + { + /// Use saturating versions - makeDayNum saturates out-of-range years, convertToDayNum saturates to 0 or 0xFFFF ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); convertToDayNum(date, ret); + return ReturnType(true); } - else + + auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); + if (!ret) { - if (!readDateTextImpl(local_date, buf, allowed_delimiters)) + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATE, "Cannot parse date"); + else return false; + } - if (saturate_on_overflow) - { - /// Use saturating versions - makeDayNum saturates out-of-range years, convertToDayNum saturates to 0 or 0xFFFF - ExtendedDayNum ret = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); - convertToDayNum(date, ret); - } + if (!tryToConvertToDayNum(date, *ret)) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type Date", ret->toUnderType()); else - { - /// Use non-saturating versions - return false for out-of-range values - auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); - if (!ret) - return false; - - if (!tryToConvertToDayNum(date, *ret)) - return false; - } - - return true; + return false; } + + return ReturnType(true); } template -inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr) +inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_delimiters = nullptr, bool saturate_on_overflow = true) { static constexpr bool throw_exception = std::is_same_v; @@ -688,6 +695,22 @@ inline ReturnType readDateTextImpl(ExtendedDayNum & date, ReadBuffer & buf, cons else if (!readDateTextImpl(local_date, buf, allowed_delimiters)) return false; + if (!saturate_on_overflow) + { + /// Every four-digit year fits into Date32, so only a calendar-invalid date can fail here + auto ret = tryToMakeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day()); + if (!ret) + { + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATE, "Cannot parse date"); + else + return false; + } + + date = *ret; + return ReturnType(true); + } + /// A calendar-invalid date (e.g. month 13) yields 1900-01-01 (-getDayNumOffsetEpoch(), -25567) for Date32 and 1970-01-01 for Date. date = makeDayNum(date_lut, local_date.year(), local_date.month(), local_date.day(), -static_cast(getDayNumOffsetEpoch())); return ReturnType(true); @@ -699,14 +722,14 @@ inline void readDateText(LocalDate & date, ReadBuffer & buf) readDateTextImpl(date, buf); } -inline void readDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), bool saturate_on_overflow = true) { - readDateTextImpl(date, buf, date_lut); + readDateTextImpl(date, buf, date_lut, nullptr, saturate_on_overflow); } -inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance()) +inline void readDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & date_lut = DateLUT::instance(), bool saturate_on_overflow = true) { - readDateTextImpl(date, buf, date_lut); + readDateTextImpl(date, buf, date_lut, nullptr, saturate_on_overflow); } inline bool tryReadDateText(LocalDate & date, ReadBuffer & buf, const char * allowed_delimiters = nullptr) @@ -719,9 +742,9 @@ inline bool tryReadDateText(DayNum & date, ReadBuffer & buf, const DateLUTImpl & return readDateTextImpl(date, buf, time_zone, allowed_delimiters, saturate_on_overflow); } -inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr) +inline bool tryReadDateText(ExtendedDayNum & date, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), const char * allowed_delimiters = nullptr, bool saturate_on_overflow = true) { - return readDateTextImpl(date, buf, time_zone, allowed_delimiters); + return readDateTextImpl(date, buf, time_zone, allowed_delimiters, saturate_on_overflow); } UUID parseUUID(std::span src); @@ -856,6 +879,23 @@ inline T parseFromStringWithoutAssertEOF(std::string_view str) template ReturnType readDateTimeTextFallback(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr, bool saturate_on_overflow = true); +/// A digit-only timestamp is read as a plain integer, so its range has to be checked separately +template +inline ReturnType checkParsedDateTimeRange(time_t datetime [[maybe_unused]], bool saturate_on_overflow [[maybe_unused]]) +{ + if constexpr (!dt64_mode) + { + if (!saturate_on_overflow && (datetime < 0 || datetime > static_cast(UINT32_MAX))) + { + if constexpr (std::is_same_v) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", datetime); + else + return false; + } + } + return ReturnType(true); +} + template ReturnType readTimeTextFallback(time_t & time, ReadBuffer & buf, const DateLUTImpl & date_lut, const char * allowed_date_delimiters = nullptr, const char * allowed_time_delimiters = nullptr); @@ -928,8 +968,9 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons second = (s[17] - '0') * 10 + (s[18] - '0'); } - if constexpr (throw_exception) + if (saturate_on_overflow) { + /// Use saturating version - makeDateTime saturates out-of-range years if (unlikely(year == 0)) datetime = 0; else @@ -937,30 +978,29 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons } else { - if (saturate_on_overflow) + /// Use non-saturating version - report out-of-range values instead of clamping them + auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); + if (!datetime_maybe) { - /// Use saturating version - makeDateTime saturates out-of-range years - if (unlikely(year == 0)) - datetime = 0; + if constexpr (throw_exception) + throw Exception(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot parse datetime"); else - datetime = makeDateTime(date_lut, year, month, day, hour, minute, second); - } - else - { - /// Use non-saturating version - return false for out-of-range values - auto datetime_maybe = tryToMakeDateTime(date_lut, year, month, day, hour, minute, second); - if (!datetime_maybe) return false; + } - /// For usual DateTime check if value is within supported range - if constexpr (!dt64_mode) + /// For usual DateTime check if value is within supported range + if constexpr (!dt64_mode) + { + if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) { - if (*datetime_maybe < 0 || *datetime_maybe > static_cast(UINT32_MAX)) + if constexpr (throw_exception) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", *datetime_maybe); + else return false; } - - datetime = *datetime_maybe; } + + datetime = *datetime_maybe; } if (dt_long) @@ -971,7 +1011,12 @@ inline ReturnType readDateTimeTextImpl(time_t & datetime, ReadBuffer & buf, cons return ReturnType(true); } /// Why not readIntTextUnsafe? Because for needs of AdFox, parsing of unix timestamp with leading zeros is supported: 000...NNNN. - return readIntTextImpl(datetime, buf); + if constexpr (throw_exception) + readIntTextImpl(datetime, buf); + else if (!readIntTextImpl(datetime, buf)) + return false; + + return checkParsedDateTimeRange(datetime, saturate_on_overflow); } return readDateTimeTextFallback(datetime, buf, date_lut, allowed_date_delimiters, allowed_time_delimiters, saturate_on_overflow); } @@ -1396,9 +1441,9 @@ inline ReturnType readTimeTextImpl(Time64 & time64, UInt32 scale, ReadBuffer & b return ReturnType(is_ok); } -inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) +inline void readDateTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), bool saturate_on_overflow = true) { - readDateTimeTextImpl(datetime, buf, time_zone); + readDateTimeTextImpl(datetime, buf, time_zone, nullptr, nullptr, saturate_on_overflow); } inline void readTimeText(time_t & datetime, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) @@ -1442,10 +1487,10 @@ inline bool tryReadTime64Text(Time64 & time64, UInt32 scale, ReadBuffer & buf, c /// `toDateTime64` and the `Values` format. Parsing stops at the first character that is not part of the /// number (e.g. the `,` or `}` following the value in JSON). The `AsRawValue` variants implement the legacy /// behavior, where the number is the raw underlying value. -void readDateTimeAsNumber(time_t & x, ReadBuffer & buf); -bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf); -void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf); -bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf); +void readDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); +bool tryReadDateTimeAsNumber(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); +void readDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); +bool tryReadDateTimeAsRawValue(time_t & x, ReadBuffer & buf, bool saturate_on_overflow = true); void readDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf); bool tryReadDateTime64AsNumber(DateTime64 & x, UInt32 scale, ReadBuffer & buf); @@ -1667,8 +1712,8 @@ inline void readText(T & x, ReadBuffer & buf) { readFloatTextPrecise(x, buf); } inline void readText(String & x, ReadBuffer & buf) { readEscapedString(x, buf); } -inline void readText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { readDateText(x, buf, time_zone); } -inline bool tryReadText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance()) { return tryReadDateText(x, buf, time_zone); } +inline void readText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), bool saturate_on_overflow = true) { readDateText(x, buf, time_zone, saturate_on_overflow); } +inline bool tryReadText(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone = DateLUT::instance(), bool saturate_on_overflow = true) { return tryReadDateText(x, buf, time_zone, nullptr, saturate_on_overflow); } inline void readText(LocalDate & x, ReadBuffer & buf) { readDateText(x, buf); } inline bool tryReadText(LocalDate & x, ReadBuffer & buf) { return tryReadDateText(x, buf); } @@ -1794,7 +1839,7 @@ inline ReturnType readCSVSimple(T & x, ReadBuffer & buf) // standalone overload for dates: to avoid instantiating DateLUTs while parsing other types template -inline ReturnType readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone) +inline ReturnType readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & time_zone, bool saturate_on_overflow = true) { static constexpr bool throw_exception = std::is_same_v; @@ -1811,8 +1856,8 @@ inline ReturnType readCSVSimple(T & x, ReadBuffer & buf, const DateLUTImpl & tim ++buf.position(); if constexpr (throw_exception) - readText(x, buf, time_zone); - else if (!tryReadText(x, buf, time_zone)) + readText(x, buf, time_zone, saturate_on_overflow); + else if (!tryReadText(x, buf, time_zone, saturate_on_overflow)) return ReturnType(false); if (maybe_quote == '\'' || maybe_quote == '\"') @@ -1853,8 +1898,8 @@ inline bool tryReadCSV(LocalDate & x, ReadBuffer & buf) { return readCSVSimple(x, buf); } -inline void readCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { readCSVSimple(x, buf, time_zone); } -inline bool tryReadCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone) { return readCSVSimple(x, buf, time_zone); } +inline void readCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone, bool saturate_on_overflow = true) { readCSVSimple(x, buf, time_zone, saturate_on_overflow); } +inline bool tryReadCSV(DayNum & x, ReadBuffer & buf, const DateLUTImpl & time_zone, bool saturate_on_overflow = true) { return readCSVSimple(x, buf, time_zone, saturate_on_overflow); } inline void readCSV(LocalDateTime & x, ReadBuffer & buf) { readCSVSimple(x, buf); } inline bool tryReadCSV(LocalDateTime & x, ReadBuffer & buf) { return readCSVSimple(x, buf); } diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp index b671cb447531..43d5c47f271f 100644 --- a/src/IO/S3/URI.cpp +++ b/src/IO/S3/URI.cpp @@ -4,11 +4,13 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -33,6 +35,34 @@ namespace ErrorCodes namespace S3 { +namespace +{ + +/// `Poco::URI::toString` renders the userinfo (`user:password@`) and the query parameters of a presigned +/// URL verbatim. Exception messages reach `system.query_log` and the server log, which, unlike the query +/// text, are not masked, so a URI must not be put into them as is. +String maskedURIString(const Poco::URI & uri) +{ + String result = uri.toString(); + maskURIUserinfo(result); + /// With `compatibility_s3_presigned_url_query_in_path` the constructor folds the query of a presigned URL + /// into the path by percent-encoding its '?', which `toString` renders as `%3F`, so `maskPresignedURLParameters` + /// would not see the parameters. Put the '?' back: `toString` encodes '%' itself as `%25`, so a `%3F` can only + /// come from a '?'. + boost::replace_all(result, "%3F", "?"); + maskPresignedURLParameters(result); + return result; +} + +/// With `compatibility_s3_presigned_url_query_in_path` the query of a presigned URL ends up in the bucket or the key. +String maskedQuotedString(String value) +{ + maskPresignedURLParameters(value); + return quoteString(value); +} + +} + URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_presigned_query_parameters, S3UriStyle uri_style) { /// Case when AWS Private Link Interface is being used @@ -143,13 +173,13 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax, bool keep_pre case S3UriStyle::VIRTUAL_HOSTED: { if (!tryInitVirtualHostedStyle(is_using_aws_private_link_interface, false)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 virtual-hosted-style uri: {}", !uri.empty() ? uri.toString() : ""); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 virtual-hosted-style uri: {}", !uri.empty() ? maskedURIString(uri) : ""); break; } case S3UriStyle::PATH: { if (!tryInitPathStyle()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 path-style uri: {}", !uri.empty() ? uri.toString() : ""); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid S3 path-style uri: {}", !uri.empty() ? maskedURIString(uri) : ""); break; } } @@ -229,8 +259,8 @@ void URI::validateBucket(const String & bucket, const Poco::URI & uri) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Bucket name length is out of bounds in virtual hosted style S3 URI: {}{}", - quoteString(bucket), - !uri.empty() ? " (" + uri.toString() + ")" : ""); + maskedQuotedString(bucket), + !uri.empty() ? " (" + maskedURIString(uri) + ")" : ""); } void URI::validateKey(const String & key, const Poco::URI & uri) @@ -240,8 +270,8 @@ void URI::validateKey(const String & key, const Poco::URI & uri) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Invalid S3 key: {}{}", - quoteString(key), - !uri.empty() ? " (" + uri.toString() + ")" : ""); + maskedQuotedString(key), + !uri.empty() ? " (" + maskedURIString(uri) + ")" : ""); }; diff --git a/src/IO/SilkFiberStreamSocketImpl.h b/src/IO/SilkFiberStreamSocketImpl.h index efbeaf042de2..b6662336e4ef 100644 --- a/src/IO/SilkFiberStreamSocketImpl.h +++ b/src/IO/SilkFiberStreamSocketImpl.h @@ -24,11 +24,6 @@ class FiberStreamSocketImpl final : public Poco::Net::StreamSocketImpl void setBlocking(bool flag) override; bool supportsExternalPolling() const override { return false; } - bool getDontWait() const { return dont_wait; } - void setDontWait(bool flag) { dont_wait = flag; } - -private: - bool dont_wait = false; }; } diff --git a/src/IO/SilkSecureFiberStreamSocketImpl.cpp b/src/IO/SilkSecureFiberStreamSocketImpl.cpp index 871baaefc896..78416e37310a 100644 --- a/src/IO/SilkSecureFiberStreamSocketImpl.cpp +++ b/src/IO/SilkSecureFiberStreamSocketImpl.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -16,12 +15,10 @@ #include #include #include -#include #include #include #include -#include namespace Silk @@ -30,11 +27,6 @@ namespace Silk namespace { -uint64_t timeoutNs(const Poco::Timespan & timeout) -{ - return static_cast(timeout.totalMicroseconds()) * 1000ULL; -} - FiberStreamSocketImpl * getUnderlyingSocket(BIO * bio) { return static_cast(static_cast(BIO_get_data(bio))); @@ -45,68 +37,30 @@ int silkBioRead(BIO * bio, char * buf, int len) auto * socket_impl = getUnderlyingSocket(bio); const int fd = socket_impl->sockfd(); - if (socket_impl->getDontWait()) - { - /// Honor non-blocking mode instead of parking the caller on an io_uring read: a - /// non-blocking caller (e.g. `SSL_peek` from the pool's staleness probe) expects an - /// immediate `EAGAIN`/`SSL_ERROR_WANT_READ`, not a wait for `getReceiveTimeout()`. - BIO_clear_retry_flags(bio); - const ssize_t n = ::recv(fd, buf, len, MSG_DONTWAIT); - if (n < 0) - { - /// `recv` left the error in `errno`; the flag call below cannot touch it - /// (`BIO_set_retry_read` is a pure in-struct flag set), so it stays valid for the - /// caller through `return`. Read it into a local for the fatality test per the - /// errno rule in silk's docs/tls.md. - const int err = errno; - if (BIO_sock_non_fatal_error(err)) - BIO_set_retry_read(bio); - return -1; - } - - /// TODO(mstetsyuk): should be done at Silk level. - __msan_unpoison(buf, static_cast(n)); - - if (n == 0) - BIO_set_flags(bio, BIO_FLAGS_IN_EOF); - return static_cast(n); - } - - const uint64_t timeout_ns = timeoutNs(socket_impl->getReceiveTimeout()); - - uint64_t bytes_read = 0; - silk::FiberScheduler::IoFuture future; - iovec iov{buf, static_cast(len)}; - silk::FiberScheduler::read(fd, &iov, 1, 0, &bytes_read, &future); - - int r = timeout_ns > 0 - ? silk::FiberFuture::waitWithTimeout(&future, timeout_ns) - : future.wait(); - - if (r == ETIMEDOUT) - { - future.cancel(); - r = future.wait(); - if (r == ECANCELED) - r = ETIMEDOUT; - } - + /// Do not schedule a fiber-aware read or wait for its future here. This callback runs + /// inside an OpenSSL operation, and suspending the fiber could resume it on another + /// OS thread before `SSL_get_error` reads that thread's error queue. Use a non-blocking + /// syscall and report retry through the `BIO` flags instead. `SecureSocketImpl` saves + /// the OpenSSL result first, then `SecureSocketImpl::mustRetry` performs the timed, + /// fiber-aware wait through `pollImpl`. + /// https://docs.openssl.org/3.5/man3/SSL_get_error/ BIO_clear_retry_flags(bio); - - if (r == 0) + const ssize_t n = ::recv(fd, buf, len, MSG_DONTWAIT); + if (n < 0) { - /// TODO(mstetsyuk): should be done at Silk level. - __msan_unpoison(buf, bytes_read); - - if (bytes_read == 0) - BIO_set_flags(bio, BIO_FLAGS_IN_EOF); - return static_cast(bytes_read); + /// Capture `errno` immediately; fiber migration makes a later read invalid. + const int err = errno; + if (BIO_sock_non_fatal_error(err)) + BIO_set_retry_read(bio); + return -1; } - errno = r; - if (BIO_sock_non_fatal_error(r) || r == ETIMEDOUT) - BIO_set_retry_read(bio); - return -1; + /// TODO(mstetsyuk): should be done at Silk level. + __msan_unpoison(buf, static_cast(n)); + + if (n == 0) + BIO_set_flags(bio, BIO_FLAGS_IN_EOF); + return static_cast(n); } int silkBioWrite(BIO * bio, const char * buf, int len) @@ -114,52 +68,17 @@ int silkBioWrite(BIO * bio, const char * buf, int len) auto * socket_impl = getUnderlyingSocket(bio); const int fd = socket_impl->sockfd(); - if (socket_impl->getDontWait()) - { - /// See the matching branch in silkBioRead: honor non-blocking mode rather than parking - /// the caller in a fiber wait for getSendTimeout(). - BIO_clear_retry_flags(bio); - const ssize_t n = ::send(fd, buf, len, MSG_DONTWAIT | MSG_NOSIGNAL); - if (n < 0) - { - /// See the matching note in silkBioRead: `send` left the error in `errno`, the flag - /// call cannot clobber it, so it survives to the caller. - const int err = errno; - if (BIO_sock_non_fatal_error(err)) - BIO_set_retry_write(bio); - return -1; - } - return static_cast(n); - } - - const uint64_t timeout_ns = timeoutNs(socket_impl->getSendTimeout()); - - uint64_t bytes_written = 0; - silk::FiberScheduler::IoFuture future; - iovec iov{const_cast(buf), static_cast(len)}; - silk::FiberScheduler::write(fd, &iov, 1, 0, &bytes_written, &future); - - int r = timeout_ns > 0 - ? silk::FiberFuture::waitWithTimeout(&future, timeout_ns) - : future.wait(); - - if (r == ETIMEDOUT) + /// See `silkBioRead`: never suspend from inside an OpenSSL operation. + BIO_clear_retry_flags(bio); + const ssize_t n = ::send(fd, buf, len, MSG_DONTWAIT | MSG_NOSIGNAL); + if (n < 0) { - future.cancel(); - r = future.wait(); - if (r == ECANCELED) - r = ETIMEDOUT; + const int err = errno; + if (BIO_sock_non_fatal_error(err)) + BIO_set_retry_write(bio); + return -1; } - - BIO_clear_retry_flags(bio); - - if (r == 0) - return static_cast(bytes_written); - - errno = r; - if (BIO_sock_non_fatal_error(r) || r == ETIMEDOUT) - BIO_set_retry_write(bio); - return -1; + return static_cast(n); } long silkBioCtrl(BIO * bio, int cmd, [[maybe_unused]] long larg, void * parg) // NOLINT(google-runtime-int) @@ -259,22 +178,11 @@ SecureFiberStreamSocketImpl::SecureFiberStreamSocketImpl(Poco::Net::Context::Ptr SecureFiberStreamSocketImpl::SecureFiberStreamSocketImpl(FiberStreamSocketImpl * underlying_, Poco::Net::Context::Ptr context) : Poco::Net::SecureStreamSocketImpl(underlying_, context) - , underlying(underlying_) { setBioMethod(silkBioMethod()); setMutex(std::make_unique()); } -bool SecureFiberStreamSocketImpl::getDontWait() const -{ - return underlying->getDontWait(); -} - -void SecureFiberStreamSocketImpl::setDontWait(bool flag) -{ - underlying->setDontWait(flag); -} - bool SecureFiberStreamSocketImpl::pollImpl(Poco::Timespan & timeout, int mode) { uint32_t events = 0; diff --git a/src/IO/SilkSecureFiberStreamSocketImpl.h b/src/IO/SilkSecureFiberStreamSocketImpl.h index ccf8383413a0..1afd28d4a574 100644 --- a/src/IO/SilkSecureFiberStreamSocketImpl.h +++ b/src/IO/SilkSecureFiberStreamSocketImpl.h @@ -17,16 +17,11 @@ class SecureFiberStreamSocketImpl final : public Poco::Net::SecureStreamSocketIm public: explicit SecureFiberStreamSocketImpl(Poco::Net::Context::Ptr context); - bool getDontWait() const; - void setDontWait(bool flag); - bool pollImpl(Poco::Timespan & timeout, int mode) override; bool supportsExternalPolling() const override { return false; } private: SecureFiberStreamSocketImpl(FiberStreamSocketImpl * underlying_, Poco::Net::Context::Ptr context); - - FiberStreamSocketImpl * underlying; }; } diff --git a/src/IO/SocketPeerClosed.cpp b/src/IO/SocketPeerClosed.cpp index c82b138b7ede..af0017fa7494 100644 --- a/src/IO/SocketPeerClosed.cpp +++ b/src/IO/SocketPeerClosed.cpp @@ -44,7 +44,16 @@ SocketState getSocketState(int fd) #if USE_SSL -SocketState getSSLSocketState(ssl_st * ssl) +namespace +{ + +struct SSLSocketStateResult +{ + SocketState state; + bool fatal_error; +}; + +SSLSocketStateResult getSSLSocketStateImpl(ssl_st * ssl) { /// `SSL_peek` decrypts just enough of the pending records to tell real application data and /// harmless post-handshake messages (session tickets, `KeyUpdate`) apart from a `close_notify`. @@ -54,32 +63,47 @@ SocketState getSSLSocketState(ssl_st * ssl) ERR_clear_error(); char c = 0; int res = SSL_peek(ssl, &c, 1); + SSLSocketStateResult result{SocketState::Closed, false}; if (res > 0) - return SocketState::DataPending; /// Application data is waiting to be read; the peer is alive. - - switch (SSL_get_error(ssl, res)) { - case SSL_ERROR_WANT_READ: [[fallthrough]]; - case SSL_ERROR_WANT_WRITE: - /// `SSL_peek` found no complete application-data record, but that alone does not prove - /// the connection is idle: the bytes of a record that has only partially arrived (e.g. - /// the first fragment of a queued response) are buffered inside the SSL object too, and - /// look identical from here - both end in `SSL_ERROR_WANT_READ`. `SSL_has_pending` - /// reports on that internal buffer regardless of whether the record is complete, so a - /// session ticket / `KeyUpdate` that was fully consumed reads as idle (nothing left - /// buffered), while a partial record correctly reads as pending. - return SSL_has_pending(ssl) ? SocketState::DataPending : SocketState::Idle; - case SSL_ERROR_ZERO_RETURN: - return SocketState::Closed; /// The peer sent `close_notify`: an orderly TLS shutdown. - default: - /// A FIN without `close_notify` (`SSL_ERROR_SYSCALL`), a protocol error (`SSL_ERROR_SSL`), - /// or anything else: treat as closed/broken. - return SocketState::Closed; + result = {SocketState::DataPending, false}; /// Application data is waiting to be read; the peer is alive. + } + else + { + switch (SSL_get_error(ssl, res)) + { + case SSL_ERROR_WANT_READ: [[fallthrough]]; + case SSL_ERROR_WANT_WRITE: + /// `SSL_peek` found no complete application-data record, but that alone does not prove + /// the connection is idle: the bytes of a record that has only partially arrived (e.g. + /// the first fragment of a queued response) are buffered inside the SSL object too, and + /// look identical from here - both end in `SSL_ERROR_WANT_READ`. `SSL_has_pending` + /// reports on that internal buffer regardless of whether the record is complete, so a + /// session ticket / `KeyUpdate` that was fully consumed reads as idle (nothing left + /// buffered), while a partial record correctly reads as pending. + result = {SSL_has_pending(ssl) ? SocketState::DataPending : SocketState::Idle, false}; + break; + case SSL_ERROR_ZERO_RETURN: + result = {SocketState::Closed, false}; /// The peer sent `close_notify`: an orderly TLS shutdown. + break; + case SSL_ERROR_SYSCALL: [[fallthrough]]; + case SSL_ERROR_SSL: + /// A FIN without `close_notify` or a protocol error is fatal. OpenSSL forbids + /// `SSL_shutdown` afterwards. + result = {SocketState::Closed, true}; + break; + default: + /// Any other unexpected result is treated as closed/broken, but only + /// `SSL_ERROR_SYSCALL` and `SSL_ERROR_SSL` make the connection fatal. + result = {SocketState::Closed, false}; + break; + } } -} -namespace -{ + /// Do not leak errors from this diagnostic probe into subsequent operations on this thread. + ERR_clear_error(); + return result; +} /// Force the socket into non-blocking mode for the duration of a call, restoring the original /// mode afterwards, so that `SSL_peek` on an idle pooled connection can never block. @@ -90,13 +114,10 @@ class ScopedNonBlocking : socket_impl(socket_impl_) { #if USE_SILK - if (auto * fiber_socket_impl = dynamic_cast(&socket_impl)) - { - was_blocking = !fiber_socket_impl->getDontWait(); - if (was_blocking) - fiber_socket_impl->setDontWait(true); + /// The Silk TLS BIO is always non-blocking so that an OpenSSL operation cannot + /// suspend and migrate between the operation and `SSL_get_error`. + if (dynamic_cast(&socket_impl)) return; - } #endif was_blocking = socket_impl.getBlocking(); if (was_blocking) @@ -110,13 +131,6 @@ class ScopedNonBlocking try { -#if USE_SILK - if (auto * fiber_socket_impl = dynamic_cast(&socket_impl)) - { - fiber_socket_impl->setDontWait(false); - return; - } -#endif socket_impl.setBlocking(true); } catch (...) @@ -130,13 +144,17 @@ class ScopedNonBlocking private: Poco::Net::SocketImpl & socket_impl; - /// For regular (non-silk) socket: whether it was blocking before. - /// For silk socket: whether it was dont-wait before. + /// Whether a regular (non-Silk) socket was blocking before the probe. bool was_blocking = false; }; } +SocketState getSSLSocketState(ssl_st * ssl) +{ + return getSSLSocketStateImpl(ssl).state; +} + #endif SocketState getSocketState(const Poco::Net::StreamSocket & socket) @@ -149,7 +167,10 @@ SocketState getSocketState(const Poco::Net::StreamSocket & socket) if (auto * ssl = secure->ssl()) { ScopedNonBlocking non_blocking(*secure); - return getSSLSocketState(ssl); + auto result = getSSLSocketStateImpl(ssl); + if (result.fatal_error) + secure->markFatalError(); + return result.state; } } #endif diff --git a/src/IO/SocketPeerClosed.h b/src/IO/SocketPeerClosed.h index 0d50cc64e307..fb7ad792164f 100644 --- a/src/IO/SocketPeerClosed.h +++ b/src/IO/SocketPeerClosed.h @@ -52,8 +52,8 @@ SocketState getSocketState(const Poco::Net::StreamSocket & socket); #if USE_SSL /// TLS-aware core of the check, exposed for testing. The socket underlying `ssl` MUST be in -/// non-blocking mode - don't-wait mode for a fiber socket - (the caller guarantees this) so that -/// `SSL_peek` cannot block. +/// non-blocking mode (the caller guarantees this) so that `SSL_peek` cannot block. The Silk TLS +/// BIO is always non-blocking. SocketState getSSLSocketState(ssl_st * ssl); #endif diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 185c22b38caf..1afee1a496e0 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -642,6 +642,19 @@ inline void writeQuotedStringSQLite(std::string_view ref, WriteBuffer & buf) writeChar('\'', buf); } +/// SQLite identifiers: a " is escaped by doubling it; every other byte, backslash included, is literal. +inline void writeDoubleQuotedStringSQLite(std::string_view ref, WriteBuffer & buf) +{ + writeChar('"', buf); + for (char c : ref) + { + if (c == '"') + writeChar('"', buf); + writeChar(c, buf); + } + writeChar('"', buf); +} + inline void writeDoubleQuotedString(const String & s, WriteBuffer & buf) { writeAnyQuotedString<'"'>(s, buf); diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 6a41e80d165d..1fb167e7267d 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int CANNOT_PARSE_DATETIME; +extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; } @@ -99,7 +100,8 @@ ReturnType parseDateTimeBestEffortImpl( const DateLUTImpl & utc_time_zone, DateTimeSubsecondPart * fractional, const char * allowed_date_delimiters = nullptr, - bool * has_explicit_zero_year = nullptr) + bool * has_explicit_zero_year = nullptr, + DateTimeOverflow overflow = DateTimeOverflow::Saturate) { auto on_error = [&]( int error_code [[maybe_unused]], @@ -323,7 +325,7 @@ ReturnType parseDateTimeBestEffortImpl( /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 16 && !year && !has_time) { @@ -339,7 +341,7 @@ ReturnType parseDateTimeBestEffortImpl( /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 19 && !year && !has_time) { @@ -355,7 +357,7 @@ ReturnType parseDateTimeBestEffortImpl( /// Fractional part is not allowed. return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: unexpected fractional part"); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 10 && !year && !has_time) { @@ -374,7 +376,7 @@ ReturnType parseDateTimeBestEffortImpl( readDigits(digits, sizeof(digits), in))); readDecimalNumber(fractional->value, fractional->digits, digits); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 9 && !year && !has_time) { @@ -393,7 +395,7 @@ ReturnType parseDateTimeBestEffortImpl( readDigits(digits, sizeof(digits), in))); readDecimalNumber(fractional->value, fractional->digits, digits); } - return ReturnType(true); + return checkParsedDateTimeRange(res, overflow == DateTimeOverflow::Saturate); } if (num_digits == 14 && !year && !has_time) { @@ -935,6 +937,10 @@ ReturnType parseDateTimeBestEffortImpl( if (has_explicit_zero_year) *has_explicit_zero_year = zero_year_was_read; + /// Year 0000 is outside DateTime and the substitution below would hide that, which `throw` forbids + if (!is_64 && zero_year_was_read && overflow == DateTimeOverflow::Report) + return on_error(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Year 0000 is out of bounds of type DateTime"); + if constexpr (strict) return on_error(ErrorCodes::CANNOT_PARSE_DATETIME, "Cannot read DateTime: year is required"); @@ -996,7 +1002,8 @@ ReturnType parseDateTimeBestEffortImpl( } }; - if constexpr (!strict || std::is_same_v) + /// `strict` always range-checks, otherwise only when the caller asked not to saturate + if (overflow == DateTimeOverflow::Saturate && !(strict && std::is_same_v)) { if (has_time_zone_offset) { @@ -1008,52 +1015,33 @@ ReturnType parseDateTimeBestEffortImpl( res = local_time_zone.makeDateTime(year, month, day_of_month, hour, minute, second); } - if constexpr (std::is_same_v) - return true; + return ReturnType(true); } - else - { - if (has_time_zone_offset) - { - auto res_maybe = utc_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second); - if (!res_maybe) - return false; - /// For usual DateTime check if value is within supported range - if constexpr (!is_64) - { - if (*res_maybe < 0 || *res_maybe > UINT32_MAX) - return false; - } - res = *res_maybe; - adjust_time_zone(); + const DateLUTImpl & time_zone = has_time_zone_offset ? utc_time_zone : local_time_zone; + auto res_maybe = time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second); + if (!res_maybe) + return on_error( + ErrorCodes::CANNOT_PARSE_DATETIME, + "Cannot read DateTime: unexpected date: {}-{}-{}", + year, + static_cast(month), + static_cast(day_of_month)); - /// After timezone adjustment, the value may have shifted outside the valid range. - /// For example, "2106-02-07 06:28:15-01:00" is within range before adjustment, - /// but after converting to UTC it exceeds UINT32_MAX. - if constexpr (!is_64) - { - if (res < 0 || static_cast(res) > UINT32_MAX) - return false; - } - } - else - { - auto res_maybe = local_time_zone.tryToMakeDateTime(year, month, day_of_month, hour, minute, second); - if (!res_maybe) - return false; + res = *res_maybe; - /// For usual DateTime check if value is within supported range - if constexpr (!is_64) - { - if (*res_maybe < 0 || *res_maybe > UINT32_MAX) - return false; - } - res = *res_maybe; - } + if (has_time_zone_offset) + adjust_time_zone(); - return true; + /// Only the adjusted value has to be in range: "2106-02-07 07:28:15+01:00" is past the maximum before the + /// offset is applied and is exactly the maximum after it, and the same holds at the lower bound. + if constexpr (!is_64) + { + if (res < 0 || res > UINT32_MAX) + return on_error(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "Value {} is out of bounds of type DateTime", res); } + + return ReturnType(true); } template @@ -1094,7 +1082,12 @@ ReturnType parseDateTime64BestEffortImpl(DateTime64 & res, UInt32 scale, ReadBuf void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) { - parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + parseDateTimeBestEffort(res, in, local_time_zone, utc_time_zone, DateTimeOverflow::Saturate); +} + +void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) +{ + parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, bool & has_explicit_zero_year) @@ -1103,19 +1096,19 @@ void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, &has_explicit_zero_year); } -void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) { - parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } -bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) { - return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } -bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) +bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow) { - return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr); + return parseDateTimeBestEffortImpl(res, in, local_time_zone, utc_time_zone, nullptr, nullptr, nullptr, overflow); } void parseDateTime64BestEffort(DateTime64 & res, UInt32 scale, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone) diff --git a/src/IO/parseDateTimeBestEffort.h b/src/IO/parseDateTimeBestEffort.h index c42243b1efc0..a09d44de1482 100644 --- a/src/IO/parseDateTimeBestEffort.h +++ b/src/IO/parseDateTimeBestEffort.h @@ -55,8 +55,14 @@ class ReadBuffer; * Mon/Tue/Wed/Thu/Fri/Sat/Sun - simply ignored. */ +/// Whether an out-of-range result saturates to the bounds of the target type or is reported as an error +enum class DateTimeOverflow : uint8_t { Saturate, Report }; + void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +/// Not defaulted, because the 4-argument form above would then be ambiguous with this one +void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow); + /// The same, but additionally reports whether the input contained an explicitly written year of `0000`. /// Such a year cannot be represented: internally a year field of `0` means "the year is not specified", /// so it is silently replaced with the current (or previous) year, and the returned value is then not the @@ -66,9 +72,9 @@ void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & void parseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, bool & has_explicit_zero_year); void parseTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); -bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); -void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); -bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); +bool tryParseDateTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow = DateTimeOverflow::Saturate); +void parseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow = DateTimeOverflow::Saturate); +bool tryParseDateTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone, DateTimeOverflow overflow = DateTimeOverflow::Saturate); bool tryParseTimeBestEffort(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); void parseTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); bool tryParseTimeBestEffortUS(time_t & res, ReadBuffer & in, const DateLUTImpl & local_time_zone, const DateLUTImpl & utc_time_zone); diff --git a/src/IO/tests/gtest_limit_read_buffer.cpp b/src/IO/tests/gtest_limit_read_buffer.cpp new file mode 100644 index 000000000000..04dde06885f1 --- /dev/null +++ b/src/IO/tests/gtest_limit_read_buffer.cpp @@ -0,0 +1,162 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ +extern const int LIMIT_EXCEEDED; +extern const int CANNOT_READ_ALL_DATA; +} + +namespace +{ + +String readAll(ReadBuffer & in) +{ + String result; + WriteBufferFromString out(result); + copyData(in, out); + out.finalize(); + return result; +} + +int codeOfThrown(ReadBuffer & in) +{ + try + { + readAll(in); + } + catch (const Exception & e) + { + return e.code(); + } + return 0; +} + +} + +TEST(LimitReadBuffer, StreamEndingAtTheLimitIsNotAnError) +{ + ReadBufferFromString nested(std::string_view("0123456789")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(readAll(limited), "0123456789"); +} + +TEST(LimitReadBuffer, ExpectEofRejectsDataPastTheLimit) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(codeOfThrown(limited), ErrorCodes::LIMIT_EXCEEDED); +} + +TEST(LimitReadBuffer, ZeroLimitRejectsANonEmptyStream) +{ + ReadBufferFromString nested(std::string_view("a")); + LimitReadBuffer limited(nested, {.read_no_more = 0, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(codeOfThrown(limited), ErrorCodes::LIMIT_EXCEEDED); +} + +TEST(LimitReadBuffer, WithoutExpectEofDataPastTheLimitIsCutOff) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); +} + +/// The check runs in `nextImpl`, so a consumer that reads exactly the limit and stops never asks for +/// the byte that would reveal the overflow. `expect_eof` is therefore best-effort, not a guarantee. +TEST(LimitReadBuffer, ExpectEofIsNotCheckedWhenTheConsumerStopsAtTheLimit) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + char buf[10] = {}; + limited.readStrict(buf, sizeof(buf)); + EXPECT_EQ(String(buf, sizeof(buf)), "0123456789"); +} + +TEST(LimitReadBuffer, ExpectEofShortStreamIsNotAnError) +{ + ReadBufferFromString nested(std::string_view("012")); + LimitReadBuffer limited(nested, {.read_no_more = 10, .expect_eof = true, .excetion_hint = "hint"}); + EXPECT_EQ(readAll(limited), "012"); +} + +TEST(LimitReadBuffer, StreamShorterThanReadNoLessIsAnError) +{ + ReadBufferFromString nested(std::string_view("012")); + LimitReadBuffer limited(nested, {.read_no_less = 10, .read_no_more = 10}); + EXPECT_EQ(codeOfThrown(limited), ErrorCodes::CANNOT_READ_ALL_DATA); +} + +TEST(LimitReadBuffer, WithoutExpectEofStreamEndingAtTheLimitIsFine) +{ + ReadBufferFromString nested(std::string_view("0123456789")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); +} + +TEST(LimitReadBuffer, WithoutExpectEofShortStreamIsFine) +{ + ReadBufferFromString nested(std::string_view("012")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "012"); +} + +TEST(LimitReadBuffer, WithoutExpectEofEmptyStreamIsFine) +{ + ReadBufferFromString nested(std::string_view("")); + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), ""); +} + +TEST(LimitReadBuffer, WithoutExpectEofZeroLimitReadsNothing) +{ + ReadBufferFromString nested(std::string_view("abc")); + { + LimitReadBuffer limited(nested, {.read_no_more = 0}); + EXPECT_EQ(readAll(limited), ""); + } + EXPECT_EQ(readAll(nested), "abc"); +} + +TEST(LimitReadBuffer, WithoutExpectEofNestedBufferContinuesPastTheLimit) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + { + LimitReadBuffer limited(nested, {.read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); + } + EXPECT_EQ(readAll(nested), "abc"); +} + +/// The `Content-Length` shape of `HTTPServerRequest`. With keep-alive the nested buffer holds the next +/// request, so the bytes past the limit have to stay readable. +TEST(LimitReadBuffer, ExactLengthLeavesTheRestForTheNextReader) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + { + LimitReadBuffer limited(nested, {.read_no_less = 10, .read_no_more = 10}); + EXPECT_EQ(readAll(limited), "0123456789"); + } + EXPECT_EQ(readAll(nested), "abc"); +} + +TEST(LimitReadBuffer, PartialReadLeavesNestedBufferAtTheConsumedOffset) +{ + ReadBufferFromString nested(std::string_view("0123456789abc")); + { + LimitReadBuffer limited(nested, {.read_no_more = 10}); + char buf[4] = {}; + limited.readStrict(buf, sizeof(buf)); + EXPECT_EQ(String(buf, sizeof(buf)), "0123"); + } + EXPECT_EQ(readAll(nested), "456789abc"); +} diff --git a/src/IO/tests/gtest_silk_fiber_stream_socket.cpp b/src/IO/tests/gtest_silk_fiber_stream_socket.cpp index 8a64a8e91bcd..b0dfeb5038c1 100644 --- a/src/IO/tests/gtest_silk_fiber_stream_socket.cpp +++ b/src/IO/tests/gtest_silk_fiber_stream_socket.cpp @@ -303,8 +303,7 @@ TYPED_TEST(SilkFiberSocketTest, ThrottlerLimitEnforced) } -/// Secure-only: the bug is TLS-specific (a blocking-only `silkBioRead`/`silkBioWrite` surfaces -/// through `SSL_peek`, not through a raw, BIO-less socket read). Reuses the +/// Secure-only tests for the TLS BIO and direct OpenSSL operations. Reuses the /// `SecurePolicy policy` member from the typed fixture rather than redeclaring it. using SilkFiberSecureSocketTest = SilkFiberSocketTest; @@ -338,15 +337,12 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingPeekDoesNotBlockOnIdleConnection) char pong[1] = {}; EXPECT_EQ(socket.receiveBytes(pong, sizeof(pong)), 1); - /// A long receive timeout. Pre-fix, `silkBioRead` has no non-blocking mode and always - /// parks the caller in a fiber wait up to this timeout, so a slow probe below proves - /// the bug; a fast one proves the fix. + /// A long receive timeout. The probe must remain non-blocking regardless of it. socket.setReceiveTimeout(Poco::Timespan(5, 0)); /// The actual production sequence (`DB::getSocketState(StreamSocket)`, the core of the - /// connection pool's staleness check in `HTTPConnectionPool.cpp`): it puts the silk - /// socket into don't-wait mode with `setDontWait` - `Socket::setBlocking(false)` is rejected - /// by silk sockets - and calls `SSL_peek`, which reaches OpenSSL's socket BIO, i.e. `silkBioRead`. + /// connection pool's staleness check in `HTTPConnectionPool.cpp`) calls `SSL_peek`, + /// which reaches the always non-blocking Silk TLS BIO. Stopwatch watch; *p->state = DB::getSocketState(socket); *p->elapsed_us = watch.elapsedMicroseconds(); @@ -371,17 +367,13 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingPeekDoesNotBlockOnIdleConnection) EXPECT_EQ(state, DB::SocketState::Idle); EXPECT_LT(elapsed_us, 500'000U) << "getSocketState() took " << elapsed_us - << "us: silkBioRead ignored non-blocking mode and blocked on the receive timeout instead " - "of returning EAGAIN immediately"; + << "us: the Silk TLS BIO blocked on the receive timeout instead of returning EAGAIN immediately"; } -/// The same bug at the raw level, without any ClickHouse helper: a plain `SSL_peek` on a -/// non-blocking TLS connection with no data pending must return `SSL_ERROR_WANT_READ` -/// immediately. This is exactly how a non-blocking consumer uses the socket - and the only way, -/// since silk sockets reject `Socket::setBlocking(false)`: the socket is put into don't-wait -/// mode with `setDontWait` and only `silkBioRead` ever observes it. Pre-fix, the BIO has no -/// non-blocking mode and parks the caller for the full receive timeout. +/// At the raw level, without a ClickHouse helper, a plain `SSL_peek` on an idle TLS connection +/// must return `SSL_ERROR_WANT_READ` immediately. Silk sockets reject `Socket::setBlocking(false)`, +/// so the BIO itself has to be non-blocking. TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) { auto listener = policy.makeListener(); @@ -412,11 +404,10 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) socket.setReceiveTimeout(Poco::Timespan(5, 0)); - /// Put the socket into don't-wait mode - what any non-blocking user must do here, - /// since `Socket::setBlocking(false)` throws on a silk socket. + /// The Silk TLS BIO is always non-blocking. OpenSSL operations return WANT_READ or + /// WANT_WRITE and `SecureSocketImpl` performs the fiber-aware wait outside OpenSSL. auto * secure = dynamic_cast(socket.impl()); SSL * ssl = secure->ssl(); - secure->setDontWait(true); char c = 0; ERR_clear_error(); @@ -425,7 +416,6 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) *p->ssl_error = SSL_get_error(ssl, res); *p->elapsed_us = watch.elapsedMicroseconds(); - secure->setDontWait(false); socket.close(); return 0; }, @@ -445,8 +435,8 @@ TEST_F(SilkFiberSecureSocketTest, NonBlockingSslPeekReturnsWantReadImmediately) EXPECT_EQ(ssl_error, SSL_ERROR_WANT_READ); EXPECT_LT(elapsed_us, 500'000U) << "SSL_peek() took " << elapsed_us - << "us on an idle non-blocking connection: silkBioRead ignored non-blocking mode and " - "blocked on the receive timeout instead of returning EAGAIN immediately"; + << "us on an idle connection: the Silk TLS BIO blocked on the receive timeout instead " + "of returning EAGAIN immediately"; } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index ea637f7688a3..48484dd0f4ea 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -3430,7 +3430,8 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu const Block & right_stream_header, const Names & equivalent_columns_to_push_down, const std::unordered_map & equivalent_left_stream_column_to_right_stream_column, - const std::unordered_map & equivalent_right_stream_column_to_left_stream_column) + const std::unordered_map & equivalent_right_stream_column_to_left_stream_column, + bool filter_is_always_false) { Node * predicate = const_cast(tryFindInOutputs(filter_name)); if (!predicate) @@ -3501,14 +3502,19 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu const bool left_stream_push_down_enabled = !left_stream_allowed_nodes.empty(); const bool right_stream_push_down_enabled = !right_stream_allowed_nodes.empty(); - if (!left_stream_push_down_enabled) - keep_conjuncts_depending_on_allowed_input(left_stream_push_down_conjunctions, left_stream_allowed_nodes); - if (!right_stream_push_down_enabled) - keep_conjuncts_depending_on_allowed_input(right_stream_push_down_conjunctions, right_stream_allowed_nodes); - /// A both-streams conjunct is pushed to BOTH sides, so a no-input conjunct here is unsafe if - /// EITHER side is disabled. - if (!left_stream_push_down_enabled || !right_stream_push_down_enabled) - keep_conjuncts_depending_on_allowed_input(both_streams_push_down_conjunctions, both_streams_allowed_nodes); + /// If no row passes the filter, the join output is already empty once a side that may be filtered + /// receives it, so a disabled side can receive the no-input conjuncts too and is not read in vain. + if (!filter_is_always_false) + { + if (!left_stream_push_down_enabled) + keep_conjuncts_depending_on_allowed_input(left_stream_push_down_conjunctions, left_stream_allowed_nodes); + if (!right_stream_push_down_enabled) + keep_conjuncts_depending_on_allowed_input(right_stream_push_down_conjunctions, right_stream_allowed_nodes); + /// A both-streams conjunct is pushed to BOTH sides, so a no-input conjunct here is unsafe if + /// EITHER side is disabled. + if (!left_stream_push_down_enabled || !right_stream_push_down_enabled) + keep_conjuncts_depending_on_allowed_input(both_streams_push_down_conjunctions, both_streams_allowed_nodes); + } NodeRawConstPtrs left_stream_allowed_conjunctions = std::move(left_stream_push_down_conjunctions.allowed); NodeRawConstPtrs right_stream_allowed_conjunctions = std::move(right_stream_push_down_conjunctions.allowed); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 4182c9f950eb..f9f60ca8aa4d 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -507,6 +507,7 @@ class ActionsDAG * to left and right streams. * @param equivalent_left_stream_column_to_right_stream_column - equivalent left stream column name to right stream column map. * @param equivalent_right_stream_column_to_left_stream_column - equivalent right stream column name to left stream column map. + * @param filter_is_always_false - no row passes the filter, and a side whose emptiness empties the join output receives it. */ ActionsForJOINFilterPushDown splitActionsForJOINFilterPushDown( const std::string & filter_name, @@ -517,7 +518,8 @@ class ActionsDAG const Block & right_stream_header, const Names & equivalent_columns_to_push_down, const std::unordered_map & equivalent_left_stream_column_to_right_stream_column, - const std::unordered_map & equivalent_right_stream_column_to_left_stream_column); + const std::unordered_map & equivalent_right_stream_column_to_left_stream_column, + bool filter_is_always_false); /** Build filter dag from multiple filter dags. * diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 8d1fc4ae00d7..3be625694b4d 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -36,6 +36,7 @@ #include #include +#include #if CLICKHOUSE_CLOUD #include #endif @@ -1842,11 +1843,11 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool if (auto set = data.prepared_sets->findStorage(set_key)) return set; #if CLICKHOUSE_CLOUD - if (StorageSharedSet * storage_shared_set = dynamic_cast(table.get())) + if (StorageSharedSet * storage_shared_set = castStorage(table, DeferredTable::Load).get()) return data.prepared_sets->addFromStorage(set_key, right_in_operand, storage_shared_set->getSet(data.getContext()), table_id); #endif - if (StorageSet * storage_set = dynamic_cast(table.get())) + if (StorageSet * storage_set = castStorage(table, DeferredTable::Load).get()) return data.prepared_sets->addFromStorage(set_key, right_in_operand, storage_set->getSet(), table_id); } diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 0d5901bff0a5..b526a8ac129d 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -117,6 +117,21 @@ bool worthConvertToTwoLevel( || (group_by_two_level_threshold_bytes && result_size_bytes >= static_cast(group_by_two_level_threshold_bytes)); } +/// The row capacity of each chunk that `convertToBlockImpl` emits. +1 for `nullKeyData`: if the table +/// doesn't have it, that's not a problem, just memory for one excessive row is preallocated. +/// A non-zero `max_rows_per_block` lowers the `max_block_size` bound so that a table smaller than +/// one block can still be emitted as several chunks. +size_t convertedBlockSize(size_t table_size, size_t max_block_size, size_t max_rows_per_block, bool return_single_block) +{ + if (return_single_block) + return table_size + 1; + + if (max_rows_per_block) + max_block_size = std::min(max_block_size, max_rows_per_block); + + return std::min(max_block_size, table_size) + 1; +} + void initDataVariantsWithSizeHint( DB::AggregatedDataVariants & result, DB::AggregatedDataVariants::Type method_chosen, const DB::Aggregator::Params & params) { @@ -3334,7 +3349,7 @@ void Aggregator::disableMinMaxOptimizationForFixedHashMaps(ManyAggregatedDataVar template requires SetAggregationMethod Chunks -Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const +Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block) const { if (data.empty()) { @@ -3344,7 +3359,7 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & return result; } - Chunks res = convertToBlockImplKeysOnly(method, data, aggregates_pools, final, return_single_block); + Chunks res = convertToBlockImplKeysOnly(method, data, aggregates_pools, final, return_single_block, max_rows_per_block); /// In order to release memory early. data.clearAndShrink(); @@ -3355,7 +3370,7 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena *, Arenas & template requires MapAggregationMethod Chunks -Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final,size_t rows, bool return_single_block) const +Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block) const { if (data.empty()) { @@ -3368,8 +3383,7 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are if (is_simple_count) { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); std::optional out_cols; std::optional shuffled_key_sizes; @@ -3483,11 +3497,11 @@ Aggregator::convertToBlockImpl(Method & method, Table & data, Arena * arena, Are #if USE_EMBEDDED_COMPILER use_compiled_functions = compiled_aggregate_functions_holder != nullptr && !Method::low_cardinality_optimization; #endif - res = convertToBlockImplFinal(method, data, arena, aggregates_pools, use_compiled_functions, return_single_block); + res = convertToBlockImplFinal(method, data, arena, aggregates_pools, use_compiled_functions, return_single_block, max_rows_per_block); } else { - res = convertToBlockImplNotFinal(method, data, aggregates_pools, rows, return_single_block); + res = convertToBlockImplNotFinal(method, data, aggregates_pools, rows, return_single_block, max_rows_per_block); } /// In order to release memory early. @@ -3661,10 +3675,9 @@ Chunk Aggregator::insertResultsIntoColumns( template requires SetAggregationMethod Chunks Aggregator::convertToBlockImplKeysOnly( - Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block) const + Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block, size_t max_rows_per_block) const { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); std::optional out_cols; std::optional shuffled_key_sizes; @@ -3730,10 +3743,10 @@ Chunks Aggregator::convertToBlockImplFinal( Arena * arena, Arenas & aggregates_pools, bool use_compiled_functions [[maybe_unused]], - bool return_single_block) const + bool return_single_block, + size_t max_rows_per_block) const { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); const bool final = true; std::optional out_cols; @@ -3811,10 +3824,9 @@ Chunks Aggregator::convertToBlockImplFinal( template Chunks NO_INLINE -Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t, bool return_single_block) const +Aggregator::convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t, bool return_single_block, size_t max_rows_per_block) const { - /// +1 for nullKeyData, if `data` doesn't have it - not a problem, just some memory for one excessive row will be preallocated - const size_t max_block_size = (return_single_block ? data.size() : std::min(params.max_block_size, data.size())) + 1; + const size_t max_block_size = convertedBlockSize(data.size(), params.max_block_size, max_rows_per_block, return_single_block); const bool final = false; Chunks res_chunks; @@ -3989,7 +4001,7 @@ Aggregator::AggregatedChunk Aggregator::prepareChunkAndFillWithoutKey(Aggregated template std::conditional_t -Aggregator::prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const +Aggregator::prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block) const { Chunks res_variant; const size_t rows = data_variants.sizeWithoutOverflowRow(); @@ -3997,7 +4009,7 @@ Aggregator::prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variant else if (data_variants.type == AggregatedDataVariants::Type::NAME) \ { \ res_variant = convertToBlockImpl( \ - *data_variants.NAME, data_variants.NAME->data, data_variants.aggregates_pool, data_variants.aggregates_pools, final, rows, return_single_block); \ + *data_variants.NAME, data_variants.NAME->data, data_variants.aggregates_pool, data_variants.aggregates_pools, final, rows, return_single_block, max_rows_per_block); \ } if (false) {} // NOLINT @@ -4092,7 +4104,7 @@ Aggregator::AggregatedChunks Aggregator::prepareChunksAndFillTwoLevelImpl(Aggreg } -Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants & data_variants, bool final) const +Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block) const { LOG_TRACE(log, "Converting aggregated data to chunks"); @@ -4111,7 +4123,7 @@ Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants if (data_variants.type != AggregatedDataVariants::Type::without_key) { if (!data_variants.isTwoLevel()) - chunks.splice(chunks.end(), prepareChunkAndFillSingleLevel(data_variants, final)); + chunks.splice(chunks.end(), prepareChunkAndFillSingleLevel(data_variants, final, max_rows_per_block)); else chunks.splice(chunks.end(), prepareChunksAndFillTwoLevel(data_variants, final)); } @@ -4142,6 +4154,16 @@ Aggregator::AggregatedChunks Aggregator::convertToChunks(AggregatedDataVariants return chunks; } +size_t Aggregator::singleLevelChunkRowsForFanOut(size_t rows, size_t output_streams) +{ + static constexpr size_t MIN_ROWS_PER_CHUNK{512}; + const size_t num_chunks = std::clamp(rows / MIN_ROWS_PER_CHUNK, 1, std::max(output_streams, 1)); + if (num_chunks <= 1) + return 0; + + return (rows + num_chunks - 1) / num_chunks; +} + template void NO_INLINE Aggregator::mergeDataNullKey( diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 2ae75015b007..48c2e5c8a9f2 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -460,8 +460,15 @@ class Aggregator final * If final = false, then ColumnAggregateFunction is created as the aggregation columns with the state of the calculations, * which can then be combined with other states (for distributed query processing). * If final = true, then columns with ready values are created as aggregate columns. + * A non-zero `max_rows_per_block` caps the size of the emitted single-level chunks below `max_block_size`. */ - AggregatedChunks convertToChunks(AggregatedDataVariants & data_variants, bool final) const; + AggregatedChunks convertToChunks(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block = 0) const; + + /// A single-level result smaller than `max_block_size` is converted to one chunk, and a `Resize` + /// hands out whole chunks, so everything downstream of it runs in one thread. Returns a chunk size + /// that splits `rows` into about one chunk per output stream, never below 512 rows per chunk, or 0 + /// to leave the result as is. + static size_t singleLevelChunkRowsForFanOut(size_t rows, size_t output_streams); /// `adaptive_session` (or nullptr when the adaptive aggregation is off) feeds the /// thaw verdict into the hash-table statistics next to the observed sizes. @@ -1034,13 +1041,13 @@ class Aggregator final template requires MapAggregationMethod Chunks - convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const; + convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block = 0) const; /// A set method skips the inline-count and compiled-function paths; it only emits keys. template requires SetAggregationMethod Chunks - convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block) const; + convertToBlockImpl(Method & method, Table & data, Arena * arena, Arenas & aggregates_pools, bool final, size_t rows, bool return_single_block, size_t max_rows_per_block = 0) const; template void insertAggregatesIntoColumns( @@ -1060,7 +1067,7 @@ class Aggregator final template requires SetAggregationMethod Chunks convertToBlockImplKeysOnly( - Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block) const; + Method & method, Table & data, Arenas & aggregates_pools, bool final, bool return_single_block, size_t max_rows_per_block) const; template Chunks convertToBlockImplFinal( @@ -1069,11 +1076,12 @@ class Aggregator final Arena * arena, Arenas & aggregates_pools, bool use_compiled_functions, - bool return_single_block) const; + bool return_single_block, + size_t max_rows_per_block) const; template Chunks - convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t rows, bool return_single_block) const; + convertToBlockImplNotFinal(Method & method, Table & data, Arenas & aggregates_pools, size_t rows, bool return_single_block, size_t max_rows_per_block) const; /// `topk_full_key_bytes`, when non-null and the bucket goes through the Top-K conversion, /// receives the byte size all of the bucket's keys would occupy materialized: the runtime @@ -1124,9 +1132,10 @@ class Aggregator final AggregatedChunk prepareChunkAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final, bool is_overflows) const; AggregatedChunks prepareChunksAndFillTwoLevel(AggregatedDataVariants & data_variants, bool final) const; + /// A non-zero `max_rows_per_block` caps the size of the emitted chunks below `max_block_size`. template std::conditional_t - prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const; + prepareChunkAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final, size_t max_rows_per_block = 0) const; template AggregatedChunks prepareChunksAndFillTwoLevelImpl(AggregatedDataVariants & data_variants, Method & method, bool final) const; diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.cpp b/src/Interpreters/ApplyWithSubqueryVisitor.cpp index c3c922348a8d..fa32a281190f 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.cpp +++ b/src/Interpreters/ApplyWithSubqueryVisitor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6,16 +7,46 @@ #include #include #include +#include #include #include #include #include +#include #include +#include namespace DB { +namespace Setting +{ + extern const SettingsBool enable_global_with_statement; +} + +namespace +{ + +/// A name is looked up in an enclosing scope only while `enable_global_with_statement` holds in the +/// subquery's own context, so a CTE name that a subquery does not see is a table name there. +/// The clause is clamped rather than rejected, so a subquery cannot widen the reader's constraints, +/// and it is applied to a copy, so the AST keeps the clause as written. +ContextPtr getSubqueryContext(const ASTSelectQuery & select, const ContextPtr & context) +{ + auto settings_ast = select.settings(); + if (!settings_ast) + return context; + + auto changes = settings_ast->as().changes; + auto subquery_context = Context::createCopy(context); + subquery_context->clampToSettingsConstraints(changes, SettingSource::QUERY); + subquery_context->applySettingsChanges(changes); + return subquery_context; +} + +} + void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) { checkStackSize(); @@ -35,18 +66,30 @@ void ApplyWithSubqueryVisitor::visit(ASTPtr & ast, const Data & data) void ApplyWithSubqueryVisitor::visit(ASTSelectQuery & ast, const Data & data) { + /// The CTEs this select declares itself are registered below either way: only the inherited + /// ones are out of scope here. + std::optional scope_data; + if (data.context) + { + scope_data = data; + scope_data->context = getSubqueryContext(ast, data.context); + if (!scope_data->context->getSettingsRef()[Setting::enable_global_with_statement]) + scope_data->subqueries.clear(); + } + const Data & scope = scope_data ? *scope_data : data; + std::optional new_data; if (auto with = ast.with()) { for (auto & child : with->children) { - visit(child, new_data ? *new_data : data); + visit(child, new_data ? *new_data : scope); auto * ast_with_elem = child->as(); auto child_alias = child->tryGetAlias(); if (ast_with_elem || !child_alias.empty()) { if (!new_data) - new_data = data; + new_data = scope; if (ast_with_elem) new_data->subqueries[ast_with_elem->name] = ast_with_elem->subquery; else @@ -58,7 +101,7 @@ void ApplyWithSubqueryVisitor::visit(ASTSelectQuery & ast, const Data & data) for (auto & child : ast.children) { if (child != ast.with()) - visit(child, new_data ? *new_data : data); + visit(child, new_data ? *new_data : scope); } } diff --git a/src/Interpreters/ApplyWithSubqueryVisitor.h b/src/Interpreters/ApplyWithSubqueryVisitor.h index 72fc07c71483..36ce500dff77 100644 --- a/src/Interpreters/ApplyWithSubqueryVisitor.h +++ b/src/Interpreters/ApplyWithSubqueryVisitor.h @@ -3,6 +3,7 @@ #include #include +#include #include @@ -20,9 +21,19 @@ class ApplyWithSubqueryVisitor { std::map subqueries; std::map literals; + /// When set, each subquery's own settings are applied while descending, so that an inherited + /// `subqueries` element is not substituted into a subquery whose settings hide it. Inherited + /// `literals` are substituted either way. + ContextPtr context; }; - static void visit(ASTPtr & ast) { visit(ast, {}); } + static void visit(ASTPtr & ast) { visit(ast, Data{}); } + static void visit(ASTPtr & ast, ContextPtr context) + { + Data data; + data.context = std::move(context); + visit(ast, data); + } static void visit(ASTSelectQuery & select) { visit(select, {}); } static void visit(ASTSelectWithUnionQuery & select) { visit(select, {}); } diff --git a/src/Interpreters/ArrayJoinAction.cpp b/src/Interpreters/ArrayJoinAction.cpp index ef87afc35c30..1b5126a77070 100644 --- a/src/Interpreters/ArrayJoinAction.cpp +++ b/src/Interpreters/ArrayJoinAction.cpp @@ -42,6 +42,15 @@ static ColumnPtr getArrayJoinColumn(const ColumnPtr & column) return column; if (const auto * map = typeid_cast(column.get())) return map->getNestedColumnPtr(); + /// Keep replicated arrays lazy, only unwrap maps. + if (const auto * replicated = typeid_cast(column.get())) + { + const auto & nested = replicated->getNestedColumn(); + if (typeid_cast(nested.get())) + return column; + if (const auto * map = typeid_cast(nested.get())) + return ColumnReplicated::create(map->getNestedColumnPtr(), replicated->getIndexesColumn()); + } return nullptr; } @@ -151,11 +160,8 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j const auto & function_array_resize = array_join->function_array_resize; const auto & function_builder = array_join->function_builder; - /// TODO: avoid convertToFullColumnIfReplicated - any_array_map_ptr = block.getByName(*columns.begin()).column->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); - any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); - if (!any_array) - throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + any_array_map_ptr = block.getByName(*columns.begin()).column->convertToFullColumnIfConst(); + initAnyArray(); if (is_unaligned) { @@ -187,27 +193,60 @@ ArrayJoinResultIterator::ArrayJoinResultIterator(const ArrayJoinAction * array_j any_array_map_ptr = src_col.column->convertToFullColumnIfConst(); } - any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); - if (!any_array) - throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + initAnyArray(); } else if (is_left) { for (const auto & name : columns) { const auto & src_col = block.getByName(name); - ColumnWithTypeAndName array_col = convertArrayJoinColumn(src_col); + /// emptyArrayToSingle is fine with a replicated input, no need to materialize it. + ColumnWithTypeAndName array_col{getArrayJoinColumn(src_col.column->convertToFullColumnIfConst()), getArrayJoinDataType(src_col.type), src_col.name}; ColumnsWithTypeAndName tmp_block{array_col}; non_empty_array_columns[name] = function_builder->build(tmp_block)->execute(tmp_block, array_col.type, array_col.column->size(), /* dry_run = */ false); } any_array_map_ptr = non_empty_array_columns.begin()->second->convertToFullColumnIfConst(); - any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); - if (!any_array) - throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + initAnyArray(); } } +void ArrayJoinResultIterator::initAnyArray() +{ + any_array = getArrayJoinColumnRawPtr(any_array_map_ptr); + if (any_array) + return; + + /// Replicated arrays are materialized per window, here we only need the row sizes. + const auto * replicated = typeid_cast(any_array_map_ptr.get()); + const auto * nested_array = replicated ? getArrayJoinColumnRawPtr(replicated->getNestedColumn()) : nullptr; + if (!nested_array) + throw Exception(ErrorCodes::TYPE_MISMATCH, "ARRAY JOIN requires array or map argument"); + + const auto & nested_offsets = nested_array->getOffsets(); + const auto & indexes = replicated->getIndexes(); + replicated_offsets.resize(replicated->size()); + size_t accumulated = 0; + for (size_t row = 0; row < replicated_offsets.size(); ++row) + { + size_t index = indexes.getIndexAt(row); + accumulated += nested_offsets[index] - nested_offsets[index - 1]; + replicated_offsets[row] = accumulated; + } +} + +const IColumn::Offsets & ArrayJoinResultIterator::anyOffsets() const +{ + return any_array ? any_array->getOffsets() : replicated_offsets; +} + +ColumnPtr ArrayJoinResultIterator::cutAnyArray(size_t start, size_t length) const +{ + if (any_array) + return any_array->cut(start, length); + return getArrayJoinColumn(any_array_map_ptr->cut(start, length)->convertToFullColumnIfReplicated()); +} + bool ArrayJoinResultIterator::hasNext() const { return total_rows != 0 && current_row < total_rows; @@ -220,7 +259,7 @@ Block ArrayJoinResultIterator::next() throw Exception(ErrorCodes::LOGICAL_ERROR, "No more elements in ArrayJoinResultIterator."); size_t max_block_size = array_join->max_block_size; - const auto & offsets = any_array->getOffsets(); + const auto & offsets = anyOffsets(); /// Make sure output block rows do not exceed max_block_size. size_t next_row = current_row; @@ -237,7 +276,7 @@ Block ArrayJoinResultIterator::next() const auto & columns = array_join->columns; bool is_unaligned = array_join->is_unaligned; bool is_left = array_join->is_left; - auto cut_any_col = any_array->cut(current_row, next_row - current_row); + auto cut_any_col = cutAnyArray(current_row, next_row - current_row); const auto * cut_any_array = typeid_cast(cut_any_col.get()); ColumnPtr indexes_for_lazy_replication; @@ -258,20 +297,9 @@ Block ArrayJoinResultIterator::next() { if (const auto & type = getArrayJoinDataType(current.type)) { - ColumnPtr array_ptr; - if (typeid_cast(current.type.get())) - { - array_ptr = (is_left && !is_unaligned) ? non_empty_array_columns[current.name]->cut(current_row, next_row - current_row) - : current.column; - array_ptr = array_ptr->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); - } - else - { - ColumnPtr map_ptr = current.column->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); - const ColumnMap & map = typeid_cast(*map_ptr); - array_ptr = (is_left && !is_unaligned) ? non_empty_array_columns[current.name]->cut(current_row, next_row - current_row) - : map.getNestedColumnPtr(); - } + ColumnPtr array_ptr = (is_left && !is_unaligned) ? non_empty_array_columns[current.name]->cut(current_row, next_row - current_row) + : getArrayJoinColumn(current.column->convertToFullColumnIfConst()->convertToFullColumnIfReplicated()); + array_ptr = array_ptr->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); const ColumnArray & array = typeid_cast(*array_ptr); if (!is_unaligned && !array.hasEqualOffsets(*cut_any_array)) diff --git a/src/Interpreters/ArrayJoinAction.h b/src/Interpreters/ArrayJoinAction.h index aac5aac7b9db..141c9e440d5b 100644 --- a/src/Interpreters/ArrayJoinAction.h +++ b/src/Interpreters/ArrayJoinAction.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -58,12 +59,18 @@ class ArrayJoinResultIterator bool hasNext() const; private: + void initAnyArray(); + const PaddedPODArray & anyOffsets() const; + ColumnPtr cutAnyArray(size_t start, size_t length) const; + const ArrayJoinAction * array_join; Block block; bool enable_lazy_columns_replication; ColumnPtr any_array_map_ptr; - const ColumnArray * any_array; + /// Null if the joined column is replicated, then replicated_offsets is used instead. + const ColumnArray * any_array = nullptr; + PaddedPODArray replicated_offsets; /// If LEFT ARRAY JOIN, then we create columns in which empty arrays are replaced by arrays with one element - the default value. std::map non_empty_array_columns; diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 38cef6d0e993..a68b1e466234 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,11 @@ namespace ProfileEvents namespace DB { +namespace FailPoints +{ + extern const char async_insert_pause_before_schedule[]; +} + namespace Setting { extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; @@ -319,8 +325,8 @@ void AsynchronousInsertQueue::flushAndShutdown() { try { - LOG_TRACE(log, "Shutting down the asynchronous insertion queue"); shutdown = true; + LOG_TRACE(log, "Shutting down the asynchronous insertion queue"); if (flush_on_shutdown) { @@ -389,6 +395,8 @@ void AsynchronousInsertQueue::clear() void AsynchronousInsertQueue::scheduleDataProcessingJob( const InsertQuery & key, InsertDataPtr data, ContextPtr global_context, size_t shard_num, ThreadGroupPtr current_query_thread_group) { + FailPointInjection::pauseFailPoint(FailPoints::async_insert_pause_before_schedule); + /// Intuitively it seems reasonable to process first inserted blocks first. /// We add new chunks in the end of entries list, so they are automatically ordered by creation time chassert(!data->entries.empty()); @@ -676,6 +684,7 @@ AsynchronousInsertQueue::PushResult AsynchronousInsertQueue::pushDataChunk(ASTPt has_enough_queries ? "enough queries accumulated" : "maximum busy wait timeout exceeded"); data->timeout_ms = Milliseconds::zero(); + data->trackFlush(shard.in_flight_flushes); data_to_process = std::move(data); NOEXCEPT_SCOPE({ @@ -704,12 +713,15 @@ AsynchronousInsertQueue::PushResult AsynchronousInsertQueue::pushDataChunk(ASTPt CurrentMetrics::add(CurrentMetrics::AsynchronousInsertQueueBytes, entry_data_size); } - if (data_to_process) - scheduleDataProcessingJob(key, std::move(data_to_process), getContext(), shard_num); - else + if (!data_to_process) shard.are_tasks_available.notify_one(); } + /// Pool admission can wait for a running flush to finish. Keep that backpressure on this + /// producer, but allow other inserts to append to their buffers in the same queue shard. + if (data_to_process) + scheduleDataProcessingJob(key, std::move(data_to_process), getContext(), shard_num); + return PushResult { .status = PushResult::OK, @@ -861,9 +873,11 @@ void AsynchronousInsertQueue::flush(const std::vector & tables) futures_to_wait.size(), total_entries, total_bytes, total_queries, fmt::join(affected_set, ", ")); } - /// Wait until all jobs are finished. That includes only jobs - /// that were scheduled for this 'flush' call. - /// Other pending inserts are not blocked and can be processed concurrently. + /// Wait only for batches collected and scheduled by this `flush` call. + /// Batches already removed by producers or deadline workers are not included, + /// even if they are still waiting for pool admission. Unlike `flushAll`, this + /// does not wait on `in_flight_flushes`, which also counts unrelated tables. + /// Other pending inserts can continue concurrently. for (auto & future : futures_to_wait) future.wait(); @@ -910,6 +924,20 @@ void AsynchronousInsertQueue::flushAll() "Will wait for finishing of {} flushing jobs (about {} inserts, {} bytes, {} distinct queries)", pool.active(), total_entries, total_bytes, total_queries); + /// A removed batch can contain already acknowledged inserts while still waiting for + /// pool admission. `flush_stopped` and the shard locks above ensure no new batches + /// can enter this state until the forced flush finishes. Wait without the shard mutex + /// so producers can continue buffering and submitting the batches already removed. + for (auto & shard : queue_shards) + { + auto in_flight = shard.in_flight_flushes.load(); + while (in_flight) + { + shard.in_flight_flushes.wait(in_flight); + in_flight = shard.in_flight_flushes.load(); + } + } + /// Wait until all jobs are finished. That includes also jobs /// that were scheduled before the call of 'flushAll'. /// All other pending inserts are blocked by 'flush_stopped'. @@ -968,6 +996,7 @@ void AsynchronousInsertQueue::processBatchDeadlines(size_t shard_num) TSA_NO_THR shard.iterators.erase(it->second.key.hash); + it->second.data->trackFlush(shard.in_flight_flushes); entries_to_flush.emplace_back(std::move(it->second)); shard.queue.erase(it); @@ -1041,9 +1070,14 @@ try SCOPE_EXIT(CurrentMetrics::sub(CurrentMetrics::PendingAsyncInsert, data->entries.size())); + /// A batch may have left the shard queue before shutdown and waited for pool admission. + /// Check when the worker starts, after that wait, so the non-flushing shutdown path + /// also cancels detached batches through the normal exception and accounting cleanup. + if (shutdown && !flush_on_shutdown) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Async insert cancelled during shutdown"); + DB::setThreadName(ThreadName::ASYNC_INSERT_QUEUE); - const auto log = getLogger("AsynchronousInsertQueue"); const auto & insert_query = assert_cast(*key.query); bool internal = true; diff --git a/src/Interpreters/AsynchronousInsertQueue.h b/src/Interpreters/AsynchronousInsertQueue.h index 633b03ad2c15..3530e230996e 100644 --- a/src/Interpreters/AsynchronousInsertQueue.h +++ b/src/Interpreters/AsynchronousInsertQueue.h @@ -215,6 +215,16 @@ class AsynchronousInsertQueue : public WithContext } ready_promise.set_value(); + + if (in_flight_flushes && in_flight_flushes->fetch_sub(1) == 1) + in_flight_flushes->notify_all(); + } + + void trackFlush(std::atomic & counter) + { + chassert(!in_flight_flushes); + in_flight_flushes = &counter; + ++counter; } using EntryPtr = std::shared_ptr; @@ -224,6 +234,7 @@ class AsynchronousInsertQueue : public WithContext std::shared_future ready_future; size_t size_in_bytes = 0; Milliseconds timeout_ms = Milliseconds::zero(); + std::atomic * in_flight_flushes = nullptr; }; using InsertDataPtr = std::unique_ptr; @@ -248,6 +259,9 @@ class AsynchronousInsertQueue : public WithContext { mutable std::mutex mutex; mutable std::condition_variable are_tasks_available; + /// Counts batches removed by producers or the deadline worker, including those + /// still waiting for pool admission. Released when the batch is destroyed. + std::atomic in_flight_flushes{0}; Queue queue TSA_GUARDED_BY(mutex); QueueIteratorByKey iterators TSA_GUARDED_BY(mutex); @@ -273,6 +287,9 @@ class AsynchronousInsertQueue : public WithContext const size_t pool_size; const bool flush_on_shutdown; + /// Batches and jobs point into these vectors: `InsertData::in_flight_flushes` refers to a shard, and + /// `processData` receives the shard's flush time history by reference. Keep them declared before + /// `pool` and `dump_by_first_update_threads`, so they are destroyed after the threads that use them. std::vector queue_shards; std::vector flush_time_history_per_queue_shard; @@ -311,7 +328,7 @@ class AsynchronousInsertQueue : public WithContext void processBatchDeadlines(size_t shard_num); void scheduleDataProcessingJob(const InsertQuery & key, InsertDataPtr data, ContextPtr global_context, size_t shard_num, ThreadGroupPtr current_query_thread_group = nullptr); - static void processData( + void processData( InsertQuery key, InsertDataPtr data, ContextPtr global_context, ThreadGroupPtr current_query_thread_group, QueueShardFlushTimeHistory & queue_shard_flush_time_history); template diff --git a/src/Interpreters/BlobStorageLog.cpp b/src/Interpreters/BlobStorageLog.cpp index 6ed63b50b8f4..2595b66ffc72 100644 --- a/src/Interpreters/BlobStorageLog.cpp +++ b/src/Interpreters/BlobStorageLog.cpp @@ -14,6 +14,7 @@ #include #include +#include #include namespace DB @@ -103,7 +104,7 @@ static std::string_view normalizePath(std::string_view path) void BlobStorageLog::prepareTable() { SystemLog::prepareTable(); - if (auto merge_tree_table = std::dynamic_pointer_cast(getStorage())) + if (auto merge_tree_table = castStorage(getStorage(), DeferredTable::Skip)) { std::unique_lock lock{prepare_mutex}; const auto & relative_data_path = merge_tree_table->getRelativeDataPath(); diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index a2e6bbca35cd..b1bb65960ec7 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -264,7 +265,7 @@ void SelectStreamFactory::createForShardImpl( return; } - const auto * replicated_storage = dynamic_cast(main_table_storage.get()); + const auto * replicated_storage = castStorage(main_table_storage, DeferredTable::Load).get(); if (!replicated_storage) { diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 756c5ee47bcd..54cc24c6f638 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -1369,6 +1370,7 @@ ContextData::ContextData(const ContextData &o) : input_blocks_reader(o.input_blocks_reader), user_id(o.user_id), current_roles(o.current_roles), + external_roles(o.external_roles), settings_constraints_and_current_profiles(o.settings_constraints_and_current_profiles), access(o.access), need_recalculate_access(o.need_recalculate_access), @@ -2231,15 +2233,16 @@ void Context::setCurrentRolesWithLock(const std::vector & new_current_role void Context::setExternalRolesWithLock(const std::vector & new_external_roles, const std::lock_guard &) { - // External roles are roles received from other node, current roles is a collection of roles that were assigned locally - if (!new_external_roles.empty()) - { - if (external_roles) - external_roles->insert(external_roles->end(), new_external_roles.begin(), new_external_roles.end()); - else - external_roles = std::make_shared>(new_external_roles); - need_recalculate_access = true; - } + // External roles are roles received from another node; current roles is a collection of roles that were assigned locally. + // Replace them unconditionally (rather than append) so that switching the principal via `setUser` clears any external + // roles carried over from a previous principal on the same or a copied context. `ContextData`'s copy constructor + // preserves `external_roles`, so without this reset a context authenticated with pushed roles would keep them after + // `setUser(target_user)` (e.g. `EXECUTE AS target_user`), silently widening the target's privileges. + if (new_external_roles.empty()) + external_roles = nullptr; + else + external_roles = std::make_shared>(new_external_roles); + need_recalculate_access = true; } void Context::setCurrentRolesImpl(const std::vector & new_current_roles, bool throw_if_not_granted, bool skip_if_not_granted, const std::shared_ptr & user) @@ -3385,6 +3388,12 @@ StoragePtr Context::getViewSource() const return view_source; } + +void Context::clearViewSource() +{ + view_source.reset(); +} + bool Context::displaySecretsInShowAndSelect() const { return shared->server_settings[ServerSetting::display_secrets_in_show_and_select]; @@ -3396,6 +3405,41 @@ Settings Context::getSettingsCopy() const return *settings; } +namespace +{ +bool isProfileChange(const SettingChange & change) +{ + return change.name == "profile"; +} + +/// Enforces the constraints on `changes` the way `applySettingsChanges` applies them: a `profile` change +/// installs a new constraint set for the changes after it. Each run of changes up to the next `profile` +/// change is enforced against the constraints in force before it, then applied together with that `profile` +/// change to a scratch copy of `context`, so a rejected list leaves `context` untouched. Returns the enforced list. +template +SettingsChanges enforceConstraintsAlongProfileChanges(const ContextPtr & context, const SettingsChanges & changes, Enforce && enforce) +{ + auto scratch_context = Context::createCopy(context); + SettingsChanges enforced; + for (auto begin = changes.begin(); begin != changes.end();) + { + auto profile = std::find_if(begin, changes.end(), isProfileChange); + SettingsChanges segment(begin, profile); + enforce(*scratch_context, segment); + begin = profile; + if (profile != changes.end()) + { + segment.push_back(*profile); + ++begin; + } + /// `setCurrentProfile` checks the profile's own settings against the constraints in force before it. + scratch_context->applySettingsChanges(segment); + enforced.insert(enforced.end(), segment.begin(), segment.end()); + } + return enforced; +} +} + void Context::setSettings(const Settings & settings_) { std::lock_guard lock(mutex); @@ -3556,6 +3600,15 @@ void Context::checkSettingsConstraints(const SettingChange & change, SettingSour void Context::checkSettingsConstraints(const SettingsChanges & changes, SettingSource source) { + if (std::ranges::any_of(changes, isProfileChange)) + { + enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.checkSettingsConstraints(std::as_const(segment), source); + }); + return; + } + SharedLockGuard lock(mutex); settings->checkShorthandChanges(changes); getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(*settings, changes, source); @@ -3568,14 +3621,46 @@ void Context::checkSettingsConstraintsForSettingsReset(const std::vector getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.checkResetToDefault(*settings, names, source); } +void Context::checkSettingsConstraintsForSettingsReset( + const std::vector & names, const SettingsChanges & changes_applied_first, SettingSource source) +{ + if (std::ranges::none_of(changes_applied_first, isProfileChange)) + { + checkSettingsConstraintsForSettingsReset(names, source); + return; + } + /// The resets take effect after the rest of the statement, so a `profile` change in it decides the constraints. + auto scratch_context = Context::createCopy(shared_from_this()); + scratch_context->applySettingsChanges(changes_applied_first); + scratch_context->checkSettingsConstraintsForSettingsReset(names, source); +} + void Context::checkSettingsConstraints(SettingsChanges & changes, SettingSource source) { + if (std::ranges::any_of(changes, isProfileChange)) + { + changes = enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.checkSettingsConstraints(segment, source); + }); + return; + } + SharedLockGuard lock(mutex); checkSettingsConstraintsWithLock(changes, source); } void Context::clampToSettingsConstraints(SettingsChanges & changes, SettingSource source) { + if (std::ranges::any_of(changes, isProfileChange)) + { + changes = enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.clampToSettingsConstraints(segment, source); + }); + return; + } + SharedLockGuard lock(mutex); clampToSettingsConstraintsWithLock(changes, source); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index f40b65b73f91..522eef1fa017 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1156,6 +1156,7 @@ class Context: public ContextData, public std::enable_shared_from_this void addViewSource(const StoragePtr & storage); StoragePtr getViewSource() const; + void clearViewSource(); String getCurrentDatabase() const; String getCurrentQueryId() const { return client_info.current_query_id; } @@ -1228,6 +1229,8 @@ class Context: public ContextData, public std::enable_shared_from_this void checkSettingsConstraints(const SettingsChanges & changes, SettingSource source); void checkSettingsConstraints(SettingsChanges & changes, SettingSource source); void checkSettingsConstraintsForSettingsReset(const std::vector & names, SettingSource source); + /// For the resets of a statement that also changes `profile`: `changes_applied_first` decides their constraints. + void checkSettingsConstraintsForSettingsReset(const std::vector & names, const SettingsChanges & changes_applied_first, SettingSource source); void clampToSettingsConstraints(SettingsChanges & changes, SettingSource source); void checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index fc9da0b55af6..a5289588bb1c 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1744,7 +1745,7 @@ void DatabaseCatalog::dropTableFinally(const TableMarkedAsDropped & table) for (const auto & [disk_name, disk] : getContext()->getDisksMap()) { String data_path = getStoreDirPath(table.table_id.uuid); - auto table_merge_tree = std::dynamic_pointer_cast(table.table); + auto table_merge_tree = castStorage(table.table, DeferredTable::Skip); if (!is_disk_eligible_for_search(disk, table_merge_tree) || !disk->existsDirectory(data_path)) continue; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 27359b9199bb..ab5cb9cb7470 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -70,6 +70,7 @@ #include #include #include +#include #include #include #include @@ -482,7 +483,7 @@ SetPtr ExpressionAnalyzer::isPlainStorageSetInSubquery(const ASTPtr & subquery_o const auto storage = DatabaseCatalog::instance().getTable(table_id, getContext()); if (storage->getName() != "Set") return nullptr; - const auto storage_set = std::dynamic_pointer_cast(storage); + const auto storage_set = castStorage(storage, DeferredTable::Load); return storage_set->getSet(); } diff --git a/src/Interpreters/FileCache/FileCache.cpp b/src/Interpreters/FileCache/FileCache.cpp index 39bd735013dd..1e4d9d014038 100644 --- a/src/Interpreters/FileCache/FileCache.cpp +++ b/src/Interpreters/FileCache/FileCache.cpp @@ -1387,6 +1387,7 @@ bool FileCache::doTryReserve( file_segment.key(), file_segment.offset(), size, query_priority->getStateInfoForLog(lock)); failure_reason = "query limit exceeded"; + reserve_stat.not_enough_space = true; return false; } query_eviction_info = query_priority->collectEvictionInfo( @@ -1434,6 +1435,7 @@ bool FileCache::doTryReserve( invalidated_entries, query_priority, failure_reason)) { chassert(!failure_reason.empty()); + reserve_stat.not_enough_space = true; return false; } @@ -1493,6 +1495,19 @@ bool FileCache::doTryReserve( throw; } + /// After eviction, so a full cache disk can still admit. + if (auto ec = file_segment.getKeyMetadata()->createBaseDirectory(); ec) + { + /// Undo the reservation, so `main_priority` stays consistent with `FileSegment::reserved_size`. + if (added_new_main_entry) + main_priority_iterator->invalidate(); + else + main_priority_iterator->decrementSize(size); + + failure_reason = "Failed to create base directory for key, error: " + ec.message(); + return false; + } + /// Mark that size was successfully updated. if (added_new_main_entry) file_segment.setQueueIterator(main_priority_iterator); @@ -1500,12 +1515,6 @@ bool FileCache::doTryReserve( file_segment.reserved_size += size; chassert(file_segment.reserved_size == main_priority_iterator->getEntry()->size); - if (auto ec = file_segment.getKeyMetadata()->createBaseDirectory(); ec) - { - failure_reason = "Failed to create base directory for key, error: " + ec.message(); - return false; - } - return true; } diff --git a/src/Interpreters/FileCache/FileCache.h b/src/Interpreters/FileCache/FileCache.h index 02848269dcfb..1b6915101d67 100644 --- a/src/Interpreters/FileCache/FileCache.h +++ b/src/Interpreters/FileCache/FileCache.h @@ -70,6 +70,8 @@ struct FileCacheReserveStat Stat total_stat; std::array()> stat_by_kind{}; + /// Set if the reservation failed because the cache or the query limit had no room for it. + bool not_enough_space = false; Stat & getStatByKind(FileSegmentKind kind) { return stat_by_kind[static_cast(kind)]; } const Stat & getStatByKind(FileSegmentKind kind) const { return stat_by_kind[static_cast(kind)]; } @@ -89,6 +91,7 @@ struct FileCacheReserveStat total_stat += other.total_stat; for (size_t i = 0; i < stat_by_kind.size(); ++i) stat_by_kind[i] += other.stat_by_kind[i]; + not_enough_space |= other.not_enough_space; return *this; } }; diff --git a/src/Interpreters/FileCache/FileCacheSettings.cpp b/src/Interpreters/FileCache/FileCacheSettings.cpp index 307b40717851..cbabbf15affd 100644 --- a/src/Interpreters/FileCache/FileCacheSettings.cpp +++ b/src/Interpreters/FileCache/FileCacheSettings.cpp @@ -38,7 +38,7 @@ namespace ErrorCodes DECLARE(UInt64, max_elements, FILECACHE_DEFAULT_MAX_ELEMENTS, "Maximum number of cache elements, e.g. file segments (limits number of files on filesystem)", 0) \ DECLARE(UInt64, max_file_segment_size, FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE, "Maximum size of a single file segment", 0) \ DECLARE(UInt64, boundary_alignment, FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT, "File segment alignment", 0) \ - DECLARE(UInt64, reserve_granularity, FILECACHE_DEFAULT_RESERVE_GRANULARITY, "When reserving space for a file segment download, reserve at least this many bytes ahead of the downloaded size (capped at the file segment size). Coarser granularity reduces the rate of cache state lock acquisitions on the reservation hot path, at the cost of holding some reserved-but-not-yet-downloaded space (reclaimed on file segment completion). Value 0 disables reserve-ahead: space is reserved exactly as requested.", 0) \ + DECLARE(UInt64, reserve_granularity, FILECACHE_DEFAULT_MAX_RESERVE_GRANULARITY, "Maximum size to reserve ahead of the downloaded size of a file segment, to take the cache lock less often. Starts from 0 for each reader/writer and grows with its repeated reservations up to this value. 0 disables reserve-ahead.", 0) \ DECLARE(Bool, cache_on_write_operations, false, "Enables write-through cache (cache on INSERT and MERGE)", 0) \ DECLARE(FileCachePolicy, cache_policy, FILECACHE_DEFAULT_CACHE_POLICY, "Cache eviction policy", 0) \ DECLARE(Double, slru_size_ratio, FILECACHE_DEFAULT_SLRU_RATIO, "SLRU cache policy size ratio of protected to probationary elements", 0) \ diff --git a/src/Interpreters/FileCache/FileCache_fwd.h b/src/Interpreters/FileCache/FileCache_fwd.h index 7c8df3d4a586..76a864a3f98b 100644 --- a/src/Interpreters/FileCache/FileCache_fwd.h +++ b/src/Interpreters/FileCache/FileCache_fwd.h @@ -7,7 +7,7 @@ namespace DB static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE = 32 * 1024 * 1024; /// 32Mi static constexpr int FILECACHE_DEFAULT_FILE_SEGMENT_ALIGNMENT = 4 * 1024 * 1024; /// 4Mi -static constexpr int FILECACHE_DEFAULT_RESERVE_GRANULARITY = 4 * 1024 * 1024; /// 4Mi. 0 disables reserve-ahead (reserve exactly the requested size). +static constexpr int FILECACHE_DEFAULT_MAX_RESERVE_GRANULARITY = 4 * 1024 * 1024; /// 4Mi. 0 disables reserve-ahead. static constexpr int FILECACHE_DEFAULT_MAX_FILE_SEGMENT_SIZE_WITH_BACKGROUND_DOWLOAD = 4 * 1024 * 1024; /// 4Mi static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_THREADS = 5; static constexpr int FILECACHE_DEFAULT_BACKGROUND_DOWNLOAD_QUEUE_SIZE_LIMIT = 5000; diff --git a/src/Interpreters/FileCache/FileSegment.cpp b/src/Interpreters/FileCache/FileSegment.cpp index fa0c71d3311b..d820d7ea15d7 100644 --- a/src/Interpreters/FileCache/FileSegment.cpp +++ b/src/Interpreters/FileCache/FileSegment.cpp @@ -34,6 +34,7 @@ namespace ProfileEvents extern const Event FilesystemCacheHoldFileSegments; extern const Event FilesystemCacheUnusedHoldFileSegments; extern const Event FilesystemCacheBackgroundDownloadQueuePush; + extern const Event FilesystemCacheReserveAheadRetries; } namespace CurrentMetrics @@ -668,7 +669,8 @@ bool FileSegment::reserve( size_t lock_wait_timeout_milliseconds, std::string & failure_reason, FileCacheReserveStat * reserve_stat, - size_t reserve_hint) + std::optional reserve_hint, + FileCacheReserveAhead * reserve_ahead) { if (!size_to_reserve) throw Exception(ErrorCodes::LOGICAL_ERROR, "Zero space reservation is not allowed"); @@ -708,29 +710,18 @@ bool FileSegment::reserve( const size_t minimum_reserve_size = size_to_reserve; - if (!is_unbound) + if (!is_unbound && reserve_ahead) { - const auto reserve_granularity = cache->getReserveGranularity(); - if (reserve_granularity && reserve_granularity > size_to_reserve) + /// Don't reserve ahead past the segment end or the end of the read. + size_t segment_reserve_limit = range().size() - reserved_size; + if (reserve_hint) { - size_to_reserve = reserved_size + reserve_granularity > range().size() - ? range().size() - reserved_size - : reserve_granularity; - - /// `reserve_hint` is measured from the current download offset, so the read ends at - /// `read_horizon` in segment-relative terms. Don't reserve ahead past it. - const size_t read_horizon = current_downloaded_size + reserve_hint; - if (reserve_hint - && read_horizon > reserved_size - && read_horizon < reserved_size + size_to_reserve) - size_to_reserve = read_horizon - reserved_size; + const size_t read_horizon = current_downloaded_size + *reserve_hint; + segment_reserve_limit = std::min(segment_reserve_limit, read_horizon > reserved_size ? read_horizon - reserved_size : 0); } - } - /// The reserve-ahead caps above (segment range, read horizon) are only an upper bound; they - /// must never reserve less than the current write needs, otherwise the write would exceed the - /// reservation. A bare assert would not protect release builds, so clamp explicitly. - size_to_reserve = std::max(size_to_reserve, minimum_reserve_size); + size_to_reserve = reserve_ahead->getReserveSize(size_to_reserve, segment_reserve_limit, cache->getReserveGranularity()); + } /// This (resizable file segments) is allowed only for single threaded use of file segment. /// Currently it is used only for temporary files through cache. @@ -742,10 +733,23 @@ bool FileSegment::reserve( FileCacheReserveStat dummy_stat; if (!reserve_stat) reserve_stat = &dummy_stat; + reserve_stat->not_enough_space = false; bool reserved = cache->tryReserve( *this, size_to_reserve, *reserve_stat, *getKeyMetadata()->origin, lock_wait_timeout_milliseconds, failure_reason); + if (!reserved && reserve_ahead) + reserve_ahead->reset(); + + /// Reserve-ahead is best-effort: retry with the exact size if it did not fit. + if (!reserved && size_to_reserve > minimum_reserve_size && reserve_stat->not_enough_space) + { + ProfileEvents::increment(ProfileEvents::FilesystemCacheReserveAheadRetries); + *reserve_stat = FileCacheReserveStat{}; + reserved = cache->tryReserve( + *this, minimum_reserve_size, *reserve_stat, *getKeyMetadata()->origin, lock_wait_timeout_milliseconds, failure_reason); + } + if (!reserved) setDownloadFailedUnlocked(lock()); diff --git a/src/Interpreters/FileCache/FileSegment.h b/src/Interpreters/FileCache/FileSegment.h index 6a53fde3184c..c4d3fe5b17cb 100644 --- a/src/Interpreters/FileCache/FileSegment.h +++ b/src/Interpreters/FileCache/FileSegment.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include #include @@ -24,6 +26,24 @@ namespace DB class ReadBufferFromFileBase; struct FileCacheReserveStat; +/// Reserve-ahead state of one reader/writer: the first reservation is exact, then the reserve-ahead +/// doubles up to `max_reserve_granularity` (the cache's `reserve_granularity`). Reset on a failed reservation. +struct FileCacheReserveAhead +{ + /// Returns the size to reserve, at least `size_to_reserve`, and grows the reserve-ahead. + /// `segment_reserve_limit` is the room left in the segment, capped by the end of the read if known. + size_t getReserveSize(size_t size_to_reserve, size_t segment_reserve_limit, size_t max_reserve_granularity) + { + const size_t result = std::max(size_to_reserve, std::min({granularity, max_reserve_granularity, segment_reserve_limit})); + granularity = std::min(max_reserve_granularity, std::max(granularity, size_to_reserve) * 2); + return result; + } + + void reset() { granularity = 0; } + + size_t granularity = 0; +}; + struct CreateFileSegmentSettings { @@ -211,18 +231,15 @@ friend class FileCache; /// Because of reserved_size in tryReserve(). * ========== Methods for _only_ file segment's `downloader` ================== */ - /// Try to reserve exactly `size` bytes (in addition to the getDownloadedSize() bytes already downloaded). - /// Returns true if reservation was successful, false otherwise. - /// - /// `reserve_hint`, if non-zero, bounds the reserve-ahead to the bytes left to read from the - /// current download offset (e.g. up to read_until_position), so the segment is never reserved - /// ahead past what the read will consume. + /// Try to reserve `size` bytes on top of getDownloadedSize(). Reserves ahead if `reserve_ahead` is + /// set, but not past `reserve_hint` bytes from the current download offset. bool reserve( size_t size_to_reserve, size_t lock_wait_timeout_milliseconds, std::string & failure_reason, FileCacheReserveStat * reserve_stat = nullptr, - size_t reserve_hint = 0); + std::optional reserve_hint = std::nullopt, + FileCacheReserveAhead * reserve_ahead = nullptr); /// Write data into reserved space. void write(char * from, size_t size, size_t offset_in_file); diff --git a/src/Interpreters/FileCache/Metadata.cpp b/src/Interpreters/FileCache/Metadata.cpp index 7e94b3851734..8cb4d3c3b6c7 100644 --- a/src/Interpreters/FileCache/Metadata.cpp +++ b/src/Interpreters/FileCache/Metadata.cpp @@ -995,6 +995,7 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optional(buf->getPosition())) buf->seek(offset, SEEK_SET); + FileCacheReserveAhead reserve_ahead; while (size_to_download && !buf->eof()) { const auto available = buf->available(); @@ -1004,7 +1005,11 @@ void CacheMetadata::downloadImpl(FileSegment & file_segment, std::optionalreserve(bytes_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason, &reserve_stat); + bool ok = file_segment->reserve( + bytes_to_write, reserve_space_lock_wait_timeout_milliseconds, failure_reason, + &reserve_stat, /* reserve_hint */std::nullopt, &reserve_ahead); if (!ok) { diff --git a/src/Interpreters/FileCache/WriteBufferToFileSegment.h b/src/Interpreters/FileCache/WriteBufferToFileSegment.h index b4d2e944f924..d6d576044d10 100644 --- a/src/Interpreters/FileCache/WriteBufferToFileSegment.h +++ b/src/Interpreters/FileCache/WriteBufferToFileSegment.h @@ -51,6 +51,7 @@ class WriteBufferToFileSegment : public WriteBufferFromFileBase, public IReadabl FileSegmentsHolderPtr segment_holder; const size_t reserve_space_lock_wait_timeout_milliseconds; + FileCacheReserveAhead reserve_ahead; size_t written_bytes = 0; }; diff --git a/src/Interpreters/GetAggregatesVisitor.cpp b/src/Interpreters/GetAggregatesVisitor.cpp index 7c13cb7fe4db..3123f60a48a2 100644 --- a/src/Interpreters/GetAggregatesVisitor.cpp +++ b/src/Interpreters/GetAggregatesVisitor.cpp @@ -63,10 +63,13 @@ struct WindowExpressionsCollectorMatcher return { .window_function_in_subtree = true }; WindowExpressionsCollectorChildInfo result; - for (auto & arg : func->arguments->children) + if (func->arguments) { - auto subtree_result = visitNode(arg, ast); - result.update(subtree_result); + for (auto & arg : func->arguments->children) + { + auto subtree_result = visitNode(arg, ast); + result.update(subtree_result); + } } // We mark functions if they should be computed after WindowStep diff --git a/src/Interpreters/IInterpreter.cpp b/src/Interpreters/IInterpreter.cpp index be42c0ca91af..3c7093d87ef1 100644 --- a/src/Interpreters/IInterpreter.cpp +++ b/src/Interpreters/IInterpreter.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -55,7 +56,7 @@ void IInterpreter::checkStorageSupportsTransactionsIfNeeded(const StoragePtr & s /// Do not allow transactions with replicated tables or MergeTree tables anyway (unless it's a readonly SELECT query) /// because it may try to process transaction on MergeTreeData-level, /// but then fail with a logical error or something on Storage{Replicated,Shared}MergeTree-level. - if (storage->supportsReplication() || dynamic_cast(storage.get()) != nullptr) + if (storage->supportsReplication() || castStorage(storage, DeferredTable::Load) != nullptr) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} (table {}) does not support transactions", storage->getName(), storage->getStorageID().getNameForLogs()); } diff --git a/src/Interpreters/InsertDependenciesBuilder.cpp b/src/Interpreters/InsertDependenciesBuilder.cpp index b1479620e72f..9dddbbbedf1b 100644 --- a/src/Interpreters/InsertDependenciesBuilder.cpp +++ b/src/Interpreters/InsertDependenciesBuilder.cpp @@ -542,7 +542,7 @@ static DB::ConstraintsDescription buildConstraints(StorageMetadataPtr metadata, { auto constraints = metadata->getConstraints(); - auto storage_merge_tree = std::dynamic_pointer_cast(storage); + auto storage_merge_tree = castStorage(storage, DeferredTable::Load); if (storage_merge_tree && (storage_merge_tree->merging_params.mode == MergeTreeData::MergingParams::Collapsing || storage_merge_tree->merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing) @@ -722,7 +722,7 @@ class ExecutingInnerQueryFromViewTransform final : public ExceptionKeepingTransf local_context); bool inner_share_nested_offsets = true; - if (auto * merge_tree = dynamic_cast(inner_storage.get())) + if (auto * merge_tree = castStorage(inner_storage, DeferredTable::Load).get()) inner_share_nested_offsets = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; auto adding_missing_defaults_dag = addMissingDefaults( @@ -790,7 +790,7 @@ bool InsertDependenciesBuilder::storageDeduplicatesBlocksOnInsert(const StorageP /// MergeTree-family engines deduplicate inserted blocks when their (synchronous) deduplication /// window is enabled. This mirrors how `MergeTreeSink` / `ReplicatedMergeTreeSink` compute their /// own `deduplicate` flag. - if (const auto * merge_tree = dynamic_cast(storage.get())) + if (const auto * merge_tree = castStorage(storage, DeferredTable::Load).get()) { const auto merge_tree_settings = merge_tree->getSettings(); if (storage->supportsReplication()) @@ -1448,7 +1448,7 @@ bool InsertDependenciesBuilder::observePath(const DependencyPath & path) const auto & parent = path.parent(1); const auto & current = path.current(); - auto storage = current == init_table_id ? init_storage : DatabaseCatalog::instance().tryGetTable(current, init_context); + auto storage = current == init_table_id ? init_storage : resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(current, init_context)); auto lock = storage ? storage->tryLockForShare(init_context->getInitialQueryId(), init_context->getSettingsRef()[Setting::lock_acquire_timeout]) : nullptr; if (!lock) { @@ -1783,7 +1783,7 @@ Chain InsertDependenciesBuilder::createPreSink(StorageIDMaybeEmpty view_id) cons auto insert_context = insert_contexts.at(view_id); bool inner_share_nested_offsets = true; - if (auto * merge_tree = dynamic_cast(storages.at(inner_table_id).get())) + if (auto * merge_tree = castStorage(storages.at(inner_table_id), DeferredTable::Load).get()) inner_share_nested_offsets = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; /// Widen Enum columns to their target type before adding defaults, so the valid Enum-widening @@ -1877,7 +1877,7 @@ Chain InsertDependenciesBuilder::createSinkImpl(StorageIDMaybeEmpty view_id) con /// but currently we don't have methods for serialization of nested structures "as a whole". { bool skip_nested_validation = false; - if (auto * merge_tree = dynamic_cast(inner_storage.get())) + if (auto * merge_tree = castStorage(inner_storage, DeferredTable::Load).get()) skip_nested_validation = !(*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; if (!skip_nested_validation) result.addSink(std::make_shared(header)); diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index ed545cf51aca..d9faa8bfa31a 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -2,7 +2,10 @@ #include #include +#include #include +#include +#include #include #include #include @@ -35,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -328,7 +332,7 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, alter_commands->validate(table, context); bool share_nested = true; - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) share_nested = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; alter_commands->prepare(*metadata_snapshot, share_nested); @@ -425,11 +429,14 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) if (table_id) { query_ptr->as().setDatabase(table_id.database_name); - table = DatabaseCatalog::instance().tryGetTable(table_id, getContext()); + /// Resolve once here so every branch below validates against the real structure instead of + /// the columns-only metadata a lazily loaded table reports. + table = resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(table_id, getContext())); } if (!alter.cluster.empty() && !maybeRemoveOnCluster(query_ptr, getContext())) { + /// NOLINT(storage-cast): `table` is resolved above. if (table && table->as()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Mutations with ON CLUSTER are not allowed for KeeperMap tables"); @@ -445,6 +452,14 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) visitor.substituteDatabaseInTableFunctions(*alter.command_list); } + /// The hosts run the entry without the user, unless `distributed_ddl_use_initial_user_and_roles` is on, + /// so the view's SQL security is authorized here, which needs the view on this host. + if (modify_query && !table) + throw Exception(ErrorCodes::UNKNOWN_TABLE, + "Table {}.{} does not exist on this host. `ALTER TABLE ... ON CLUSTER ... MODIFY QUERY` is authorized " + "on the initiator, so it must be run from a host that has the view", + backQuoteIfNeed(alter.getDatabase()), backQuoteIfNeed(alter.getTable())); + DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccess(table); return executeDDLQueryOnCluster(query_ptr, getContext(), params); @@ -599,14 +614,70 @@ bool InterpreterAlterQuery::isRowExistsLightweightDeleteMarker(const StoragePtr return metadata_snapshot->isVirtualColumn(RowExistsColumn::name); } +/** `MODIFY QUERY` replaces the body a view executes, and for `SQL SECURITY DEFINER` or `NONE` that body does + * not run with the caller's privileges. Writing it is the same act of impersonation that `CREATE` and + * `MODIFY SQL SECURITY` gate behind `SET DEFINER` and `ALLOW SQL SECURITY NONE` in + * `processSQLSecurityOption`, so it takes the same grants. + */ +void InterpreterAlterQuery::addRequiredAccessForModifyQuerySQLSecurity( + AccessRightsElements & required_access, const StoragePtr & storage) const +{ + /// Without the view there is no body to replace: the local path fails on the missing table, and an + /// `ON CLUSTER` statement refuses to dispatch a `MODIFY QUERY` for a view this host does not have. + if (!storage) + return; + + const auto metadata_snapshot = storage->getInMemoryMetadataPtr(getContext(), /*bypass_metadata_cache=*/ false); + if (!metadata_snapshot->sql_security_type) + return; + + if (*metadata_snapshot->sql_security_type == SQLSecurityType::NONE) + { + required_access.emplace_back(AccessType::ALLOW_SQL_SECURITY_NONE); + return; + } + + if (*metadata_snapshot->sql_security_type != SQLSecurityType::DEFINER || !metadata_snapshot->definer) + return; + + /// `processSQLSecurityOption` stores an ephemeral definer as a clone named `:definer`, and the grant is + /// held on the user it was cloned from. A real user can have the same suffix, so strip it only when the base + /// user is ephemeral, which is the condition under which the clone is made. + String definer_name = *metadata_snapshot->definer; + static constexpr std::string_view ephemeral_suffix = ":definer"; + if (definer_name.ends_with(ephemeral_suffix)) + { + String base_name = definer_name.substr(0, definer_name.size() - ephemeral_suffix.size()); + const auto & access_control = getContext()->getAccessControl(); + if (auto base_id = access_control.find(base_name); base_id && access_control.isEphemeral(*base_id)) + definer_name = std::move(base_name); + } + + if (definer_name != getContext()->getUserName()) + required_access.emplace_back(AccessType::SET_DEFINER, definer_name); +} + AccessRightsElements InterpreterAlterQuery::getRequiredAccess(const StoragePtr & storage) const { AccessRightsElements required_access; const auto & alter = query_ptr->as(); const bool row_exists_is_marker = isRowExistsLightweightDeleteMarker(storage, getContext()); + /// A `MODIFY SQL SECURITY` in the same statement decides what the new body will execute as, and + /// `processSQLSecurityOption` has already authorized exactly that. The stored security then says + /// nothing about the body being written, so it must not add a requirement of its own. + const bool sql_security_is_being_replaced = std::ranges::any_of( + alter.command_list->children, + [](const auto & child) { return child->template as().sql_security != nullptr; }); + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); required_access.append_range( - getRequiredAccessForCommand(child->as(), alter.getDatabase(), alter.getTable(), row_exists_is_marker)); + getRequiredAccessForCommand(command, alter.getDatabase(), alter.getTable(), row_exists_is_marker)); + + if (command.type == ASTAlterCommand::MODIFY_QUERY && !sql_security_is_being_replaced) + addRequiredAccessForModifyQuerySQLSecurity(required_access, storage); + } return required_access; } diff --git a/src/Interpreters/InterpreterAlterQuery.h b/src/Interpreters/InterpreterAlterQuery.h index 541c2793ab4f..ebb9deddba45 100644 --- a/src/Interpreters/InterpreterAlterQuery.h +++ b/src/Interpreters/InterpreterAlterQuery.h @@ -42,6 +42,10 @@ class InterpreterAlterQuery : public IInterpreter, WithMutableContext private: AccessRightsElements getRequiredAccess(const StoragePtr & storage) const; + /// Adds the grants `processSQLSecurityOption` demands for the view's stored SQL security, so replacing the + /// body of a `DEFINER` or `NONE` view takes the same authority as declaring that security in the first place. + void addRequiredAccessForModifyQuerySQLSecurity(AccessRightsElements & required_access, const StoragePtr & storage) const; + BlockIO executeToTable(const ASTAlterQuery & alter); BlockIO executeToDatabase(const ASTAlterQuery & alter); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index bac302032c8a..84344019e717 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -831,7 +832,10 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti { String as_database_name = getContext()->resolveDatabase(create.as_database); getContext()->checkAccess(AccessType::SHOW_COLUMNS, as_database_name, create.as_table); - StoragePtr as_storage = DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, getContext()); + /// A lazily loaded source reports only its columns, so the indices, projections, constraints + /// and comment copied below would silently come out empty. + StoragePtr as_storage = resolveStorageProxyLoading( + DatabaseCatalog::instance().getTable({as_database_name, create.as_table}, getContext())); /// as_storage->getColumns() and setEngine(...) must be called under structure lock of other_table for CREATE ... AS other_table. as_storage_lock = as_storage->lockForShare(getContext()->getCurrentQueryId(), getContext()->getSettingsRef()[Setting::lock_acquire_timeout]); @@ -857,7 +861,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti /// CREATE TABLE AS should copy PRIMARY KEY, ORDER BY, and similar clauses. /// Note: only supports the source table engine is using the new syntax. - if (const auto * merge_tree_data = dynamic_cast(as_storage.get())) + if (const auto * merge_tree_data = castStorage(as_storage, DeferredTable::Load).get()) { if (merge_tree_data->format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { @@ -1668,6 +1672,16 @@ bool isReplicated(const ASTStorage & storage) return storage_name.starts_with("Replicated") || storage_name.starts_with("Shared"); } +/// The drop privilege matching the kind of an existing table. +AccessType getDropAccessType(const IStorage & table) +{ + if (table.isView()) + return AccessType::DROP_VIEW; + if (table.isDictionary()) + return AccessType::DROP_DICTIONARY; + return AccessType::DROP_TABLE; +} + } BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) @@ -2442,7 +2456,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, if (!create.attach && getContext()->getSettingsRef()[Setting::database_replicated_allow_only_replicated_engine]) { - bool is_replicated_storage = typeid_cast(res.get()) != nullptr; + bool is_replicated_storage = castStorage(res, DeferredTable::Skip) != nullptr; if (!is_replicated_storage && res->storesDataOnDisk() && database && database->getEngineName() == "Replicated") throw Exception(ErrorCodes::UNKNOWN_STORAGE, "Only tables with a Replicated engine " @@ -2454,7 +2468,9 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, "ATTACH ... FROM ... query is not supported for {} table engine, " "because such tables do not store any data on disk. Use CREATE instead.", res->getName()); - auto * replicated_storage = typeid_cast(res.get()); + /// `res` is the storage this query just built, and for a table function it is a proxy that + /// resolving would run during CREATE. + auto * replicated_storage = castStorage(res, DeferredTable::Skip).get(); if (replicated_storage) { const auto probability = getContext()->getSettingsRef()[Setting::create_replicated_merge_tree_fault_injection_probability]; @@ -2840,12 +2856,7 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTable(ASTCreateQuery & create, { /// The replaced table is dropped after the swap, under an internal temporary name that /// grants cannot cover, so check the drop privilege for its kind here, on its real name. - AccessType drop_access = AccessType::DROP_TABLE; - if (to_drop->isView()) - drop_access = AccessType::DROP_VIEW; - else if (to_drop->isDictionary()) - drop_access = AccessType::DROP_DICTIONARY; - current_context->checkAccess(drop_access, to_drop_id); + current_context->checkAccess(getDropAccessType(*to_drop), to_drop_id); to_drop->checkTableSizeBelowDropLimit(current_context); } }); @@ -3095,7 +3106,8 @@ StoragePtr InterpreterCreateQuery::getValidatedAtomicPopulateSource(const ASTCre if (context->hasQueryContext()) context->getQueryContext()->dropStorageCacheEntry(*ref_dependencies.mv_from_dependency); - auto source = DatabaseCatalog::instance().tryGetTable(*ref_dependencies.mv_from_dependency, context); + auto source = resolveStorageProxyLoading( + DatabaseCatalog::instance().tryGetTable(*ref_dependencies.mv_from_dependency, context)); /// The view's SELECT was validated against the source before the view was published, so the source /// existed then; not finding it now means it was dropped, renamed or exchanged away in the window @@ -3510,6 +3522,16 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const } } + /// Replicated and ON CLUSTER replays run with full access, so the drop privilege for the replaced + /// table's kind must be required here, on its real name, while the query still runs as the user. + if ((create.replace_table || create.create_or_replace || create.replace_view) && !create.isTemporary()) + { + String database_name = getContext()->resolveDatabase(create.getDatabase()); + if (auto database = DatabaseCatalog::instance().tryGetDatabase(database_name)) + if (auto table = database->tryGetTable(create.getTable(), getContext())) + required_access.emplace_back(getDropAccessType(*table), database_name, create.getTable()); + } + if (create.targets) { for (const auto & target : create.targets->targets) diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 256113146ad3..8d8b2b07e693 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include @@ -95,7 +96,10 @@ BlockIO InterpreterDeleteQuery::execute() /// metadata is not loaded until the first access. Initialize it now so that /// supportsDelete() and subsequent mutation checks see valid metadata. table->updateExternalDynamicMetadataIfExists(getContext()); - auto metadata_snapshot = table->getInMemoryMetadataPtr(getContext(), false); + /// A lazily loaded table reports only its columns, so the checks below would see no projections + /// and validate the mutation against metadata that has no keys. + auto resolved_table = resolveStorageProxyLoading(table); + auto metadata_snapshot = resolved_table->getInMemoryMetadataPtr(getContext(), false); if (table->supportsDelete()) { @@ -137,7 +141,8 @@ BlockIO InterpreterDeleteQuery::execute() if (metadata_snapshot->hasProjections()) { - if (const auto * merge_tree_data = dynamic_cast(table.get())) + /// `MutateTask` treats THROW like DROP, so missing this check drops the projections. + if (const auto * merge_tree_data = castStorage(resolved_table, DeferredTable::Load).get()) if ((*merge_tree_data->getSettings())[MergeTreeSetting::lightweight_mutation_projection_mode] == LightweightMutationProjectionMode::THROW) throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "DELETE query is not allowed for table {} because as it has projections and setting " diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index e3c37585e60b..9e16ed6cc722 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -330,7 +331,7 @@ BlockIO InterpreterDropQuery::executeToTableImpl(const ContextPtr & context_, AS TableExclusiveLockHolder table_excl_lock; /// We don't need any lock for ReplicatedMergeTree and for simple MergeTree /// For the rest of tables types exclusive lock is needed - if (!std::dynamic_pointer_cast(table)) + if (!castStorage(table, DeferredTable::Load)) table_excl_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef()[Setting::lock_acquire_timeout]); auto metadata_snapshot = table->getInMemoryMetadataPtr(context_, false); diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 1f2b0e376ad3..e2459d3f5314 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -28,9 +28,12 @@ #include #include #include +#include +#include #include #include #include +#include #include #include @@ -61,10 +64,13 @@ #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -336,6 +342,161 @@ namespace } }; + /// Replace a node with a single `'[HIDDEN]'` literal, keeping its alias. + void hideWholeNode(ASTPtr & node) + { + auto hidden = make_intrusive(Field("[HIDDEN]")); + hidden->setAlias(node->tryGetAlias()); + node = std::move(hidden); + } + + /// Replace every literal inside a node with `'[HIDDEN]'`, keeping the expression structure. Only + /// for the secret arguments of `encrypt` / `HMAC`, where the shape is not a secret (a key built as + /// `leftPad('...', 16, '*')` stays readable as such); every other secret slot is hidden whole. + void hideLiteralsInSubtree(ASTPtr & node) + { + if (node->as()) + { + hideWholeNode(node); + return; + } + for (auto & child : node->children) + hideLiteralsInSubtree(child); + } + + /// Keep in sync with the names `FunctionSecretArgumentsFinder` sends to `findEncryptionFunctionSecretArguments` + /// and `findHMACSecretArguments`. A name missing here only makes the dump stricter: its span is hidden whole. + bool isEncryptionOrHMACFunction(const ASTFunction & function) + { + return function.name == "encrypt" || function.name == "decrypt" || function.name == "aes_encrypt_mysql" + || function.name == "aes_decrypt_mysql" || function.name == "tryDecrypt" || equalsCaseInsensitive(function.name, "HMAC"); + } + + bool isKeyValueArgument(const IAST & node) + { + const auto * function = node.as(); + return function && function->name == "equals" && function->arguments && function->arguments->children.size() == 2; + } + + /// The secret value of a `key = value` argument is its second child; anything else carries the + /// secret in the node itself. + ASTPtr & secretValueSlot(ASTPtr & node) + { + if (isKeyValueArgument(*node)) + return node->as()->arguments->children[1]; + return node; + } + + /// Replace an argument with the partially masked SQL the formatter prints for it: a URL with its + /// credentials removed, or the masked locator of a `Backup` database. The original node must not + /// stay in the tree, so text that does not parse, or parses into a node that cannot take the + /// argument's place (a `COLUMNS(...)` matcher has no alias), hides the argument whole. + void replaceWithMaskedText(ASTPtr & node, const String & text) + { + ParserExpression parser; + const char * pos = text.data(); + String error; + ASTPtr parsed = tryParseQuery( + parser, + pos, + text.data() + text.size(), + error, + /* hilite= */ false, + "masked secret argument", + /* allow_multi_statements= */ false, + /* max_query_size= */ 0, + DBMS_DEFAULT_MAX_PARSER_DEPTH, + DBMS_DEFAULT_MAX_PARSER_BACKTRACKS, + /* skip_insignificant= */ true); + if (!parsed || !dynamic_cast(parsed.get())) + { + hideWholeNode(node); + return; + } + parsed->setAlias(node->tryGetAlias()); + node = std::move(parsed); + } + + /// `DumpASTNode` prints a literal through `IAST::getID`, value included, so the dump cannot hide + /// secrets while formatting as `ASTFunction::formatImpl` does. Hide them in the tree instead. As in the + /// formatter, a secret slot becomes one `'[HIDDEN]'` literal. That includes the slots the finder could + /// not inspect (a url built by `concat(...)`, an identifier in a password slot): their expression is + /// part of the secret and must not be dumped node by node. Only the `encrypt` / `HMAC` span keeps its + /// structure (see `hideLiteralsInSubtree`). All values of a nested map (`headers(...)`, + /// `extra_credentials(...)`) are hidden; the formatter keeps the non-secret `extra_credentials` + /// values, so the dump is stricter. + struct HideSecretArgumentsMatcher + { + struct Data + { + }; + + static bool needChildVisit(const ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data &) + { + auto * function = ast->as(); + if (!function || !function->arguments) + return; + + auto secret_arguments = FunctionSecretArgumentsFinderAST(*function).getResult(); + if (!secret_arguments.hasSecrets()) + return; + + auto & arguments = function->arguments->children; + for (size_t i = 0; i < arguments.size(); ++i) + { + if (auto * map = arguments[i]->as(); + map && map->arguments && std::ranges::contains(secret_arguments.nested_maps, map->name)) + { + for (auto & entry : map->arguments->children) + hideWholeNode(secretValueSlot(entry)); + continue; + } + + if (auto replaced = secret_arguments.replaced_arguments.find(i); replaced != secret_arguments.replaced_arguments.end()) + { + replaceWithMaskedText(arguments[i], replaced->second); + continue; + } + + /// An individually masked argument: only the named `key = value` form keeps its key. + if (auto masked = secret_arguments.masked_arguments.find(i); masked != secret_arguments.masked_arguments.end()) + { + hideWholeNode(masked->second ? secretValueSlot(arguments[i]) : arguments[i]); + continue; + } + + if (!(secret_arguments.start <= i && i < secret_arguments.start + secret_arguments.count)) + continue; + + if (!secret_arguments.replacement.empty()) + { + const auto text + = secret_arguments.quote_replacement ? quoteString(secret_arguments.replacement) : secret_arguments.replacement; + replaceWithMaskedText(secret_arguments.are_named ? secretValueSlot(arguments[i]) : arguments[i], text); + continue; + } + + if (secret_arguments.are_named) + { + hideWholeNode(secretValueSlot(arguments[i])); + continue; + } + + /// Only the span of `encrypt` / `HMAC` keeps its structure. Any other unnamed span without a + /// replacement, such as an unreadable url in `mongodb(concat(...), 'c')`, is hidden whole. So is a + /// `key = value` in the span: it is a positional secret written as a comparison. + if (isEncryptionOrHMACFunction(*function) && !isKeyValueArgument(*arguments[i])) + hideLiteralsInSubtree(arguments[i]); + else + hideWholeNode(arguments[i]); + } + } + }; + + using HideSecretArgumentsVisitor = InDepthNodeVisitor; + bool hasSecretsInActionsDAG(const ActionsDAG & dag) { for (const auto & node : dag.getNodes()) @@ -996,6 +1157,14 @@ QueryPipeline InterpreterExplainQuery::executeImpl() ExplainAnalyzedSyntaxVisitor(data).visit(query); } + /// `optimize = 1` inlines views the user may read but whose secrets they may not see. + /// Hide them under the same gate as `SHOW CREATE`. + if (!canDisplaySecrets(query_context)) + { + HideSecretArgumentsVisitor::Data data; + HideSecretArgumentsVisitor(data).visit(query); + } + if (settings.graph) dumpASTInDotFormat(*ast.getExplainedQuery(), buf); else diff --git a/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp b/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp index 4e05d0399fbb..0cbd5e978c65 100644 --- a/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp +++ b/src/Interpreters/InterpreterHypotheticalIndexQuery.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -49,8 +50,20 @@ BlockIO InterpreterHypotheticalIndexQuery::execute() auto table_id = context->resolveStorageID(StorageID(query.getDatabase(), query.getTable())); auto table = DatabaseCatalog::instance().getTable(table_id, context); + auto & store = context->getHypotheticalObjectStore(); - const auto * merge_tree = dynamic_cast(table.get()); + /// Dropping only forgets a session entry, so it must not load a lazily loaded table. + if (query.kind == ASTHypotheticalIndexQuery::Drop) + { + /// No access check. a session-local drop leaks nothing (unlike CREATE), and every entry + /// in the session store already passed that check at creation + auto index_name = query.index_name->as().name(); + store.remove(table_id, index_name, query.if_exists); + return {}; + } + + table = resolveStorageProxyLoading(table); + const auto * merge_tree = castStorage(table, DeferredTable::Load).get(); if (!merge_tree) throw Exception( ErrorCodes::NOT_IMPLEMENTED, @@ -66,17 +79,6 @@ BlockIO InterpreterHypotheticalIndexQuery::execute() table_id.getDatabaseName(), table_id.getTableName()); - auto & store = context->getHypotheticalObjectStore(); - - if (query.kind == ASTHypotheticalIndexQuery::Drop) - { - /// No access check. a session-local drop leaks nothing (unlike CREATE), and every entry - /// in the session store already passed that check at creation - auto index_name = query.index_name->as().name(); - store.remove(table_id, index_name, query.if_exists); - return {}; - } - /// CREATE HYPOTHETICAL INDEX const auto & index_ast = query.index_decl->as(); auto metadata = table->getInMemoryMetadataPtr(context, /* bypass_metadata_cache = */ false); diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index afbd4a86e552..a158eaf3ba07 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,8 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query) query.table_id = current_context->resolveStorageID(local_table_id); } - return DatabaseCatalog::instance().getTable(query.table_id, current_context); + /// The insert path reads engine facts the proxy of an unloaded table cannot answer. + return resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(query.table_id, current_context)); } Block InterpreterInsertQuery::getSampleBlock( @@ -1064,7 +1066,7 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica if (query.table_id.empty()) return {}; - StoragePtr dst_storage = DatabaseCatalog::instance().getTable(query.table_id, local_context); + StoragePtr dst_storage = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(query.table_id, local_context)); if (!(dst_storage->isMergeTree() || dst_storage->isDataLake()) || !dst_storage->supportsReplication()) return {}; @@ -1107,12 +1109,30 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica /// query will be executed on all nodes of the cluster auto src_cluster = src_storage_cluster->getCluster(local_context); - /// Actually the query doesn't change, we just serialize it to string. Strip the initiator-only - /// settings from the forwarded query text (both `changes` and `default_settings`, across the INSERT - /// and its source SELECT) so those names — including the new HTTP table-as-file settings — do not reach - /// the shards and trip `UNKNOWN_SETTING` on a rolling upgrade; the per-shard context is stripped below. + src_storage_cluster->updateExternalDynamicMetadataIfExists(local_context); + + const auto src_metadata_snapshot = src_storage_cluster->getInMemoryMetadataPtr(local_context, false); + const auto src_snapshot = src_storage_cluster->getStorageSnapshot(src_metadata_snapshot, local_context); + + /// Strip the initiator-only settings from the forwarded query text (both `changes` and `default_settings`, + /// across the INSERT and its source SELECT) so those names — including the new HTTP table-as-file settings — + /// do not reach the shards and trip `UNKNOWN_SETTING` on a rolling upgrade; the per-shard context is + /// stripped below. auto query_to_send = query.clone(); ClusterProxy::stripInitiatorOnlySettingsFromQuery(query_to_send); + + /// The source storage may have been created by `parallel_replicas_for_cluster_engines` from a plain table + /// function (`url`, `s3`, ...), while the query text still names that plain function. A node that runs + /// the forwarded query as a secondary query does not convert it again: it creates a plain storage that + /// expands the globs and reads every file on its own instead of taking its share of the read tasks from + /// the initiator, so N nodes insert the data N times. Rewrite the source the same way `IStorageCluster::read` + /// does for a `SELECT`: the function becomes its `*Cluster` variant with the cluster name argument, and the + /// structure and format arguments are added so that the nodes do not infer the schema again. + { + auto & select_to_send = query_to_send->as().select->as(); + src_storage_cluster->updateQueryToSendIfNeeded(select_to_send.list_of_selects->children.at(0), src_snapshot, local_context); + } + String query_str; { WriteBufferFromOwnString buf; @@ -1134,8 +1154,6 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica query_context->setSettings(stripped_settings); } - src_storage_cluster->updateExternalDynamicMetadataIfExists(local_context); - std::optional filter_dag; const ActionsDAG::Node * predicate = nullptr; if (select_query) @@ -1171,7 +1189,6 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica } } } - const auto src_metadata_snapshot = src_storage_cluster->getInMemoryMetadataPtr(local_context, false); auto extension = src_storage_cluster->getTaskIteratorExtension( predicate, filter_dag ? &*filter_dag : nullptr, local_context, src_cluster, src_metadata_snapshot); diff --git a/src/Interpreters/InterpreterKillQueryQuery.cpp b/src/Interpreters/InterpreterKillQueryQuery.cpp index 177ca1a051c0..32b725ce3474 100644 --- a/src/Interpreters/InterpreterKillQueryQuery.cpp +++ b/src/Interpreters/InterpreterKillQueryQuery.cpp @@ -474,17 +474,31 @@ AccessRightsElements InterpreterKillQueryQuery::getRequiredAccessForDDLOnCluster { const auto & query = query_ptr->as(); AccessRightsElements required_access; - if (query.type == ASTKillQueryQuery::Type::Query) - required_access.emplace_back(AccessType::KILL_QUERY); - else if (query.type == ASTKillQueryQuery::Type::Mutation) - required_access.emplace_back( - AccessType::ALTER_UPDATE - | AccessType::ALTER_DELETE - | AccessType::ALTER_MATERIALIZE_INDEX - | AccessType::ALTER_MATERIALIZE_COLUMN - | AccessType::ALTER_MATERIALIZE_TTL - | AccessType::ALTER_REWRITE_PARTS - ); + /// This switch has no `default:`, so a new Type has to be mapped here to compile. + switch (query.type) + { + case ASTKillQueryQuery::Type::Query: + required_access.emplace_back(AccessType::KILL_QUERY); + break; + case ASTKillQueryQuery::Type::Mutation: + required_access.emplace_back( + AccessType::ALTER_UPDATE + | AccessType::ALTER_DELETE + | AccessType::ALTER_MATERIALIZE_INDEX + | AccessType::ALTER_MATERIALIZE_COLUMN + | AccessType::ALTER_MATERIALIZE_TTL + | AccessType::ALTER_REWRITE_PARTS + ); + break; + case ASTKillQueryQuery::Type::PartMoveToShard: + required_access.emplace_back(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "part_moves_between_shards"); + required_access.emplace_back(AccessType::ALTER_MOVE_PARTITION | AccessType::MOVE_PARTITION_BETWEEN_SHARDS); + break; + case ASTKillQueryQuery::Type::Transaction: + required_access.emplace_back(AccessType::KILL_TRANSACTION); + required_access.emplace_back(AccessType::SELECT, DatabaseCatalog::SYSTEM_DATABASE, "transactions"); + break; + } return required_access; } diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp index 756b35a9727e..be8738e8ac33 100644 --- a/src/Interpreters/InterpreterOptimizeQuery.cpp +++ b/src/Interpreters/InterpreterOptimizeQuery.cpp @@ -1,6 +1,7 @@ #include "config.h" #include +#include #include #include #include @@ -47,7 +48,9 @@ BlockIO InterpreterOptimizeQuery::execute() getContext()->checkAccess(getRequiredAccess()); auto table_id = getContext()->resolveStorageID(ast); - StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); + /// Resolve before reading the metadata, so the checks below and `optimizeDryRun` see the real + /// structure rather than the columns-only one a lazily loaded table reports. + StoragePtr table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(table_id, getContext())); checkStorageSupportsTransactionsIfNeeded(table, getContext()); auto metadata_snapshot = table->getInMemoryMetadataPtr(getContext(), false); auto storage_snapshot = table->getStorageSnapshotWithoutData(metadata_snapshot, getContext()); @@ -59,7 +62,7 @@ BlockIO InterpreterOptimizeQuery::execute() throw Exception(ErrorCodes::BAD_ARGUMENTS, "OPTIMIZE MANIFEST is incompatible with FINAL, PARTITION, DEDUPLICATE, CLEANUP, and DRY RUN options"); #if USE_AVRO - auto * object_storage_table = dynamic_cast(table.get()); + auto object_storage_table = castStorage(table, DeferredTable::Skip); if (!object_storage_table) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "OPTIMIZE MANIFEST is only supported for Iceberg tables"); @@ -112,7 +115,7 @@ BlockIO InterpreterOptimizeQuery::execute() if (ast.dry_run) { - auto * merge_tree_data = dynamic_cast(table.get()); + auto * merge_tree_data = castStorage(table, DeferredTable::Load).get(); if (!merge_tree_data) throw Exception(ErrorCodes::BAD_ARGUMENTS, "OPTIMIZE DRY RUN is only supported for MergeTree family tables"); diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 4de94dd46ea0..f3589d75b5b1 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -139,6 +140,7 @@ namespace Setting extern const SettingsUInt64 aggregation_in_order_max_block_bytes; extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; extern const SettingsBool allow_calculating_subcolumns_sizes_for_merge_tree_reading; + extern const SettingsBool allow_experimental_analyzer; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; extern const SettingsUInt64 automatic_parallel_replicas_mode; extern const SettingsBool async_socket_for_remote; @@ -303,10 +305,13 @@ try ASTs select_expressions; - /// The first column is our filter expression. - /// the row_policy_filter_expression should be cloned, because it may be changed by TreeRewriter. - /// which make it possible an invalid expression, although it may be valid in whole select. - select_expressions.push_back(row_policy_filter_expression->clone()); + /// The first column is our filter expression. Clone it because `TreeRewriter` can change the AST. + auto filter_expression = row_policy_filter_expression->clone(); + + /// `TreeRewriter` expands table aliases only on the initiator, but these filters are also + /// created on shards. An `ALIAS` used by a row policy must be evaluated before the read filter. + replaceAliasColumnsInQuery(filter_expression, metadata_snapshot->getColumns(), {}, context); + select_expressions.push_back(std::move(filter_expression)); /// Keep columns that are required after the filter actions. for (const auto & column_str : prerequisite_columns) @@ -1216,7 +1221,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis() return true; } - auto storage_merge_tree = std::dynamic_pointer_cast(storage); + auto storage_merge_tree = castStorage(storage, DeferredTable::Load); if (!storage_merge_tree || settings[Setting::parallel_replicas_min_number_of_rows_per_replica] == 0) return false; @@ -2626,7 +2631,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() } /// Set of all (including ALIAS) required columns for PREWHERE - auto get_prewhere_columns = [&]() + auto get_prewhere_columns = [&](bool include_row_level_filter) { NameSet columns; @@ -2639,7 +2644,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() /// A row policy that will not be pushed into the storage read is applied as an ordinary /// FilterStep above it, so its columns are read normally and are not PREWHERE columns. - if (row_level_filter && shouldPushRowLevelFilterToStorage()) + if (row_level_filter && include_row_level_filter && shouldPushRowLevelFilterToStorage()) { auto row_level_required_columns = row_level_filter->actions.getRequiredColumns().getNames(); columns.insert(row_level_required_columns.begin(), row_level_required_columns.end()); @@ -2657,15 +2662,14 @@ void InterpreterSelectQuery::addPrewhereAliasActions() /// before any other executions. if (alias_columns_required) { - NameSet required_columns_from_prewhere = get_prewhere_columns(); + /// The row-level filter runs before `PREWHERE`, but its inputs are not produced by + /// `PREWHERE`. Keep them in the alias actions for queries that still need them. + NameSet required_columns_from_prewhere = get_prewhere_columns(/*include_row_level_filter=*/ false); NameSet required_aliases_from_prewhere; /// Set of ALIAS required columns for PREWHERE /// Expression, that contains all raw required columns ASTPtr required_columns_all_expr = make_intrusive(); - /// Expression, that contains raw required columns for PREWHERE - ASTPtr required_columns_from_prewhere_expr = make_intrusive(); - /// Sort out already known required columns between expressions, /// also populate `required_aliases_from_prewhere`. for (const auto & column : required_columns) @@ -2690,8 +2694,6 @@ void InterpreterSelectQuery::addPrewhereAliasActions() if (required_columns_from_prewhere.contains(column)) { - required_columns_from_prewhere_expr->children.emplace_back(std::move(column_expr)); - if (is_alias) required_aliases_from_prewhere.insert(column); } @@ -2759,7 +2761,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions() const auto & supported_prewhere_columns = storage->supportedPrewhereColumns(); if (supported_prewhere_columns.has_value()) { - NameSet required_columns_from_prewhere = get_prewhere_columns(); + NameSet required_columns_from_prewhere = get_prewhere_columns(/*include_row_level_filter=*/ true); const auto & table_columns = metadata_snapshot->getColumns(); const bool include_subcolumns = storage->supportedPrewhereColumnsIncludeSubcolumns(); @@ -3949,8 +3951,16 @@ void InterpreterSelectQuery::initSettings() { auto & query = getSelectQuery(); if (query.settings()) + { InterpreterSetQuery(query.settings(), context).executeForCurrentContext(options.ignore_setting_constraints); + /// The old interpreter disabled the analyzer in `IInterpreterUnionOrSelectQuery`, but a `SELECT` + /// stored in a `VIEW` may enable it again here. Storages must not use analyzer-only query tree + /// paths (for example, `StorageDistributed::read`) when this interpreter built the query. + if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) + context->setSetting("allow_experimental_analyzer", false); + } + const auto & client_info = context->getClientInfo(); if (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && diff --git a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp index 4f39711e5f35..d8355d065b16 100644 --- a/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp +++ b/src/Interpreters/InterpreterSelectQueryAnalyzer.cpp @@ -219,9 +219,44 @@ QueryPlanPtr buildQueryPlanForAutomaticParallelReplicas( } } +/// Like `extractAllTableReferences`, but does not descend into the inner queries of views inlined +/// by the analyzer (`analyzer_inline_views`) into a query that is not itself inside a view: +/// they read their own tables, just like a view that is not inlined. +static bool isViewInnerQueryNode(const QueryTreeNodePtr & node) +{ + if (const auto * query_node = node->as()) + return query_node->getContext()->isViewInnerQuery(); + if (const auto * union_node = node->as()) + return union_node->getContext()->isViewInnerQuery(); + return false; +} + +static void extractTableReferencesOutsideViews(const QueryTreeNodePtr & node, bool outer_is_view_inner, QueryTreeNodes & result) +{ + bool is_view_inner = isViewInnerQueryNode(node); + if (is_view_inner && !outer_is_view_inner) + return; + + if (node->getNodeType() == QueryTreeNodeType::TABLE) + { + result.push_back(node); + } + else if (const auto * query_node = node->as()) + { + for (const auto & table_expression : extractTableExpressions(query_node->getJoinTreeNodeTyped(), /*add_array_join=*/ false, /*recursive=*/ false)) + extractTableReferencesOutsideViews(table_expression, is_view_inner, result); + } + else if (const auto * union_node = node->as()) + { + for (const auto & query : union_node->getQueries().getNodes()) + extractTableReferencesOutsideViews(query, is_view_inner, result); + } +} + void replaceStorageInQueryTree(QueryTreeNodePtr & query_tree, const ContextPtr & context, const StoragePtr & storage) { - auto nodes = extractAllTableReferences(query_tree); + QueryTreeNodes nodes; + extractTableReferencesOutsideViews(query_tree, isViewInnerQueryNode(query_tree), nodes); IQueryTreeNode::ReplacementMap replacement_map; for (auto & node : nodes) diff --git a/src/Interpreters/InterpreterSetQuery.cpp b/src/Interpreters/InterpreterSetQuery.cpp index fd7ff81b7a77..cef2ade6d97d 100644 --- a/src/Interpreters/InterpreterSetQuery.cpp +++ b/src/Interpreters/InterpreterSetQuery.cpp @@ -89,7 +89,7 @@ BlockIO InterpreterSetQuery::execute() /// explicitly set to its current value. The original code applies const `ast.changes`. getContext()->checkSettingsConstraints(std::as_const(changes), SettingSource::QUERY); /// Checked before anything is applied, so that a violation leaves the whole statement without effect. - getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, SettingSource::QUERY); + getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, changes, SettingSource::QUERY); auto session_context = getContext()->getSessionContext(); session_context->applySettingsChanges(changes); session_context->addQueryParameters(NameToNameMap{ast.query_parameters.begin(), ast.query_parameters.end()}); @@ -109,7 +109,7 @@ void InterpreterSetQuery::executeForCurrentContext(bool ignore_setting_constrain if (!ignore_setting_constraints) { getContext()->checkSettingsConstraints(std::as_const(changes), SettingSource::QUERY); - getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, SettingSource::QUERY); + getContext()->checkSettingsConstraintsForSettingsReset(ast.default_settings, changes, SettingSource::QUERY); rejectHTTPOnlyConstructionSettings(ast); } getContext()->applySettingsChanges(changes); diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index 9efabe1c7877..a8b7f75ad33d 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include #include #include @@ -1374,7 +1375,7 @@ void InterpreterSystemQuery::restoreReplica() const StoragePtr table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto * const table_replicated_ptr = dynamic_cast(table_ptr.get()); + auto * const table_replicated_ptr = castStorage(table_ptr, DeferredTable::Load).get(); if (table_replicated_ptr == nullptr) throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), table_id.getNameForLogs()); @@ -1447,7 +1448,9 @@ StoragePtr InterpreterSystemQuery::doRestartReplica(const StorageID & replica, C return nullptr; } - if (!dynamic_cast(table.get())) + /// The resolved pointer must not outlive this check. `waitDetachedTableNotInUse` below waits + /// for the last reference to the detached table to be released. + if (!castStorage(table, DeferredTable::Load)) { if (throw_on_error) throw Exception(ErrorCodes::BAD_ARGUMENTS, table_is_not_replicated.data(), replica.getNameForLogs()); @@ -1626,7 +1629,7 @@ void InterpreterSystemQuery::restartReplicas(ContextMutablePtr system_context) for (auto it = elem.second->getTablesIterator(getContext()); it->isValid(); it->next()) { - if (dynamic_cast(it->table().get())) + if (castStorage(it->table(), DeferredTable::Skip)) { if (!access_is_granted_globally && !access->isGranted(AccessType::SYSTEM_RESTART_REPLICA, elem.first, it->name())) { @@ -1730,7 +1733,7 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) DatabasePtr & database = elem.second; for (auto iterator = database->getTablesIterator(getContext()); iterator->isValid(); iterator->next()) { - if (auto * storage_replicated = dynamic_cast(iterator->table().get())) + if (auto * storage_replicated = castStorage(iterator->table(), DeferredTable::Skip).get()) { /// getReplicaPath() is built from getZooKeeperPath(), which strips only a single trailing /// slash, so a table created from "/a///" metadata keeps "/a//replicas/..." and would slip @@ -1775,7 +1778,8 @@ void InterpreterSystemQuery::dropReplica(ASTSystemQuery & query) bool InterpreterSystemQuery::dropStorageReplica(const String & query_replica, const StoragePtr & storage) { - auto * storage_replicated = dynamic_cast(storage.get()); + /// Dropping a replica from Keeper is what the command asks for, so loading the table is warranted. + auto * storage_replicated = castStorage(storage, DeferredTable::Load).get(); if (!storage_replicated) return false; @@ -2189,7 +2193,8 @@ bool InterpreterSystemQuery::trySyncReplica(StoragePtr table, SyncReplicaMode sy break; } - if (auto * storage_replicated = dynamic_cast(table.get())) + + if (auto * storage_replicated = castStorage(table, DeferredTable::Load).get()) { auto log = getLogger("InterpreterSystemQuery"); LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for current last entry to be processed"); @@ -2237,7 +2242,7 @@ void InterpreterSystemQuery::waitLoadingParts() getContext()->checkAccess(AccessType::SYSTEM_WAIT_LOADING_PARTS, table_id); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) { LOG_TRACE(log, "Waiting for loading of parts of table {}", table_id.getFullTableName()); merge_tree->waitForOutdatedPartsToBeLoaded(); @@ -2283,7 +2288,7 @@ void InterpreterSystemQuery::restartDisk(const String & disk_name) /// skip_not_loaded: act only on already-loaded tables, do not block on async loading. for (auto it = elem.second->getTablesIterator(getContext(), {}, /*skip_not_loaded=*/ true); it->isValid(); it->next()) { - auto * merge_tree = dynamic_cast(it->table().get()); + auto * merge_tree = castStorage(it->table(), DeferredTable::Skip).get(); if (!merge_tree) continue; @@ -2319,7 +2324,8 @@ namespace MergeTreeData & getMergeTreeWithManualSelector(const StoragePtr & table, const StorageID & table_id, const char * action) { - auto * merge_tree = dynamic_cast(table.get()); + auto resolved = resolveStorageProxyLoading(table); + auto * merge_tree = castStorage(resolved, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Command {} is supported only for MergeTree-family tables, but got: {}", @@ -2413,7 +2419,7 @@ void InterpreterSystemQuery::loadOrUnloadPrimaryKeysImpl(bool load) getContext()->checkAccess(load ? AccessType::SYSTEM_LOAD_PRIMARY_KEY : AccessType::SYSTEM_UNLOAD_PRIMARY_KEY, table_id.database_name, table_id.table_name); StoragePtr table = DatabaseCatalog::instance().getTable(table_id, getContext()); - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) { LOG_TRACE(log, "{} primary keys for table {}", load ? "Loading" : "Unloading", table_id.getFullTableName()); load ? merge_tree->loadPrimaryKeys() : merge_tree->unloadPrimaryKeys(); @@ -2433,7 +2439,7 @@ void InterpreterSystemQuery::loadOrUnloadPrimaryKeysImpl(bool load) { for (auto it = database.second->getTablesIterator(getContext()); it->isValid(); it->next()) { - if (auto * merge_tree = dynamic_cast(it->table().get())) + if (auto * merge_tree = castStorage(it->table(), DeferredTable::Skip).get()) { load ? merge_tree->loadPrimaryKeys() : merge_tree->unloadPrimaryKeys(); } @@ -2544,6 +2550,7 @@ void InterpreterSystemQuery::syncReplicatedDatabase(ASTSystemQuery & query) void InterpreterSystemQuery::syncTransactionLog() { + getContext()->checkAccess(AccessType::SYSTEM_SYNC_TRANSACTION_LOG); getContext()->checkTransactionsAreAllowed(/* explicit_tcl_query */ true); TransactionLog::instance().sync(); } @@ -2577,7 +2584,7 @@ void InterpreterSystemQuery::flushObjectStorageQueue(ASTSystemQuery & query) throw Exception(ErrorCodes::BAD_ARGUMENTS, "PATH must be specified for SYSTEM FLUSH OBJECT STORAGE QUEUE"); auto table = DatabaseCatalog::instance().getTable(table_id, context); - auto * queue = dynamic_cast(table.get()); + auto queue = castStorage(table, DeferredTable::Load); if (!queue) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table {} is not an S3Queue or AzureQueue table", table_id.getNameForLogs()); @@ -2638,7 +2645,7 @@ void InterpreterSystemQuery::controlBackgroundActivity(const ASTSystemQuery & qu const bool can_views = access->isGranted(AccessType::SYSTEM_VIEWS, table_id.database_name, table_id.table_name); const bool can_streaming = access->isGranted(AccessType::SYSTEM_STREAMING_ENGINES, table_id.database_name, table_id.table_name); - auto storage = DatabaseCatalog::instance().tryGetTable(table_id, getContext()); + auto storage = resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(table_id, getContext())); const bool is_streaming = storage && storage->isStreamingStorage(); const auto * mv = storage ? dynamic_cast(storage.get()) : nullptr; const bool is_refreshable_view = mv && mv->isRefreshable(); @@ -2720,8 +2727,8 @@ void InterpreterSystemQuery::prewarmMarkCache() getContext()->checkAccess(AccessType::SYSTEM_PREWARM_MARK_CACHE, table_id); - auto table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto * merge_tree = dynamic_cast(table_ptr.get()); + auto table_ptr = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(table_id, getContext())); + auto * merge_tree = castStorage(table_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Command PREWARM MARK CACHE is supported only for MergeTree table, but got: {}", table_ptr->getName()); @@ -2744,8 +2751,8 @@ void InterpreterSystemQuery::prewarmPrimaryIndexCache() getContext()->checkAccess(AccessType::SYSTEM_PREWARM_PRIMARY_INDEX_CACHE, table_id); - auto table_ptr = DatabaseCatalog::instance().getTable(table_id, getContext()); - auto * merge_tree = dynamic_cast(table_ptr.get()); + auto table_ptr = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(table_id, getContext())); + auto * merge_tree = castStorage(table_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Command PREWARM PRIMARY INDEX CACHE is supported only for MergeTree table, but got: {}", table_ptr->getName()); @@ -3229,11 +3236,22 @@ AccessRightsElements InterpreterSystemQuery::getRequiredAccessForDDLOnCluster() } case Type::STOP_THREAD_FUZZER: case Type::START_THREAD_FUZZER: + { + required_access.emplace_back(AccessType::SYSTEM_THREAD_FUZZER); + break; + } + case Type::RESET_COVERAGE: + { + required_access.emplace_back(AccessType::SYSTEM); + break; + } + /// The parser cases of the failpoint statements and of SYSTEM SET COVERAGE TEST never read an + /// ON CLUSTER clause, so those cluster spellings do not parse and reach no host. UNKNOWN and + /// END are not statements. case Type::ENABLE_FAILPOINT: case Type::WAIT_FAILPOINT: case Type::NOTIFY_FAILPOINT: case Type::DISABLE_FAILPOINT: - case Type::RESET_COVERAGE: case Type::SET_COVERAGE_TEST: case Type::UNKNOWN: case Type::END: break; diff --git a/src/Interpreters/InterpreterUpdateQuery.cpp b/src/Interpreters/InterpreterUpdateQuery.cpp index e03f38901d07..f6393967ee71 100644 --- a/src/Interpreters/InterpreterUpdateQuery.cpp +++ b/src/Interpreters/InterpreterUpdateQuery.cpp @@ -174,16 +174,17 @@ BlockIO InterpreterUpdateQuery::execute() } /// Expand CTEs before filling the default database, otherwise a CTE alias is qualified as if it - /// were a table. + /// were a table. The context makes CTE expansion respect `enable_global_with_statement`: a CTE + /// name a subquery does not see is a table name there, and has to be qualified. if (update_query.predicate) { ASTPtr predicate = update_query.predicate->ptr(); - ApplyWithSubqueryVisitor::visit(predicate); + ApplyWithSubqueryVisitor::visit(predicate, getContext()); } if (update_query.assignments) { ASTPtr assignments = update_query.assignments->ptr(); - ApplyWithSubqueryVisitor::visit(assignments); + ApplyWithSubqueryVisitor::visit(assignments, getContext()); } /// Add default database to table identifiers that we can encounter in the update expression. diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp index ebdc9bd48d15..f3d671cf38c1 100644 --- a/src/Interpreters/JoinedTables.cpp +++ b/src/Interpreters/JoinedTables.cpp @@ -1,5 +1,7 @@ #include +#include +#include #include #include @@ -22,6 +24,7 @@ #include #include +#include #include #include #include @@ -344,9 +347,17 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se { auto joined_table_id = context->resolveStorageID(table_to_join.database_and_table_name); StoragePtr storage = DatabaseCatalog::instance().tryGetTable(joined_table_id, context); + + /// A special storage replaces the right-side plan, and with it the `FilterStep` carrying the + /// table's row policy, so such a table has to be joined as an ordinary stream. + auto joined_table_row_policy = context->getRowPolicyFilter( + joined_table_id.getDatabaseName(), joined_table_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + if (joined_table_row_policy && !joined_table_row_policy->isAlwaysTrue()) + storage = nullptr; + if (storage) { - if (auto storage_join = std::dynamic_pointer_cast(storage); storage_join) + if (auto storage_join = castStorage(storage, DeferredTable::Load); storage_join) { table_join->setStorageJoin(storage_join); } @@ -369,11 +380,12 @@ std::shared_ptr JoinedTables::makeTableJoin(const ASTSelectQuery & se return nullptr; } + /// NOLINT(storage-cast): a dictionary, which the catalog never hands out behind a proxy. auto dictionary_kv = std::dynamic_pointer_cast(dictionary); table_join->setStorageJoin(dictionary_kv); } - if (auto storage_kv = std::dynamic_pointer_cast(storage); storage_kv && try_use_direct_join) + if (auto storage_kv = castStorage(storage, DeferredTable::Load); storage_kv && try_use_direct_join) { table_join->setStorageJoin(storage_kv); } diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index d267f9c4f08d..f286a7af274b 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -338,8 +339,9 @@ ASTPtr getPartitionAndPredicateExpressionForMutationCommand( { String partition_id; - auto storage_merge_tree = std::dynamic_pointer_cast(storage); - auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(storage); + auto resolved_storage = resolveStorageProxyLoading(storage); + auto storage_merge_tree = castStorage(resolved_storage, DeferredTable::Load); + auto storage_from_merge_tree_data_part = std::dynamic_pointer_cast(resolved_storage); if (storage_merge_tree) partition_id = storage_merge_tree->getPartitionIDFromQuery(ASTPtr(alter->partition), context); else if (storage_from_merge_tree_data_part) @@ -367,7 +369,9 @@ ASTPtr getPartitionAndPredicateExpressionForMutationCommand( return predicate_ast; } -MutationsInterpreter::Source::Source(StoragePtr storage_) : storage(std::move(storage_)) +/// A mutation reads and rewrites parts, so it needs the real storage rather than the proxy a lazily +/// loaded table is reached through. +MutationsInterpreter::Source::Source(StoragePtr storage_) : storage(resolveStorageProxyLoading(storage_)) { } @@ -409,7 +413,7 @@ const MergeTreeData * MutationsInterpreter::Source::getMergeTreeData() const if (data) return data; - return dynamic_cast(storage.get()); + return castStorage(storage, DeferredTable::Load).get(); } MergeTreeData::DataPartPtr MutationsInterpreter::Source::getMergeTreeDataPart() const @@ -490,7 +494,7 @@ MutationsInterpreter::MutationsInterpreter( std::move(available_columns_), std::move(context_), std::move(settings_)) { - if (settings.can_execute && !settings.return_mutated_rows && dynamic_cast(source.getStorage().get())) + if (settings.can_execute && !settings.return_mutated_rows && castStorage(source.getStorage(), DeferredTable::Load)) { throw Exception( ErrorCodes::LOGICAL_ERROR, @@ -768,6 +772,9 @@ void MutationsInterpreter::prepare(bool dry_run) /// MATERIALIZED columns (e.g. m2 MATERIALIZED m1 MATERIALIZED src) so a change of a /// base column recalculates every MATERIALIZED column transitively derived from it. std::unordered_map materialized_column_dependencies; + /// MATERIALIZED columns reading an EPHEMERAL column, with their readable dependencies. They are + /// skipped by the recompute, so they are checked for staleness after the graph is built. + std::vector> ephemeral_reading_materialized; /// The MATERIALIZED-chain analysis is needed for classical UPDATE, for materializing /// patch parts (APPLY PATCHES) and for CLEAR COLUMN, since all three can change a @@ -803,16 +810,15 @@ void MutationsInterpreter::prepare(bool dry_run) if (std::ranges::any_of(required_columns, [&](const auto & dep) { return ephemeral_columns.contains(dep); })) { - /// Warn if the mutation also updates a non-ephemeral dependency - /// of this MATERIALIZED column — the on-disk value will become stale. - if (std::ranges::any_of(required_columns, [&](const auto & dep) - { return !ephemeral_columns.contains(dep) && updated_columns.contains(dep); })) - LOG_WARNING(logger, - "MATERIALIZED column '{}' depends on both EPHEMERAL and regular " - "columns that are being updated. Its value will NOT be recalculated " - "during this mutation — the on-disk value may become inconsistent. " - "To fix this, re-INSERT the affected rows.", - column.name); + /// Such a column is never recomputed, so its on-disk value goes stale as soon as + /// one of its regular inputs changes. Collect it and warn below, once the readable + /// dependency graph is complete: whether an input changed can only be answered + /// over the transitive closure, and a hop of the chain may still be unvisited here. + Names readable_dependencies; + for (const auto & dep : required_columns) + if (!ephemeral_columns.contains(dep)) + readable_dependencies.push_back(dep); + ephemeral_reading_materialized.emplace_back(column.name, std::move(readable_dependencies)); continue; } @@ -882,6 +888,30 @@ void MutationsInterpreter::prepare(bool dry_run) return affected; }; + /// A MATERIALIZED column reading an EPHEMERAL column cannot be recomputed by a mutation, so if + /// any of its readable inputs is changed by this mutation — directly, or through another + /// MATERIALIZED hop that this mutation does recompute — its on-disk value becomes stale. + if (!ephemeral_reading_materialized.empty()) + { + NameSet changed_base_columns = updated_columns; + changed_base_columns.insert(clear_column_names.begin(), clear_column_names.end()); + changed_base_columns.insert(patch_updated_columns.begin(), patch_updated_columns.end()); + + NameSet stale_columns = affected_materialized_closure(changed_base_columns); + stale_columns.insert(changed_base_columns.begin(), changed_base_columns.end()); + + for (const auto & [name, readable_dependencies] : ephemeral_reading_materialized) + { + if (std::ranges::any_of(readable_dependencies, [&](const auto & dep) { return stale_columns.contains(dep); })) + LOG_WARNING(logger, + "MATERIALIZED column '{}' depends on both EPHEMERAL and regular " + "columns that are being updated or cleared. Its value will NOT be " + "recalculated during this mutation — the on-disk value may become " + "inconsistent. To fix this, re-INSERT the affected rows.", + name); + } + } + /// Emit recompute stages for a set of affected MATERIALIZED columns. A MATERIALIZED /// column may read another affected MATERIALIZED column (e.g. m2 MATERIALIZED m1, where /// m1 is itself recomputed). Recomputing them all in a single stage would evaluate m2 @@ -964,14 +994,21 @@ void MutationsInterpreter::prepare(bool dry_run) if (!patch_updated_columns.empty()) patch_affected_materialized = affected_materialized_closure(patch_updated_columns); + NameSet materialized_with_ephemeral_dependency; + for (const auto & [name, readable_dependencies] : ephemeral_reading_materialized) + materialized_with_ephemeral_dependency.insert(name); + /// MATERIALIZED columns rewritten by a CLEAR COLUMN. Must stay equal to the set the recompute - /// below writes, otherwise a rewritten column keeps stale dependent artifacts. + /// below writes, otherwise a rewritten column keeps stale dependent artifacts. A column reading + /// an EPHEMERAL one is not recomputable outside INSERT and must be left out, or the recompute + /// stage fails to resolve the EPHEMERAL name and the whole mutation fails. NameSet clear_affected_materialized; if (!clear_column_names.empty() && !affected_materialized_closure(clear_column_names).empty()) { for (const auto & column : columns_desc) { - if (column.default_desc.kind == ColumnDefaultKind::Materialized && column.default_desc.expression) + if (column.default_desc.kind == ColumnDefaultKind::Materialized && column.default_desc.expression + && !materialized_with_ephemeral_dependency.contains(column.name)) clear_affected_materialized.insert(column.name); } } @@ -1544,9 +1581,12 @@ void MutationsInterpreter::prepare(bool dry_run) bool has_dependent_materialized = false; for (const auto & column : columns_desc) { + /// A column reading an EPHEMERAL one is not recomputed, and its expression cannot + /// even be analyzed without the EPHEMERAL columns, which are absent from `all_columns`. if (column.default_desc.kind != ColumnDefaultKind::Materialized || !available_columns_set.contains(column.name) - || !column.default_desc.expression) + || !column.default_desc.expression + || materialized_with_ephemeral_dependency.contains(column.name)) continue; auto query = column.default_desc.expression->clone(); diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp index 466a92cd7cfd..4a7d75fdd827 100644 --- a/src/Interpreters/PreparedSets.cpp +++ b/src/Interpreters/PreparedSets.cpp @@ -666,14 +666,10 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) } /// Runs the speculative pipeline in its own scope so that `executor`, `pipeline`, and the pipeline - /// builder are destroyed before `source` is reset below. On the non-destructive path the cloned plan - /// carries an *empty* `QueryPlanResourceHolder` (`QueryPlan::clone` copies the plan nodes only, not the - /// resources), so the speculative pipeline relies on the original `source` plan to keep the interpreter - /// contexts, storage holders, and table locks alive: processors may use them implicitly, including in - /// their destructors — this is exactly what the resource holder normally guarantees for the lifetime of - /// the pipeline. Resetting `source` while the pipeline is still alive would release them too early. On - /// the destructive fallback `build` moved the resources into the plan, which outlives this call, so the - /// ordering is safe there too. + /// builder are destroyed before `source` is reset below. `QueryPlan::clone` copies the shared resource + /// handles onto the cloned plan, so the speculative pipeline holds the interpreter contexts, storage + /// holders, and table locks itself rather than depending on `source` outliving it. On the destructive + /// fallback `build` moved the resources into the plan, which outlives this call. /// /// Returns false when the pipeline stopped without creating the set. auto run_plan = [&](QueryPlan & plan_to_run) @@ -722,8 +718,7 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context) /// In-place build succeeded. On the non-destructive path, publish the fully-created temporary set into /// the canonical `set_and_key`; the deferred build is then skipped (it checks `isCreated()` / `get()`), /// so the original `source` plan is no longer needed. On the destructive fallback `source` was already - /// consumed by `build`, so `reset` is a no-op there. Reset only now, after the pipeline and executor have - /// been destroyed, so the resources held by `source` outlive every processor. + /// consumed by `build`, so `reset` is a no-op there. if (tmp_set_and_key) set_and_key->set = tmp_set_and_key->set; source.reset(); diff --git a/src/Interpreters/ServerAsynchronousMetrics.cpp b/src/Interpreters/ServerAsynchronousMetrics.cpp index 3ca6fc71beb9..c24ca42a48f4 100644 --- a/src/Interpreters/ServerAsynchronousMetrics.cpp +++ b/src/Interpreters/ServerAsynchronousMetrics.cpp @@ -34,6 +34,7 @@ #include #endif +#include #include #include #include @@ -495,11 +496,11 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr if (is_system) ++total_number_of_tables_system; - const auto & table = iterator->table(); + auto table = iterator->table(); if (!table) continue; - if (MergeTreeData * table_merge_tree = dynamic_cast(table.get())) + if (auto table_merge_tree = castStorage(table, DeferredTable::Skip)) { calculateMax(max_part_count_for_partition, table_merge_tree->getMaxPartsCountAndSizeForPartition().first); @@ -541,7 +542,7 @@ void ServerAsynchronousMetrics::updateImpl(TimePoint update_time, TimePoint curr } } - if (StorageReplicatedMergeTree * table_replicated_merge_tree = typeid_cast(table.get())) + if (StorageReplicatedMergeTree * table_replicated_merge_tree = castStorage(table, DeferredTable::Skip).get()) { StorageReplicatedMergeTree::ReplicatedStatus status; table_replicated_merge_tree->getStatus(status, false); @@ -684,11 +685,7 @@ void ServerAsynchronousMetrics::updateMutationAndDetachedPartsStats() for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); - if (!table) - continue; - - if (MergeTreeData * table_merge_tree = dynamic_cast(table.get())) + if (auto table_merge_tree = castStorage(iterator->table(), DeferredTable::Skip)) { for (const auto & detached_part: table_merge_tree->getDetachedParts()) { diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index 6b6cc000b61f..17521ee3d1bc 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -413,7 +413,7 @@ void Session::authenticate(const Credentials & credentials_, const Poco::Net::So prepared_client_info->connection_address = Poco::Net::SocketAddress(connection_address ? *connection_address : address); } -void Session::checkIfUserIsStillValid() +void Session::checkIfUserIsStillValid() const { if (const auto valid_until = user_authenticated_with.getValidUntil()) { @@ -696,6 +696,13 @@ ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_t if (!user_id && getClientInfo().interface != ClientInfo::Interface::TCP_INTERSERVER) throw Exception(ErrorCodes::LOGICAL_ERROR, "Query context must be created after authentication"); + /// The authentication method's `VALID UNTIL` must be enforced per query, not only at login: + /// stateful protocols (MySQL, PostgreSQL, native TCP, gRPC, Arrow Flight) authenticate once and + /// then create a query context per command, so an expired credential must stop working here. + /// Interserver connections replay an already-checked initiator identity, so they are exempt. + if (getClientInfo().interface != ClientInfo::Interface::TCP_INTERSERVER) + checkIfUserIsStillValid(); + /// We can create a query context either from a session context or from a global context. const bool from_session_context = static_cast(session_context) && !detached; diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 055c8b8bcbd9..4a8bf55154f3 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -62,7 +62,7 @@ class Session // Verifies whether the user's validity extends beyond the current time. // Throws an exception if the user's validity has expired. - void checkIfUserIsStillValid(); + void checkIfUserIsStillValid() const; /// Writes a row about login failure into session log (if enabled) void onAuthenticationFailure(const std::optional & user_name, const Poco::Net::SocketAddress & address_, const Exception & e); diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 8615932970d4..b27dbf94cbbc 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -523,6 +523,27 @@ static String httpRequestURLForLogging(const ContextPtr & context) return url.substr(0, url.find_first_of("?#")); } +String formatQueryForLogging(const String & query, const Settings & settings) +{ + const char * pos = query.data(); + const char * end = pos + query.size(); + ParserQuery parser(end, settings[Setting::allow_settings_after_format_in_insert], settings[Setting::implicit_select]); + String parse_error; + const ASTPtr ast = tryParseQuery( + parser, + pos, + end, + parse_error, + /*hilite*/ false, + "", + /*allow_multi_statements*/ false, + settings[Setting::max_query_size], + settings[Setting::max_parser_depth], + settings[Setting::max_parser_backtracks], + /*skip_insignificant*/ true); + return ast ? ast->formatForLogging(settings[Setting::log_queries_cut_to_length]) : ""; +} + QueryLogElement logQueryStart( const std::chrono::time_point & query_start_time, const ContextMutablePtr & context, diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 8870bbb6c2a5..62214bcf9af8 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -23,6 +23,7 @@ class WriteBuffer; class IOutputFormat; struct QueryStatusInfo; struct QueryPlanAndSets; +struct Settings; struct QueryResultDetails { @@ -126,6 +127,9 @@ void finishExecutedQuery(BlockIO & io, const QueryFinishCallback & query_finish_ /// also used for generated queries that bypass `executeQuery`, such as in the `eval` table function. void validateAnalyzerSettings(ASTPtr ast, bool context_value); +/// Parses the query only to produce text with secrets hidden; returns an empty string if the query does not parse. +String formatQueryForLogging(const String & query, const Settings & settings); + /// Prepares a QueryLogElement and, if enabled, logs it to system.query_log QueryLogElement logQueryStart( const std::chrono::time_point & query_start_time, diff --git a/src/Interpreters/tests/gtest_filecache.cpp b/src/Interpreters/tests/gtest_filecache.cpp index 51cd47f464ca..9ddd0e382259 100644 --- a/src/Interpreters/tests/gtest_filecache.cpp +++ b/src/Interpreters/tests/gtest_filecache.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -74,6 +75,7 @@ namespace ProfileEvents { extern const Event FilesystemCacheDowngradedFileSegments; extern const Event FilesystemCacheEvictedFileSegments; + extern const Event FilesystemCacheReserveAheadRetries; } using namespace std::chrono_literals; @@ -107,6 +109,7 @@ namespace DB::FileCacheSetting extern const FileCacheSettingsUInt64 max_elements; extern const FileCacheSettingsUInt64 max_file_segment_size; extern const FileCacheSettingsUInt64 boundary_alignment; + extern const FileCacheSettingsUInt64 reserve_granularity; extern const FileCacheSettingsFileCachePolicy cache_policy; extern const FileCacheSettingsDouble slru_size_ratio; extern const FileCacheSettingsDouble keep_free_space_elements_ratio; @@ -3755,6 +3758,209 @@ TEST_F(FileCacheTest, RenameToIncludeSizeInNameFailureKeepsSegmentConsistent) ASSERT_EQ((*reloaded_holder->begin())->state(), State::DOWNLOADED); } +TEST_F(FileCacheTest, ReserveUndoneWhenKeyDirectoryCannotBeCreated) +{ + ServerUUID::setRandomForUnitTests(); + DB::ThreadStatus thread_status; + + Poco::XML::DOMParser dom_parser; + std::string xml(R"CONFIG()CONFIG"); + Poco::AutoPtr document = dom_parser.parseString(xml); + Poco::AutoPtr config = new Poco::Util::XMLConfiguration(document); + getMutableContext().context->setConfig(config); + + auto query_context = DB::Context::createCopy(getContext().context); + query_context->makeQueryContext(); + query_context->setCurrentQueryId("reserve_key_directory_failure"); + chassert(&DB::CurrentThread::get() == &thread_status); + auto query_scope_holder = DB::QueryScope::create(query_context); + + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_size] = 16; + settings[FileCacheSetting::max_elements] = 4; + settings[FileCacheSetting::max_file_segment_size] = 8; + settings[FileCacheSetting::boundary_alignment] = 8; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("reserve_key_directory_failure", settings); + cache->initialize(); + + const auto & user = FileCache::getCommonOrigin(); + auto key = DB::FileCacheKey::fromPath("reserve_key_directory_failure_key"); + + /// A regular file at the key directory path makes `create_directories` fail, even as root. + const fs::path key_path = cache->getKeyPath(key, user); + fs::create_directories(key_path.parent_path()); + std::ofstream(key_path) << "x"; + + auto holder = cache->getOrSet(key, 0, 8, /*file_size=*/8, {}, 0, user); + ASSERT_EQ(holder->size(), 1u); + auto seg = *holder->begin(); + ASSERT_EQ(seg->getOrSetDownloader(), FileSegment::getCallerId()); + + std::string failure_reason; + ASSERT_FALSE(seg->reserve(8, 1000, failure_reason)); + ASSERT_TRUE(failure_reason.contains("base directory")) << failure_reason; + ASSERT_EQ(seg->getReservedSize(), 0u); + ASSERT_EQ(cache->getUsedCacheSize(), 0u); + + /// The failed segment is `PARTIALLY_DOWNLOADED_NO_CONTINUATION`; releasing its last holder removes it, + /// so the same offset gets a new segment, which caches normally once the directory can be created. + seg.reset(); + holder = nullptr; + fs::remove(key_path); + auto next_holder = cache->getOrSet(key, 0, 8, /*file_size=*/8, {}, 0, user); + ASSERT_EQ(next_holder->size(), 1u); + download(*next_holder->begin()); + ASSERT_EQ(cache->getUsedCacheSize(), 8u); +} + +TEST(FileCacheReserveAhead, GrowsUpToLimitAndResets) +{ + DB::FileCacheReserveAhead reserve_ahead; + + /// Exact first, then doubling up to the limit. + ASSERT_EQ(reserve_ahead.getReserveSize(/* size_to_reserve */ 2, /* max_reserve_size */ 100, /* limit */ 16), 2u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 4u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 8u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 16u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 16u); + + /// Capped by `max_reserve_size`, but never less than the request. + ASSERT_EQ(reserve_ahead.getReserveSize(2, 10, 16), 10u); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 1, 16), 2u); + ASSERT_EQ(reserve_ahead.getReserveSize(32, 100, 16), 32u); + + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 4), 4u); + + /// Limit 0 disables reserve-ahead. + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 0), 2u); + + reserve_ahead.reset(); + ASSERT_EQ(reserve_ahead.getReserveSize(6, 100, 16), 6u); + ASSERT_EQ(reserve_ahead.getReserveSize(6, 100, 16), 12u); + + reserve_ahead.reset(); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 100, 16), 2u); + + /// A short first request does not keep the reserve-ahead small for later, larger requests. + reserve_ahead.reset(); + ASSERT_EQ(reserve_ahead.getReserveSize(2, 1000, 1024), 2u); + ASSERT_EQ(reserve_ahead.getReserveSize(100, 1000, 1024), 100u); + ASSERT_EQ(reserve_ahead.getReserveSize(100, 1000, 1024), 200u); +} + +TEST_F(FileCacheTest, DynamicReserveGranularity) +{ + ServerUUID::setRandomForUnitTests(); + DB::ThreadStatus thread_status; + + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_size] = 1000; + settings[FileCacheSetting::max_elements] = 10; + settings[FileCacheSetting::max_file_segment_size] = 100; + settings[FileCacheSetting::boundary_alignment] = 100; + settings[FileCacheSetting::reserve_granularity] = 16; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("dynamic_reserve_granularity", settings); + cache->initialize(); + + const auto & user = FileCache::getCommonOrigin(); + std::string failure_reason; + std::string data(100, '0'); + + auto reserve_and_write = [&](DB::FileSegment & segment, size_t size, DB::FileCacheReserveAhead * reserve_ahead) + { + EXPECT_TRUE(segment.reserve(size, 1000, failure_reason, nullptr, std::nullopt, reserve_ahead)) << failure_reason; + segment.write(data.data(), size, segment.getCurrentWriteOffset()); + return segment.getReservedSize(); + }; + + { + auto holder = cache->getOrSet(DB::FileCacheKey::fromPath("no_reserve_ahead"), 0, 100, /*file_size=*/100, {}, 0, user); + auto segment = *holder->begin(); + ASSERT_EQ(segment->getOrSetDownloader(), FileSegment::getCallerId()); + ASSERT_EQ(reserve_and_write(*segment, 2, nullptr), 2u); + ASSERT_EQ(reserve_and_write(*segment, 2, nullptr), 4u); + ASSERT_EQ(reserve_and_write(*segment, 2, nullptr), 6u); + } + + { + auto holder = cache->getOrSet(DB::FileCacheKey::fromPath("reserve_ahead"), 0, 100, /*file_size=*/100, {}, 0, user); + auto segment = *holder->begin(); + ASSERT_EQ(segment->getOrSetDownloader(), FileSegment::getCallerId()); + + DB::FileCacheReserveAhead reserve_ahead; + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 2u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 2u + 4); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 6u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 6u + 8); + for (size_t downloaded = 10; downloaded <= 14; downloaded += 2) + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 14u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 14u + 16); + for (size_t downloaded = 18; downloaded <= 30; downloaded += 2) + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 30u); + ASSERT_EQ(reserve_and_write(*segment, 2, &reserve_ahead), 30u + 16); + } +} + +TEST_F(FileCacheTest, ReserveAheadFallsBackToExactSize) +{ + ServerUUID::setRandomForUnitTests(); + DB::ThreadStatus thread_status; + + /// The cache is smaller than the reserve-ahead limit. + DB::FileCacheSettings settings; + settings[FileCacheSetting::path] = cache_base_path; + settings[FileCacheSetting::max_size] = 12; + settings[FileCacheSetting::max_elements] = 10; + settings[FileCacheSetting::max_file_segment_size] = 20; + settings[FileCacheSetting::boundary_alignment] = 20; + settings[FileCacheSetting::reserve_granularity] = 16; + settings[FileCacheSetting::load_metadata_asynchronously] = false; + settings[FileCacheSetting::cache_policy] = FileCachePolicy::LRU; + + auto cache = std::make_shared("reserve_ahead_fallback", settings); + cache->initialize(); + + const auto & user = FileCache::getCommonOrigin(); + auto holder = cache->getOrSet(DB::FileCacheKey::fromPath("reserve_ahead_fallback"), 0, 20, /*file_size=*/20, {}, 0, user); + auto segment = *holder->begin(); + ASSERT_EQ(segment->getOrSetDownloader(), FileSegment::getCallerId()); + + std::string failure_reason; + std::string data(20, '0'); + DB::FileCacheReserveAhead reserve_ahead; + auto & events = CurrentThread::getProfileEvents(); + const auto retries_before = events[ProfileEvents::FilesystemCacheReserveAheadRetries]; + auto reserve_and_write = [&]() + { + EXPECT_TRUE(segment->reserve(2, 1000, failure_reason, nullptr, std::nullopt, &reserve_ahead)) << failure_reason; + segment->write(data.data(), 2, segment->getCurrentWriteOffset()); + return segment->getReservedSize(); + }; + + ASSERT_EQ(reserve_and_write(), 2u); + ASSERT_EQ(reserve_and_write(), 6u); + ASSERT_EQ(reserve_and_write(), 6u); + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before); + ASSERT_EQ(reserve_and_write(), 8u); /// +8 does not fit, falls back to exact +2 and resets + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before + 1); + ASSERT_EQ(reserve_and_write(), 10u); + ASSERT_EQ(reserve_and_write(), 12u); /// +4 does not fit, falls back to exact +2, the cache is full + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before + 2); + + ASSERT_FALSE(segment->reserve(2, 1000, failure_reason, nullptr, std::nullopt, &reserve_ahead)); + ASSERT_EQ(events[ProfileEvents::FilesystemCacheReserveAheadRetries], retries_before + 2); + ASSERT_EQ(segment->getReservedSize(), 12u); + ASSERT_EQ(cache->getUsedCacheSize(), 12u); +} + TEST_F(FileCacheTest, QueryLimitContextRevivedDuringRelease) { /// Regression for STID 4192-71db: a holder for some query_id decides it is the last one and diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp index f35fbb8f8b65..547fdd7f4f73 100644 --- a/src/Loggers/OwnSplitChannel.cpp +++ b/src/Loggers/OwnSplitChannel.cpp @@ -65,7 +65,8 @@ void OwnSplitChannel::log(Poco::Message && msg) return; const auto & logs_queue = CurrentThread::getInternalTextLogsQueue(); - if (channels.empty() && (logs_queue == nullptr && !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) + if (channels.empty() && !text_log_max_priority.load(std::memory_order_relaxed) + && (logs_queue == nullptr || !logs_queue->isNeeded(msg.getPriority(), msg.getSource()))) return; if (const auto & masker = SensitiveDataMasker::getInstance()) diff --git a/src/Parsers/ASTFromJSON.cpp b/src/Parsers/ASTFromJSON.cpp index 40ceb1cf9583..bb4d0b6dd8f1 100644 --- a/src/Parsers/ASTFromJSON.cpp +++ b/src/Parsers/ASTFromJSON.cpp @@ -78,12 +78,14 @@ #include #include +#include #include #include #include #include +#include #include #include @@ -309,14 +311,27 @@ size_t computeJSONNestingDepth(const String & json) return max_depth; } +/// `Poco::JSON::Parser` descends recursively over the whole document before any AST node is built. +class StackCheckingParseHandler : public Poco::JSON::ParseHandler +{ +public: + void startObject() override + { + checkStackSize(); + Poco::JSON::ParseHandler::startObject(); + } + + void startArray() override + { + checkStackSize(); + Poco::JSON::ParseHandler::startArray(); + } +}; + } ASTPtr IAST::createFromJSON(const String & json) { - /// `Poco::JSON::Parser::setDepth` does not actually bound recursion in our Poco fork - /// (`ParserImpl` stores `_depth` but `handle`/`handleObject`/`handleArray` never read it), - /// so a hostile deeply-nested payload would recurse through the parser and overflow the - /// stack before any AST-level depth check runs. Enforce a raw-text bracket budget first. /// The budget is a safe multiple of the effective depth limit (the JSON encoding adds bracket /// levels per AST/`Field` level), not the limit itself, so a valid serialized AST is never /// rejected here — the constructed AST depth is still bounded by the counter check below. @@ -328,14 +343,19 @@ ASTPtr IAST::createFromJSON(const String & json) "JSON nesting depth exceeds the limit derived from max_ast_depth ({}) during JSON AST deserialization", json_nesting_budget); Poco::JSON::Parser parser; - /// Also request the parser-level bound (kept for forward compatibility if the fork starts - /// honouring it); the pre-scan above is the actual enforcement. + parser.setHandler(new StackCheckingParseHandler); + /// Poco's own default bound is 1000 levels (JSON_DEFAULT_DEPTH). parser.setDepth(json_nesting_budget); Poco::Dynamic::Var result; try { result = parser.parse(json); } + /// `DB::Exception` derives from `Poco::Exception`, so this clause must precede the one below. + catch (const Exception &) + { + throw; + } catch (const Poco::Exception & e) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Failed to parse JSON for AST deserialization: {}", e.displayText()); @@ -365,6 +385,9 @@ ASTPtr IAST::createFromJSON(const Poco::JSON::Object & json) throw Exception(ErrorCodes::TOO_DEEP_AST, "JSON AST deserialization exceeded maximum depth limit ({})", max_depth); + /// The limit above counts nodes, which is not a stack budget at any value. + checkStackSize(); + /// Check element count limit. if (json_deser_max_elements && json_deser_current_elements >= json_deser_max_elements) throw Exception(ErrorCodes::TOO_BIG_AST, diff --git a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp index ea1b63f82b41..17156bf476ef 100644 --- a/src/Parsers/ASTFunctionWithKeyValueArguments.cpp +++ b/src/Parsers/ASTFunctionWithKeyValueArguments.cpp @@ -20,12 +20,16 @@ namespace { /// Keys of a dictionary source whose value must not be shown. Besides the password, this covers /// the TLS credentials that are given as the contents of a certificate or a key file (a path is - /// not accepted from a `CREATE DICTIONARY` query in the first place). + /// not accepted from a `CREATE DICTIONARY` query in the first place), and the custom HTTP headers + /// of the `HTTP` source, whose values often carry API tokens. The headers are hidden as a whole, + /// names included: the query is logged before the dictionary source validates its structure, + /// so a malformed definition must not leak either. bool isSecretKey(const String & key) { return key == "password" || key == "ssl_ca_pem" || key == "ssl_cert_pem" || key == "ssl_key_pem" - || key == "sslrootcert_pem" || key == "sslcert_pem" || key == "sslkey_pem"; + || key == "sslrootcert_pem" || key == "sslcert_pem" || key == "sslkey_pem" + || key == "headers" || key == "header"; } } @@ -56,7 +60,9 @@ void ASTPair::readJSON(const Poco::JSON::Object & json) { JSONObjectReader r(json); - first = r.getString("first"); + /// The SQL parser lower-cases the key (see `ParserKeyValuePair`), and the checks for secret keys in + /// `formatImpl` and `hasSecretParts` rely on it, so canonicalize it the same way here. + first = Poco::toLower(r.getString("first")); if (first.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Missing or empty 'first' in ASTPair during AST JSON deserialization"); diff --git a/src/Parsers/ASTJSONHelpers.cpp b/src/Parsers/ASTJSONHelpers.cpp index 1d8e3e28cdad..63cad18b82c8 100644 --- a/src/Parsers/ASTJSONHelpers.cpp +++ b/src/Parsers/ASTJSONHelpers.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -22,6 +23,7 @@ void JSONObjectWriter::writeAlias(const ASTWithAlias & node) static void writeFieldJSON(WriteBuffer & out, const FormatSettings & fs, const Field & field) { + checkStackSize(); out << "{\"field_type\":"; writeJSONString(field.getTypeName(), out, fs); diff --git a/src/Parsers/ASTJSONHelpers.h b/src/Parsers/ASTJSONHelpers.h index 68afb51b2aae..7cc6c5d7d467 100644 --- a/src/Parsers/ASTJSONHelpers.h +++ b/src/Parsers/ASTJSONHelpers.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -30,6 +31,8 @@ class JSONObjectWriter JSONObjectWriter(WriteBuffer & out_, const char * type_name) : out(out_) { + /// One of these is constructed per node by the AST-to-JSON walk, which recurses as deep as the query nests. + checkStackSize(); out << "{\"type\":"; writeJSONString(std::string_view(type_name), out, fs); } diff --git a/src/Parsers/ASTJSONReadHelpers.cpp b/src/Parsers/ASTJSONReadHelpers.cpp index db816fc4d5f4..ae4111a79448 100644 --- a/src/Parsers/ASTJSONReadHelpers.cpp +++ b/src/Parsers/ASTJSONReadHelpers.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -128,6 +129,9 @@ Field JSONObjectReader::readFieldFromObjectImpl(const Poco::JSON::Object & obj, "Structured Field value exceeds maximum AST depth limit ({}) during JSON AST deserialization", max_depth); + /// The limit above counts `Field` levels, which is not a stack budget at any value. + checkStackSize(); + /// Count every `Field` value (scalar or structured) against the element-count budget too, so a /// wide literal payload (e.g. one huge `Array`) cannot bypass `max_ast_elements` while adding no /// AST nodes. diff --git a/src/Parsers/ASTSetQuery.cpp b/src/Parsers/ASTSetQuery.cpp index 4c440a87852a..43eb30fcc937 100644 --- a/src/Parsers/ASTSetQuery.cpp +++ b/src/Parsers/ASTSetQuery.cpp @@ -58,12 +58,16 @@ static std::array engineSettingsToHide() /// disagree on what is secret. static std::optional renderSecretChangeValue(const SettingChange & change) { - if (auto masked = CoreSettings::renderSecretSettingValue(change.name, change.value)) + /// The queue engines also take every setting, a format setting included, with the legacy `s3queue_` prefix. + static constexpr std::string_view s3queue_prefix = "s3queue_"; + const String setting_name = change.name.starts_with(s3queue_prefix) ? change.name.substr(s3queue_prefix.size()) : change.name; + + if (auto masked = CoreSettings::renderSecretSettingValue(setting_name, change.value)) return masked; for (const auto * settings_to_hide : engineSettingsToHide()) { - auto it = settings_to_hide->find(change.name); + auto it = settings_to_hide->find(setting_name); if (it != settings_to_hide->end()) return it->second(change.value); } diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 6215a2933f43..8c4855a06d2b 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -26,6 +26,89 @@ namespace changed |= maskPresignedURLParameters(url); return changed; } + + /// How an Azure destination reads a connection value, and whether the masking here can show it. + enum class AzureConnectionValue + { + /// An http(s) scheme and a host, with no userinfo, query or fragment: each of those carries a + /// credential of its own (`http://user:key@host`, a SAS `?sig=`). + PlainStorageAccountURL, + /// A connection string, whose secret keys `maskAzureConnectionString` masks in place. + ConnectionString, + /// A value that can carry a credential no rule here masks. + Unmaskable, + }; + + AzureConnectionValue classifyAzureConnectionValue(const String & value) + { + static constexpr std::string_view SEPARATOR = "://"; + const size_t separator = value.find(SEPARATOR); + const std::string_view scheme = std::string_view(value).substr(0, std::min(separator, value.length())); + /// The scheme grammar `maskURIUserinfo` reads. A connection string does not match it, even when + /// one of its values embeds an endpoint URL. + const bool is_url = separator != String::npos && !scheme.empty() && isAlphaASCII(scheme.front()) + && std::all_of( + scheme.begin(), scheme.end(), [](char c) { return isAlphaNumericASCII(c) || c == '+' || c == '.' || c == '-'; }); + /// `maskAzureConnectionString` masks nothing in a value starting with `http`, so one that is no + /// URL either would be left as written. + if (!is_url) + return value.starts_with("http") ? AzureConnectionValue::Unmaskable : AzureConnectionValue::ConnectionString; + + if ((!equalsCaseInsensitive(scheme, "http") && !equalsCaseInsensitive(scheme, "https")) + || value.find_first_of("?#") != String::npos) + return AzureConnectionValue::Unmaskable; + + const size_t authority_begin = separator + SEPARATOR.length(); + const size_t authority_end = std::min(value.find('/', authority_begin), value.length()); + if (authority_end == authority_begin || value.find('@', authority_begin) < authority_end) + return AzureConnectionValue::Unmaskable; + return AzureConnectionValue::PlainStorageAccountURL; + } + + /// The backup engines whose locator names a destination with no credential in it, each with the + /// argument count it accepts. `BackupFactory` registers exactly these plus `S3` and `AzureBlobStorage`. + std::optional credentialFreeBackupEngineArity(const String & engine_name) + { + if (engine_name == "File" || engine_name == "Memory") + return 1; + if (engine_name == "Disk") + return 2; + if (engine_name == "Null") + return 0; + return {}; + } +} + +void FunctionSecretArgumentsFinder::maskEveryArgument() +{ + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + markSecretArgument(i); +} + +bool FunctionSecretArgumentsFinder::hasOnlyLiteralArguments(const AbstractFunction & function) +{ + if (!function.hasArguments()) + return true; + for (size_t i = 0, size = function.arguments->size(); i < size; ++i) + if (!function.arguments->at(i)->tryGetLiteralText(nullptr)) + return false; + return true; +} + +bool FunctionSecretArgumentsFinder::isCredentialFreeBackupLocator(const AbstractFunction & function) +{ + auto arity = credentialFreeBackupEngineArity(function.name()); + if (!arity) + return false; + const size_t count = function.hasArguments() ? function.arguments->size() : 0; + if (count != *arity) + return false; + /// Each of these reads every argument of its own as a string, so another shape - an array among + /// them - is read by none of them and can carry a string of its own. + for (size_t i = 0; i < count; ++i) + if (!tryGetStringFromArgument(*function.arguments->at(i), nullptr, /* allow_identifier= */ false)) + return false; + return true; } void FunctionSecretArgumentsFinder::markSecretArgument(size_t index, bool argument_is_named) @@ -308,16 +391,41 @@ void FunctionSecretArgumentsFinder::findMySQLFunctionSecretArguments() else { /// mysql('host:port', 'database', 'table', 'user', 'password', ...) - markSecretArgument(4); - findTLSCredentialsSecretArguments(5); + const auto positional = classifyPositionalArguments(); + if (positional.size() > 4) + markSecretArgument(positional[4]); + findSecretNamedArgument("password", 0); + findTLSCredentialsSecretArguments(0); } } -void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t start) +std::vector FunctionSecretArgumentsFinder::classifyPositionalArguments(size_t start) { - for (const auto & key : tls_credentials_secret_keys) - findSecretNamedArgument(key, start); + std::vector positional; + bool seen_named = false; + for (size_t i = start; i < function->arguments->size(); ++i) + { + const auto equals_func = function->arguments->at(i)->getFunction(); + if (equals_func && equals_func->name() == "equals" && equals_func->hasArguments() + && equals_func->arguments->size() == 2) + { + seen_named = true; + continue; + } + + if (seen_named) + { + markSecretArgument(i); + continue; + } + positional.push_back(i); + } + return positional; +} + +void FunctionSecretArgumentsFinder::markNamedArgumentsWithUnreadableKeys(size_t start) +{ /// The named-collection parser does not require the key of a `key = value` argument to be a plain /// literal or identifier: `getKeyValueFromASTImpl` evaluates it as a constant expression, so /// `mysql(creds, concat('ssl_ca', '_pem') = 'SECRET', table = 't')` passes a TLS credential too. @@ -340,6 +448,14 @@ void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t sta } } +void FunctionSecretArgumentsFinder::findTLSCredentialsSecretArguments(size_t start) +{ + for (const auto & key : tls_credentials_secret_keys) + findSecretNamedArgument(key, start); + + markNamedArgumentsWithUnreadableKeys(start); +} + void FunctionSecretArgumentsFinder::findMongoDBSecretArguments() { String uri; @@ -410,12 +526,16 @@ void FunctionSecretArgumentsFinder::findArrowFlightSecretArguments() void FunctionSecretArgumentsFinder::findXDBCSecretArguments() { + /// The connection string goes verbatim to the bridge, so its grammar is the JDBC/ODBC driver's: the + /// password can sit in a query parameter (`?password=`) or as `Pwd=` in a `KEY=value;` list. + /// An invalid call is formatted for logging before validation rejects it, so both branches below + /// fail closed: after a collection name a positional argument can be the connection string, and a + /// named argument means the call is not the positional form at all. if (isNamedCollectionName(0)) { /// jdbc(named_collection, ..., datasource = 'DSN', ...) /// odbc(named_collection, ..., connection_settings = 'DSN', ...) /// `datasource` and `connection_settings` are mutually exclusive aliases. - /// If the value is a URI, mask only the password; otherwise hide the whole value. /// If somehow both are present (invalid query), hide all named arguments. ssize_t ds_idx = findNamedArgument(nullptr, "datasource", 1); ssize_t cs_idx = findNamedArgument(nullptr, "connection_settings", 1); @@ -426,53 +546,31 @@ void FunctionSecretArgumentsFinder::findXDBCSecretArguments() result.start = 1; result.count = function->arguments->size() - 1; result.are_named = true; + return; + } + + findSecretNamedArgument("datasource", 1); + findSecretNamedArgument("connection_settings", 1); + markNamedArgumentsWithUnreadableKeys(1); + + for (size_t i = 1; i < function->arguments->size(); ++i) + { + const auto equals_func = function->arguments->at(i)->getFunction(); + if (!equals_func || equals_func->name() != "equals" || !equals_func->hasArguments() + || equals_func->arguments->size() != 2) + markSecretArgument(i, /* argument_is_named= */ false); } - else if (ds_idx >= 0) - maskXDBCSecretNamedArgument("datasource", 1); - else if (cs_idx >= 0) - maskXDBCSecretNamedArgument("connection_settings", 1); } else { /// jdbc('DSN', schema, table) / jdbc('DSN', table) /// odbc('DSN', schema, table) / odbc('DSN', table) /// JDBC('DSN', database, table) / ODBC('DSN', database, table) - /// The connection string may be a URI with credentials embedded, - /// e.g. scheme://username:password@host:port/dbname - /// If so, mask only the password part; otherwise hide the whole argument. - String uri; - if (tryGetStringFromArgument(0, &uri)) - { - if (maskURIPassword(&uri)) - { - chassert(result.count == 0); - result.start = 0; - result.count = 1; - result.replacement = std::move(uri); - return; - } - } markSecretArgument(0, false); - } -} -void FunctionSecretArgumentsFinder::maskXDBCSecretNamedArgument(std::string_view key, size_t start) -{ - String value; - ssize_t arg_idx = findNamedArgument(&value, key, start); - if (arg_idx < 0) - return; - - if (!value.empty() && maskURIPassword(&value)) - { - result.are_named = true; - result.start = arg_idx; - result.count = 1; - result.replacement = std::move(value); - } - else - { - markSecretArgument(arg_idx, /* argument_is_named= */ true); + findSecretNamedArgument("datasource", 1); + findSecretNamedArgument("connection_settings", 1); + markNamedArgumentsWithUnreadableKeys(1); } } @@ -599,6 +697,54 @@ bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_id return false; } +bool FunctionSecretArgumentsFinder::azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit) +{ + size_t positionals = 0; + for (size_t i = start, size = function->arguments->size(); i < size; ++i) + { + const auto argument_function = function->arguments->at(i)->getFunction(); + if (argument_function && argument_function->name() == "equals") + { + /// A key this rule cannot read hides which credential the override carries; a value that is + /// no plain literal or identifier can nest one (`headers('Authorization' = '...')`). + if (argument_function->arguments && argument_function->arguments->size() == 2 + && tryGetStringFromArgument(*argument_function->arguments->at(0), nullptr) + && (tryGetStringFromArgument(*argument_function->arguments->at(1), nullptr) + || argument_function->arguments->at(1)->tryGetLiteralText(nullptr))) + continue; + return false; + } + if (++positionals > positional_limit || !function->arguments->at(i)->tryGetLiteralText(nullptr)) + return false; + } + + /// A destination reads at most one of the two mutually exclusive connection keys, and rejects a + /// second one only after the statement has been formatted, so a surplus one stays as written. + size_t connection_overrides = 0; + for (const auto & key : {"connection_string", "storage_account_url"}) + for (ssize_t i = findNamedArgument(nullptr, key, start); i >= 0; + i = findNamedArgument(nullptr, key, static_cast(i) + 1)) + ++connection_overrides; + + if (connection_overrides > 1) + return false; + + for (const auto & key : {"connection_string", "storage_account_url"}) + { + String value; + if (findNamedArgument(&value, key, start) < 0) + continue; + /// Hiding a connection string replaces its whole argument, which cannot be combined with + /// hiding `account_key`. + const auto shape = classifyAzureConnectionValue(value); + if (value.empty() || shape == AzureConnectionValue::Unmaskable + || (shape == AzureConnectionValue::ConnectionString + && findNamedArgument(nullptr, "account_key", start) >= 0)) + return false; + } + return true; +} + void FunctionSecretArgumentsFinder::findURLSecretArguments(size_t url_offset) { /// `headers(...)` can appear at any position in every url form (function, cluster function, engine, @@ -680,7 +826,8 @@ void FunctionSecretArgumentsFinder::findRemoteFunctionSecretArguments() { /// remote(named_collection, ..., password = 'password', ...) findSecretNamedArgument("password", 1); - return; + /// An identifier is also a cluster name when no such collection exists, and that form keeps the + /// password in a positional slot, so the walk below has to run for it too. } /// We're going to replace 'password' with '[HIDDEN'] for the following signatures: @@ -864,6 +1011,18 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() /// NATS(named_collection, nats_password = 'password', nats_credentials = '...', ...) findNATSTableEngineSecretArguments(); } + else if (engine_name == "RabbitMQ") + { + /// RabbitMQ(named_collection, rabbitmq_address = '...', rabbitmq_password = '...') + findRabbitMQTableEngineSecretArguments(); + } + else if (engine_name == "Kafka") + { + /// Kafka(named_collection, kafka_sasl_password = '...'); the legacy positional form carries no + /// secret and makes the collection name optional, so a named argument can be the first one. + findSecretNamedArgument("kafka_sasl_password", 0); + markNamedArgumentsWithUnreadableKeys(0); + } else if ((engine_name == "JDBC") || (engine_name == "ODBC")) { /// JDBC('DSN', database, table) @@ -873,17 +1032,18 @@ void FunctionSecretArgumentsFinder::findTableEngineSecretArguments() } } -void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() +void FunctionSecretArgumentsFinder::findBrokerTableEngineSecretArguments( + std::span secret_keys, std::string_view address_key) { /// NATS(named_collection [, nats_password = 'password'] [, nats_token = 'token'] /// [, nats_credential_file = '/path'] [, nats_credentials = 'user JWT and seed'] /// [, nats_url = 'nats://user:password@host:4222'] /// [, nats_server_list = 'nats://user:password@host:4222,...'], ...) - /// The only positional argument the engine accepts is the name of a named collection, so the + /// RabbitMQ(named_collection [, rabbitmq_password = '...'] [, rabbitmq_address = 'amqp://user:pass@host'], ...) + /// The only positional argument these engines accept is the name of a named collection, so the /// credentials can only appear as named overrides. The `SETTINGS` clause form is masked - /// separately by `NATS::SETTINGS_TO_HIDE`, and this function masks the same keys the same way: - /// the secrets are hidden whole, while `nats_url` keeps everything but its userinfo password. - /// `nats_server_list` is hidden whole because each list entry can carry userinfo credentials. + /// separately by the engine's own `SETTINGS_TO_HIDE`, which this function must stay in sync with. + /// A destination key (`nats_server_list`) is hidden whole: each list entry can carry userinfo. /// Fail closed on a key we cannot read as a plain literal: it can name a secret setting. for (size_t i = 0; i < function->arguments->size(); ++i) { @@ -905,13 +1065,14 @@ void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() { markSecretArgument(i, /* argument_is_named= */ true); } - else if (key == "nats_url") + else if (key == address_key) { String url; if (equals_func->arguments->at(1)->tryGetString(&url, /* allow_identifier= */ false)) { - if (maskURIPassword(&url)) - result.replaced_arguments[i] = "nats_url = " + quoteString(url); + /// An '@' is the only reliable sign of a credential here; see the engine's `_fwd.h`. + if (url.contains('@')) + markSecretArgument(i, /* argument_is_named= */ true); } else { @@ -920,13 +1081,23 @@ void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() markSecretArgument(i, /* argument_is_named= */ true); } } - else if (std::find(std::begin(nats_secret_keys), std::end(nats_secret_keys), key) != std::end(nats_secret_keys)) + else if (std::find(secret_keys.begin(), secret_keys.end(), key) != secret_keys.end()) { markSecretArgument(i, /* argument_is_named= */ true); } } } +void FunctionSecretArgumentsFinder::findNATSTableEngineSecretArguments() +{ + findBrokerTableEngineSecretArguments(nats_secret_keys, "nats_url"); +} + +void FunctionSecretArgumentsFinder::findRabbitMQTableEngineSecretArguments() +{ + findBrokerTableEngineSecretArguments(rabbitmq_secret_keys, "rabbitmq_address"); +} + void FunctionSecretArgumentsFinder::findExternalDistributedTableEngineSecretArguments() { if (isNamedCollectionName(1)) @@ -965,31 +1136,52 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen if (isNamedCollectionName(url_arg_idx)) { /// AzureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + if (!azureCollectionArgumentsAreShowable(url_arg_idx + 1, /* positional_limit= */ 0)) + { + maskEveryArgument(); + return; + } if (maskAzureConnectionString(-1, true, 1)) return; findSecretNamedArgument("account_key", 1); return; } - if (maskAzureConnectionString(url_arg_idx)) - return; - /// We should check other arguments first because we don't need to do any replacement in case of /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument size_t count = function->arguments->size(); + bool fourth_argument_is_format = false; if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) { String fourth_arg; if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) - { - if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) - return; - } + fourth_argument_is_format = fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg); + } + /// Which argument holds a credential: the two-argument shape takes a shared access signature beside + /// the url (`endpoint.sas_auth`), the longer ones an `account_key` - unless the fourth names a format. + std::optional credential_arg_idx; + if (count == url_arg_idx + 2) + credential_arg_idx = url_arg_idx + 1; + else if (!fourth_argument_is_format && (url_arg_idx + 4 < count)) + credential_arg_idx = url_arg_idx + 4; + + /// The engine reads this argument as a connection string or as a plain account url; a value of + /// another shape is read by neither rule below, and a hidden connection string replaces it whole. + String connection_value; + const auto shape = tryGetStringFromArgument(url_arg_idx, &connection_value) + ? classifyAzureConnectionValue(connection_value) + : AzureConnectionValue::Unmaskable; + if (shape == AzureConnectionValue::Unmaskable || (shape == AzureConnectionValue::ConnectionString && credential_arg_idx)) + { + maskEveryArgument(); + return; } - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); + if (maskAzureConnectionString(url_arg_idx)) + return; + + if (credential_arg_idx) + markSecretArgument(*credential_arg_idx); } void FunctionSecretArgumentsFinder::findRedisFunctionSecretArguments() @@ -1141,8 +1333,11 @@ void FunctionSecretArgumentsFinder::findMySQLDatabaseSecretArguments() else { /// MySQL('host:port', 'database', 'user', 'password') - markSecretArgument(3); - findTLSCredentialsSecretArguments(4); + const auto positional = classifyPositionalArguments(); + if (positional.size() > 3) + markSecretArgument(positional[3]); + findSecretNamedArgument("password", 0); + findTLSCredentialsSecretArguments(0); } } @@ -1174,18 +1369,56 @@ void FunctionSecretArgumentsFinder::findDataLakeCatalogSecretArguments() void FunctionSecretArgumentsFinder::findBackupDatabaseSecretArguments() { - if (function->arguments->size() < 2) + /// `Backup(database_name, locator)` is the only valid shape, a locator carrying credentials can be + /// written in either position, and the query is formatted for logging before validation rejects it. + if (function->arguments->size() != 2 || !function->arguments->at(0)->tryGetLiteralText(nullptr)) + { + maskEveryArgument(); return; + } auto storage_arg = function->arguments->at(1); auto storage_function = storage_arg->getFunction(); + /// A locator that is not a function - a string literal holding its text, or an expression - carries + /// the destination as text this finder cannot parse, and that text can hold an access key, a secret + /// access key or a presigned URL. The engine accepts such a locator only while replaying its own + /// metadata, but a statement carrying it is formatted before the engine rejects it: by `PARALLEL WITH`, + /// by the distributed DDL queue, and by `query_log`. Hide it whole rather than let it through verbatim. + if (!storage_function) + { + result.start = 1; + result.count = 1; + result.replacement = "'[HIDDEN]'"; + result.quote_replacement = false; + return; + } + /// The nested S3 destination is not recognized as an S3 engine when the formatter recurses into it, /// so its secrets must be masked here. Handle both forms: /// Backup('', S3('url', 'access_key_id', 'secret_access_key' [, ...])) /// Backup('', S3(named_collection, ..., secret_access_key = '...', session_token = '...', ...)) /// by reconstructing the nested `S3(...)` with the secret arguments replaced by `[HIDDEN]`. - if (!storage_function || storage_function->name() != "S3" || !storage_function->hasArguments()) + if (storage_function->name() != "S3") + { + if (isCredentialFreeBackupLocator(*storage_function)) + return; + + /// Any other locator holds a credential no rule below reconstructs (`AzureBlobStorage` holds + /// `account_key` and connection-string material); its engine name and arity are not secrets. + std::string replacement = backQuoteIfNeed(storage_function->name()) + "("; + for (size_t i = 0, size = storage_function->hasArguments() ? storage_function->arguments->size() : 0; i < size; ++i) + replacement += i > 0 ? ", '[HIDDEN]'" : "'[HIDDEN]'"; + replacement += ")"; + + result.start = 1; + result.count = 1; + result.replacement = std::move(replacement); + result.quote_replacement = false; + return; + } + + if (!storage_function->hasArguments()) return; const auto & nested_args = *storage_function->arguments; @@ -1368,10 +1601,70 @@ void FunctionSecretArgumentsFinder::findBackupNameSecretArguments() maskS3UrlArgument(positional, 0); maskS3PositionalsFrom(positional, positional.size() == 3 ? 2 : 1); } - else if (engine_name == "AzureBlobStorage" || engine_name == "AzureQueue") + else if (engine_name == "AzureBlobStorage") { - findAzureBlobStorageTableEngineSecretArguments(); + findAzureBlobStorageBackupSecretArguments(); + } + else if (!isCredentialFreeBackupLocator(*function)) + { + /// Everything else either is an engine no rule here reconstructs, or has arguments the named + /// engine does not read (an override, a nested map, a surplus slot), which can carry a credential. + /// `AzureQueue` reaches this branch: it is a table engine, not a registered backup engine. + maskEveryArgument(); + } +} + +void FunctionSecretArgumentsFinder::findAzureBlobStorageBackupSecretArguments() +{ + /// The destination reads AzureBlobStorage(named_collection [, 'filename'] [, key = value, ...]), + /// ('connection_string|storage_account_url', 'container', 'path'), or those three followed by + /// ('account_name', 'account_key'). An argument no shape reads holds whatever was written in it. + const size_t count = function->arguments->size(); + + if (isNamedCollectionName(0)) + { + if (!azureCollectionArgumentsAreShowable(1, /* positional_limit= */ 1)) + { + maskEveryArgument(); + return; + } + if (maskAzureConnectionString(-1, /* argument_is_named= */ true, 1)) + return; + findSecretNamedArgument("account_key", 1); + return; + } + + if ((count != 3 && count != 5) || !hasOnlyLiteralArguments(*function)) + { + maskEveryArgument(); + return; + } + + if (count == 3) + { + /// Only this shape accepts a connection string, which can embed `AccountKey`. A value that is no + /// string is read by neither the classification below nor the destination. + String connection_value; + if (!tryGetStringFromArgument(0, &connection_value) + || classifyAzureConnectionValue(connection_value) == AzureConnectionValue::Unmaskable) + { + maskEveryArgument(); + return; + } + maskAzureConnectionString(0); + return; } + + String storage_account_url; + if (!tryGetStringFromArgument(0, &storage_account_url) + || classifyAzureConnectionValue(storage_account_url) != AzureConnectionValue::PlainStorageAccountURL) + { + /// This shape requires a plain account URL. A connection string here can only be hidden whole, + /// which cannot be combined with hiding `account_key`. + maskEveryArgument(); + return; + } + markSecretArgument(4); } bool FunctionSecretArgumentsFinder::isNamedCollectionName(size_t arg_idx) const diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index 10342f18cc6b..79894454c833 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -87,11 +88,17 @@ class FunctionSecretArgumentsFinder FunctionSecretArgumentsFinder::Result getResult() const { return result; } /// Whether a key of the `extra_credentials(..)` nested map carries a non-secret identifier whose - /// value stays visible when the map is masked (`role_arn` and `role_session_name`; the map's - /// secret is `external_id`). Any other key - unknown, malformed or an expression - fails closed. + /// value stays visible when the map is masked. Only `role_arn` qualifies: it names the role to + /// assume, like `access_key_id` names a key. The other two keys of the assume-role triple are + /// secrets: `external_id` is its shared secret, and `role_session_name` can be one too, because a + /// trust policy can require a specific value through the `sts:RoleSessionName` condition (the + /// ClickHouse Cloud guide documents exactly this use). Any other key - unknown, malformed or an + /// expression - fails closed. + /// The `.backup` metadata is a different matter: its `` locator keeps `role_session_name` + /// on purpose, so that a role-authenticated backup chain stays restorable (see `BackupInfo.cpp`). static bool isNonSecretExtraCredentialsKey(std::string_view key) { - return key == "role_arn" || key == "role_session_name"; + return key == "role_arn"; } protected: @@ -99,11 +106,12 @@ class FunctionSecretArgumentsFinder Result result; /// Named arguments carrying S3 secrets, shared by every S3 form (explicit-url and named-collection). - /// `external_id` is the shared secret of the assume-role triple; the other two (`role_arn`, - /// `role_session_name`) are non-secret identifiers passed inside `extra_credentials` and stay - /// visible (see isNonSecretExtraCredentialsKey). + /// `external_id` and `role_session_name` are the secrets of the assume-role triple; the third key + /// (`role_arn`) is a non-secret identifier passed inside `extra_credentials` and stays visible + /// (see isNonSecretExtraCredentialsKey). static constexpr std::string_view s3_secret_keys[] - = {"secret_access_key", "session_token", "google_adc_client_secret", "google_adc_refresh_token", "external_id"}; + = {"secret_access_key", "session_token", "google_adc_client_secret", "google_adc_refresh_token", "external_id", + "role_session_name"}; /// Named arguments carrying TLS credentials as the literal contents of a certificate or a key file, /// rather than as a path to it. They are secret and have to be hidden the same way a password is. @@ -113,16 +121,22 @@ class FunctionSecretArgumentsFinder /// Named arguments carrying NATS credentials. They are the setting names, because the `NATS` engine /// takes its arguments as overrides of a named collection (`NATS(collection, nats_token = '...')`). /// `nats_server_list` is a destination and can carry URI userinfo credentials, so hide it whole. + /// `nats_url` is not here: it is hidden only when its value carries an '@'. /// Keep in sync with `NATS::SETTINGS_TO_HIDE`, which masks the same secrets in the `SETTINGS` clause. static constexpr std::string_view nats_secret_keys[] = {"nats_password", "nats_token", "nats_credential_file", "nats_credentials", "nats_server_list"}; + /// As `nats_secret_keys`, for RabbitMQ; `rabbitmq_address` is hidden only when it carries an '@'. + /// Keep in sync with `RabbitMQ::SETTINGS_TO_HIDE`. + static constexpr std::string_view rabbitmq_secret_keys[] = {"rabbitmq_password"}; + void markSecretArgument(size_t index, bool argument_is_named = false); /// `headers(..)` and `extra_credentials(..)` are nested maps whose values are secret auth material - /// (`extra_credentials` carries the assume-role secret `external_id`; its non-secret identifiers - /// stay visible, see isNonSecretExtraCredentialsKey). The parsers accept them at any position, not - /// just at the tail. Record them so their values are hidden with the keys kept. + /// (`extra_credentials` carries the assume-role secrets `external_id` and `role_session_name`; its + /// non-secret identifier `role_arn` stays visible, see isNonSecretExtraCredentialsKey). The parsers + /// accept them at any position, not just at the tail. Record them so their values are hidden with + /// the keys kept. /// Idempotent: each map is recorded at most once. void maskNestedSecretMaps(); @@ -163,14 +177,14 @@ class FunctionSecretArgumentsFinder void findRedisTableEngineSecretArguments(); void findArrowFlightSecretArguments(); void findXDBCSecretArguments(); - - /// Similar to `findSecretNamedArgument`, but if the value is a URI with credentials, - /// masks only the password part instead of hiding the entire value. - void maskXDBCSecretNamedArgument(std::string_view key, size_t start); - void findS3FunctionSecretArguments(bool is_cluster_function); void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0); + /// Whether the arguments an `AzureBlobStorage(named_collection, ...)` destination or table takes + /// from `start` can be shown: only an argument written here can carry a credential, and each has + /// to be readable enough to tell that it does not. `positional_limit` bounds the plain literals + /// read beside the overrides: one filename for a backup locator, none for a table engine. + bool azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit); /// Masks the secrets of every URL form (`url`/`urlCluster` table functions, the `URL` table /// engine, and their named-collection variants): the userinfo password of the url positional or a /// named `url = ...` override, and the `headers(...)` values at any position. `url` is at @@ -180,6 +194,18 @@ class FunctionSecretArgumentsFinder bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const; static bool tryGetStringFromArgument(const AbstractFunction::Argument & argument, String * res, bool allow_identifier = true); + /// `BackupInfo` keeps named overrides and a trailing map for every backup engine, including the + /// ones that read neither, so an argument that is not a plain literal can carry a credential. + static bool hasOnlyLiteralArguments(const AbstractFunction & function); + + /// Whether a backup locator names its destination with exactly the literal arguments its engine + /// accepts, and therefore holds no credential. An engine that takes fewer rejects the rest only + /// after the statement has been formatted for logging, so the count has to be checked here too. + static bool isCredentialFreeBackupLocator(const AbstractFunction & function); + + /// Hides every argument, for a shape whose valid slots cannot be established. + void maskEveryArgument(); + void findRemoteFunctionSecretArguments(); /// Tries to get either a database name or a qualified table name from an argument. @@ -199,7 +225,10 @@ class FunctionSecretArgumentsFinder void findRedisFunctionSecretArguments(); void findYTsaurusStorageTableEngineSecretArguments(); void findBigQuerySecretArguments(); + void findBrokerTableEngineSecretArguments( + std::span secret_keys, std::string_view address_key); void findNATSTableEngineSecretArguments(); + void findRabbitMQTableEngineSecretArguments(); void findDatabaseEngineSecretArguments(); void findMySQLDatabaseSecretArguments(); void findS3DatabaseSecretArguments(); @@ -207,6 +236,10 @@ class FunctionSecretArgumentsFinder void findBackupDatabaseSecretArguments(); void findBackupNameSecretArguments(); + /// A backup destination reads a different signature than the table engine of the same name, so the + /// table-engine rule leaves an argument it does not model visible. + void findAzureBlobStorageBackupSecretArguments(); + /// Whether a specified argument can be the name of a named collection? bool isNamedCollectionName(size_t arg_idx) const; @@ -219,6 +252,13 @@ class FunctionSecretArgumentsFinder /// duplicate-key validation runs, so `session_token = 'a', session_token = 'b'` must hide both. bool findSecretNamedArgument(std::string_view key, size_t start = 0); + /// Hides the value of every `key = value` argument from `start` on whose key is not a plain literal. + void markNamedArgumentsWithUnreadableKeys(size_t start); + + /// The raw indexes of the arguments from `start` on that are not `key = value` pairs, in order. A + /// positional argument after the first named one is hidden instead of listed: its slot is unknowable. + std::vector classifyPositionalArguments(size_t start = 0); + /// Masks the secrets of an S3 named-collection form: the secret named overrides (every occurrence, /// in any order; the span covering them may hide a non-secret argument in between, which is safe) /// and the `headers(...)` / `extra_credentials(...)` map overrides. diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 588067e1751f..634465edcde6 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -296,8 +296,8 @@ TEST(ParserCreateQuery, MaskNATSTableEngineCredentials) TEST(ParserCreateQuery, MaskNATSTableEngineURLPassword) { - /// A `nats_url` override can carry the credentials in its userinfo. Only the password is hidden, - /// keeping the rest of the url visible, the same way the `SETTINGS` clause form is masked. + /// A `nats_url` override carrying an '@' is hidden whole, the same way the `SETTINGS` clause form + /// is masked: libnats reads a credential that no URI masker can bound. const String query = "CREATE TABLE test_nats (key UInt64) " "ENGINE = NATS(nats1, nats_url = 'nats://plain_user:plain_password@example.com:4222')"; @@ -308,7 +308,8 @@ TEST(ParserCreateQuery, MaskNATSTableEngineURLPassword) const String masked = ast->formatForLogging(); EXPECT_EQ(masked.find("plain_password"), String::npos); - EXPECT_NE(masked.find("nats://plain_user:[HIDDEN]@example.com:4222"), String::npos); + EXPECT_EQ(masked.find("plain_user"), String::npos); + EXPECT_NE(masked.find("nats_url = '[HIDDEN]'"), String::npos); } TEST(ParserCreateQuery, MaskNATSTableEngineServerListPassword) @@ -379,6 +380,158 @@ TEST(ParserCreateQuery, MaskNATSTableEnginePositionalArguments) EXPECT_NE(masked.find("[HIDDEN]"), String::npos); } +TEST(ParserCreateQuery, MaskXDBCTableEnginePositionalAfterCollection) +{ + /// After a collection name every XDBC argument must be a named override, but the statement is + /// formatted for logging before validation rejects a positional one, and that positional can be + /// the connection string itself. The engine spelling takes more positional arguments than the + /// table function does, so it is asserted separately. + const String query = + "CREATE TABLE test_jdbc (key UInt64) " + "ENGINE = JDBC(jdbc1, 'DSN=mydb;Uid=user;Pwd=plain_password', 'mydb', 'mytable')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_EQ(masked.find("Uid=user"), String::npos); + /// The collection name is the one legitimate positional argument and stays visible. + EXPECT_NE(masked.find("jdbc1"), String::npos); + EXPECT_NE(masked.find("[HIDDEN]"), String::npos); + + /// A named override of the same connection string keeps its key visible, as before. + const String named_query = + "CREATE TABLE test_jdbc (key UInt64) " + "ENGINE = JDBC(jdbc1, datasource = 'DSN=mydb;Uid=user;Pwd=plain_named_password', " + "external_database = 'mydb', external_table = 'mytable')"; + + DB::ASTPtr named_ast = DB::parseQuery(parser, named_query, 0, 0, 0); + const String named_masked = named_ast->formatForLogging(); + + EXPECT_EQ(named_masked.find("plain_named_password"), String::npos); + EXPECT_NE(named_masked.find("datasource = '[HIDDEN]'"), String::npos); + /// The non-secret named arguments stay visible: the positional scan must not widen to them. + EXPECT_NE(named_masked.find("external_table = 'mytable'"), String::npos); +} + +TEST(ParserCreateQuery, MaskXDBCNamedArgumentsWithoutCollection) +{ + /// A named argument at index 0 is not a collection name, so this call is not the positional form: + /// the connection string can be under either alias at any index, and the statement is formatted + /// for logging before validation rejects it. + const String query = + "CREATE TABLE test_jdbc (key UInt64) ENGINE = JDBC(external_database = 'mydb', " + "datasource = 'DSN=mydb;Uid=user;Pwd=plain_password')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_EQ(masked.find("Uid=user"), String::npos); + EXPECT_NE(masked.find("datasource = '[HIDDEN]'"), String::npos); +} + +TEST(ParserCreateQuery, MaskRabbitMQTableEngineCredentials) +{ + /// `RabbitMQ` also takes its settings as overrides of a named collection, so the same credentials + /// reach `SHOW CREATE TABLE` through the engine arguments and through the `SETTINGS` clause. + const String query = + "CREATE TABLE test_rabbitmq (key UInt64) ENGINE = RabbitMQ(rabbitmq1, " + "rabbitmq_password = 'plain_password', " + "rabbitmq_address = 'amqp://plain_user:plain_address_password@example.com:5672/vhost')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_EQ(masked.find("plain_address_password"), String::npos); + EXPECT_EQ(masked.find("plain_user"), String::npos); + /// The keys of the named overrides are not secrets and stay visible, as does the collection name. + EXPECT_NE(masked.find("rabbitmq1"), String::npos); + EXPECT_NE(masked.find("rabbitmq_password = '[HIDDEN]'"), String::npos); + EXPECT_NE(masked.find("rabbitmq_address = '[HIDDEN]'"), String::npos); + + /// An address with no '@' carries no credential and stays fully visible. + const String control_query = + "CREATE TABLE test_rabbitmq (key UInt64) " + "ENGINE = RabbitMQ(rabbitmq1, rabbitmq_address = 'amqp://example.com:5672/vhost')"; + + DB::ASTPtr control_ast = DB::parseQuery(parser, control_query, 0, 0, 0); + EXPECT_NE(control_ast->formatForLogging().find("amqp://example.com:5672/vhost"), String::npos); + + /// The `SETTINGS` clause form is masked by `RabbitMQ::SETTINGS_TO_HIDE` and must agree. + const String settings_query = + "CREATE TABLE test_rabbitmq_settings (key UInt64) ENGINE = RabbitMQ " + "SETTINGS rabbitmq_password = 'plain_settings_password'"; + + DB::ASTPtr settings_ast = DB::parseQuery(parser, settings_query, 0, 0, 0); + const String settings_masked = settings_ast->formatForLogging(); + + EXPECT_EQ(settings_masked.find("plain_settings_password"), String::npos); + EXPECT_NE(settings_masked.find("rabbitmq_password = '[HIDDEN]'"), String::npos); +} + +TEST(ParserCreateQuery, MaskKafkaTableEngineCredentials) +{ + /// `Kafka` reads named overrides of a collection too, so `kafka_sasl_password` needs masking in the + /// engine arguments and not only in the `SETTINGS` clause. + const String query = + "CREATE TABLE test_kafka (key UInt64) ENGINE = Kafka(kafka1, kafka_sasl_password = 'plain_password')"; + + DB::ParserCreateQuery parser; + DB::ASTPtr ast = DB::parseQuery(parser, query, 0, 0, 0); + + const String masked = ast->formatForLogging(); + + EXPECT_EQ(masked.find("plain_password"), String::npos); + EXPECT_NE(masked.find("kafka1"), String::npos); + EXPECT_NE(masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); + + /// Unlike `NATS` and `RabbitMQ`, `Kafka` accepts a legacy positional form whose arguments are all + /// non-secret, so a positional argument must stay visible rather than fail closed. + const String positional_query = + "CREATE TABLE test_kafka (key UInt64) " + "ENGINE = Kafka('broker:9092', 'topic', 'group', 'JSONEachRow')"; + + DB::ASTPtr positional_ast = DB::parseQuery(parser, positional_query, 0, 0, 0); + const String positional_masked = positional_ast->formatForLogging(); + + EXPECT_NE(positional_masked.find("broker:9092"), String::npos); + EXPECT_NE(positional_masked.find("group"), String::npos); + EXPECT_EQ(positional_masked.find("[HIDDEN]"), String::npos); + + /// The legacy positional form makes the collection name optional, so a named argument can be the + /// first one, and the statement is formatted for logging before it is rejected. + const String first_arg_query = + "CREATE TABLE test_kafka (key UInt64) " + "ENGINE = Kafka(kafka_sasl_password = 'plain_first_password', 'clickhouse')"; + + DB::ASTPtr first_arg_ast = DB::parseQuery(parser, first_arg_query, 0, 0, 0); + const String first_arg_masked = first_arg_ast->formatForLogging(); + + EXPECT_EQ(first_arg_masked.find("plain_first_password"), String::npos); + EXPECT_NE(first_arg_masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); + /// The positional argument beside it is not a secret and stays visible. + EXPECT_NE(first_arg_masked.find("'clickhouse'"), String::npos); + + /// The `SETTINGS` clause form is masked by `Kafka::SETTINGS_TO_HIDE` and must agree. + const String settings_query = + "CREATE TABLE test_kafka_settings (key UInt64) ENGINE = Kafka " + "SETTINGS kafka_sasl_password = 'plain_settings_password'"; + + DB::ASTPtr settings_ast = DB::parseQuery(parser, settings_query, 0, 0, 0); + const String settings_masked = settings_ast->formatForLogging(); + + EXPECT_EQ(settings_masked.find("plain_settings_password"), String::npos); + EXPECT_NE(settings_masked.find("kafka_sasl_password = '[HIDDEN]'"), String::npos); +} + TEST_P(ParserTest, parseQuery) { const auto & parser = std::get<0>(GetParam()); diff --git a/src/Planner/CollectSets.cpp b/src/Planner/CollectSets.cpp index 2e32f08f8e6f..01c4d564f147 100644 --- a/src/Planner/CollectSets.cpp +++ b/src/Planner/CollectSets.cpp @@ -2,6 +2,7 @@ #include #include +#include #if CLICKHOUSE_CLOUD #include #endif @@ -74,7 +75,7 @@ class CollectSetsVisitor : public InDepthQueryTreeVisitorWithContextas(); - StorageSet * storage_set = second_argument_table != nullptr ? dynamic_cast(second_argument_table->getStorage().get()) : nullptr; + StorageSet * storage_set = second_argument_table != nullptr ? castStorage(second_argument_table->getStorage(), DeferredTable::Load).get() : nullptr; if (storage_set) { diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp index 996b518cb17f..c3d7831aeeaa 100644 --- a/src/Planner/CollectTableExpressionData.cpp +++ b/src/Planner/CollectTableExpressionData.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include namespace DB @@ -62,7 +65,16 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextgetOrCreateTableExpressionData(column_node->getColumnSource()); + index_hint_table_expression_data.markColumnForAccessCheck(column_node->getColumnName()); + } return; + } auto column_source_node = column_node->getColumnSource(); auto column_source_node_type = column_source_node->getNodeType(); @@ -106,6 +118,9 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextgetExpression(); + /// The visitor above has registered the expression's source columns as read but not selected, and the + /// children walk that follows must not select them either: a grant on the ALIAS name is sufficient. + inlined_alias_expressions.insert(node.get()); return; } @@ -174,6 +189,8 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContextas()) return child_node != table_node->getMaterializedCTESubquery(); - return !(checkSubquery(child_node) || isAliasColumn(parent_node)); + return !(checkSubquery(child_node) || isAliasColumn(parent_node) || inlined_alias_expressions.contains(parent_node.get())); } static bool isIndexHintFunction(const QueryTreeNodePtr & node) @@ -263,6 +280,9 @@ class CollectSourceColumnsVisitor : public InDepthQueryTreeVisitorWithContext inlined_alias_expressions; }; class CollectPrewhereTableExpressionVisitor : public ConstInDepthQueryTreeVisitor @@ -440,6 +460,11 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr const auto & selected_column_names = table_expression_data.getSelectedColumnsNames(); required_column_names_without_prewhere.insert(selected_column_names.begin(), selected_column_names.end()); + /// The visit below inlines ALIAS columns, which would hide their names from the access check. + /// Record what PREWHERE references first, so the same names are checked as for WHERE. + for (const auto & column_name : collectReferencedColumnNames(query_node_typed.getPrewhere(), prewhere_table_expression)) + table_expression_data.markColumnForAccessCheck(column_name); + collect_source_columns_visitor.setKeepAliasColumns(false); collect_source_columns_visitor.visit(query_node_typed.getPrewhere()); diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 5fba02502364..c1f673221fd1 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -64,6 +64,7 @@ #include #include #include +#include #include #include @@ -277,7 +278,7 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr & const auto * raw = storage_ptr.get(); if (typeid_cast(raw)) return true; - if (parallel_replicas_estimation_enabled && std::dynamic_pointer_cast(storage_ptr)) + if (parallel_replicas_estimation_enabled && castStorage(storage_ptr, DeferredTable::Load)) return true; if (typeid_cast(raw)) return true; diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp index 67ca7fb5945a..52e09cafb0b3 100644 --- a/src/Planner/PlannerJoinTree.cpp +++ b/src/Planner/PlannerJoinTree.cpp @@ -722,6 +722,21 @@ bool applyTrivialCountWithSparsityFilterIfPossible( return true; } +/** Check the SELECT privilege for the columns that the planner resolved "away": `indexHint` arguments and ALIAS + * columns inlined into PREWHERE. Checked separately from the selected columns on purpose: a trivial query such as + * `SELECT count() FROM t` passes with a grant on any one column, while these names are always required. + */ +void checkAccessRightsForColumnsResolvedAway( + const TableNode & table_node, const TableExpressionData & table_expression_data, const ContextPtr & query_context) +{ + const auto & column_names = table_expression_data.getAccessCheckedColumnsNames(); + if (column_names.empty()) + return; + + checkAccessRights( + table_node.getStorage(), table_node.getStorageID(), table_node.getStorageSnapshot(), column_names, query_context); +} + void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expression, const SelectQueryOptions & select_query_options, PlannerContextPtr & planner_context) { const auto & query_context = planner_context->getQueryContext(); @@ -744,6 +759,8 @@ void prepareBuildQueryPlanForTableExpression(const QueryTreeNodePtr & table_expr const auto & column_names_with_aliases = table_expression_data.getSelectedColumnsNames(); columns_names_allowed_to_select = checkAccessRights( table_node->getStorage(), table_node->getStorageID(), table_node->getStorageSnapshot(), column_names_with_aliases, query_context); + + checkAccessRightsForColumnsResolvedAway(*table_node, table_expression_data, query_context); } else if (table_function_node) { diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp index 9503f927d31d..5ac521c6209f 100644 --- a/src/Planner/PlannerJoins.cpp +++ b/src/Planner/PlannerJoins.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -831,7 +832,7 @@ static JoinClausesAndActions buildJoinClausesAndActions( bool is_join_with_special_storage = false; if (const auto * right_table_node = join_node.getRightTableExpressionNode()->as()) { - is_join_with_special_storage = dynamic_cast(right_table_node->getStorage().get()); + is_join_with_special_storage = castStorage(right_table_node->getStorage(), DeferredTable::Load).get(); } for (auto & join_clause : result.join_clauses) @@ -1028,7 +1029,7 @@ void trySetStorageInTableJoin(const QueryTreeNodePtr & table_expression, std::sh else if (auto * table_function = table_expression->as()) storage = table_function->getStorage(); - auto storage_join = std::dynamic_pointer_cast(storage); + auto storage_join = castStorage(storage, DeferredTable::Load); if (storage_join) { table_join->setStorageJoin(storage_join); @@ -1040,8 +1041,9 @@ void trySetStorageInTableJoin(const QueryTreeNodePtr & table_expression, std::sh if (auto storage_dictionary = std::dynamic_pointer_cast(storage); storage_dictionary && storage_dictionary->getDictionary()->getSpecialKeyType() != DictionarySpecialKeyType::Range) + /// NOLINT(storage-cast): a dictionary, which the catalog never hands out behind a proxy. table_join->setStorageJoin(std::dynamic_pointer_cast(storage_dictionary->getDictionary())); - else if (auto storage_key_value = std::dynamic_pointer_cast(storage); storage_key_value) + else if (auto storage_key_value = castStorage(storage, DeferredTable::Load); storage_key_value) table_join->setStorageJoin(storage_key_value); } diff --git a/src/Planner/PlannerJoinsLogical.cpp b/src/Planner/PlannerJoinsLogical.cpp index 1d64aee2bc9f..d0582e099162 100644 --- a/src/Planner/PlannerJoinsLogical.cpp +++ b/src/Planner/PlannerJoinsLogical.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -596,18 +597,19 @@ PreparedJoinStorage tryGetStorageInTableJoin(const QueryTreeNodePtr & table_expr const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression); result.column_mapping = table_expression_data.getColumnIdentifierToColumnName(); - result.storage_join = std::dynamic_pointer_cast(storage); + result.storage_join = castStorage(storage, DeferredTable::Load); if (result.storage_join) return result; auto storage_dictionary = std::dynamic_pointer_cast(storage); if (storage_dictionary && storage_dictionary->getDictionary()->getSpecialKeyType() != DictionarySpecialKeyType::Range) { + /// NOLINT(storage-cast): a dictionary, which the catalog never hands out behind a proxy. result.storage_key_value = std::dynamic_pointer_cast(storage_dictionary->getDictionary()); return result; } - result.storage_key_value = std::dynamic_pointer_cast(storage); + result.storage_key_value = castStorage(storage, DeferredTable::Load); if (result.storage_key_value) return result; diff --git a/src/Planner/TableExpressionData.h b/src/Planner/TableExpressionData.h index 29b765a27dc7..b3943d38feb4 100644 --- a/src/Planner/TableExpressionData.h +++ b/src/Planner/TableExpressionData.h @@ -93,6 +93,24 @@ class TableExpressionData selected_column_names.push_back(column_name); } + /** Mark a column that the user references explicitly, but that never becomes a selected column. + * + * This is needed for columns that the planner resolves away before the access check runs : + * an ALIAS column inlined into PREWHERE and a column used only as an indexHint argument. + */ + void markColumnForAccessCheck(const std::string & column_name) + { + auto [_, inserted] = access_checked_column_names_set.emplace(column_name); + if (inserted) + access_checked_column_names.push_back(column_name); + } + + /// Get columns that are not selected, but still require a SELECT privilege check + const Names & getAccessCheckedColumnsNames() const + { + return access_checked_column_names; + } + /// Get columns that are requested from table expression, including ALIAS columns const Names & getSelectedColumnsNames() const { @@ -292,6 +310,11 @@ class TableExpressionData /// To deduplicate columns in `selected_column_names` NameSet selected_column_names_set; + /// Columns that the user references explicitly, but that are resolved away before access check. + Names access_checked_column_names; + /// To deduplicate columns in above + NameSet access_checked_column_names_set; + /// Expression to calculate ALIAS columns /// Keep alias name (String) + expression (ActionsDAG) pairs; vector preserves insertion order. AliasColumnExpressions alias_column_expressions; diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp index a54f859f69fb..32c307f61773 100644 --- a/src/Planner/Utils.cpp +++ b/src/Planner/Utils.cpp @@ -620,6 +620,29 @@ NameSet checkAccessRights( return {}; } +NameSet collectReferencedColumnNames(const QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression) +{ + NameSet column_names; + traverseQueryTree( + node, + [](const QueryTreeNodePtr & parent, const QueryTreeNodePtr &) + { + /// Don't go inside an ALIAS column expression: a grant on the alias name is sufficient. + const auto * column_node = parent->as(); + if (!column_node || !column_node->hasExpression()) + return true; + const auto & column_source = column_node->getColumnSourceOrNull(); + return !(column_source && column_source->getNodeType() == QueryTreeNodeType::TABLE); + }, + [&](const QueryTreeNodePtr & current) + { + const auto * column_node = current->as(); + if (column_node && column_node->getColumnSourceOrNull().get() == table_expression.get()) + column_names.insert(column_node->getColumnName()); + }); + return column_names; +} + static void checkAccessRightsForFilter(const QueryTreeNodePtr & filter_query_tree, const QueryTreeNodePtr & table_expression, const ContextPtr & query_context) @@ -650,24 +673,7 @@ static void checkAccessRightsForFilter(const QueryTreeNodePtr & filter_query_tre return; } - NameSet column_names; - traverseQueryTree( - filter_query_tree, - [](const QueryTreeNodePtr & parent, const QueryTreeNodePtr &) - { - /// Don't go inside an ALIAS column expression: a grant on the alias name is sufficient. - const auto * column_node = parent->as(); - if (!column_node || !column_node->hasExpression()) - return true; - const auto & column_source = column_node->getColumnSourceOrNull(); - return !(column_source && column_source->getNodeType() == QueryTreeNodeType::TABLE); - }, - [&](const QueryTreeNodePtr & node) - { - const auto * column_node = node->as(); - if (column_node && column_node->getColumnSourceOrNull().get() == table_expression.get()) - column_names.insert(column_node->getColumnName()); - }); + NameSet column_names = collectReferencedColumnNames(filter_query_tree, table_expression); if (column_names.empty()) return; diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h index 3b504afb5286..032a614a73d7 100644 --- a/src/Planner/Utils.h +++ b/src/Planner/Utils.h @@ -92,6 +92,10 @@ QueryTreeNodePtr replaceTableExpressionsWithDummyTables( SelectQueryInfo buildSelectQueryInfo(const QueryTreeNodePtr & query_tree, const PlannerContextPtr & planner_context); +/// Names of `table_expression` columns referenced from `node`, with ALIAS columns under their own name: +/// their expressions are not entered, because a grant on the alias name is sufficient to use it. +NameSet collectReferencedColumnNames(const QueryTreeNodePtr & node, const QueryTreeNodePtr & table_expression); + /// Check if current user has privileges to SELECT columns from table /// Throws an exception if access to any column from `column_names` is not granted /// If `column_names` is empty, check access to any columns and return names of accessible columns diff --git a/src/Planner/collectSelectedColumnsFromTable.cpp b/src/Planner/collectSelectedColumnsFromTable.cpp index 25d1ae549e96..6fecac2fd34b 100644 --- a/src/Planner/collectSelectedColumnsFromTable.cpp +++ b/src/Planner/collectSelectedColumnsFromTable.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include @@ -21,12 +20,6 @@ class CollectSelectedColumnsFromTableVisitor : public InDepthQueryTreeVisitorWit void enterImpl(QueryTreeNodePtr & node) { - if (isIndexHintFunction(node)) - { - is_inside_index_hint_function = true; - return; - } - auto * column_node = node->as(); if (!column_node) return; @@ -38,24 +31,10 @@ class CollectSelectedColumnsFromTableVisitor : public InDepthQueryTreeVisitorWit if (!source_table || source_table->getStorageID() != storage_id) return; - /// A special case for the "indexHint" function. We don't need its arguments for execution if column's source table is MergeTree. - /// Instead, we prepare an ActionsDAG for its arguments and store it inside a function (see ActionsDAG::buildFilterActionsDAG). - /// So this optimization allows not to read arguments of "indexHint" (if not needed in other contexts) but only to use index analysis for them. - if (is_inside_index_hint_function && source_table->getStorage()->isMergeTree()) - return; - + /// Note that arguments of the "indexHint" function need to be checked for SELECT privilege selected_columns.insert(column_node->getColumnName()); } - void leaveImpl(QueryTreeNodePtr & node) - { - if (isIndexHintFunction(node)) - { - is_inside_index_hint_function = false; - return; - } - } - bool isAliasColumn(const QueryTreeNodePtr & node) const { const auto * column_node = node->as(); @@ -73,19 +52,12 @@ class CollectSelectedColumnsFromTableVisitor : public InDepthQueryTreeVisitorWit return !isAliasColumn(parent_node); } - bool isIndexHintFunction(const QueryTreeNodePtr & node) const - { - return node->as() && node->as()->getFunctionName() == "indexHint"; - } - std::vector getSelectedColumns() const { return std::vector(selected_columns.begin(), selected_columns.end()); } private: - /// True if we are traversing arguments of function "indexHint". - bool is_inside_index_hint_function = false; const StorageID & storage_id; std::unordered_set selected_columns; }; diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index a86ca31cf01d..6edf87caef91 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -86,6 +87,8 @@ class IOutputFormat : public IProcessor virtual bool supportsWritingException() const { return false; } virtual void setException(const String & /*exception_message*/) {} + virtual std::unordered_map getColumnSizesOnDisk() const { return {}; } + /// A framing format (see IFramingFormat.h) multiplexes the formatted data along with auxiliary /// packets (progress, logs, profile events, exceptions) in the output stream. The format must /// have been created over the framing format's payload buffer. When set, the format notifies diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp index 3b10b98a2783..bc380233d0cc 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include namespace DB @@ -14,8 +15,12 @@ namespace DB namespace ErrorCodes { extern const int CANNOT_SKIP_UNKNOWN_FIELD; + extern const int TOO_LARGE_ARRAY_SIZE; } +/// Bound number of columns in header so user cannot reserve() arbitrarily large amount of memory +static constexpr auto TOO_MANY_COLUMNS_MESSAGE = "Suspiciously many columns in RowBinary header: {}"; + template BinaryRowInputFormat::BinaryRowInputFormat(ReadBuffer & in_, SharedHeader header, IRowInputFormat::Params params_, bool with_names_, bool with_types_, const FormatSettings & format_settings_) : RowInputFormatWithNamesAndTypes>( @@ -88,6 +93,8 @@ template std::vector BinaryFormatReader::readNames() { readVarUInt(read_columns, *in); + if (read_columns > DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, TOO_MANY_COLUMNS_MESSAGE, read_columns); return readHeaderRow(); } @@ -150,6 +157,8 @@ template void BinaryFormatReader::skipNames() { readVarUInt(read_columns, *in); + if (read_columns > DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, TOO_MANY_COLUMNS_MESSAGE, read_columns); skipHeaderRow(); } @@ -160,6 +169,8 @@ void BinaryFormatReader::skipTypes() { /// It's possible only when with_names = false and with_types = true readVarUInt(read_columns, *in); + if (read_columns > DEFAULT_NATIVE_BINARY_MAX_NUM_COLUMNS) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, TOO_MANY_COLUMNS_MESSAGE, read_columns); } skipHeaderRow(); diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 20096b6f91d1..aa7115021e73 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -1015,6 +1015,17 @@ std::unique_ptr buildORCSearchArgument( return builder->build(); } +std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options) +{ + auto reader = orc::createReader(std::move(stream), options); + if (reader->getType().getKind() != orc::STRUCT) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "ORC files whose root type is not a struct are not supported, the file has root type {}", + reader->getType().toString()); + return reader; +} + static void getFileReader( ReadBuffer & in, std::unique_ptr & file_reader, @@ -1036,7 +1047,7 @@ static void getFileReader( options.setCacheOptions(orc::CacheOptions{.holeSizeLimit = hole_size_limit, .rangeSizeLimit = range_size_limit}); auto input_stream = asORCInputStream(in, format_settings, use_prefetch, is_stopped); - file_reader = orc::createReader(std::move(input_stream), options); + file_reader = createORCReader(std::move(input_stream), options); } static const orc::Type * @@ -1519,7 +1530,7 @@ void ORCColumnToCHColumn::orcTableToCHChunk( { const auto * struct_batch = dynamic_cast(table); if (!struct_batch) - throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", struct_batch->toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", table->toString()); if (schema->getSubtypeCount() != struct_batch->fields.size()) throw Exception( diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index 70df42d54ead..49a29a7d8517 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -63,6 +63,9 @@ std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in /// instead of returning a null pointer that the library dereferences. orc::MemoryPool & getORCMemoryPool(); +/// Creates an ORC file reader; throws INCORRECT_DATA if the root type of the file is not a struct. +std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options); + std::unique_ptr buildORCSearchArgument( const KeyCondition & key_condition, const Block & header, const orc::Type & schema, const FormatSettings & format_settings); diff --git a/src/Processors/Formats/Impl/Parquet/Reader.cpp b/src/Processors/Formats/Impl/Parquet/Reader.cpp index 1ee01ab2bbd5..d5ba590006c6 100644 --- a/src/Processors/Formats/Impl/Parquet/Reader.cpp +++ b/src/Processors/Formats/Impl/Parquet/Reader.cpp @@ -432,16 +432,25 @@ void Reader::prefilterAndInitRowGroups(const std::optional clickhouse_to_parquet_name; - const auto * query_side_column_mapper = format_filter_info->current_schema_column_mapper - ? format_filter_info->current_schema_column_mapper.get() - : format_filter_info->column_mapper.get(); - if (query_side_column_mapper && format_filter_info->column_mapper) - clickhouse_to_parquet_name = - query_side_column_mapper->makeMapping(format_filter_info->column_mapper->getFieldIdToClickHouseName()).first; + std::optional> clickhouse_to_parquet_name; + auto get_clickhouse_to_parquet_name = [&]() -> const std::unordered_map & + { + if (!clickhouse_to_parquet_name) + { + clickhouse_to_parquet_name.emplace(); + const auto * query_side_column_mapper = format_filter_info->current_schema_column_mapper + ? format_filter_info->current_schema_column_mapper.get() + : format_filter_info->column_mapper.get(); + if (query_side_column_mapper && format_filter_info->column_mapper) + *clickhouse_to_parquet_name + = query_side_column_mapper->makeMapping(format_filter_info->column_mapper->getFieldIdToClickHouseName()).first; + } + return *clickhouse_to_parquet_name; + }; auto resolve_geo_meta = [&](const String & ch_name) -> std::unordered_map::const_iterator { - if (auto it = clickhouse_to_parquet_name.find(ch_name); it != clickhouse_to_parquet_name.end()) + const auto & mapping = get_clickhouse_to_parquet_name(); + if (auto it = mapping.find(ch_name); it != mapping.end()) return geo_meta->find(it->second); return geo_meta->find(ch_name); }; @@ -453,7 +462,8 @@ void Reader::prefilterAndInitRowGroups(const std::optional String { - if (auto it = clickhouse_to_parquet_name.find(ch_name); it != clickhouse_to_parquet_name.end()) + const auto & mapping = get_clickhouse_to_parquet_name(); + if (auto it = mapping.find(ch_name); it != mapping.end()) return it->second; return ch_name; }; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 7d5ea6dd2941..28c3c0816d54 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -21,6 +21,11 @@ namespace CurrentMetrics namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + using namespace Parquet; ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHeader header_, const FormatSettings & format_settings_, FormatFilterInfoPtr format_filter_info_) @@ -198,10 +203,62 @@ void ParquetBlockOutputFormat::finalizeImpl() writeFileHeader(file_state, out); } Block header = materializeBlock(getPort(PortKind::Main).getHeader()); + collectColumnSizesOnDisk(header); writeFileFooter(file_state, schema, options, out, header); chassert(out.count() - base_offset == file_state.offset); } +static size_t countSchemaLeaves(const SchemaElements & schema, size_t & index) +{ + if (index >= schema.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Parquet schema of {} elements is truncated", schema.size()); + + const auto & element = schema[index]; + ++index; + + if (!element.__isset.num_children || element.num_children == 0) + return 1; + + size_t leaves = 0; + for (Int32 i = 0; i < element.num_children; ++i) + leaves += countSchemaLeaves(schema, index); + return leaves; +} + +void ParquetBlockOutputFormat::collectColumnSizesOnDisk(const Block & header) +{ + std::vector leaves_per_column; + leaves_per_column.reserve(header.columns()); + size_t schema_index = 1; + size_t num_leaves = 0; + for (size_t i = 0; i < header.columns(); ++i) + { + leaves_per_column.push_back(countSchemaLeaves(schema, schema_index)); + num_leaves += leaves_per_column.back(); + } + + column_sizes_on_disk.clear(); + for (const auto & row_group : file_state.completed_row_groups) + { + const auto & column_chunks = row_group.row_group.columns; + if (column_chunks.size() != num_leaves) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Parquet row group has {} column chunks while the schema has {} leaf columns", + column_chunks.size(), + num_leaves); + + size_t leaf_index = 0; + for (size_t i = 0; i < header.columns(); ++i) + { + size_t column_size = 0; + for (size_t j = 0; j < leaves_per_column[i]; ++j, ++leaf_index) + column_size += static_cast(column_chunks[leaf_index].meta_data.total_compressed_size); + column_sizes_on_disk[header.getByPosition(i).name] += column_size; + } + } +} + void ParquetBlockOutputFormat::resetFormatterImpl() { if (pool) @@ -216,6 +273,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl() task_queue.clear(); row_groups.clear(); file_state = {}; + column_sizes_on_disk.clear(); staging_chunks.clear(); staging_rows = 0; staging_bytes = 0; diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index 07322de00c29..c38b1e5d0ce7 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -20,6 +20,8 @@ class ParquetBlockOutputFormat final : public IOutputFormat String getName() const override { return "ParquetBlockOutputFormat"; } + std::unordered_map getColumnSizesOnDisk() const override { return column_sizes_on_disk; } + private: struct MemoryToken { @@ -94,6 +96,7 @@ class ParquetBlockOutputFormat final : public IOutputFormat void consume(Chunk) override; void finalizeImpl() override; + void collectColumnSizesOnDisk(const Block & header); void resetFormatterImpl() override; void onCancel() noexcept override; @@ -119,6 +122,7 @@ class ParquetBlockOutputFormat final : public IOutputFormat Parquet::IcebergOptionality iceberg_optionality; Parquet::SchemaElements schema; Parquet::FileWriteState file_state; + std::unordered_map column_sizes_on_disk; size_t base_offset = 0; // initial out.count(), just for assert std::mutex mutex; diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp index 272d0f4f0e35..08fe3c68f92f 100644 --- a/src/Processors/QueryPlan/AggregatingStep.cpp +++ b/src/Processors/QueryPlan/AggregatingStep.cpp @@ -563,6 +563,10 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B }); } + /// The per-set results are spread over `max_threads` streams below, so split a small single-level + /// result of each set for that width as for an ordinary aggregation. + const size_t grouping_sets_output_streams = should_produce_results_in_order_of_bucket_number ? 1 : params.max_threads; + pipeline.transform([&](OutputPortRawPtrs ports) { chassert(streams * grouping_sets_size == ports.size()); @@ -586,7 +590,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B new_temporary_data_merge_threads, should_produce_results_in_order_of_bucket_number, skip_merging, - nullptr); + nullptr, + grouping_sets_output_streams); // For each input stream we have `grouping_sets_size` copies, so port index // for transform #j should skip ports of first (j-1) streams. connect(*ports[i + grouping_sets_size * j], aggregation_for_set->getInputs().front()); @@ -597,7 +602,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B else { auto aggregation_for_set - = std::make_shared(input_header, transform_params_for_set, dataflow_cache_updater); + = std::make_shared(input_header, transform_params_for_set, dataflow_cache_updater, grouping_sets_output_streams); connect(*ports[i], aggregation_for_set->getInputs().front()); ports[i] = &aggregation_for_set->getOutputs().front(); processors.push_back(aggregation_for_set); @@ -859,7 +864,8 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B new_temporary_data_merge_threads, should_produce_results_in_order_of_bucket_number, skip_merging, - dataflow_cache_updater); + dataflow_cache_updater, + streams_after_aggregation); }); pipeline.resize(streams_after_aggregation, false, settings.min_outstreams_per_resize_after_split); @@ -869,7 +875,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B else { pipeline.addSimpleTransform([&](const SharedHeader & header) - { return std::make_shared(header, transform_params, dataflow_cache_updater); }); + { return std::make_shared(header, transform_params, dataflow_cache_updater, streams_after_aggregation); }); pipeline.resize(streams_after_aggregation); diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp index 7697b0e5ed0c..e98c055c074a 100644 --- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp +++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp @@ -29,6 +29,11 @@ std::unique_ptr createLocalPlan( auto query_plan = std::make_unique(); auto new_context = Context::createCopy(context); + /// The local shard reads the table itself, like the remote shards do: the inserted block + /// of a materialized view (whose query contains this `remote` or `Distributed` read) must not + /// replace the source table here. + new_context->clearViewSource(); + if (build_logical_plan && !default_database.empty()) new_context->setCurrentDatabase(default_database); diff --git a/src/Processors/QueryPlan/MergingAggregatedStep.cpp b/src/Processors/QueryPlan/MergingAggregatedStep.cpp index 95528696962e..3f64bfea764a 100644 --- a/src/Processors/QueryPlan/MergingAggregatedStep.cpp +++ b/src/Processors/QueryPlan/MergingAggregatedStep.cpp @@ -151,7 +151,8 @@ void MergingAggregatedStep::transformPipeline(QueryPipelineBuilder & pipeline, c pipeline.resize(1); /// Now merge the aggregated blocks - auto transform = std::make_shared(pipeline.getSharedHeader(), params, final, grouping_sets_params); + auto transform = std::make_shared(pipeline.getSharedHeader(), params, final, grouping_sets_params, + should_produce_results_in_order_of_bucket_number ? 1 : max_threads); pipeline.addTransform(std::move(transform)); } else diff --git a/src/Processors/QueryPlan/Optimizations/Optimizations.h b/src/Processors/QueryPlan/Optimizations/Optimizations.h index c512274db8fe..e54b0544287e 100644 --- a/src/Processors/QueryPlan/Optimizations/Optimizations.h +++ b/src/Processors/QueryPlan/Optimizations/Optimizations.h @@ -85,6 +85,7 @@ struct Optimization /// optimization when the plan is going to be distributed or serialized. bool make_distributed_plan = false; bool serialize_query_plan = false; + bool filter_push_down_below_limit_by = true; }; using Function = size_t (*)(QueryPlan::Node *, QueryPlan::Nodes &, const ExtraSettings &); diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index a1450a3dbc2e..ee68091b256d 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -60,6 +60,7 @@ namespace Setting extern const SettingsBool query_plan_enable_optimizations; extern const SettingsBool query_plan_execute_functions_after_sorting; extern const SettingsBool query_plan_filter_push_down; + extern const SettingsBool query_plan_filter_push_down_below_limit_by; extern const SettingsBool query_plan_join_shard_by_pk_ranges; extern const SettingsBool query_plan_lift_up_array_join; extern const SettingsBool query_plan_lift_up_union; @@ -178,6 +179,8 @@ QueryPlanOptimizationSettings::QueryPlanOptimizationSettings( merge_filters = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_merge_filters]; push_limit_by_into_sort = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_push_limit_by_into_sort]; filter_push_down = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_filter_push_down]; + /// The filter push-down pass also runs with `filter_push_down` off, once a JOIN runtime filter has been added. + filter_push_down_below_limit_by = from[Setting::query_plan_filter_push_down_below_limit_by]; push_down_volume_reducing_functions = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_push_down_volume_reducing_functions]; convert_outer_join_to_inner_join = from[Setting::query_plan_enable_optimizations] && from[Setting::query_plan_convert_outer_join_to_inner_join]; diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index e22022e41e4a..7923e0cced7d 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -55,6 +55,7 @@ struct QueryPlanOptimizationSettings bool merge_expressions; bool merge_filters; bool filter_push_down; + bool filter_push_down_below_limit_by; bool push_down_volume_reducing_functions; bool convert_outer_join_to_inner_join; bool short_circuit_constant_false_join; diff --git a/src/Processors/QueryPlan/Optimizations/Utils.cpp b/src/Processors/QueryPlan/Optimizations/Utils.cpp index b408d902edd4..e65f389683cd 100644 --- a/src/Processors/QueryPlan/Optimizations/Utils.cpp +++ b/src/Processors/QueryPlan/Optimizations/Utils.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -69,20 +70,25 @@ FilterResult getFilterResult(const ColumnWithTypeAndName & column) return column.column->getBool(0) ? FilterResult::TRUE : FilterResult::FALSE; } +static bool isNotReadySetColumn(const ActionsDAG::Node & node) +{ + if (node.type != ActionsDAG::ActionType::COLUMN || !node.column) + return false; + + const ColumnSet * column_set = checkAndGetColumn(&node.column->getDataColumn()); + if (!column_set) + return false; + + auto future_set = column_set->getData(); + return !future_set || !future_set->get(); +} + bool dagContainsNonReadySet(const ActionsDAG & dag) { for (const auto & node : dag.getNodes()) { - if (node.type == ActionsDAG::ActionType::COLUMN && node.column) - { - const ColumnSet * column_set = checkAndGetColumn(&node.column->getDataColumn()); - if (column_set) - { - auto future_set = column_set->getData(); - if (!future_set || !future_set->get()) - return true; - } - } + if (isNotReadySetColumn(node)) + return true; } return false; } @@ -108,6 +114,23 @@ bool dagContainsNonDeterministicFunction(const ActionsDAG & dag) return false; } +bool isSensitiveToEvaluationCount(const ActionsDAG & dag) +{ + /// `sleep` and `sleepEachRow` are the functions with observable side effects. A lambda hides the + /// functions of its body, so it counts as sensitive. + for (const auto & node : dag.getNodes()) + { + if (node.result_type && WhichDataType(node.result_type).isFunction()) + return true; + if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base + && (!node.function_base->isDeterministicInScopeOfQuery() || node.function_base->isStateful() + || node.function_base->getName() == "sleep" || node.function_base->getName() == "sleepEachRow")) + return true; + } + + return false; +} + FilterResult filterResultForNotMatchedRows( const ActionsDAG & filter_dag, const String & filter_column_name, @@ -115,10 +138,6 @@ FilterResult filterResultForNotMatchedRows( bool allow_unknown_function_arguments ) { - /// If the filter DAG contains IN subquery sets that are not yet built - we cannot evaluate the filter result - if (dagContainsNonReadySet(filter_dag)) - return FilterResult::UNKNOWN; - /// `ActionsDAG::evaluatePartialResult` (called below) routes every function node through /// `IFunction::executeImplDryRun` with `input_rows_count=1`. For functions that are not /// deterministic within a single query (`rand`, `nowInBlock`, `rowNumberInAllBlocks`, @@ -140,6 +159,13 @@ FilterResult filterResultForNotMatchedRows( ActionsDAG::IntermediateExecutionResult filter_input; + /// A set that is not built yet is an unknown argument: `in` must never be dry-run on it (that yields a fake 0). + for (const auto & node : filter_dag.getNodes()) + { + if (isNotReadySetColumn(node)) + filter_input.emplace(&node, ColumnWithTypeAndName{nullptr, node.result_type, node.result_name}); + } + /// Create constant columns with default values for inputs of the filter DAG for (const auto * input : filter_dag.getInputs()) { diff --git a/src/Processors/QueryPlan/Optimizations/Utils.h b/src/Processors/QueryPlan/Optimizations/Utils.h index d3d17199b54d..297fcce20290 100644 --- a/src/Processors/QueryPlan/Optimizations/Utils.h +++ b/src/Processors/QueryPlan/Optimizations/Utils.h @@ -76,6 +76,11 @@ enum class FilterResult [[nodiscard]] bool dagContainsNonDeterministicFunction(const ActionsDAG & dag); +/// True when the expression holds a function that is non-deterministic within the query (`rand`), stateful +/// (`rowNumberInAllBlocks`, `aiEmbed`), or has observable side effects (`sleep`), so an observer can see how +/// many times and on which rows it was evaluated. +[[nodiscard]] bool isSensitiveToEvaluationCount(const ActionsDAG & dag); + [[nodiscard]] FilterResult filterResultForNotMatchedRows( const ActionsDAG & filter_dag, const String & filter_column_name, diff --git a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp index fa04c059cff6..a4462d6004c3 100644 --- a/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp +++ b/src/Processors/QueryPlan/Optimizations/actionsDAGUtils.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -555,12 +556,14 @@ bool isInjectiveFunction(const ActionsDAG::Node * node) if (node->function_base->isInjective({})) return true; - size_t fixed_args = 0; - for (const auto & child : node->children) - if (child->type == ActionsDAG::ActionType::COLUMN) - ++fixed_args; - static const std::vector injective = {"plus", "minus", "negate", "tuple"}; - return (fixed_args + 1 >= node->children.size()) && (std::ranges::find(injective, node->function_base->getName()) != injective.end()); + const auto & name = node->function_base->getName(); + if (node->children.size() != 2 || (name != "plus" && name != "minus")) + return false; + + const auto & left = *node->children[0]; + const auto & right = *node->children[1]; + return plusMinusWithConstantsIsInjective( + {left.column, left.result_type, left.result_name}, {right.column, right.result_type, right.result_name}, node->result_type); } NodeSet removeInjectiveFunctionsFromResultsRecursively(const ActionsDAG & actions) diff --git a/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp b/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp index 79f12db5f7a7..c934e9278492 100644 --- a/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp +++ b/src/Processors/QueryPlan/Optimizations/applyParallelReplicas.cpp @@ -432,7 +432,7 @@ class ConvertToDistributedVisitor : public QueryPlanVisitor(QueryPlan::cloneSubtree(split_node->children.front())); + auto plan_fragment = std::make_unique(QueryPlan::cloneSubtree(split_node->children.front(), query_plan)); ContextPtr context; /// Mark only the coordinated reads (collectReadsToDistribute follows a join's coordinated side) so they diff --git a/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp b/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp index bfc21ca6f7ee..511ae302c14f 100644 --- a/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/convertAnyJoinToSemiOrAntiJoin.cpp @@ -151,6 +151,11 @@ size_t tryConvertAnyJoinToSemiOrAntiJoin(QueryPlan::Node * parent_node, QueryPla return 0; const auto & filter_dag = filter->getExpression(); + + /// Not-built sets keep the join ANY: not every join algorithm can run SEMI. + if (dagContainsNonReadySet(filter_dag)) + return 0; + const auto & filter_column_name = filter->getFilterColumnName(); const auto & left_stream_input_header = join->getInputHeaders().front(); const auto & right_stream_input_header = join->getInputHeaders().back(); diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp index 79c3786a7b0e..e875c9d06701 100644 --- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp +++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -819,6 +820,12 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: if (is_filter_column_const_before) original_filter_const_column = filter->getOutputHeader()->getByName(filter->getFilterColumnName()).column; + /// With no input given only the conjuncts that read no column are evaluated, so FALSE means no row passes. + const bool filter_is_always_false + = (left_stream_filter_push_down_input_columns_available || right_stream_filter_push_down_input_columns_available) + && !isSensitiveToEvaluationCount(filter->getExpression()) + && filterResultForNotMatchedRows(filter->getExpression(), filter->getFilterColumnName(), Block{}) == FilterResult::FALSE; + auto join_filter_push_down_actions = filter->getExpression().splitActionsForJOINFilterPushDown( filter->getFilterColumnName(), filter->removesFilterColumn(), @@ -828,7 +835,8 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan:: *right_stream_input_header, equivalent_columns_to_push_down, equivalent_left_stream_column_to_right_stream_column, - equivalent_right_stream_column_to_left_stream_column); + equivalent_right_stream_column_to_left_stream_column, + filter_is_always_false); if (is_filter_column_const_before && !join_filter_push_down_actions.is_filter_const_after_all_push_downs) { @@ -1170,6 +1178,9 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes if (const auto * limit_by = typeid_cast(child.get())) { + if (!settings.filter_push_down_below_limit_by) + return 0; + /// A predicate on the LIMIT BY key columns removes whole groups, so the surviving /// per-group rows (and therefore the result) are identical whether it runs above or /// below the LIMIT BY. But it is only safe to push when every non-empty input group diff --git a/src/Processors/QueryPlan/Optimizations/joinOrder.cpp b/src/Processors/QueryPlan/Optimizations/joinOrder.cpp index a7f654ac37ed..0083567295b9 100644 --- a/src/Processors/QueryPlan/Optimizations/joinOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/joinOrder.cpp @@ -842,9 +842,10 @@ const std::vector & JoinOrderOptimizer::collectJoinEdgesMask(UI if (dpsub_data.edge_pinned[i] && (dpsub_data.edge_pin_mask[i] & ~joined)) continue; - /// Relations that must all be present before the predicate is applicable: the relations it - /// references (`sources`) plus any relations it is pinned to. For a plain equi-predicate the - /// pin is empty, so this is just `sources`. For a single-table conjunct of an outer join's ON + /// Works much like Extended Eligibility List (EEL) in case of outerjoins: + /// encoding relations that must be present for the predicate to be applicable (in `pin` mask) + /// For innerjoins its just the sources of the predicate, i.e., NEL, here pin is empty. + /// For a single-table conjunct of an outer join's ON /// clause (e.g. `t2.value = 'x'` in `... LEFT JOIN t3 ON t2.id = t3.id AND t2.value = 'x'`), /// `sources` is only `{t2}` but the pin is `{t3}`: the predicate belongs to the ON condition of /// the join that brings in `t3`, not to `t2` as a base-table filter. Placing it by `sources` @@ -856,17 +857,17 @@ const std::vector & JoinOrderOptimizer::collectJoinEdgesMask(UI if (std::popcount(applicable) <= 1) { - /// Base-relation filter or constant predicate: it becomes applicable as soon as its single - /// relation is present, so attach it at the earliest (two-relation) join to filter as low - /// as possible. - if (two_relations && (edge.fromLeft() || edge.fromRight() || edge.fromNone())) + /// Base-relation filter or constant predicate (the edge references at most one relation). + const bool relation_introduced = applicable != 0 && (left_mask == applicable || right_mask == applicable); + const bool constant_at_earliest_join = applicable == 0 && two_relations; + if (relation_introduced || constant_at_earliest_join) out.push_back(&edge); } else if ((applicable & ~left_mask) && (applicable & ~right_mask)) { /// The predicate spans the split (a connecting equi-predicate, or a single-table ON-clause /// conjunct pinned to the opposite side): neither side alone contains all the relations it - /// needs. This join is the lowest one that makes it applicable, so attach it here — into the + /// needs. This join is the lowest one that makes it applicable, so attach it here: into the /// correct join's ON condition. out.push_back(&edge); } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp index a1f99567b656..765a0ffde1dc 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeDirectReadFromTextIndex.cpp @@ -703,11 +703,15 @@ class TextIndexDAGReplacer const auto & preprocessor_dag = preprocessor->getOriginalActionsDAG(); chassert(preprocessor_dag.getOutputs().size() == 1); const auto & preprocessor_output = preprocessor_dag.getOutputs().front(); - auto haystack_name = getNameWithoutAliases(arg_haystack); + /// The index was analyzed on the expression under lossless conversions, e.g. `s` in `hasToken(toNullable(s), 'Foo')`. + const auto * haystack = unwrapLosslessConversion(arg_haystack); + auto haystack_name = getNameWithoutAliases(haystack); /// Check that preprocessor contains current expression as its argument. if (hasSubexpression(preprocessor_output, haystack_name)) { + new_children[0] = haystack; + if (apply_postprocessor) { preprocessor_source_ast = preprocessor->getExpressionAST(new_children[0]->result_name); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index e48e683728e7..9556da9e9550 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -102,6 +102,7 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & optimization_se optimization_settings.push_down_volume_reducing_functions, optimization_settings.make_distributed_plan, optimization_settings.serialize_query_plan, + optimization_settings.filter_push_down_below_limit_by, }; while (!stack.empty()) @@ -232,6 +233,7 @@ void optimizeTreeSecondPass( optimization_settings.push_down_volume_reducing_functions, optimization_settings.make_distributed_plan, optimization_settings.serialize_query_plan, + optimization_settings.filter_push_down_below_limit_by, }; Stack stack; @@ -307,6 +309,11 @@ void optimizeTreeSecondPass( /// to move newly added runtime filter as deep in the tree as possible if (join_runtime_filters_were_added) { + /// The re-run passes default `ExtraSettings`, except for the `LIMIT BY` gate: this pass also pushes + /// filters below `LIMIT BY`, so it must follow `query_plan_filter_push_down_below_limit_by` too. + Optimization::ExtraSettings push_down_settings{}; + push_down_settings.filter_push_down_below_limit_by = optimization_settings.filter_push_down_below_limit_by; + traverseQueryPlan(stack, root, [&](auto & frame_node) { @@ -316,7 +323,7 @@ void optimizeTreeSecondPass( size_t changed_nodes = 0; changed_nodes += tryMergeExpressions(&frame_node, nodes, {}); changed_nodes += tryMergeFilters(&frame_node, nodes, {}); - changed_nodes += tryPushDownFilter(&frame_node, nodes, {}); + changed_nodes += tryPushDownFilter(&frame_node, nodes, push_down_settings); if (!changed_nodes) break; diff --git a/src/Processors/QueryPlan/QueryPlan.cpp b/src/Processors/QueryPlan/QueryPlan.cpp index 6eead25e4a2a..cb3a85558dca 100644 --- a/src/Processors/QueryPlan/QueryPlan.cpp +++ b/src/Processors/QueryPlan/QueryPlan.cpp @@ -176,6 +176,7 @@ void QueryPlan::unitePlans(QueryPlanStepPtr step, std::vectormax_threads); + concurrency_control = concurrency_control || plan->concurrency_control; resources = std::move(plan->resources); } } @@ -1215,6 +1216,14 @@ QueryPlan QueryPlan::extractSubplan(Node * subplan_root) new_plan.nodes.splice(new_plan.nodes.end(), nodes, curr); } + /// A subplan extracted from this plan inherits the same execution limits and resource holder. + /// The splice above moves only the node tree; without this the extracted subplan would run with the + /// default thread fan-out and no concurrency control instead of this plan's caps. append copies the + /// shared handles, so ownership is only shared, never moved out of this plan. + new_plan.max_threads = max_threads; + new_plan.concurrency_control = concurrency_control; + new_plan.resources.append(resources); + return new_plan; } @@ -1232,6 +1241,18 @@ QueryPlan QueryPlan::clone() const result.cloneInplace(current_subplan_copy_root, root); result.root = current_subplan_copy_root; + /// Preserve the plan-level execution limits. They are not part of the node tree, so cloneInplace + /// does not copy them; without this a cloned plan runs with the default thread fan-out and no + /// concurrency control instead of the caps the source plan carries. + result.max_threads = max_threads; + result.concurrency_control = concurrency_control; + + /// Preserve the resource holder (storage holders, table locks, interpreter contexts, etc.). + /// These keep the objects a cloned plan reads (e.g. MergeTree parts in the direct-join lookup + /// path) alive for as long as the clone lives. append copies the shared handles, so ownership is + /// only ever shared, never moved out of the source: strictly a lifetime extension for the clone. + result.resources.append(resources); + return result; } @@ -1247,6 +1268,15 @@ QueryPlan QueryPlan::cloneSubtree(Node * subplan_root) return result; } +QueryPlan QueryPlan::cloneSubtree(Node * subplan_root, const QueryPlan & source_plan) +{ + auto result = cloneSubtree(subplan_root); + result.max_threads = source_plan.max_threads; + result.concurrency_control = source_plan.concurrency_control; + result.resources.append(source_plan.resources); + return result; +} + void QueryPlan::cloneSubplanAndReplace(Node * node_to_replace, Node * subplan_root, Nodes & nodes) { if (!subplan_root) @@ -1344,6 +1374,7 @@ void QueryPlan::replaceNodeWithPlan(Node * node, QueryPlan plan, SharedHeader ex node->children = std::move(plan.getRootNode()->children); max_threads = std::max(max_threads, plan.max_threads); + concurrency_control = concurrency_control || plan.concurrency_control; resources = std::move(plan.resources); } diff --git a/src/Processors/QueryPlan/QueryPlan.h b/src/Processors/QueryPlan/QueryPlan.h index daa386181014..e465bb92bed4 100644 --- a/src/Processors/QueryPlan/QueryPlan.h +++ b/src/Processors/QueryPlan/QueryPlan.h @@ -220,6 +220,9 @@ class QueryPlan /// (multiple sources / multi-input steps). static QueryPlan cloneSubtree(Node * subplan_root); + /// Same as above, preserving the execution limits and resources from the plan that owns the subtree. + static QueryPlan cloneSubtree(Node * subplan_root, const QueryPlan & source_plan); + static void cloneSubplanAndReplace(Node * node_to_replace, Node * subplan_root, Nodes & nodes); private: diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index db565127c152..748dfddbb1cf 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -6555,7 +6556,7 @@ std::unique_ptr ReadFromMergeTree::deserialize(Deserialization & StorageID table_id(database_name, table_name); auto storage_ptr = DatabaseCatalog::instance().getTable(table_id, ctx.context); - auto * merge_tree = dynamic_cast(storage_ptr.get()); + auto * merge_tree = castStorage(storage_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {} is not a MergeTree table", table_id.getNameForLogs()); diff --git a/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp b/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp index a36a43bbbcc3..a27873b024ca 100644 --- a/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp +++ b/src/Processors/QueryPlan/ReadFromObjectStorageStep.cpp @@ -35,6 +35,7 @@ namespace Setting { extern const SettingsBool parallelize_output_from_storages; extern const SettingsBool s3_validate_etag_on_read; + extern const SettingsBool azure_validate_etag_on_read; } @@ -216,8 +217,7 @@ bool ReadFromObjectStorageStep::canUseLazyMaterialization() const return false; /// Even when the two generations are comparable, on most backends the second pass opens an - /// unconditional read: `AzureObjectStorage`, `HDFSObjectStorage` and the local disk ignore - /// `StoredObject::etag`, so a concurrent in-place overwrite between the metadata probe and the + /// unconditional read: `HDFSObjectStorage` and the local disk ignore `StoredObject::etag`, so a concurrent in-place overwrite between the metadata probe and the /// read could still stitch together rows of two versions of the file. The reread is only /// generation-safe when either: /// - the data files are immutable by the format's contract — a data lake never overwrites a @@ -225,7 +225,8 @@ bool ReadFromObjectStorageStep::canUseLazyMaterialization() const /// - the backend pins the actual read to the captured generation — S3 with /// `s3_validate_etag_on_read` issues the GET with an `If-Match` on the captured ETag and /// rejects a response whose ETag drifted from it (see `ReadBufferFromS3`), which is atomic - /// with respect to an overwrite. + /// with respect to an overwrite; Azure with `azure_validate_etag_on_read` does the same + /// (see `ReadBufferFromAzureBlobStorage`). /// The pin only takes effect when the captured metadata actually carries a non-empty `ETag` /// (see `createReadBuffer`), and `GCS` accessed through the S3 API is documented to legitimately /// return objects without one — so a `GCS`-provider client is not pinned even with the setting @@ -241,6 +242,9 @@ bool ReadFromObjectStorageStep::canUseLazyMaterialization() const reread_is_generation_pinned = s3_client && s3_client->getProviderType() != S3::ProviderType::GCS; } #endif + if (object_storage->getType() == ObjectStorageType::Azure + && getContext()->getSettingsRef()[Setting::azure_validate_etag_on_read]) + reread_is_generation_pinned = true; if (!configuration->dataFilesAreImmutable() && !reread_is_generation_pinned) return false; diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp index aace7e4a7f94..12544a025ef4 100644 --- a/src/Processors/QueryPlan/ReadFromRemote.cpp +++ b/src/Processors/QueryPlan/ReadFromRemote.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -658,7 +659,7 @@ void ReadFromRemote::addLazyPipe( // has no local storage and reaches a lazy shard only via the failpoint, so it always reads remotely. if (!use_delayed_remote_source && !my_table_func_ptr) { - const auto replicated_storage = std::dynamic_pointer_cast(my_storage); + const auto replicated_storage = castStorage(my_storage, DeferredTable::Load); if (!replicated_storage) { throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected lazy remote read from a non-replicated table: {}", my_storage->getName()); @@ -719,6 +720,7 @@ void ReadFromRemote::addLazyPipe( std::move(connections), query_string, header, my_context, my_throttler, my_scalars, my_external_tables, stage_to_use, my_shard.query_plan, /*extension=*/std::nullopt, my_shard.shard_info.pool); remote_query_executor->setLogger(my_log); + remote_query_executor->setQueryPlanFallbackStage(my_stage); remote_query_executor->setDistributedFanout(my_distributed_fanout); /// Attach the shared tracker so exception-based shard skips on the lazy path are also bounded by /// `max_skip_unavailable_shards_num` / `max_skip_unavailable_shards_ratio`, like the non-lazy path. @@ -813,6 +815,7 @@ void ReadFromRemote::addPipe( std::nullopt, priority_func); remote_query_executor->setLogger(log); + remote_query_executor->setQueryPlanFallbackStage(stage); remote_query_executor->setPoolMode(PoolMode::GET_ONE); remote_query_executor->setDistributedFanout(shards.size() * shard.shard_info.per_replica_pools.size()); remote_query_executor->setUnavailableShardTracker(unavailable_shard_tracker); @@ -844,6 +847,7 @@ void ReadFromRemote::addPipe( stage_to_use, shard.query_plan); remote_query_executor->setLogger(log); + remote_query_executor->setQueryPlanFallbackStage(stage); remote_query_executor->setDistributedFanout(shards.size()); remote_query_executor->setUnavailableShardTracker(unavailable_shard_tracker); diff --git a/src/Processors/QueryPlan/Serialization.cpp b/src/Processors/QueryPlan/Serialization.cpp index a2c1ec552bc2..fc2ae2932e91 100644 --- a/src/Processors/QueryPlan/Serialization.cpp +++ b/src/Processors/QueryPlan/Serialization.cpp @@ -71,6 +71,14 @@ static Block deserializeHeader(ReadBuffer & in, size_t max_type_complexity) void QueryPlan::serialize(WriteBuffer & out, size_t max_supported_version) const { UInt64 version = std::min(max_supported_version, DBMS_QUERY_PLAN_SERIALIZATION_VERSION); + + if (version < DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS && (max_threads || concurrency_control)) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Cannot serialize a query plan with execution limits for serialization version {}; version {} or newer is required", + version, + DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS); + writeVarUInt(version, out); SerializationFlags flags; @@ -81,6 +89,14 @@ void QueryPlan::serialize(WriteBuffer & out, size_t max_supported_version) const void QueryPlan::serializeForDistributedTask(WriteBuffer & out, size_t max_supported_version, const SizeLimits & sets_transfer_limits) const { UInt64 version = std::min(max_supported_version, DBMS_QUERY_PLAN_SERIALIZATION_VERSION); + + if (version < DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS && (max_threads || concurrency_control)) + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Cannot serialize a query plan with execution limits for serialization version {}; version {} or newer is required", + version, + DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS); + writeVarUInt(version, out); SerializationFlags flags; @@ -94,6 +110,12 @@ void QueryPlan::serialize(WriteBuffer & out, const SerializationFlags & flags) c { checkInitialized(); + if (flags.version >= DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS) + { + writeVarUInt(max_threads, out); + writeBinary(concurrency_control, out); + } + SerializedSetsRegistry registry; struct Frame @@ -206,6 +228,12 @@ QueryPlanAndSets QueryPlan::deserialize(ReadBuffer & in, const ContextPtr & cont std::stack stack; QueryPlan plan; + if (flags.version >= DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS) + { + readVarUInt(plan.max_threads, in); + readBinary(plan.concurrency_control, in); + } + stack.push(Frame{.to_fill = plan.root}); while (!stack.empty()) diff --git a/src/Processors/QueryPlan/SetsSerialization.cpp b/src/Processors/QueryPlan/SetsSerialization.cpp index 92a8a932224c..c4bab839c4cd 100644 --- a/src/Processors/QueryPlan/SetsSerialization.cpp +++ b/src/Processors/QueryPlan/SetsSerialization.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB { @@ -261,7 +262,7 @@ static void makeSetsFromStorage(std::list sets { Identifier identifier = parseTableIdentifier(set.storage_name, context); auto table_node = resolveTable(identifier, context); - const auto * storage_set = typeid_cast(table_node->getStorage().get()); + const auto * storage_set = castStorage(table_node->getStorage(), DeferredTable::Load).get(); if (!storage_set) throw Exception(ErrorCodes::INCORRECT_DATA, "Table {} is not a StorageSet", set.storage_name); diff --git a/src/Processors/QueryPlan/tests/gtest_query_plan_execution_limits.cpp b/src/Processors/QueryPlan/tests/gtest_query_plan_execution_limits.cpp new file mode 100644 index 000000000000..61edb277ecb6 --- /dev/null +++ b/src/Processors/QueryPlan/tests/gtest_query_plan_execution_limits.cpp @@ -0,0 +1,200 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace DB +{ +void registerReadNothingStep(QueryPlanStepRegistry & registry); +} + +namespace +{ + +SharedHeader makeHeader() +{ + auto type = std::make_shared(); + return std::make_shared(Block({ColumnWithTypeAndName(type->createColumn(), type, "k")})); +} + +void tryRegisterReadNothingStep() +{ + static struct Register + { + Register() + { + registerReadNothingStep(QueryPlanStepRegistry::instance()); + } + } registered; +} + +/// Smallest plan that owns a root node: a single source step. +QueryPlan makeSourcePlan() +{ + /// `registerStep` rejects duplicate names, and other tests in this binary register + /// overlapping subsets. Register only the step deserialized in this test file. + tryRegisterReadNothingStep(); + + QueryPlan plan; + plan.addStep(std::make_unique(makeHeader())); + return plan; +} + +} + +/// `max_threads` and `concurrency_control` are plan-level, not part of the node tree, so they are +/// invisible in query results and have to be asserted directly. + +TEST(QueryPlanExecutionLimits, ClonePreservesLimitsAndResources) +{ + auto source = makeSourcePlan(); + source.setMaxThreads(4); + source.setConcurrencyControl(true); + source.addInterpreterContext(Context::createCopy(getContext().context)); + + auto clone = source.clone(); + + EXPECT_EQ(clone.getMaxThreads(), 4u); + EXPECT_TRUE(clone.getConcurrencyControl()); + /// `append` shares the handle rather than moving it, so the source keeps its own copy. + EXPECT_EQ(clone.getInterpretersContexts().size(), 1u); + EXPECT_EQ(source.getInterpretersContexts().size(), 1u); +} + +TEST(QueryPlanExecutionLimits, CloneSubtreePreservesSourceLimitsAndResources) +{ + auto source = makeSourcePlan(); + source.setMaxThreads(4); + source.setConcurrencyControl(true); + source.addInterpreterContext(Context::createCopy(getContext().context)); + + auto subtree = QueryPlan::cloneSubtree(source.getRootNode(), source); + + EXPECT_EQ(subtree.getMaxThreads(), 4u); + EXPECT_TRUE(subtree.getConcurrencyControl()); + EXPECT_EQ(subtree.getInterpretersContexts().size(), 1u); + EXPECT_EQ(source.getInterpretersContexts().size(), 1u); +} + +TEST(QueryPlanExecutionLimits, SerializationPreservesLimits) +{ + auto source = makeSourcePlan(); + source.setMaxThreads(4); + source.setConcurrencyControl(true); + + WriteBufferFromOwnString out; + source.serialize(out, DBMS_QUERY_PLAN_SERIALIZATION_VERSION); + + ReadBufferFromString in(out.str()); + auto deserialized = QueryPlan::deserialize(in, getContext().context, /*max_type_complexity=*/0); + auto restored = QueryPlan::makeSets(std::move(deserialized), getContext().context); + + EXPECT_EQ(restored.getMaxThreads(), 4u); + EXPECT_TRUE(restored.getConcurrencyControl()); +} + +TEST(QueryPlanExecutionLimits, SerializationRejectsOlderPeerWhenLimitsAreSet) +{ + auto source = makeSourcePlan(); + source.setMaxThreads(4); + + WriteBufferFromOwnString out; + EXPECT_THROW(source.serialize(out, DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS - 1), Exception); +} + +TEST(QueryPlanExecutionLimits, ReplaceNodeWithPlanMergesConcurrencyControl) +{ + auto destination = makeSourcePlan(); + destination.setMaxThreads(2); + + auto replacement = makeSourcePlan(); + replacement.setMaxThreads(4); + replacement.setConcurrencyControl(true); + + destination.replaceNodeWithPlan(destination.getRootNode(), std::move(replacement)); + + EXPECT_EQ(destination.getMaxThreads(), 4u); + EXPECT_TRUE(destination.getConcurrencyControl()); +} + +TEST(QueryPlanExecutionLimits, ExtractSubplanPreservesLimitsAndResources) +{ + auto source = makeSourcePlan(); + source.setMaxThreads(4); + source.setConcurrencyControl(true); + source.addInterpreterContext(Context::createCopy(getContext().context)); + + auto subplan = source.extractSubplan(source.getRootNode()); + + EXPECT_EQ(subplan.getMaxThreads(), 4u); + EXPECT_TRUE(subplan.getConcurrencyControl()); + EXPECT_EQ(subplan.getInterpretersContexts().size(), 1u); + EXPECT_EQ(source.getInterpretersContexts().size(), 1u); +} + +/// `unitePlans` already merged `max_threads` across the children; `concurrency_control` has to be +/// OR-ed the same way, so a united plan honours it whenever any child asked for it. Both child +/// orders are asserted: the folds accumulate over every child, so the result must not depend on +/// which child happens to come last. +TEST(QueryPlanExecutionLimits, UnitePlansMergesLimitsFromChildren) +{ + for (bool asking_child_first : {false, true}) + { + auto without = std::make_unique(makeSourcePlan()); + without->setMaxThreads(2); + without->setConcurrencyControl(false); + + auto with = std::make_unique(makeSourcePlan()); + with->setMaxThreads(4); + with->setConcurrencyControl(true); + + std::vector plans; + if (asking_child_first) + { + plans.emplace_back(std::move(with)); + plans.emplace_back(std::move(without)); + } + else + { + plans.emplace_back(std::move(without)); + plans.emplace_back(std::move(with)); + } + + SharedHeaders input_headers{makeHeader(), makeHeader()}; + QueryPlan united; + united.unitePlans(std::make_unique(input_headers), std::move(plans)); + + EXPECT_EQ(united.getMaxThreads(), 4u) << "asking_child_first=" << asking_child_first; + EXPECT_TRUE(united.getConcurrencyControl()) << "asking_child_first=" << asking_child_first; + } +} + +/// The merge must not invent a limit no child asked for: a plan united from children that all run +/// uncapped stays uncapped. +TEST(QueryPlanExecutionLimits, UnitePlansKeepsDefaultsWhenNoChildAsks) +{ + auto left = std::make_unique(makeSourcePlan()); + auto right = std::make_unique(makeSourcePlan()); + + SharedHeaders input_headers{makeHeader(), makeHeader()}; + std::vector plans; + plans.emplace_back(std::move(left)); + plans.emplace_back(std::move(right)); + + QueryPlan united; + united.unitePlans(std::make_unique(input_headers), std::move(plans)); + + EXPECT_EQ(united.getMaxThreads(), 0u); + EXPECT_FALSE(united.getConcurrencyControl()); +} diff --git a/src/Processors/Sources/LazyReadFromObjectStorageSource.cpp b/src/Processors/Sources/LazyReadFromObjectStorageSource.cpp index a45e15002cc7..4ff2440b8286 100644 --- a/src/Processors/Sources/LazyReadFromObjectStorageSource.cpp +++ b/src/Processors/Sources/LazyReadFromObjectStorageSource.cpp @@ -17,6 +17,7 @@ namespace ErrorCodes namespace Setting { extern const SettingsBool s3_validate_etag_on_read; + extern const SettingsBool azure_validate_etag_on_read; } namespace @@ -68,8 +69,9 @@ class LazyRowsObjectIterator : public IObjectIterator "Lazy materialization: no object metadata was captured for {} in the main reading pass", object.getPath()); - /// For S3 with `s3_validate_etag_on_read`, the GET itself is pinned to the captured ETag - /// (see `ReadBufferFromS3`), which is race-free; an extra HEAD here would buy nothing. + /// For S3 with `s3_validate_etag_on_read` (and Azure with `azure_validate_etag_on_read`), the GET + /// itself is pinned to the captured ETag (see `ReadBufferFromS3`, `ReadBufferFromAzureBlobStorage`), + /// which is race-free; an extra HEAD here would buy nothing. if (etag_validated_on_read && !captured->etag.empty()) return; @@ -215,8 +217,10 @@ IProcessor::PipelineUpdate LazyReadFromObjectStorageSource::updatePipeline() if (rows->rows_in_files.empty()) return {}; - const bool etag_validated_on_read = object_storage->getType() == ObjectStorageType::S3 - && context->getSettingsRef()[Setting::s3_validate_etag_on_read]; + const auto & settings = context->getSettingsRef(); + const bool etag_validated_on_read + = (object_storage->getType() == ObjectStorageType::S3 && settings[Setting::s3_validate_etag_on_read]) + || (object_storage->getType() == ObjectStorageType::Azure && settings[Setting::azure_validate_etag_on_read]); auto iterator = std::make_shared( std::move(rows->rows_in_files), object_storage, etag_validated_on_read, configuration->dataFilesAreImmutable()); diff --git a/src/Processors/Sources/MongoDBSource.cpp b/src/Processors/Sources/MongoDBSource.cpp index 4dca8b937265..e5993dde6597 100644 --- a/src/Processors/Sources/MongoDBSource.cpp +++ b/src/Processors/Sources/MongoDBSource.cpp @@ -160,7 +160,7 @@ MongoDBSource::MongoDBSource( const mongocxx::options::find & options, SharedHeader sample_block_, const UInt64 & max_block_size_) - : ISource{sample_block_} + : ISource{std::make_shared(sample_block_->cloneEmpty())} , client{uri} , database{client.database(uri.database())} , collection{database.collection(collection_name)} diff --git a/src/Processors/Sources/ShellCommandSource.cpp b/src/Processors/Sources/ShellCommandSource.cpp index b39668905a7f..01a0eca5d00e 100644 --- a/src/Processors/Sources/ShellCommandSource.cpp +++ b/src/Processors/Sources/ShellCommandSource.cpp @@ -503,7 +503,7 @@ namespace const ShellCommandSourceConfiguration & configuration_ = {}, std::unique_ptr && command_holder_ = nullptr, std::shared_ptr process_pool_ = nullptr) - : ISource(sample_block_) + : ISource(std::make_shared(sample_block_->cloneEmpty())) , context(context_) , format(format_) , sample_block(sample_block_) diff --git a/src/Processors/Sources/YTsaurusSource.cpp b/src/Processors/Sources/YTsaurusSource.cpp index 64e1b1250626..48e8faa9a12d 100644 --- a/src/Processors/Sources/YTsaurusSource.cpp +++ b/src/Processors/Sources/YTsaurusSource.cpp @@ -31,7 +31,7 @@ namespace YTsaurusSetting YTsaurusTableSourceStaticTable::YTsaurusTableSourceStaticTable( YTsaurusClientPtr client_, const String & cypress_path_, std::pair rows_range_, const YTsaurusTableSourceOptions & source_options_, const SharedHeader & sample_block_, const UInt64 & max_block_size_) - : ISource(sample_block_) + : ISource(std::make_shared(sample_block_->cloneEmpty())) , client(std::move(client_)) , cypress_path(cypress_path_) , rows_range(std::move(rows_range_)) @@ -62,7 +62,7 @@ Chunk YTsaurusTableSourceStaticTable::generate() YTsaurusTableSourceDynamicTable::YTsaurusTableSourceDynamicTable( YTsaurusClientPtr client_, const String & cypress_path, const YTsaurusTableSourceOptions & source_options_, const SharedHeader & sample_block_, const UInt64 & max_block_size_) - : ISource(sample_block_) + : ISource(std::make_shared(sample_block_->cloneEmpty())) , client(std::move(client_)) , sample_block(sample_block_) , max_block_size(max_block_size_) diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp index e03ad193b77e..6a7c49764e0f 100644 --- a/src/Processors/Transforms/AggregatingTransform.cpp +++ b/src/Processors/Transforms/AggregatingTransform.cpp @@ -546,6 +546,7 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor AggregatingTransformParamsPtr params_, ManyAggregatedDataVariantsPtr data_, size_t num_threads_, + size_t output_streams_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_, AdaptiveAggregationSessionPtr adaptive_session_) : IProcessor({}, {params_->getHeader()}) @@ -553,6 +554,7 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor , data(std::move(data_)) , shared_data(std::make_shared()) , num_threads(num_threads_) + , output_streams(output_streams_) , updater(std::move(updater_)) , adaptive_session(std::move(adaptive_session_)) { @@ -907,6 +909,11 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor size_t num_threads; + /// How many streams the output is spread over downstream. It is not `num_threads`. That is capped by the + /// number of aggregating streams (1 for a single input stream), while the `Resize` after the aggregation + /// fans out to `max_threads`. 1 when the results must go out in bucket order. + size_t output_streams; + RuntimeDataflowStatisticsCacheUpdaterPtr updater; AdaptiveAggregationSessionPtr adaptive_session; @@ -1003,7 +1010,11 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor throw Exception(ErrorCodes::UNKNOWN_AGGREGATED_DATA_VARIANT, "Unknown aggregated data variant."); } - auto agg_chunks = params->aggregator.prepareChunkAndFillSingleLevel(*first, params->final); + const size_t max_rows_per_block = Aggregator::singleLevelChunkRowsForFanOut(first->sizeWithoutOverflowRow(), output_streams); + if (max_rows_per_block) + LOG_TRACE(getLogger("AggregatingTransform"), "Split single level result into chunks of at most {} rows.", max_rows_per_block); + + auto agg_chunks = params->aggregator.prepareChunkAndFillSingleLevel(*first, params->final, max_rows_per_block); for (auto & agg_chunk : agg_chunks) { if (agg_chunk.chunk.getNumRows() > 0) @@ -1086,7 +1097,7 @@ class ConvertingAggregatedToChunksTransform final : public IProcessor }; AggregatingTransform::AggregatingTransform( - SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_) + SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_, size_t output_streams_) : AggregatingTransform( std::move(header), std::move(params_), @@ -1096,7 +1107,8 @@ AggregatingTransform::AggregatingTransform( 1, true /* should_produce_results_in_order_of_bucket_number */, false /* skip_merging */, - updater_) + updater_, + output_streams_) { } @@ -1109,7 +1121,8 @@ AggregatingTransform::AggregatingTransform( size_t temporary_data_merge_threads_, bool should_produce_results_in_order_of_bucket_number_, bool skip_merging_, - RuntimeDataflowStatisticsCacheUpdaterPtr updater_) + RuntimeDataflowStatisticsCacheUpdaterPtr updater_, + size_t output_streams_) : IProcessor({std::move(header)}, {params_->getHeader()}) , params(std::move(params_)) , key_columns(params->params.keys_size) @@ -1121,6 +1134,7 @@ AggregatingTransform::AggregatingTransform( , should_produce_results_in_order_of_bucket_number(should_produce_results_in_order_of_bucket_number_) , skip_merging(skip_merging_) , updater(std::move(updater_)) + , output_streams(output_streams_) { /// `AggregatingStep` leaves its engagement verdict in the flag. Without a producer nothing is ever /// staged, so the merge-time drains find empty backlogs and do nothing. @@ -1412,7 +1426,12 @@ void AggregatingTransform::initGenerate() std::move(many_data->variants), adaptive_context ? adaptive_context->session.get() : nullptr); auto prepared_data_ptr = std::make_shared(std::move(prepared_data)); processors.emplace_back(std::make_shared( - params, std::move(prepared_data_ptr), max_threads, updater, adaptive_engaged ? adaptive_context->session : nullptr)); + params, + std::move(prepared_data_ptr), + max_threads, + output_streams, + updater, + adaptive_engaged ? adaptive_context->session : nullptr)); } else { diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h index 6be36c2f7677..9842346a5e2f 100644 --- a/src/Processors/Transforms/AggregatingTransform.h +++ b/src/Processors/Transforms/AggregatingTransform.h @@ -115,7 +115,7 @@ using ManyAggregatedDataPtr = std::shared_ptr; class AggregatingTransform final : public IProcessor { public: - AggregatingTransform(SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_); + AggregatingTransform(SharedHeader header, AggregatingTransformParamsPtr params_, RuntimeDataflowStatisticsCacheUpdaterPtr updater_, size_t output_streams_ = 1); /// For Parallel aggregating. AggregatingTransform( @@ -127,7 +127,8 @@ class AggregatingTransform final : public IProcessor size_t temporary_data_merge_threads, bool should_produce_results_in_order_of_bucket_number_ = true, bool skip_merging_ = false, - RuntimeDataflowStatisticsCacheUpdaterPtr updater_ = nullptr); + RuntimeDataflowStatisticsCacheUpdaterPtr updater_ = nullptr, + size_t output_streams_ = 1); ~AggregatingTransform() override; @@ -195,6 +196,9 @@ class AggregatingTransform final : public IProcessor RuntimeDataflowStatisticsCacheUpdaterPtr updater; + /// How many streams `AggregatingStep` spreads this transform's output over; 1 when it doesn't. + size_t output_streams = 1; + void initGenerate(); }; diff --git a/src/Processors/Transforms/DistinctTransform.cpp b/src/Processors/Transforms/DistinctTransform.cpp index 5d693a474108..8ac959c1a338 100644 --- a/src/Processors/Transforms/DistinctTransform.cpp +++ b/src/Processors/Transforms/DistinctTransform.cpp @@ -394,7 +394,7 @@ void DistinctTransform::transform(Chunk & chunk) else { for (auto & column : columns) - column = column->filter(filter, -1); + column = column->filter(filter, num_selected); chunk.setColumns(std::move(columns), num_selected); } diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 6c3a11a3944f..4e080b4931ae 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -64,8 +64,9 @@ static ActionsDAG makeReorderingActions(const Block & in_header, const GroupingS MergingAggregatedTransform::~MergingAggregatedTransform() = default; MergingAggregatedTransform::MergingAggregatedTransform( - SharedHeader header_, Aggregator::Params params, bool final, GroupingSetsParamsList grouping_sets_params) + SharedHeader header_, Aggregator::Params params, bool final, GroupingSetsParamsList grouping_sets_params, size_t output_streams_) : IAccumulatingTransform(header_, std::make_shared(appendGroupingIfNeeded(*header_, params.getHeader(*header_, final)))) + , output_streams(output_streams_) { if (!grouping_sets_params.empty()) { @@ -258,7 +259,8 @@ Chunk MergingAggregatedTransform::generate() /// TODO: this operation can be made async. Add async for IAccumulatingTransform. params->aggregator.mergeBlocks(std::move(bucket_to_chunks), data_variants, is_cancelled); - auto merged_chunks = params->aggregator.convertToChunks(data_variants, params->final); + const size_t max_rows_per_block = Aggregator::singleLevelChunkRowsForFanOut(data_variants.sizeWithoutOverflowRow(), output_streams); + auto merged_chunks = params->aggregator.convertToChunks(data_variants, params->final, max_rows_per_block); if (grouping_set.creating_missing_keys_actions) { diff --git a/src/Processors/Transforms/MergingAggregatedTransform.h b/src/Processors/Transforms/MergingAggregatedTransform.h index 69388c021dd3..4ace709bafe9 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.h +++ b/src/Processors/Transforms/MergingAggregatedTransform.h @@ -15,7 +15,7 @@ using ExpressionActionsPtr = std::shared_ptr; class MergingAggregatedTransform final : public IAccumulatingTransform { public: - MergingAggregatedTransform(SharedHeader header_, Aggregator::Params params_, bool final_, GroupingSetsParamsList grouping_sets_params); + MergingAggregatedTransform(SharedHeader header_, Aggregator::Params params_, bool final_, GroupingSetsParamsList grouping_sets_params, size_t output_streams_ = 1); ~MergingAggregatedTransform() override; @@ -50,6 +50,9 @@ class MergingAggregatedTransform final : public IAccumulatingTransform bool consume_started = false; bool generate_started = false; + /// How many streams `MergingAggregatedStep` spreads the output over; 1 when it doesn't. + size_t output_streams = 1; + void addChunk(Columns columns, size_t num_rows, Int32 bucket_num, bool is_overflows); }; diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 87bce9b87531..53944d7ee9fc 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -475,6 +475,22 @@ void RemoteQueryExecutor::sendQueryUnlocked(ClientInfo::QueryKind query_kind, As connections = create_connections(async_callback); AsyncCallbackSetter async_callback_setter(connections.get(), async_callback); + /// Plan-level execution limits are serialized beginning with version 10. Before that version, + /// sending a plan would silently lose them. Use the original SQL request for old replicas: it + /// carries the query settings and lets the remote server build a plan with the same limits. + /// This keeps `serialize_query_plan` usable while a cluster is being upgraded. + if (query_plan + && (query_plan->getMaxThreads() || query_plan->getConcurrencyControl()) + && !connections->supportsQueryPlanSerializationVersion(DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS)) + { + LOG_DEBUG( + log, + "Sending query as SQL because a replica does not support query-plan serialization version {} required for execution limits", + DBMS_MIN_QUERY_PLAN_SERIALIZATION_VERSION_WITH_EXECUTION_LIMITS); + query_plan.reset(); + stage = query_plan_fallback_stage; + } + const auto & settings = context->getSettingsRef(); if (isReplicaUnavailable() || needToSkipUnavailableShard()) { diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h index cbeda47e98c9..3c212b2c44b5 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.h +++ b/src/QueryPipeline/RemoteQueryExecutor.h @@ -130,6 +130,9 @@ class RemoteQueryExecutor void sendQuery(ClientInfo::QueryKind query_kind = ClientInfo::QueryKind::SECONDARY_QUERY, AsyncCallback async_callback = {}); void sendQueryUnlocked(ClientInfo::QueryKind query_kind = ClientInfo::QueryKind::SECONDARY_QUERY, AsyncCallback async_callback = {}) TSA_REQUIRES(was_cancelled_mutex); + /// Stage used when a remote replica is too old to receive the query plan and the executor sends SQL instead. + void setQueryPlanFallbackStage(QueryProcessingStage::Enum stage_) { query_plan_fallback_stage = stage_; } + int sendQueryAsync(); struct ReadResult @@ -273,6 +276,7 @@ class RemoteQueryExecutor /// Temporary tables needed to be sent to remote servers Tables external_tables; QueryProcessingStage::Enum stage; + QueryProcessingStage::Enum query_plan_fallback_stage = QueryProcessingStage::Complete; std::optional extension; /// Initiator identifier for distributed task processing diff --git a/src/Server/ArrowFlight/ArrowFlightServer.cpp b/src/Server/ArrowFlight/ArrowFlightServer.cpp index 87627bbce6af..c560ba7838ed 100644 --- a/src/Server/ArrowFlight/ArrowFlightServer.cpp +++ b/src/Server/ArrowFlight/ArrowFlightServer.cpp @@ -1376,14 +1376,14 @@ arrow::Status ArrowFlightServer::DoAction( } }; - for (const auto & [setting, value] : request.session_options) + auto apply_option = [&](const std::string & setting, const auto & value) { if (!isValidIdentifier(setting)) { result.errors[setting] = arrow::flight::SetSessionOptionsResult::Error{ arrow::flight::SetSessionOptionErrorValue::kInvalidName }; - continue; + return; } try @@ -1391,14 +1391,14 @@ arrow::Status ArrowFlightServer::DoAction( if (std::holds_alternative(value)) { /// std::monostate means "reset to default" (SET setting = DEFAULT). - query_context->checkSettingsConstraintsForSettingsReset({setting}, SettingSource::QUERY); + session_context->checkSettingsConstraintsForSettingsReset({setting}, SettingSource::QUERY); session_context->resetSettingsToDefaultValue({setting}); } else { auto string_value = std::visit(to_string_value, value); SettingChange change{setting, Field{string_value}}; - query_context->checkSettingsConstraints(change, SettingSource::QUERY); + session_context->checkSettingsConstraints(change, SettingSource::QUERY); session_context->setSetting(setting, string_value); } } @@ -1416,6 +1416,15 @@ arrow::Status ArrowFlightServer::DoAction( result.errors[setting] = arrow::flight::SetSessionOptionsResult::Error{error_value}; } + }; + + /// The options arrive in a map with no order, so `profile` goes first for its constraints to bind the rest. + if (auto profile = request.session_options.find("profile"); profile != request.session_options.end()) + apply_option(profile->first, profile->second); + for (const auto & [setting, value] : request.session_options) + { + if (setting != "profile") + apply_option(setting, value); } ARROW_ASSIGN_OR_RAISE(auto serialized, result.SerializeToString()) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 787c4e1fba8a..543508d2c9e4 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -3,6 +3,7 @@ #if USE_SSL #include +#include #include #include #include @@ -23,6 +24,7 @@ CertificateReloader & CertificateReloader::instance() namespace ErrorCodes { extern const int INVALID_CONFIG_PARAMETER; + extern const int OPENSSL_ERROR; } namespace @@ -38,6 +40,91 @@ int callSetCertificate(SSL * ssl, void * arg) return CertificateReloader::instance().setCertificate(ssl, pdata); } +/// Called by OpenSSL instead of `X509_verify_cert` to verify the peer's certificate. +int callVerifyCertificate(X509_STORE_CTX * store_ctx, void * arg) +{ + const MultiVersion * ca_store = reinterpret_cast *>(arg); + return CertificateReloader::instance().verifyCertificate(store_ctx, ca_store); +} + +int defaultVerifyCertificate(X509_STORE_CTX * store_ctx) +{ + /// Same as libssl does when no callback is installed: an error is treated as a verification failure. + int ok = X509_verify_cert(store_ctx); + return ok < 0 ? 0 : ok; +} + +/// Verify the certificate from `original_ctx` like `X509_verify_cert(original_ctx)` would, but against the trusted certificates in `store`. +/// libssl prepares `original_ctx` in `ssl_verify_internal` (ssl/ssl_cert.c) before calling the application callback. +/// The same preparation is carried over to a new `X509_STORE_CTX` that uses `store`, and the outcome is reported back +/// through `original_ctx`, because that is what libssl looks at after the callback returns. +int verifyCertificateWithStore(X509_STORE_CTX * original_ctx, X509 * certificate, X509_STORE * store) +{ + std::unique_ptr verify_ctx(X509_STORE_CTX_new(), X509_STORE_CTX_free); + if (!verify_ctx) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_OUT_OF_MEM); + return 0; + } + + if (X509_STORE_CTX_init(verify_ctx.get(), store, certificate, X509_STORE_CTX_get0_untrusted(original_ctx)) != 1) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } + + /// The connection, for the per-connection verification callbacks (e.g. the ones of Poco and boost::asio) that look it up. + int ssl_idx = SSL_get_ex_data_X509_STORE_CTX_idx(); + SSL * ssl = static_cast(X509_STORE_CTX_get_ex_data(original_ctx, ssl_idx)); + if (ssl) + { + if (X509_STORE_CTX_set_ex_data(verify_ctx.get(), ssl_idx, ssl) != 1) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } + /// Has an effect only if DANE is enabled for the connection. + X509_STORE_CTX_set0_dane(verify_ctx.get(), SSL_get0_dane(ssl)); + } + + /// The default purpose depends on which side is verified. Everything libssl derived from the connection and its `SSL_CTX` + /// (verification depth, expected host name, security level, flags, ...) is already in the parameters of `original_ctx`. + X509_STORE_CTX_set_default(verify_ctx.get(), (ssl && SSL_is_server(ssl)) ? "ssl_client" : "ssl_server"); + if (X509_VERIFY_PARAM_set1(X509_STORE_CTX_get0_param(verify_ctx.get()), X509_STORE_CTX_get0_param(original_ctx)) != 1) + { + X509_STORE_CTX_set_error(original_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } + X509_STORE_CTX_set_verify_cb(verify_ctx.get(), X509_STORE_CTX_get_verify_cb(original_ctx)); + + int ok = defaultVerifyCertificate(verify_ctx.get()); + + X509_STORE_CTX_set_error(original_ctx, X509_STORE_CTX_get_error(verify_ctx.get())); + X509_STORE_CTX_set_error_depth(original_ctx, X509_STORE_CTX_get_error_depth(verify_ctx.get())); + if (STACK_OF(X509) * chain = X509_STORE_CTX_get1_chain(verify_ctx.get())) + X509_STORE_CTX_set0_verified_chain(original_ctx, chain); + X509_VERIFY_PARAM_move_peername(X509_STORE_CTX_get0_param(original_ctx), X509_STORE_CTX_get0_param(verify_ctx.get())); + + return ok; +} + +/// Load the trusted CA certificates the same way `Poco::Net::Context` does it for the contexts created at startup, +/// so that a reload results in the same set of trusted certificates as a restart would. +std::unique_ptr loadCAStore(const std::string & ca_path, bool load_default_cas) +{ + Poco::Net::Context::Params params; + params.caLocation = ca_path; + params.loadDefaultCAs = load_default_cas; + /// Only the certificate store is taken from this context, so the usage does not matter. + Poco::Net::Context context(Poco::Net::Context::CLIENT_USE, params); + + X509_STORE * store = SSL_CTX_get_cert_store(context.sslContext()); + if (!store || X509_STORE_up_ref(store) != 1) + throw Exception(ErrorCodes::OPENSSL_ERROR, "Cannot get CA certificates from SSL context: {}", Poco::Net::Utility::getLastError()); + + return std::make_unique(store, context.getCAPaths()); +} + } /// This is callback for OpenSSL. It will be called on every connection to obtain a certificate and private key. @@ -50,6 +137,28 @@ int CertificateReloader::setCertificate(SSL * ssl, const CertificateReloader::Mu return setCertificateCallback(ssl, current.get(), log); } +/// This is callback for OpenSSL. It will be called on every connection that verifies the certificate of the peer. +int CertificateReloader::verifyCertificate(X509_STORE_CTX * store_ctx, const MultiVersion * ca_store) const +{ + try + { + auto current = ca_store->get(); + X509 * certificate = X509_STORE_CTX_get0_cert(store_ctx); + + /// Raw public keys (RFC 7250) are verified without CA certificates. + if (!current || !certificate) + return defaultVerifyCertificate(store_ctx); + + return verifyCertificateWithStore(store_ctx, certificate, current->store); + } + catch (...) + { + LOG_ERROR(log, getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false)); + X509_STORE_CTX_set_error(store_ctx, X509_V_ERR_UNSPECIFIED); + return 0; + } +} + int setCertificateCallback(SSL * ssl, const CertificateReloader::Data * current_data, LoggerPtr log) { if (current_data->certs_chain.empty()) @@ -142,6 +251,10 @@ std::list::iterator CertificateReloader::findOrI data.push_back(MultiData(ctx)); --it; data_index[prefix] = it; + + /// Verify peer certificates against the reloadable CA certificates of this prefix. + /// Until (and unless) they are loaded, the callback does exactly what OpenSSL does without it. + SSL_CTX_set_cert_verify_callback(ctx, callVerifyCertificate, reinterpret_cast(&it->ca.store)); } return it; } @@ -179,8 +292,51 @@ void CertificateReloader::tryLoadACMECertificate(SSL_CTX * ctx, const std::strin } } +void CertificateReloader::tryLoadCAImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) +{ + std::string new_ca_path = config.getString(prefix + Poco::Net::SSLManager::CFG_CA_LOCATION, ""); + + /// Without `caConfig` the trusted certificates come only from the system locations (if at all), there is nothing to reload. + /// But if `caConfig` was there before, keep following the configuration, like a restart would. + if (new_ca_path.empty()) + { + auto index_it = data_index.find(prefix); + if (index_it == data_index.end() || index_it->second->ca_file.path.empty()) + return; + } + + try + { + auto it = findOrInsert(ctx, prefix); + bool ca_file_changed = it->ca_file.changeIfModified(std::move(new_ca_path), log); + + auto load = [&](CAData & ca, bool load_default_cas_default) + { + bool new_load_default_cas = config.getBool(prefix + Poco::Net::SSLManager::CFG_ENABLE_DEFAULT_CA, load_default_cas_default); + if (!ca_file_changed && ca.store.get() && new_load_default_cas == ca.load_default_cas) + return; + + LOG_DEBUG(log, "Reloading CA certificates ({}), load default CAs: {}.", it->ca_file.path, new_load_default_cas); + ca.store.set(loadCAStore(it->ca_file.path, new_load_default_cas)); + ca.load_default_cas = new_load_default_cas; + LOG_INFO(log, "Reloaded CA certificates ({}), load default CAs: {}.", it->ca_file.path, new_load_default_cas); + }; + + load(it->ca, Poco::Net::SSLManager::VAL_ENABLE_DEFAULT_CA); + if (it->ca_with_other_default) + load(*it->ca_with_other_default, it->other_load_default_cas_default); + } + catch (...) + { + LOG_ERROR(log, getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false)); + } +} + void CertificateReloader::tryLoadImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) { + /// Trusted CA certificates do not depend on how the own certificate is configured. + tryLoadCAImpl(config, ctx, prefix); + /// If at least one of the files is modified - recreate std::string new_cert_path = config.getString(prefix + "certificateFile", ""); std::string new_key_path = config.getString(prefix + "privateKeyFile", ""); @@ -239,7 +395,7 @@ void CertificateReloader::tryReloadAll(const Poco::Util::AbstractConfiguration & } -bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix) +bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix, bool load_default_cas_default) { if (!ctx) return false; @@ -256,9 +412,24 @@ bool CertificateReloader::registerAdditionalContext(SSL_CTX * ctx, const std::st MultiData * pdata = &*(it->second); + /// Share the reloadable CA certificates of this prefix (see `findOrInsert`). A context that assumes another default + /// for `loadDefaultCAFile` may trust other CA certificates than the primary one, so it gets its own ones. + /// They are loaded by the next `tryLoad`, until then the context keeps using the store it was created with. + CAData * ca = &pdata->ca; + if (load_default_cas_default != Poco::Net::SSLManager::VAL_ENABLE_DEFAULT_CA) + { + if (!pdata->ca_with_other_default) + { + pdata->ca_with_other_default.emplace(); + pdata->other_load_default_cas_default = load_default_cas_default; + } + ca = &*pdata->ca_with_other_default; + } + SSL_CTX_set_cert_verify_callback(ctx, callVerifyCertificate, reinterpret_cast(&ca->store)); + /// Verify that certificate data was actually loaded, not just the entry created. /// If data is null, return false so caller can use fallback (static cert loading). - /// This can happen if initial cert parsing failed in tryLoadImpl. + /// This can happen if initial cert parsing failed in tryLoadImpl or if only `caConfig` is set for the prefix. if (!pdata->data.get()) { LOG_WARNING(log, "Cannot register additional context for prefix '{}': certificate data not loaded. " @@ -292,6 +463,22 @@ std::optional CertificateReloader::getCertificate(const std::st } +std::optional CertificateReloader::getCAPaths(const std::string & prefix) const +{ + std::lock_guard lock{data_mutex}; + + auto it = data_index.find(prefix); + if (it == data_index.end()) + return {}; + + auto current = it->second->ca.store.get(); + if (!current) + return {}; + + return current->paths; +} + + CertificateReloader::Data::Data(std::string cert_path, std::string key_path, std::string pass_phrase) : certs_chain(X509Certificate::fromFile(cert_path)), key(KeyPair::fromFile(key_path, pass_phrase)) { @@ -305,6 +492,14 @@ CertificateReloader::Data::Data(KeyPair _pkey, X509Certificate::List _certs_chai bool CertificateReloader::File::changeIfModified(std::string new_path, LoggerPtr logger) { + if (new_path.empty()) + { + bool changed = !path.empty(); + path.clear(); + modification_time = {}; + return changed; + } + std::error_code ec; std::filesystem::file_time_type new_modification_time = std::filesystem::last_write_time(new_path, ec); if (ec) @@ -318,10 +513,24 @@ bool CertificateReloader::File::changeIfModified(std::string new_path, LoggerPtr return false; } - if (new_path != path || new_modification_time != modification_time) + /// `caConfig` can be a directory with certificates, replacing one of them is a change too. + UInt64 new_directory_contents_hash = 0; + if (std::filesystem::is_directory(new_path, ec)) + { + SipHash hash; + for (const auto & entry : std::filesystem::directory_iterator(new_path, ec)) + { + hash.update(entry.path().filename().string()); + hash.update(std::filesystem::last_write_time(entry.path(), ec).time_since_epoch().count()); + } + new_directory_contents_hash = hash.get64(); + } + + if (new_path != path || new_modification_time != modification_time || new_directory_contents_hash != directory_contents_hash) { path = new_path; modification_time = new_modification_time; + directory_contents_hash = new_directory_contents_hash; return true; } diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 44330a1f7007..735b40392d43 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -27,17 +28,34 @@ namespace DB { -/// The CertificateReloader singleton performs 2 functions: -/// 1. Dynamic reloading of TLS key-pair when requested by server: +/// The CertificateReloader singleton performs 3 functions: +/// 1. Dynamic reloading of TLS key-pair and of the trusted CA certificates (`caConfig`) when requested by server: /// Server config reloader notifies CertificateReloader when the config changes. /// On changed config, CertificateReloader reloads certs from disk. /// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: /// OpenSSL invokes a callback to setup a connection. +/// 3. Implement `SSL_CTX_set_cert_verify_callback` to verify the peer's certificate of a new connection +/// against the most recently loaded CA certificates. +/// +/// An `SSL_CTX` that is shared between threads must not be modified, so instead of touching the contexts on reload, +/// both callbacks apply the current immutable snapshot (`MultiVersion`) to each new connection. class CertificateReloader { public: using stat_t = struct stat; + /// Owns a reference to a set of trusted CA certificates and remembers where they were loaded from. + struct CAStore + { + CAStore(X509_STORE * store_, Poco::Net::Context::CAPaths paths_) : store(store_), paths(std::move(paths_)) {} + CAStore(const CAStore &) = delete; + CAStore & operator=(const CAStore &) = delete; + ~CAStore() { X509_STORE_free(store); } + + X509_STORE * const store; + const Poco::Net::Context::CAPaths paths; + }; + struct Data { X509Certificate::List certs_chain; @@ -56,10 +74,19 @@ class CertificateReloader std::string path; std::filesystem::file_time_type modification_time; + /// For a directory: the names and modification times of the files in it. + UInt64 directory_contents_hash = 0; bool changeIfModified(std::string new_path, LoggerPtr logger); }; + /// Trusted CA certificates from `caConfig` and, if `load_default_cas`, the default ones. + struct CAData + { + MultiVersion store; + bool load_default_cas = false; + }; + struct MultiData { SSL_CTX * ctx = nullptr; @@ -69,6 +96,14 @@ class CertificateReloader File cert_file{"certificate"}; File key_file{"key"}; + /// Empty if `caConfig` is not set for the prefix, then verification keeps using the store the context was created with. + CAData ca; + /// For additional contexts that assume another `loadDefaultCAFile` than Poco when it is not configured (the ones of Keeper). + std::optional ca_with_other_default; + bool other_load_default_cas_default = false; + + File ca_file{"CA"}; + explicit MultiData(SSL_CTX * ctx_) : ctx(ctx_) {} }; @@ -88,8 +123,11 @@ class CertificateReloader /// Handle configuration reload void tryLoad(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix); - /// Register an additional SSL_CTX to share certificates with the primary context - bool registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix); + /// Register an additional SSL_CTX to share certificates and trusted CAs with the primary context of `prefix`. + /// `load_default_cas_default` is what the caller assumes for `loadDefaultCAFile` when it is not configured. + /// Returns true if the context will get its certificate and key from CertificateReloader, + /// false if the caller has to configure them on the context itself. + bool registerAdditionalContext(SSL_CTX * ctx, const std::string & prefix, bool load_default_cas_default); /// Handle configuration reload for all contexts void tryReloadAll(const Poco::Util::AbstractConfiguration & config); @@ -97,11 +135,18 @@ class CertificateReloader /// A callback for OpenSSL int setCertificate(SSL * ssl, const MultiData * pdata); + /// A callback for OpenSSL: verify the peer certificate in `store_ctx` against the current CA certificates in `ca_store`. + int verifyCertificate(X509_STORE_CTX * store_ctx, const MultiVersion * ca_store) const; + /// The leaf certificate that is currently served for `prefix` connections, if there is one. /// It is not necessarily the certificate of the corresponding `SSL_CTX`: certificates are installed /// per connection, and with `` the context itself never gets a certificate at all. std::optional getCertificate(const std::string & prefix) const; + /// Where the CA certificates that are currently used to verify peers of `prefix` connections were loaded from, + /// if they are managed by CertificateReloader (i.e. `caConfig` is set for the prefix). + std::optional getCAPaths(const std::string & prefix) const; + private: CertificateReloader() = default; @@ -111,6 +156,7 @@ class CertificateReloader /// Unsafe implementation void tryLoadImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); void tryLoadACMECertificate(SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); + void tryLoadCAImpl(const Poco::Util::AbstractConfiguration & config, SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); std::list::iterator findOrInsert(SSL_CTX * ctx, const std::string & prefix) TSA_REQUIRES(data_mutex); diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index f8dd4d717c2a..ae5a796bfe12 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -310,6 +310,20 @@ namespace } }; + /// A protobuf map has no order, so `profile` goes first for its constraints to bind the other settings. + SettingsChanges settingsChangesFromMap(const google::protobuf::Map & map) + { + SettingsChanges changes; + for (const auto & [key, value] : map) + { + if (key == "profile") + changes.insert(changes.begin(), {key, value}); + else + changes.push_back({key, value}); + } + return changes; + } + /// Gets session's timeout from query info or from the server config. std::chrono::steady_clock::duration getSessionTimeout(const GRPCQueryInfo & query_info, const Poco::Util::AbstractConfiguration & config) { @@ -945,12 +959,7 @@ namespace query_context = session->makeQueryContext(std::move(client_info)); - /// Prepare settings. - SettingsChanges settings_changes; - for (const auto & [key, value] : query_info.settings()) - { - settings_changes.push_back({key, value}); - } + auto settings_changes = settingsChangesFromMap(query_info.settings()); query_context->checkSettingsConstraints(settings_changes, SettingSource::QUERY); query_context->applySettingsChanges(settings_changes); @@ -1277,9 +1286,7 @@ namespace { temp_context = Context::createCopy(query_context); external_table_context = temp_context; - SettingsChanges settings_changes; - for (const auto & [key, value] : external_table.settings()) - settings_changes.push_back({key, value}); + auto settings_changes = settingsChangesFromMap(external_table.settings()); external_table_context->checkSettingsConstraints(settings_changes, SettingSource::QUERY); external_table_context->applySettingsChanges(settings_changes); } diff --git a/src/Server/HTTP/HTTPServerRequest.cpp b/src/Server/HTTP/HTTPServerRequest.cpp index c580ee955f4b..a0cbe4e8a2c0 100644 --- a/src/Server/HTTP/HTTPServerRequest.cpp +++ b/src/Server/HTTP/HTTPServerRequest.cpp @@ -93,7 +93,9 @@ HTTPServerRequest::HTTPServerRequest(HTTPContextPtr context, HTTPServerResponse else if (hasContentLength()) { size_t content_length = getContentLength(); - stream = std::make_shared(std::move(in), LimitReadBuffer::Settings{.read_no_less = content_length, .read_no_more = content_length, .expect_eof = true}); + /// No `expect_eof`: `Content-Length` already defines where the body ends, and with keep-alive + /// the socket legitimately holds the bytes of the next request. + stream = std::make_shared(std::move(in), LimitReadBuffer::Settings{.read_no_less = content_length, .read_no_more = content_length}); stream_is_bounded = true; } else if (getMethod() != HTTPRequest::HTTP_GET && getMethod() != HTTPRequest::HTTP_HEAD && getMethod() != HTTPRequest::HTTP_DELETE) diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 3637fe61d4f3..df830316d51b 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -94,6 +94,7 @@ namespace Setting extern const SettingsBool http_allow_table_as_file; extern const SettingsBool http_allow_filters_as_path; extern const SettingsBool http_allow_filters_as_unrecognized_url_parameters; + extern const SettingsBool http_x_clickhouse_format_overrides_output_format; extern const SettingsString compression; extern const SettingsString filter; extern const SettingsString format; @@ -393,20 +394,15 @@ void HTTPHandler::processQuery( deferred_unrecognized_params.emplace_back(key, value); } - /// The `X-ClickHouse-Database` header is an alias for the `database` setting, and - /// `X-ClickHouse-Format` is an alias for the `output_format` setting. They override any matching - /// URL parameter (preserving the historical precedence). - /// - /// `X-ClickHouse-Format` maps to `output_format` rather than to `default_format`: sending this - /// header means the client definitely wants the response in that format, so it is an explicit - /// override (winning over the query's `FORMAT` clause and the path extension), not a fallback - /// used only when nothing else selects a format. It maps to `output_format` and not to the - /// bidirectional `format`, because the header has always described the response only: the same - /// header on `INSERT INTO t FORMAT JSONEachRow …` must not reinterpret the request body. + /// The `X-ClickHouse-Database` header is an alias for the `database` setting. It overrides a + /// matching URL parameter (preserving the historical precedence). if (auto header_value = request.get("X-ClickHouse-Database", ""); !header_value.empty()) settings_changes.setSetting("database", header_value); - if (auto header_value = request.get("X-ClickHouse-Format", ""); !header_value.empty()) - settings_changes.setSetting("output_format", header_value); + + /// The `X-ClickHouse-Format` header is applied below, once the settings from the URL and the user + /// profile are in effect: which setting it aliases depends on + /// `http_x_clickhouse_format_overrides_output_format`. + const String format_header_value = request.get("X-ClickHouse-Format", ""); ContextMutablePtr context; { @@ -480,6 +476,29 @@ void HTTPHandler::processQuery( context->checkSettingsConstraints(settings_changes, SettingSource::QUERY); context->applySettingsChanges(settings_changes); + /// The `X-ClickHouse-Format` header is an alias for the `output_format` setting, or - when + /// `http_x_clickhouse_format_overrides_output_format` is disabled - for the `default_format` + /// setting, which is what it meant before 26.8. Either way it overrides the URL parameter of the + /// same name (preserving the historical precedence). The choice is read from the context after + /// the URL parameters and the user profile have been applied, so the compatibility setting can + /// come from either of them. + /// + /// By default `X-ClickHouse-Format` maps to `output_format` rather than to `default_format`: + /// sending this header means the client definitely wants the response in that format, so it is + /// an explicit override (winning over the query's `FORMAT` clause and the path extension), not a + /// fallback used only when nothing else selects a format. It maps to `output_format` and not to + /// the bidirectional `format`, because the header has always described the response only: the + /// same header on `INSERT INTO t FORMAT JSONEachRow …` must not reinterpret the request body. + if (!format_header_value.empty()) + { + SettingsChanges format_header_changes; + format_header_changes.setSetting( + context->getSettingsRef()[Setting::http_x_clickhouse_format_overrides_output_format] ? "output_format" : "default_format", + format_header_value); + context->checkSettingsConstraints(format_header_changes, SettingSource::QUERY); + context->applySettingsChanges(format_header_changes); + } + const auto & settings = context->getSettingsRef(); /// === URL path parsing happens after settings are applied === diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 3ae104728940..1235b44bb4a2 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -276,10 +276,10 @@ KeeperTCPHandler::KeeperTCPHandler( } } -void KeeperTCPHandler::sendHandshake(bool has_leader, bool & use_compression) +void KeeperTCPHandler::sendHandshake(HandshakeResult result, bool & use_compression) { Coordination::write(Coordination::SERVER_HANDSHAKE_LENGTH, *out); - if (has_leader) + if (result != HandshakeResult::Rejected) { if (expect_opentelemetry_tracing_context) Coordination::write(Coordination::ZOOKEEPER_PROTOCOL_VERSION_WITH_TRACING, *out); @@ -298,8 +298,11 @@ void KeeperTCPHandler::sendHandshake(bool has_leader, bool & use_compression) Coordination::write(Coordination::KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT, *out); } - Coordination::write(static_cast(session_timeout.totalMilliseconds()), *out); - Coordination::write(session_id, *out); + /// A zero timeout with a zero session id tells a ZooKeeper client that its session has expired. + const bool expired = result == HandshakeResult::SessionExpired; + Coordination::write(expired ? int32_t{0} : static_cast(session_timeout.totalMilliseconds()), *out); + /// A rejected client has no session, and would send any non-zero id back as the session to continue. + Coordination::write(result == HandshakeResult::Accepted ? session_id : int64_t{0}, *out); std::array passwd{}; Coordination::write(passwd, *out); out->next(); @@ -315,7 +318,6 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length, bool int32_t protocol_version = 0; int64_t last_zxid_seen = 0; int32_t timeout_ms = 0; - int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero. std::array passwd {}; if (!isHandShake(handshake_length)) @@ -465,6 +467,14 @@ void KeeperTCPHandler::runImpl() if (keeper_dispatcher->isTCPConnectionDrainStarted() || keeper_dispatcher->isShuttingDown()) return; + /// Keeper cannot restore sessions, and a new session in place of the old one would go unnoticed by the client. + if (previous_session_id != 0) + { + LOG_INFO(log, "Client asked to continue session {}, which cannot be restored, replying that it has expired", previous_session_id); + sendHandshake(HandshakeResult::SessionExpired, use_compression); + return; + } + if (keeper_dispatcher->isServerActive()) { try @@ -476,7 +486,7 @@ void KeeperTCPHandler::runImpl() catch (const Exception & e) { LOG_WARNING(log, "Cannot receive session id {}", e.displayText()); - sendHandshake(/* has_leader */ false, use_compression); + sendHandshake(HandshakeResult::Rejected, use_compression); return; } @@ -489,12 +499,12 @@ void KeeperTCPHandler::runImpl() return; } - sendHandshake(/* has_leader */ true, use_compression); + sendHandshake(HandshakeResult::Accepted, use_compression); } else { LOG_WARNING(log, "Ignoring user request, because the server is not active yet"); - sendHandshake(/* has_leader */ false, use_compression); + sendHandshake(HandshakeResult::Rejected, use_compression); return; } diff --git a/src/Server/KeeperTCPHandler.h b/src/Server/KeeperTCPHandler.h index 820306a41fed..6e1b9e5e96b7 100644 --- a/src/Server/KeeperTCPHandler.h +++ b/src/Server/KeeperTCPHandler.h @@ -77,6 +77,8 @@ class KeeperTCPHandler : public Poco::Net::TCPServerConnection Poco::Timespan max_session_timeout; Poco::Timespan session_timeout; int64_t session_id{-1}; + /// Session the client asked to continue in its handshake, 0 for a new session. + int64_t previous_session_id{0}; Stopwatch session_stopwatch; SocketInterruptablePollWrapperPtr poll_wrapper; Poco::Timespan send_timeout; @@ -105,7 +107,13 @@ class KeeperTCPHandler : public Poco::Net::TCPServerConnection void cancelWriteBuffer() noexcept; ReadBuffer & getReadBuffer(); - void sendHandshake(bool has_leader, bool & use_compression); + enum class HandshakeResult + { + Accepted, + Rejected, + SessionExpired, + }; + void sendHandshake(HandshakeResult result, bool & use_compression); Poco::Timespan receiveHandshake(int32_t handshake_length, bool & use_compression); static bool isHandShake(int32_t handshake_length); diff --git a/src/Server/ReplicasStatusHandler.cpp b/src/Server/ReplicasStatusHandler.cpp index 9932a74c2c4e..7b7637482554 100644 --- a/src/Server/ReplicasStatusHandler.cpp +++ b/src/Server/ReplicasStatusHandler.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -68,12 +69,7 @@ void ReplicasStatusHandler::handleRequest(HTTPServerRequest & request, HTTPServe // If they have some lag it will be reflected as soon as they are load. for (auto iterator = db.second->getTablesIterator(getContext(), {}, true); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); - if (!table) - continue; - - StorageReplicatedMergeTree * table_replicated = dynamic_cast(table.get()); - + auto table_replicated = castStorage(iterator->table(), DeferredTable::Skip); if (!table_replicated) continue; diff --git a/src/Server/SSH/SSHPtyHandler.cpp b/src/Server/SSH/SSHPtyHandler.cpp index 02ecb110d3c7..8edc1d477100 100644 --- a/src/Server/SSH/SSHPtyHandler.cpp +++ b/src/Server/SSH/SSHPtyHandler.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -528,6 +529,18 @@ void SSHPtyHandler::run() } bool fds_set = false; + /// `ssh_event_add_fd` allocates a wrapper that is released only by `ssh_event_remove_fd`: + /// `ssh_event_free` does not know about it. The fds therefore have to be removed on every exit + /// path, including when `poll` throws because the connection with the client is already gone. + int registered_out_fd = -1; + int registered_err_fd = -1; + SCOPE_EXIT({ + if (registered_out_fd != -1) + event.removeFd(registered_out_fd); + if (registered_err_fd != -1) + event.removeFd(registered_err_fd); + }); + do { /* Poll the main event which takes care of the session, the channel and @@ -544,10 +557,16 @@ void SSHPtyHandler::run() /* If stdout valid, add stdout to be monitored by the poll event. */ if (sdata.channel_callback->client_input_output.out != -1) + { event.addFd(sdata.channel_callback->client_input_output.out, POLLIN, process_stdout, sdata.channel_callback->channel.getCChannelPtr()); + registered_out_fd = sdata.channel_callback->client_input_output.out; + } if (sdata.channel_callback->client_input_output.err != -1) + { event.addFd(sdata.channel_callback->client_input_output.err, POLLIN, process_stderr, sdata.channel_callback->channel.getCChannelPtr()); + registered_err_fd = sdata.channel_callback->client_input_output.err; + } } while (sdata.channel_callback->channel.isOpen() && !sdata.channel_callback->hasClientFinished() && !server.isCancelled()); @@ -558,8 +577,16 @@ void SSHPtyHandler::run() sdata.channel_callback->channel.isOpen(), sdata.channel_callback->hasClientFinished(), server.isCancelled() ); - event.removeFd(sdata.channel_callback->client_input_output.out); - event.removeFd(sdata.channel_callback->client_input_output.err); + if (registered_out_fd != -1) + { + event.removeFd(registered_out_fd); + registered_out_fd = -1; + } + if (registered_err_fd != -1) + { + event.removeFd(registered_err_fd); + registered_err_fd = -1; + } /// Drain any remaining data from stdout/stderr pipes before closing the channel. /// The client may have finished writing to the pipes before the event loop had a chance diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 18d9711bf0c5..938759d9205a 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include #include @@ -1750,7 +1751,8 @@ void TCPHandler::processTablesStatusRequest() continue; TableStatus status; - if (auto * replicated_table = dynamic_cast(table.get())) + /// The initiator asks about this table by name, so a lazily loaded replica is loaded to report its delay. + if (auto * replicated_table = castStorage(table, DeferredTable::Load).get()) { status.is_replicated = true; status.absolute_delay = static_cast(replicated_table->getAbsoluteDelay()); @@ -1947,7 +1949,10 @@ bool TCPHandler::receiveProxyHeader() /// Only PROXYv1 is supported. /// Validation of protocol is not fully performed. - LimitReadBuffer limit_in(*in, {.read_no_more=107, .expect_eof=true}); /// Maximum length from the specs. + /// No `expect_eof`: except for the `UNKNOWN` health check below, the client sends its handshake + /// right after the header, so the connection does not end at the limit. An over-long header is + /// rejected anyway, by carrying no `\r\n` within these 107 bytes. + LimitReadBuffer limit_in(*in, {.read_no_more=107}); /// Maximum length from the specs. assertString("PROXY ", limit_in); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 9e58d158ba0c..693e332bb674 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -1194,17 +1195,18 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context, } MergeTreeSettings effective_settings; - bool any_mt_setting = false; + SettingsChanges builtin_changes; for (const auto & change : settings_from_storage) { if (MergeTreeSettings::hasBuiltin(change.name)) - { - effective_settings.applyChange(change, context, /*is_loading_from_existing_metadata=*/true); - any_mt_setting = true; - } + builtin_changes.push_back(change); } - if (any_mt_setting) + if (!builtin_changes.empty()) { + /// Only the implicit-index settings below are read here, and this runs before the statement is + /// known to be allowed, so the `disk` setting is left unresolved rather than creating the disk. + effective_settings.applyChangesLeavingDiskUnresolved(builtin_changes); + metadata.add_minmax_index_for_numeric_columns = effective_settings[MergeTreeSetting::add_minmax_index_for_numeric_columns]; metadata.add_minmax_index_for_string_columns = effective_settings[MergeTreeSetting::add_minmax_index_for_string_columns]; metadata.add_minmax_index_for_temporal_columns = effective_settings[MergeTreeSetting::add_minmax_index_for_temporal_columns]; @@ -2026,7 +2028,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const const auto virtuals = metadata->virtuals; bool share_nested = true; - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) share_nested = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; auto all_columns = metadata->columns; @@ -2323,7 +2325,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (all_columns.hasNested(command.column_name)) { bool skip = false; - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) skip = !(*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; if (!skip) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot rename whole Nested struct"); @@ -2366,7 +2368,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// When share_nested_offsets is disabled, dotted-name columns are independent /// and not part of a Nested group, so they can be freely renamed. - if (auto * merge_tree = dynamic_cast(table.get())) + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) { if (!(*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]) { diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index 6dfd1f78e73f..378bb0840ef7 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -161,7 +161,7 @@ void HiveORCFile::prepareReader() std::atomic is_stopped{0}; orc::ReaderOptions options; options.setMemoryPool(getORCMemoryPool()); - reader = orc::createReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), options); + reader = createORCReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), options); } void HiveORCFile::prepareColumnMapping() diff --git a/src/Storages/IStorageCluster.h b/src/Storages/IStorageCluster.h index 43b2d690955d..6ecfdd22dc19 100644 --- a/src/Storages/IStorageCluster.h +++ b/src/Storages/IStorageCluster.h @@ -54,9 +54,16 @@ class IStorageCluster : public IStorage const String & getClusterName() const { return cluster_name; } + /// Prepare the `SELECT ... FROM f(...)` query (`f` is a table function) for the other nodes of the cluster: add the + /// structure and format arguments so that the nodes do not infer the schema again, and turn a plain table + /// function (`url`, `s3`, ...) that `parallel_replicas_for_cluster_engines` converted into this cluster + /// storage into its `*Cluster` variant with the cluster name argument, so that the nodes take their read + /// tasks from the initiator instead of reading every file on their own. Called by `read` and by the + /// distributed `INSERT ... SELECT` in `InterpreterInsertQuery`, which forwards the query the same way. + virtual void updateQueryToSendIfNeeded(ASTPtr & /*query*/, const StorageSnapshotPtr & /*storage_snapshot*/, const ContextPtr & /*context*/) {} + protected: virtual void updateBeforeRead(const ContextPtr &) {} - virtual void updateQueryToSendIfNeeded(ASTPtr & /*query*/, const StorageSnapshotPtr & /*storage_snapshot*/, const ContextPtr & /*context*/) {} virtual void updateConfigurationIfNeeded(ContextPtr /* context */) {} diff --git a/src/Storages/Kafka/KafkaConfigLoader.cpp b/src/Storages/Kafka/KafkaConfigLoader.cpp index ded88d65b90a..fceea40e63e1 100644 --- a/src/Storages/Kafka/KafkaConfigLoader.cpp +++ b/src/Storages/Kafka/KafkaConfigLoader.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,7 @@ #include #include #include +#include #include #include @@ -552,6 +554,36 @@ void updateConfigurationFromConfig( } +namespace +{ + +/// Sensitive properties must not be logged in cleartext: the log records can reach not only the +/// server log, but also clients that set `send_logs_level`. +bool isSensitiveProperty(std::string_view name) +{ + /// The properties librdkafka marks with the _RK_SENSITIVE flag, plus a substring safety net + /// for properties unknown to the vendored librdkafka version. + static const std::unordered_set sensitive_properties = [] + { + std::unordered_set res; + for (const char * const * prop_name = chrd_kafka_conf_sensitive_properties(); *prop_name; ++prop_name) + res.emplace(*prop_name); + return res; + }(); + return sensitive_properties.contains(name) || name.contains("password") || name.contains("secret"); +} + +/// Log all properties of a Kafka client configuration, replacing the values of sensitive +/// properties, e.g. `sasl.password` or `sasl.oauthbearer.client.secret`, with `[HIDDEN]`. +void logConfigProperties(const cppkafka::Configuration & conf, const LoggerPtr & log, std::string_view client_type) +{ + for (const auto & property : conf.get_all()) + LOG_TRACE(log, "{} set property {}:{}", client_type, property.first, + isSensitiveProperty(property.first) ? "[HIDDEN]" : property.second); +} + +} + template cppkafka::Configuration KafkaConfigLoader::getConsumerConfiguration(TKafkaStorage & storage, const ConsumerConfigParams & params, IKafkaExceptionInfoSinkPtr exception_info_sink_ptr) { @@ -577,17 +609,16 @@ cppkafka::Configuration KafkaConfigLoader::getConsumerConfiguration(TKafkaStorag updateConfigurationFromConfig(loadConsumerConfig, conf, storage, params, exception_info_sink_ptr); + /// Re-validate the broker list in case they are changed in the merged configuration + if (const auto merged_broker_list = conf.get("metadata.broker.list"); merged_broker_list != params.brokers) + conf.set("metadata.broker.list", StorageKafkaUtils::validateBrokerList(merged_broker_list, params.context)); + // those settings should not be changed by users. conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. conf.set("enable.partition.eof", "false"); // Ignore EOF messages - for (auto & property : conf.get_all()) - { - if (property.first.contains("password")) - continue; - LOG_TRACE(params.log, "Consumer set property {}:{}", property.first, property.second); - } + logConfigProperties(conf, params.log, "Consumer"); return conf; } @@ -608,8 +639,11 @@ cppkafka::Configuration KafkaConfigLoader::getProducerConfiguration(TKafkaStorag updateConfigurationFromConfig(loadProducerConfig, conf, storage, params); - for (auto & property : conf.get_all()) - LOG_TRACE(params.log, "Producer set property {}:{}", property.first, property.second); + /// See the same check in getConsumerConfiguration. + if (const auto merged_broker_list = conf.get("metadata.broker.list"); merged_broker_list != params.brokers) + conf.set("metadata.broker.list", StorageKafkaUtils::validateBrokerList(merged_broker_list, params.context)); + + logConfigProperties(conf, params.log, "Producer"); /// compression.codec is a global and topic level property, however compression.level is only a topic level property. /// cppkafka::Configuration::get_all returns the global properties only, so we need to check compression.level separately. diff --git a/src/Storages/Kafka/KafkaConfigLoader.h b/src/Storages/Kafka/KafkaConfigLoader.h index a007218723c5..9fd04c3750e9 100644 --- a/src/Storages/Kafka/KafkaConfigLoader.h +++ b/src/Storages/Kafka/KafkaConfigLoader.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -28,6 +29,7 @@ struct KafkaConfigLoader String & collection_name; const Names & topics; LoggerPtr & log; + ContextPtr context; }; struct ConsumerConfigParams : public LoadConfigParams diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index d949a038ebc9..9d2d24650c63 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -179,7 +179,8 @@ StorageKafka::StorageKafka( , kafka_settings(std::move(kafka_settings_)) , macros_info{.table_id = table_id_} , topics(StorageKafkaUtils::parseTopics(getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_topic_list].value, macros_info))) - , brokers(getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_broker_list].value, macros_info)) + , brokers(StorageKafkaUtils::validateBrokerList( + getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_broker_list].value, macros_info), context_)) , group(getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_group_name].value, macros_info)) , client_id( (*kafka_settings)[KafkaSetting::kafka_client_id].value.empty() @@ -513,7 +514,7 @@ KafkaConsumerPtr StorageKafka::createKafkaConsumer(size_t consumer_number) cppkafka::Configuration StorageKafka::getConsumerConfiguration(size_t consumer_number, IKafkaExceptionInfoSinkPtr exception_info_sink_ptr) { KafkaConfigLoader::ConsumerConfigParams params{ - {getContext()->getConfigRef(), collection_name, topics, log}, + {getContext()->getConfigRef(), collection_name, topics, log, getContext()}, brokers, group, num_consumers > 1, @@ -526,7 +527,7 @@ cppkafka::Configuration StorageKafka::getConsumerConfiguration(size_t consumer_n cppkafka::Configuration StorageKafka::getProducerConfiguration() { KafkaConfigLoader::ProducerConfigParams params{ - {getContext()->getConfigRef(), collection_name, topics, log}, + {getContext()->getConfigRef(), collection_name, topics, log, getContext()}, brokers, client_id}; return KafkaConfigLoader::getProducerConfiguration(*this, params); diff --git a/src/Storages/Kafka/StorageKafka2.cpp b/src/Storages/Kafka/StorageKafka2.cpp index f68df3bee237..6596242684b4 100644 --- a/src/Storages/Kafka/StorageKafka2.cpp +++ b/src/Storages/Kafka/StorageKafka2.cpp @@ -169,7 +169,8 @@ StorageKafka2::StorageKafka2( , kafka_settings(std::move(kafka_settings_)) , macros_info{.table_id = table_id_, .shard = getContext()->getMacros()->tryGetValue("shard")} , topics(StorageKafkaUtils::parseTopics(getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_topic_list].value, macros_info))) - , brokers(getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_broker_list].value, macros_info)) + , brokers(StorageKafkaUtils::validateBrokerList( + getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_broker_list].value, macros_info), context_)) , group(getContext()->getMacros()->expand((*kafka_settings)[KafkaSetting::kafka_group_name].value, macros_info)) , client_id( (*kafka_settings)[KafkaSetting::kafka_client_id].value.empty() @@ -688,7 +689,7 @@ KafkaConsumer2Ptr StorageKafka2::createKafkaConsumer(size_t consumer_number) cppkafka::Configuration StorageKafka2::getConsumerConfiguration(size_t consumer_number, IKafkaExceptionInfoSinkPtr exception_sink) { KafkaConfigLoader::ConsumerConfigParams params{ - {getContext()->getConfigRef(), collection_name, topics, log}, + {getContext()->getConfigRef(), collection_name, topics, log, getContext()}, brokers, group, num_consumers > 1, @@ -706,7 +707,7 @@ cppkafka::Configuration StorageKafka2::getConsumerConfiguration(size_t consumer_ cppkafka::Configuration StorageKafka2::getProducerConfiguration() { KafkaConfigLoader::ProducerConfigParams params{ - {getContext()->getConfigRef(), collection_name, topics, log}, + {getContext()->getConfigRef(), collection_name, topics, log, getContext()}, brokers, client_id}; return KafkaConfigLoader::getProducerConfiguration(*this, params); diff --git a/src/Storages/Kafka/StorageKafkaUtils.cpp b/src/Storages/Kafka/StorageKafkaUtils.cpp index 3e513f11834c..385f81b037b9 100644 --- a/src/Storages/Kafka/StorageKafkaUtils.cpp +++ b/src/Storages/Kafka/StorageKafkaUtils.cpp @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include #include #include @@ -36,6 +38,7 @@ #include #include +#include #include #include @@ -929,6 +932,53 @@ String getDefaultClientId(const StorageID & table_id) return fmt::format("{}-{}-{}-{}", VERSION_NAME, getFQDNOrHostName(), table_id.database_name, table_id.table_name); } +String validateBrokerList(const String & broker_list, const ContextPtr & context) +{ + /// The remote host filter must see exactly the host and port librdkafka will dial, so the value is + /// not passed on as it was written: every entry is parsed here, validated, and the returned list is + /// rebuilt from the parsed entries. The rebuilt entries are `[SCHEME://]host:port` with an explicit + /// port, a form librdkafka re-parses to the same host and port. + /// + /// librdkafka reads `metadata.broker.list` as a C string, splits it on `,` and ` `, cuts an entry at + /// the first `/` after the `scheme://` prefix, substitutes `localhost` for an empty host, and + /// connects to port 9092 when none is given (`rd_kafka_broker_name_parse`). An entry which such a + /// re-parse could read differently - a NUL, a `/`, an empty host, a character outside printable + /// ASCII - is rejected instead of repaired. + + if (broker_list.contains('\0')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Kafka broker list must not contain NUL characters"); + + Names brokers; + boost::split(brokers, broker_list, [](char c) { return c == ',' || c == ' '; }); + + Names canonical_brokers; + canonical_brokers.reserve(brokers.size()); + + for (String & broker : brokers) + { + boost::trim(broker); + if (broker.empty()) + continue; + + String scheme; + if (const auto scheme_end = broker.find("://"); scheme_end != String::npos) + { + scheme = broker.substr(0, scheme_end + strlen("://")); + broker = broker.substr(scheme_end + strlen("://")); + + /// The underscore appears in the librdkafka protocols `sasl_plaintext` and `sasl_ssl`. + for (const char c : scheme.substr(0, scheme_end)) + if (!isAlphaASCII(c) && c != '_') + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid protocol in Kafka broker '{}{}'", scheme, broker); + } + + canonical_brokers.push_back( + scheme + context->getRemoteHostFilter().checkAndGetCanonicalHostAndPort(broker, 9092, "Kafka broker")); + } + + return boost::algorithm::join(canonical_brokers, ","); +} + void consumerGracefulStop( cppkafka::Consumer & consumer, const std::chrono::milliseconds drain_timeout, const LoggerPtr & log, ErrorHandler error_handler) { diff --git a/src/Storages/Kafka/StorageKafkaUtils.h b/src/Storages/Kafka/StorageKafkaUtils.h index 7053d998d32f..941df8238195 100644 --- a/src/Storages/Kafka/StorageKafkaUtils.h +++ b/src/Storages/Kafka/StorageKafkaUtils.h @@ -68,6 +68,12 @@ struct ConsumerStatistics // system.kafka_consumers data Names parseTopics(String topic_list); String getDefaultClientId(const StorageID & table_id); +/// Parses a broker list value, checks every broker against the remote host filter, and returns the list +/// rebuilt from the parsed entries - the string to hand to librdkafka in place of the original value. +/// Throws `UNACCEPTABLE_URL` for a broker the filter does not allow and `BAD_ARGUMENTS` for an entry +/// whose parse could disagree with librdkafka's. +String validateBrokerList(const String & broker_list, const ContextPtr & context); + using ErrorHandler = std::function; void consumerGracefulStop( diff --git a/src/Storages/MaterializedView/RefreshSet.cpp b/src/Storages/MaterializedView/RefreshSet.cpp index c65838c16a73..9346f0c8bd35 100644 --- a/src/Storages/MaterializedView/RefreshSet.cpp +++ b/src/Storages/MaterializedView/RefreshSet.cpp @@ -85,18 +85,14 @@ RefreshSet::RefreshSet() = default; void RefreshSet::emplace(StorageID id, std::optional inner_table_id, const std::vector & dependencies, RefreshTaskPtr task) { - { - std::lock_guard guard(mutex); - const auto iter = addTaskLocked(id, task); - RefreshTaskList::iterator inner_table_iter; - if (inner_table_id) - inner_table_iter = addInnerTableLocked(*inner_table_id, task); - addDependenciesLocked(task, dependencies); - - task->setRefreshSetHandleUnlock(Handle(this, id, inner_table_id, iter, inner_table_iter, dependencies)); - } - - notifyDependents(id); + std::lock_guard guard(mutex); + const auto iter = addTaskLocked(id, task); + RefreshTaskList::iterator inner_table_iter; + if (inner_table_id) + inner_table_iter = addInnerTableLocked(*inner_table_id, task); + addDependenciesLocked(task, dependencies); + + task->setRefreshSetHandleUnlock(Handle(this, id, inner_table_id, iter, inner_table_iter, dependencies)); } RefreshTaskList::iterator RefreshSet::addTaskLocked(StorageID id, RefreshTaskPtr task) diff --git a/src/Storages/MaterializedView/RefreshSet.h b/src/Storages/MaterializedView/RefreshSet.h index a0ac15513882..7fa91d2ed244 100644 --- a/src/Storages/MaterializedView/RefreshSet.h +++ b/src/Storages/MaterializedView/RefreshSet.h @@ -52,6 +52,8 @@ class RefreshSet RefreshSet(); + /// Caller should then also call notifyDependents, because dependent views need to know when + /// their dependencies appear/disappear. void emplace(StorageID id, std::optional inner_table_id, const std::vector & dependencies, RefreshTaskPtr task); /// Finds active refreshable view(s) by database and table name. diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp index faed43facb7c..2a3d98b5af9f 100644 --- a/src/Storages/MaterializedView/RefreshTask.cpp +++ b/src/Storages/MaterializedView/RefreshTask.cpp @@ -315,13 +315,29 @@ bool RefreshTask::canCreateOrDropOtherTables() const void RefreshTask::startup() { - if (start_paused || view->getContext()->getSettingsRef()[Setting::stop_refreshable_materialized_views_on_startup]) - scheduling.stop_requested = true; - auto inner_table_id = refresh_append ? std::nullopt : std::make_optional(view->getTargetTableId()); - view->getContext()->getRefreshSet().emplace(view->getStorageID(), inner_table_id, initial_dependencies, shared_from_this()); + ContextMutablePtr context; + StorageID view_id = StorageID::createEmpty(); + { + std::lock_guard guard(mutex); - std::lock_guard guard(mutex); - scheduleRefresh(guard); + /// shutdown() is allowed to run before or during startup() (see its declaration) and nulls `view`. + if (!view) + return; + + if (start_paused || view->getContext()->getSettingsRef()[Setting::stop_refreshable_materialized_views_on_startup]) + scheduling.stop_requested = true; + context = view->getContext(); + view_id = view->getStorageID(); + auto inner_table_id = refresh_append ? std::nullopt : std::make_optional(view->getTargetTableId()); + + /// `set_handle` is not thread safe and shutdown() resets it under `mutex`. + context->getRefreshSet().emplace(view_id, inner_table_id, initial_dependencies, shared_from_this()); + + scheduleRefresh(guard); + } + + /// Outside `mutex`: notifying a dependent view locks that view's own task mutex. + context->getRefreshSet().notifyDependents(view_id); } void RefreshTask::finalizeRestoreFromBackup() diff --git a/src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp b/src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp new file mode 100644 index 000000000000..01991dac8ab3 --- /dev/null +++ b/src/Storages/MaterializedView/tests/gtest_refresh_task_startup_after_shutdown.cpp @@ -0,0 +1,105 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// `DatabaseCatalog` is process-wide and shared by every suite in `unit_tests_dbms`, so the +/// database name has to be unique across the whole binary, not just readable here. +constexpr auto database_name = "refresh_task_startup_after_shutdown_db"; + +struct State +{ + State(const State &) = delete; + + ContextMutablePtr context; + + static const State & instance() + { + static State state; + return state; + } + +private: + State() : context(Context::createCopy(getContext().context)) + { + tryRegisterFunctions(); + tryRegisterAggregateFunctions(); + + DatabasePtr database = std::make_shared(database_name, context); + const ColumnsDescription columns{{"x", std::make_shared()}}; + for (const auto * table_name : {"src", "target"}) + database->attachTable( + context, + table_name, + std::make_shared( + StorageID(database_name, table_name), columns, ConstraintsDescription{}, String{}, MemorySettings{}), + {}); + + DatabaseCatalog::instance().attachDatabase(database_name, database); + context->setCurrentDatabase(database_name); + } +}; + +/// An explicit `TO` target and `LoadingStrictnessLevel::ATTACH` keep this to a plain constructor +/// call: neither creates an inner table, and a non-Replicated database leaves the refresh +/// uncoordinated, so no Keeper is needed. +std::shared_ptr attachRefreshableView(const ContextMutablePtr & context, const String & view_name) +{ + const String db = database_name; + const String query = "ATTACH MATERIALIZED VIEW " + db + "." + view_name + " REFRESH EVERY 1 YEAR TO " + db + + ".target AS SELECT x FROM " + db + ".src"; + + ParserCreateQuery parser; + ASTPtr ast = parseQuery(parser, query, 100000, 1000, 1000000); + + return std::make_shared( + StorageID(database_name, view_name), + context, + ast->as(), + ColumnsDescription{{"x", std::make_shared()}}, + LoadingStrictnessLevel::ATTACH, + /*comment=*/String{}, + /*is_restore_from_backup=*/false); +} + +} + +/// shutdown() is documented to be callable before or during startup(), and it nulls the `view` +/// back-pointer that startup() reads, so this order is sanctioned rather than a misuse. +/// +/// The two calls are made directly because no query sequence produces that order: ATTACH TABLE +/// builds a fresh RefreshTask, and a database sweep joins the outstanding startup jobs before it +/// begins (DatabaseOnDisk::shutdown() calls stopLoading()). +TEST(RefreshTaskStartupAfterShutdown, StartupAfterShutdownDoesNotDereferenceNullView) +{ + const auto & state = State::instance(); + + auto view = attachRefreshableView(state.context, "mv"); + /// Without a refresh task there is no `view` pointer to null and the test below is vacuous. + ASSERT_TRUE(view->isRefreshable()); + const StorageID view_id = view->getStorageID(); + + view->flushAndPrepareForShutdown(); + view->startup(); + + /// startup() must also decline to register the task: `RefreshSet` membership is what + /// `system.view_refreshes` reports and what schedules refreshes, and this view is shut down. + EXPECT_TRUE(state.context->getRefreshSet().findTasks(view_id).empty()); +} diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 81ff0ca34e5b..d22df1d1f9ae 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1516,6 +1516,17 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks if (auto * constant_granularity = dynamic_cast(index_granularity.get())) constant_granularity->fixFromRowsCount(rows_count); + /// A patch part that holds rows names the parts it patches, so an index without source + /// parts is not a patch that applies to nothing - it is a file that lost its content. + /// Failing here is what keeps the acknowledged update recoverable: an empty index reports + /// data version 0, so `clearUnusedPatchParts` would find the patch materialized everywhere + /// and delete the only copy of it. An empty index belongs to an empty part alone (the + /// covering parts `cloneEmpty` creates). + if (info.isPatch() && rows_count > 0 && patch_part_index && patch_part_index->empty()) + throw Exception(ErrorCodes::CORRUPTED_DATA, + "Patch part {} has {} rows, but its index in {} references no source parts", + name, rows_count, PatchPartIndex::FILENAME); + loadExistingRowsCount(); /// Must be called after loadRowsCount() as it uses the value of `rows_count`. loadPartitionAndMinMaxIndex(); @@ -2001,7 +2012,15 @@ void IMergeTreeDataPart::loadPatchPartIndex() return; if (auto in = readFileIfExists(PatchPartIndex::FILENAME)) + { patch_part_index = PatchPartIndex::readBinary(*in); + + /// The file holds nothing but this index, so bytes left over mean its content is not what was + /// written. One corruption shape makes this check the difference between a loud and a silent + /// failure: a zeroed block parses as an index of format version `V1` with no source parts at + /// all, and everything after those nine bytes would otherwise be ignored. + assertEOF(*in); + } else throw Exception(ErrorCodes::CORRUPTED_DATA, "Missing file {} in patch part {}", PatchPartIndex::FILENAME, name); } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7138a2685bb3..fc33ad1cc3a8 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -3530,6 +3530,10 @@ MergeAlgorithm MergeTask::ExecuteAndFinalizeHorizontalPart::chooseMergeAlgorithm return MergeAlgorithm::Horizontal; if (ctx->need_remove_expired_values) { + /// `TTLTransform` stops reading after the first block when the rows TTL has expired for the whole part, + /// while a vertical merge reads every key row to write `rows_sources`. + if (global_ctx->future_part->merge_type == MergeType::TTLDrop && global_ctx->metadata_snapshot->hasRowsTTL()) + return MergeAlgorithm::Horizontal; if (!canVerticalTTLDelete(*global_ctx)) return MergeAlgorithm::Horizontal; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 592d37ed175b..24b6f10fa50f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -5071,8 +5072,9 @@ Names expressionSourceColumns(const ASTPtr & ast, const ColumnsDescription & col auto planner_context = std::make_shared(analysis_context, global_planner_context, SelectQueryOptions{}); collectSetsAndSourceColumns(expression, planner_context, /*keep_alias_columns=*/ false); + /// ALIAS columns are inlined above, so the physical columns their expressions read are the dependencies. if (const auto * table_expression_data = planner_context->getTableExpressionDataOrNull(table_node)) - return table_expression_data->getSelectedColumnsNames(); + return table_expression_data->getColumnNames(); return {}; } @@ -6022,9 +6024,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context MergeTreeSettingsPtr alter_effective_settings = getSettings(); if (new_metadata.settings_changes) { - const auto & new_changes = new_metadata.settings_changes->as().changes; + auto new_changes = new_metadata.settings_changes->as().changes; + /// The settings constraints below compare the resolved `disk`, so it is resolved here. A changed + /// `disk` is a fresh definition and is checked as one, before anything registers the disk unchecked. + MergeTreeSettings::resolveDiskSetting(new_changes, local_context, /*is_loading_from_existing_metadata=*/!disk_setting_changed); auto copy = getDefaultSettings(); - copy->applyChanges(new_changes, getContext(), /*is_loading_from_existing_metadata=*/true); + copy->applyChanges(new_changes, local_context, /*is_loading_from_existing_metadata=*/true); alter_effective_settings = std::move(copy); } @@ -6117,7 +6122,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context { /// Use default settings + new and check if doesn't affect part format settings auto copy = getDefaultSettings(); - copy->applyChanges(new_changes, local_context, /*is_loading_from_existing_metadata=*/true); + copy->applyChangesLeavingDiskUnresolved(new_changes); String reason; if (!canUsePolymorphicParts(*copy, reason) && !reason.empty()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Can't change settings. Reason: {}", reason); @@ -8452,21 +8457,36 @@ void MergeTreeData::movePartitionToVolume(const ASTPtr & partition, const String } } +/// Patch parts carry data versions allocated from the block numbers of their own table, so they cannot be copied to another table. +static void assertNotPatchPartition(const MergeTreeData & data, const ASTPtr & partition, ContextPtr query_context, std::string_view command) +{ + if (partition->as().all) + return; + + const auto partition_id = data.getPartitionIDFromQuery(partition, query_context); + if (isPatchPartitionId(partition_id)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Cannot execute {} for partition {} of patch parts, they cannot be copied to another table. " + "Apply them with `ALTER TABLE ... APPLY PATCHES IN PARTITION ID '{}'` and use that partition instead", + command, partition_id, getOriginalPartitionIdOfPatch(partition_id)); +} + void MergeTreeData::movePartitionToTable(const PartitionCommand & command, ContextPtr query_context) { String dest_database = query_context->resolveDatabase(command.to_database); - auto dest_storage = DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context); + auto dest_storage = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable({dest_database, command.to_table}, query_context)); /// The target table and the source table are the same. if (dest_storage->getStorageID() == this->getStorageID()) return; - auto * dest_storage_merge_tree = dynamic_cast(dest_storage.get()); + auto * dest_storage_merge_tree = castStorage(dest_storage, DeferredTable::Load).get(); if (!dest_storage_merge_tree) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot move partition from table {} to table {} with storage {}", getStorageID().getNameForLogs(), dest_storage->getStorageID().getNameForLogs(), dest_storage->getName()); + assertNotPatchPartition(*this, command.partition, query_context, "MOVE PARTITION TO TABLE"); dest_storage_merge_tree->waitForOutdatedPartsToBeLoaded(); movePartitionToTable(dest_storage, command.partition, query_context); } @@ -8592,13 +8612,14 @@ Pipe MergeTreeData::alterPartition( case PartitionCommand::REPLACE_PARTITION: { + assertNotPatchPartition(*this, command.partition, query_context, command.replace ? "REPLACE PARTITION" : "ATTACH PARTITION FROM"); if (command.replace) checkPartitionCanBeDropped(command.partition, query_context); auto resolved = query_context->resolveStorageID({command.from_database, command.from_table}); - auto from_storage = DatabaseCatalog::instance().getTable(resolved, query_context); + auto from_storage = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(resolved, query_context)); - auto * from_storage_merge_tree = dynamic_cast(from_storage.get()); + auto * from_storage_merge_tree = castStorage(from_storage, DeferredTable::Load).get(); if (!from_storage_merge_tree) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot replace partition from table {} with storage {} to table {}", @@ -10974,7 +10995,7 @@ void MergeTreeData::checkColumnFilenamesForCollision(const StorageInMemoryMetada if (metadata.settings_changes) { const auto & changes = metadata.settings_changes->as().changes; - settings->applyChanges(changes, getContext(), /*is_loading_from_existing_metadata=*/true); + settings->applyChangesLeavingDiskUnresolved(changes); } checkColumnFilenamesForCollision(metadata.getColumns(), *settings, throw_on_error); @@ -11039,6 +11060,7 @@ void MergeTreeData::checkColumnFilenamesForCollision(const ColumnsDescription & MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & source_table, const StorageMetadataPtr & src_snapshot, const StorageMetadataPtr & my_snapshot) const { + /// NOLINT(storage-cast): a reference, and the `StoragePtr` overload below resolves the proxy. MergeTreeData * src_data = dynamic_cast(&source_table); if (!src_data) throw Exception(ErrorCodes::NOT_IMPLEMENTED, @@ -11098,7 +11120,7 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData( const StoragePtr & source_table, const StorageMetadataPtr & src_snapshot, const StorageMetadataPtr & my_snapshot) const { - return checkStructureAndGetMergeTreeData(*source_table, src_snapshot, my_snapshot); + return checkStructureAndGetMergeTreeData(*resolveStorageProxyLoading(source_table), src_snapshot, my_snapshot); } /// must_on_same_disk=false is used only when attach partition; Both for same disk and different disk. diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 30930da722f4..862a95339e5a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -804,6 +806,15 @@ std::expected MergeTreeDataSelectExecutor::canUseInde } +static bool hasEffectiveRowPolicy(const IStorage & storage, const ContextPtr & context) +{ + auto storage_id = storage.getStorageID(); + if (!storage_id.hasDatabase()) + return false; + auto filter = context->getRowPolicyFilter(storage_id.getDatabaseName(), storage_id.getTableName(), RowPolicyFilterType::SELECT_FILTER); + return filter && !filter->isAlwaysTrue(); +} + RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByStatistics( const RangesInDataParts & parts, const StorageMetadataPtr & metadata_snapshot, @@ -821,10 +832,17 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByStatistics( /// 3. There are on-the-fly mutations or patch parts (statistics only reflects original data) /// 4. A masking policy applies: it rewrites values at read time, so the statistics (like /// the on-the-fly mutations above) no longer describe the values the query sees. + /// 5. A row policy applies: the statistics describe all rows of a part, including the ones the + /// policy hides, so the number of rows left to read after pruning by the query's predicate, + /// which is reported to the client, reveals the values of the hidden rows. The policy is + /// either pushed into this read (possibly from a wrapper such as `Alias`) or belongs to + /// this table and is applied above the read (e.g. for a child of `Merge`). if (!settings[Setting::use_statistics_for_part_pruning] || query_info.isFinal() || (mutations_snapshot && (mutations_snapshot->hasDataMutations() || mutations_snapshot->hasPatchParts())) - || (!parts.empty() && parts.front().data_part->storage.hasEnabledMaskingPolicies(context))) + || (!parts.empty() && parts.front().data_part->storage.hasEnabledMaskingPolicies(context)) + || query_info.row_level_filter + || (!parts.empty() && hasEffectiveRowPolicy(parts.front().data_part->storage, context))) { return parts; } diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index ecc0c9a6defc..87c2b6dabf7e 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -585,12 +586,13 @@ static bool bloomFilterHashDomainMatches(const DataTypePtr & value_type, const D bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( const String & function_name, - const RPNBuilderTreeNode & key_node, + const RPNBuilderTreeNode & wrapped_key_node, const ConstSetPtr & prepared_set, const DataTypePtr & type, const ColumnPtr & column, RPNElement & out) { + const auto key_node = unwrapLosslessConversion(wrapped_key_node); auto key_node_column_name = key_node.getColumnName(); if (header.has(key_node_column_name)) @@ -598,6 +600,12 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( size_t row_size = column->size(); size_t position = header.getPositionByName(key_node_column_name); const DataTypePtr & index_type = header.getByPosition(position).type; + + /// A NULL of a set built for `toNullable(key)` does not cast to the key type and matches no stored value. + const auto * nullable_column = typeid_cast(column.get()); + if (nullable_column && !index_type->isNullable() && std::ranges::any_of(nullable_column->getNullMapData(), [](UInt8 is_null) { return is_null != 0; })) + return false; + const auto & converted_column = castColumn(ColumnWithTypeAndName{column, type, ""}, index_type); /// An `Array` index holds one hash per element, so a set array is looked up by its elements @@ -964,12 +972,13 @@ static bool indexOfCanUseBloomFilter(const RPNBuilderTreeNode * parent) bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( const String & function_name, - const RPNBuilderTreeNode & key_node, + const RPNBuilderTreeNode & wrapped_key_node, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, const RPNBuilderTreeNode * parent) { + const auto key_node = unwrapLosslessConversion(wrapped_key_node); auto key_column_name = key_node.getColumnName(); /// `arrayJoin(col) = const` needs an element equal to the constant, same as `has(col, const)`. @@ -1014,8 +1023,16 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( if (function_name == "has" || indexOfCanUseBloomFilter(parent)) { out.function = RPNElement::FUNCTION_HAS; - const DataTypePtr & nested_type = array_type->getNestedType(); - const DataTypePtr actual_type = BloomFilter::getPrimitiveType(nested_type); + /// The function coerces the constant by the element type it sees, which may differ in `LowCardinality`. + DataTypePtr nested_type = array_type->getNestedType(); + + if (const auto * wrapped_dag_node = wrapped_key_node.getDAGNode()) + { + if (const auto * wrapped_array_type = typeid_cast(wrapped_dag_node->result_type.get())) + nested_type = wrapped_array_type->getNestedType(); + } + + const DataTypePtr actual_type = BloomFilter::getPrimitiveType(array_type->getNestedType()); Field converted_field = convertConstantForArrayIndexFunction(value_field, value_type, nested_type, actual_type); if (converted_field.isNull()) return false; diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h index 70061bd7110b..e0b19f513f71 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h @@ -101,7 +101,7 @@ class MergeTreeIndexConditionBloomFilter final : public IMergeTreeIndexCondition bool traverseTreeIn( const String & function_name, - const RPNBuilderTreeNode & key_node, + const RPNBuilderTreeNode & wrapped_key_node, const ConstSetPtr & prepared_set, const DataTypePtr & type, const ColumnPtr & column, @@ -109,7 +109,7 @@ class MergeTreeIndexConditionBloomFilter final : public IMergeTreeIndexCondition bool traverseTreeEquals( const String & function_name, - const RPNBuilderTreeNode & key_node, + const RPNBuilderTreeNode & wrapped_key_node, const DataTypePtr & value_type, const Field & value_field, RPNElement & out, diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index c36427c44164..0ef8f588cc09 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -387,16 +387,11 @@ bool MergeTreeConditionBloomFilterText::extractAtomFromTree(const RPNBuilderTree { if (tryPrepareSetBloomFilter(left_argument, right_argument, out)) { - if (function_name == "notIn") - { - out.function = RPNElement::FUNCTION_NOT_IN; - return true; - } - if (function_name == "in") - { - out.function = RPNElement::FUNCTION_IN; - return true; - } + /// `transform_null_in = 1` renames the family; a NULL element is refused above. + const bool negated = function_name == "notIn" || function_name == "globalNotIn" + || function_name == "notNullIn" || function_name == "globalNotNullIn"; + out.function = negated ? RPNElement::FUNCTION_NOT_IN : RPNElement::FUNCTION_IN; + return true; } } else if (function_name == "equals" || @@ -437,13 +432,52 @@ bool MergeTreeConditionBloomFilterText::extractAtomFromTree(const RPNBuilderTree return false; } +namespace +{ + +/// `String = FixedString(N)` ignores the constant's trailing zero padding, so the search terms must be taken from the value without it. +Field stripFixedStringPaddingForTerms(const Field & field, const DataTypePtr & type) +{ + auto inner_type = removeNullable(removeLowCardinality(type)); + + if (isFixedString(inner_type) && field.getType() == Field::Types::String) + { + String value = field.safeGet(); + value.resize(value.find_last_not_of('\0') + 1); + return Field(std::move(value)); + } + + if (const auto * array_type = typeid_cast(inner_type.get()); + array_type && field.getType() == Field::Types::Array) + { + Array stripped; + const auto & elements = field.safeGet(); + stripped.reserve(elements.size()); + for (const auto & element : elements) + stripped.push_back(stripFixedStringPaddingForTerms(element, array_type->getNestedType())); + return Field(std::move(stripped)); + } + + return field; +} + +/// These functions compare a `FixedString` constant through the `String` supertype, which drops the trailing zero padding. +bool functionIgnoresFixedStringPadding(const String & function_name) +{ + return function_name == "equals" || function_name == "notEquals" || function_name == "hasAny" || function_name == "hasAll"; +} + +} + bool MergeTreeConditionBloomFilterText::traverseTreeEquals( const String & function_name, - const RPNBuilderTreeNode & key_node, + const RPNBuilderTreeNode & wrapped_key_node, const DataTypePtr & value_type, const Field & value_field, RPNElement & out) { + const auto key_node = unwrapLosslessConversion(wrapped_key_node); + /// Try JSON subcolumn detection early, before the string-type check. /// JSON path comparison values may not be strings (e.g., json.a.b = 1 where value is UInt8), /// but we tokenize the *path* string against the JSONAllPaths index, not the value. @@ -467,7 +501,10 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( if (!value_data_type.isStringOrFixedString() && !value_data_type.isArray()) return false; - Field const_value = value_field; + /// Every allowed tokenizer `ngrams`, `splitByNonAlpha` and `sparseGrams` emit the terms of the unpadded value as terms of the padded one. + Field const_value = functionIgnoresFixedStringPadding(function_name) + ? stripFixedStringPaddingForTerms(value_field, value_type) + : value_field; const auto column_name = key_node.getColumnName(); auto key_index = getKeyIndex(column_name); @@ -488,7 +525,7 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( * We cannot skip keys that does not exist in map if comparison is with default type value because * that way we skip necessary granules where map key does not exist. */ - if (value_field == value_type->getDefault()) + if (const_value == value_type->getDefault()) return false; auto first_argument = key_function_node.getArgumentAt(0); @@ -531,7 +568,7 @@ bool MergeTreeConditionBloomFilterText::traverseTreeEquals( /// Same as arrayElement: skip when comparing with default value because /// the subcolumn returns default for keys that don't exist in the map. - if (value_field == value_type->getDefault()) + if (const_value == value_type->getDefault()) return false; if (const auto map_keys_index = getKeyIndex(fmt::format("mapKeys({})", map_column_name))) @@ -781,14 +818,14 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( for (size_t i = 0; i < left_argument_function_node_arguments_size; ++i) { - if (const auto key = getKeyIndex(left_argument_function_node.getArgumentAt(i).getColumnName())) + if (const auto key = getKeyIndex(unwrapLosslessConversion(left_argument_function_node.getArgumentAt(i)).getColumnName())) { key_tuple_mapping.emplace_back(i, *key); data_types.push_back(index_data_types[*key]); } } } - else if (const auto key = getKeyIndex(left_argument.getColumnName())) + else if (const auto key = getKeyIndex(unwrapLosslessConversion(left_argument).getColumnName())) { key_tuple_mapping.emplace_back(0, *key); data_types.push_back(index_data_types[*key]); @@ -807,7 +844,8 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( for (const auto & prepared_set_data_type : prepared_set->getDataTypes()) { - auto prepared_set_data_type_id = prepared_set_data_type->getTypeId(); + /// A `Nullable` key keeps the wrapper on its elements at `transform_null_in = 1`. + auto prepared_set_data_type_id = removeNullable(prepared_set_data_type)->getTypeId(); if (prepared_set_data_type_id != TypeIndex::String && prepared_set_data_type_id != TypeIndex::FixedString) return false; } @@ -826,10 +864,22 @@ bool MergeTreeConditionBloomFilterText::tryPrepareSetBloomFilter( size_t tuple_idx = elem.tuple_index; const auto & column = columns[tuple_idx]; + const DataTypePtr & element_type = prepared_set->getElementsTypes()[tuple_idx]; + const bool is_fixed_string_element = WhichDataType(removeNullable(element_type)).isFixedString(); + for (size_t row = 0; row < prepared_set_total_row_count; ++row) { + /// A NULL element also matches the column's NULL rows, which the filter cannot express. + if (column->isNullAt(row)) + return false; + bloom_filters.back().emplace_back(params); - auto ref = column->getDataAt(row); + + /// `FixedString` element carries its padding, which the comparison ignores but the tokenizer would not. + std::string_view ref = column->getDataAt(row); + if (is_fixed_string_element) + ref = ref.substr(0, ref.find_last_not_of('\0') + 1); + forEachTokenToBloomFilter(*tokenizer, ref.data(), ref.size(), bloom_filters.back().back()); } } diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h index 73150da1d62e..73d5861257b8 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h @@ -135,7 +135,7 @@ class MergeTreeConditionBloomFilterText final : public IMergeTreeIndexCondition bool traverseTreeEquals( const String & function_name, - const RPNBuilderTreeNode & key_node, + const RPNBuilderTreeNode & wrapped_key_node, const DataTypePtr & value_type, const Field & value_field, RPNElement & out); diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp index 548148d12bd7..81f576d8a6e8 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,7 +31,11 @@ #include #include #include +#include +#include +#include #include +#include #include #include #include @@ -126,6 +132,28 @@ void TextSearchQuery::initializeHash() hash = hash_state.get128(); } +/// The type under `Nullable`, `LowCardinality` and one level of `Array`. +static DataTypePtr removeArrayNullableLowCardinality(const DataTypePtr & type) +{ + auto inner_type = removeNullable(removeLowCardinality(type)); + if (const auto * array_type = typeid_cast(inner_type.get())) + inner_type = removeNullable(removeLowCardinality(array_type->getNestedType())); + return inner_type; +} + +static std::optional tryGetIndexedFixedStringSize(const Block & header) +{ + /// A text index is always defined on a single expression. + if (header.columns() != 1) + return std::nullopt; + + auto element_type = removeArrayNullableLowCardinality(header.getByPosition(0).type); + if (const auto * fixed_string_type = typeid_cast(element_type.get())) + return fixed_string_type->getN(); + + return std::nullopt; +} + MergeTreeIndexConditionText::MergeTreeIndexConditionText( const ActionsDAG::Node * predicate, ContextPtr context_, @@ -137,6 +165,7 @@ MergeTreeIndexConditionText::MergeTreeIndexConditionText( bool has_positions_) : WithContext(context_) , header(index_sample_block) + , indexed_fixed_string_size(tryGetIndexedFixedStringSize(header)) , normalized_index_column_name(normalized_index_column_name_) , owned_tokenizer(tokenizer_ && tokenizer_->isStateful() ? std::shared_ptr(tokenizer_->clone()) : nullptr) , tokenizer(owned_tokenizer ? owned_tokenizer.get() : tokenizer_) @@ -382,7 +411,6 @@ namespace /// A query in `All` mode folds postings by intersection, so a partially folded /// posting list is a superset of the result and can be used for pruning right away. bool queryMayBeTrueInRange( - const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range, TextSearchMode search_mode) @@ -391,8 +419,8 @@ bool queryMayBeTrueInRange( if (query_builder.is_failed) return false; - /// Pattern bypass means analysis is incomplete, so conservatively return true. - if (query_builder.is_bypassed && !query.getPatterns().empty()) + /// An incomplete scan may have missed matching tokens, so nothing can be pruned. + if (query_builder.is_analysis_incomplete) return true; if (!current_range.has_value()) @@ -424,7 +452,7 @@ bool hasAnyTokensInRange(const TextSearchQuery & query, const TextIndexAnalyzer: if (query.getTokens().empty()) return false; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::Any); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::Any); } bool hasAnyPatternsInRange(const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range) @@ -432,7 +460,7 @@ bool hasAnyPatternsInRange(const TextSearchQuery & query, const TextIndexAnalyze if (query.getPatterns().empty()) return false; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::Any); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::Any); } bool hasAllTokensOrEmptyInRange(const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range) @@ -440,7 +468,7 @@ bool hasAllTokensOrEmptyInRange(const TextSearchQuery & query, const TextIndexAn if (query.getTokens().empty()) return true; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::All); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::All); } bool hasAllTokensInRange(const TextSearchQuery & query, const TextIndexAnalyzer::QueryBuilder & query_builder, const std::optional & current_range) @@ -448,7 +476,7 @@ bool hasAllTokensInRange(const TextSearchQuery & query, const TextIndexAnalyzer: if (query.getTokens().empty()) return false; - return queryMayBeTrueInRange(query, query_builder, current_range, TextSearchMode::All); + return queryMayBeTrueInRange(query_builder, current_range, TextSearchMode::All); } } @@ -651,7 +679,8 @@ bool MergeTreeIndexConditionText::traverseAtomNode(const RPNBuilderTreeNode & no auto lhs_argument = function.getArgumentAt(0); auto rhs_argument = function.getArgumentAt(1); - if ((function_name == "in" || function_name == "globalIn") + if ((function_name == "in" || function_name == "globalIn" + || function_name == "nullIn" || function_name == "globalNullIn") && tryPrepareSetForTextSearch(lhs_argument, rhs_argument, function_name, out)) { out.function = RPNElement::FUNCTION_HAS_ANY_ELEMENTS; @@ -679,12 +708,16 @@ bool MergeTreeIndexConditionText::traverseAtomNode(const RPNBuilderTreeNode & no } VectorWithMemoryTracking MergeTreeIndexConditionText::stringToTokens(const Field & field) const +{ + return stringToTokens(std::string_view(field.safeGet())); +} + +VectorWithMemoryTracking MergeTreeIndexConditionText::stringToTokens(std::string_view raw) const { VectorWithMemoryTracking tokens; - const String & raw = field.safeGet(); if (has_preprocessor) { - const String processed = preprocessor->processConstant(raw); + const String processed = preprocessor->processConstant(String(raw)); tokenizer->stringToTokens(processed.data(), processed.size(), tokens); } else @@ -786,12 +819,21 @@ VectorWithMemoryTracking MergeTreeIndexConditionText::stringLikeToTokens return VectorWithMemoryTracking(unique_tokens.begin(), unique_tokens.end()); } -std::vector MergeTreeIndexConditionText::stringLikeToPatterns(const Field & field, bool case_insensitive) const +namespace +{ + +/// '%needle%' is anchored the same way in a token as in the whole value; an affix is not. +bool isInfixPattern(const String & pattern) { - /// Only handles the pure '%value%' form: one leading '%', a non-empty alphanumeric token immediately following, - /// then one trailing '%' immediately after the token, and nothing else. - /// Returns a single-element vector on success, empty on anything more complex. - /// Only this form is eligible for direct read mode. + return pattern.starts_with('%') && pattern.ends_with('%'); +} + +} + +std::vector +MergeTreeIndexConditionText::stringLikeToPatterns(const Field & field, bool case_insensitive) const +{ + /// Handles '%value%', 'value%' and '%value' with an alphanumeric needle; rejects anything more complex. const String value = preprocessor->processConstant(field.safeGet()); if (value.empty()) @@ -801,35 +843,22 @@ std::vector MergeTreeIndexConditionText::stringLikeT const size_t length = value.size(); size_t pos = 0; - const auto is_token_char = [](unsigned char c) { return isASCII(c) && isAlphaNumericASCII(static_cast(c)); }; - - const size_t min_pattern_length = getContext()->getSettingsRef()[Setting::text_index_like_min_pattern_length]; - - /// Must start with at least one '%'. - if (data[pos] != '%') - return {}; - while (pos < length && data[pos] == '%') ++pos; + const bool has_leading_wildcard = pos > 0; + /// Alphanumeric content must follow immediately. - if (pos >= length || !is_token_char(static_cast(data[pos]))) + if (pos >= length || !isAlphaNumericASCII(data[pos])) return {}; const size_t start = pos; - while (pos < length && is_token_char(static_cast(data[pos]))) + while (pos < length && isAlphaNumericASCII(data[pos])) ++pos; const size_t end = pos; - - /// Reject short needles: it might match too many dictionary tokens. - if (end - start < min_pattern_length) - return {}; - - /// Trailing '%' must follow immediately after the content. - if (pos >= length || data[pos] != '%') - return {}; + const bool has_trailing_wildcard = pos < length && data[pos] == '%'; while (pos < length && data[pos] == '%') ++pos; @@ -838,10 +867,20 @@ std::vector MergeTreeIndexConditionText::stringLikeT if (pos < length) return {}; + /// A pattern without wildcards is a plain equality comparison, which is served by the exact tokens path. + if (!has_leading_wildcard && !has_trailing_wildcard) + return {}; + + /// Reject short needles: they might match too many dictionary tokens. + if (end - start < getContext()->getSettingsRef()[Setting::text_index_like_min_pattern_length]) + return {}; + String pattern; - pattern += '%'; + if (has_leading_wildcard) + pattern += '%'; pattern.append(data + start, end - start); - pattern += '%'; + if (has_trailing_wildcard) + pattern += '%'; std::vector patterns; if (case_insensitive) @@ -889,9 +928,118 @@ static void validateRegexpPatterns(const Array & patterns, const Settings & sett #endif } +/// How a function treats the trailing zero bytes of a `FixedString` needle. +enum class FixedStringPaddingSemantics +{ + NeedleAsTyped, /// `has`, `mapContainsKey`, `mapContainsValue` + NeedleStripped, /// `hasAny`, `hasAll`: the needle is cast to `String` + BothStripped, /// `equals`, `IN`: the value loses its trailing zero bytes as well +}; + +/// What stays constant while one needle is normalized. +struct FixedStringNeedleContext +{ + FixedStringPaddingSemantics semantics; + std::optional indexed_fixed_string_size; + bool padding_never_in_terms; +}; + +static std::optional fixedStringPaddingSemantics(const String & function_name) +{ + if (function_name == "equals") + return FixedStringPaddingSemantics::BothStripped; + if (function_name == "hasAny" || function_name == "hasAll") + return FixedStringPaddingSemantics::NeedleStripped; + if (function_name == "has" || function_name == "mapContainsKey" || function_name == "mapContainsValue") + return FixedStringPaddingSemantics::NeedleAsTyped; + return std::nullopt; +} + +/// Their terms never contain a zero byte, so appending zero bytes to a value keeps all its terms. +static bool tokenizerSplitsAtZeroByte(ITokenizer::Type type) +{ + return type == ITokenizer::Type::SplitByNonAlpha + || type == ITokenizer::Type::Ngrams + || type == ITokenizer::Type::SparseGrams + || type == ITokenizer::Type::AsciiCJK; +} + +static std::string_view withoutTrailingZeros(std::string_view value) +{ + return value.substr(0, value.find_last_not_of('\0') + 1); +} + +/// Strips or re-pads the needle's trailing zero bytes in place to the form the index stores; false when no single form covers every match. +static bool tryNormalizeNeedlePadding(String & needle, bool needle_is_fixed_string, const FixedStringNeedleContext & context) +{ + const bool both_stripped = context.semantics == FixedStringPaddingSemantics::BothStripped; + const size_t stripped_size = withoutTrailingZeros(needle).size(); + + if (context.indexed_fixed_string_size) + { + /// The column stores its values padded to N and compares them without the padding. + if (both_stripped || needle_is_fixed_string) + needle.resize(stripped_size); + /// A `String` needle keeps its zero bytes for these functions and then matches nothing. + else if (needle.ends_with('\0')) + return false; + + if (needle.size() > *context.indexed_fixed_string_size) + return false; + + needle.resize(*context.indexed_fixed_string_size, '\0'); + return true; + } + + if (!needle_is_fixed_string || context.semantics == FixedStringPaddingSemantics::NeedleAsTyped) + return true; + + /// A value with any number of trailing zero bytes matches, so one lookup covers them only if none can end up in a term. + if (both_stripped && stripped_size != needle.size() && !context.padding_never_in_terms) + return false; + + needle.resize(stripped_size); + return true; +} + +/// Same, elementwise for arrays. +static bool tryNormalizeNeedlePadding(Field & value, const DataTypePtr & value_type, const FixedStringNeedleContext & context) +{ + auto inner_type = removeNullable(removeLowCardinality(value_type)); + + if (const auto * array_type = typeid_cast(inner_type.get()); + array_type && value.getType() == Field::Types::Array) + { + for (auto & element : value.safeGet()) + { + if (!tryNormalizeNeedlePadding(element, array_type->getNestedType(), context)) + return false; + } + return true; + } + + if (value.getType() != Field::Types::String) + return true; + + return tryNormalizeNeedlePadding(value.safeGet(), isFixedString(inner_type), context); +} + +/// The value an absent map key reads: `''`, or all NUL when the value type is `FixedString`. +/// `mapValues` stores neither. +static bool isMapValueDefault(std::string_view value, const Block & header) +{ + /// A text index is always defined on a single expression. + chassert(header.columns() == 1); + auto value_type = removeNullable(removeLowCardinality(header.getByPosition(0).type)); + if (const auto * array_type = typeid_cast(value_type.get())) + value_type = removeNullable(removeLowCardinality(array_type->getNestedType())); + + return value.empty() || (isFixedString(value_type) && value.find_first_not_of('\0') == std::string_view::npos); +} + bool MergeTreeIndexConditionText::traverseFunctionNode( const RPNBuilderFunctionTreeNode & function_node, - const RPNBuilderTreeNode & index_column_node, + const RPNBuilderTreeNode & argument_node, DataTypePtr value_type, Field value_field, RPNElement & out) const @@ -899,6 +1047,9 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( const String function_name = function_node.getFunctionName(); auto direct_read_mode = getDirectReadMode(function_name); + /// The index knows the expression under the conversion, e.g. `m.key_` in `equals(_CAST(m.key_, 'String'), 'value')`. + const auto index_column_node = unwrapLosslessConversion(argument_node); + auto index_column_name = index_column_node.getColumnName(); bool has_index_column = hasIndexForColumn(index_column_name); bool has_map_keys_column = hasIndexForColumn(fmt::format("mapKeys({})", index_column_name)); @@ -938,7 +1089,7 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( auto & [map_column_name, _] = *parsed; if (header.has(fmt::format("mapValues({})", map_column_name)) && value_field.getType() == Field::Types::String - && !value_field.safeGet().empty()) + && !isMapValueDefault(value_field.safeGet(), header)) { has_index_column = true; direct_read_mode = getHintOrNoneMode(); @@ -954,10 +1105,28 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( if (!value_data_type.isStringOrFixedString() && !value_data_type.isArray()) return false; + /// A `FixedString` on either side drops trailing zero bytes in the comparison. + if (auto semantics = fixedStringPaddingSemantics(function_name); + semantics && (indexed_fixed_string_size || isFixedString(removeArrayNullableLowCardinality(value_type)))) + { + const FixedStringNeedleContext context{ + .semantics = *semantics, + .indexed_fixed_string_size = indexed_fixed_string_size, + .padding_never_in_terms = !has_preprocessor && tokenizerSplitsAtZeroByte(tokenizer->getType()), + }; + if (!tryNormalizeNeedlePadding(value_field, value_type, context)) + return false; + } + const auto & settings = getContext()->getSettingsRef(); const bool is_array_tokenizer = (tokenizer->getType() == ITokenizer::Type::Array); + const auto * index_column_dag_node = index_column_node.getDAGNode(); + const DataTypePtr index_column_type = index_column_dag_node ? index_column_dag_node->result_type : nullptr; + /// A UInt8 virtual column cannot carry the NULL a predicate returns for a NULL value, which NOT flips to true. + const bool affix_patterns_allowed = index_column_type && !isNullableOrLowCardinalityNullable(index_column_type); + /// like/ilike optimization is only supported for splitByNonAlpha and array tokenizers. static const std::unordered_set like_optimization_supported_tokenizers = { ITokenizer::Type::SplitByNonAlpha, @@ -1130,50 +1299,30 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( } if (function_name == "hasToken" || function_name == "hasTokenOrNull") { - // hasToken and hasTokenOrNull are legacy functions which assume splitByNonAlpha as - /// tokenizer. The text index can answer it only correctly if this is the index tokenizer. - /// In all other cases, bypass the index. + /// `hasToken` splits by non-alphanumeric characters, so only an index with the same tokenizer can answer it. if (tokenizer->getType() != ITokenizer::Type::SplitByNonAlpha) return false; - /// Unlike hasToken, hasTokenOrNull is never rewritten to direct-read, so the pre/postprocessor - /// is also not applied to its needle. Using the index here (where stringToTokens does apply them, - /// e.g. mapping a dropped token to the empty sentinel that prunes every granule) would disagree - /// with the scan result. Bail out so the index is not used for hasTokenOrNull when a - /// pre/postprocessor is configured; the plain index path is unaffected. + /// `hasTokenOrNull` is never rewritten to a direct read, so its needle never goes through the + /// pre/postprocessor, while `stringToTokens` applies them. The index would disagree with the scan. if (function_name == "hasTokenOrNull" && (has_preprocessor || has_postprocessor)) return false; - /// A needle containing a token separator is invalid for `hasToken` and the brute-force scan raises - /// BAD_ARGUMENTS for this. hasToken uses Exact direct read, so the index would tokenize the needle and - /// silently replace the predicate (or prune the granule that would have thrown), hiding the exception. - /// Therefore bypass the index and do a brute-force scan. hasTokenOrNull is not affected: it returns NULL - /// - /// A separator is any ASCII non-alphanumeric character. - if (function_name == "hasToken" - && std::ranges::any_of(value_field.safeGet(), [](unsigned char c) { return isASCII(c) && !isAlphaNumericASCII(c); })) + /// The scan raises BAD_ARGUMENTS for a needle with a separator, and the exact direct read of `hasToken` + /// would replace the predicate and hide it. `hasTokenOrNull` returns NULL for such a needle instead. + if (function_name == "hasToken" && std::ranges::any_of(value_field.safeGet(), isTokenSeparator)) return false; auto tokens = stringToTokens(value_field); if (tokens.empty()) { + /// A needle without a word character is invalid: leave it to the scan, which raises or returns NULL. + /// Otherwise the pre/postprocessor dropped the needle (e.g. a stop word), so it is not in the index: + /// push the empty sentinel to prune every granule. const String & string_needle = value_field.safeGet(); - if (!string_needle.empty()) - { - /// hasToken uses splitByNonAlpha as its tokenizer, so: - /// - A needle without any word character (alphanumeric or non-ASCII) is invalid. - /// - Bypass the index in that case so the row-level evaluation throws BAD_ARGUMENTS (or returns NULL for hasTokenOrNull) - /// -- Consistent with the no-index behaviour. - /// If the needle does contain word characters (e.g. "abc" with ngrams(4)): - /// - It is valid but too short for the index's tokenizer: - /// -- Fall through to push "" so all granules are pruned and the query returns 0 rows. - /// If the postprocessor filters the needle (e.g. stop-word): - /// -- The needle is not in the index; push "" sentinel so the condition evaluates to false. - if (std::ranges::none_of(string_needle, [](unsigned char c) { return !isASCII(c) || isAlphaNumericASCII(c); })) - return false; - } - /// - If the needle does contain word characters (e.g. "abc" with ngrams(4)), it is valid but too short for the index's tokenizer: - /// Fall through but push "" so all granules are pruned and the query returns 0 rows. + if (!string_needle.empty() && std::ranges::all_of(string_needle, isTokenSeparator)) + return false; + tokens.push_back(""); } @@ -1256,20 +1405,39 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); return true; } - if (function_name == "startsWith" && tokenizer->supportsStringLike()) + if (function_name == "startsWith" || function_name == "endsWith") { if (!value_data_type.isStringOrFixedString()) return false; - auto tokens = substringToTokens(value_field, true, false); - out.function = RPNElement::FUNCTION_EQUALS; - out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); - return true; - } - if (function_name == "endsWith" && tokenizer->supportsStringLike()) - { - if (!value_data_type.isStringOrFixedString()) + + const bool is_prefix = (function_name == "startsWith"); + + /// A needle inside a single token yields no complete token below, so evaluate it as `LIKE 'needle%'`. + /// Safe for a literal needle: one that would need LIKE escaping is not alphanumeric and is rejected. + if (like_optimization_supported_tokenizers.contains(tokenizer->getType()) && !has_preprocessor && !has_postprocessor + && settings[Setting::use_text_index_like_evaluation_by_dictionary_scan]) + { + const auto & needle = value_field.safeGet(); + const auto affix_pattern = is_prefix ? needle + "%" : "%" + needle; + auto patterns = stringLikeToPatterns(affix_pattern, /*case_insensitive=*/ false); + if (patterns.size() == 1 && affix_patterns_allowed) + { + const auto is_exact = is_array_tokenizer && candidate_for_exact_mode; + const auto pattern_read_mode = is_exact ? TextIndexDirectReadMode::Exact : direct_read_mode; + + out.function = RPNElement::FUNCTION_LIKE; + out.text_search_queries.emplace_back( + std::make_shared( + function_name, TextSearchMode::Any, pattern_read_mode, + VectorWithMemoryTracking(), std::move(patterns))); + return true; + } + } + + if (!tokenizer->supportsStringLike()) return false; - auto tokens = substringToTokens(value_field, false, true); + + auto tokens = substringToTokens(value_field, is_prefix, !is_prefix); out.function = RPNElement::FUNCTION_EQUALS; out.text_search_queries.emplace_back(std::make_shared(function_name, TextSearchMode::All, direct_read_mode, std::move(tokens))); return true; @@ -1282,7 +1450,7 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( if (like_optimization_supported_tokenizers.contains(tokenizer->getType()) && !has_preprocessor && !has_postprocessor && settings[Setting::use_text_index_like_evaluation_by_dictionary_scan]) { - /// TODO(ahmadov): Only '%foo%' pattern is eligible for direct read mode. An empty vector means the pattern is too complex. + /// TODO(ahmadov): Only the '%foo%', 'foo%' and '%foo' patterns are eligible for a dictionary scan. /// Add support for multiple patterns later with hint mode: /// 1. Handle multiple patterns e.g. %foo bar% -> postings_pattern(%foo) && postings_pattern(bar%) && regex(%foo bar%) /// 2. Handle exact tokens and patterns e.g. %foo bar baz% -> postings_exact(bar) && postings_pattern(%foo) && postings_pattern(bar%) @@ -1290,15 +1458,21 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( /// 4. Fall-back to the brute-force search for other cases for now. /// Follow-up: /// 1. Handle more complex patterns e.g. %foo%bar% -> (postings_pattern(%foo%) && postings_pattern(%bar%)) || postings_pattern(%foo%bar%) - auto patterns = stringLikeToPatterns(value_field, false); - if (patterns.size() == 1) + /// 2. Seek the sorted dictionary to the matching range of 'foo%' instead of scanning every block. + /// 3. Bypass a non-selective hint: it prunes nothing and still costs a dictionary scan. + const auto & like_pattern = value_field.safeGet(); + auto patterns = stringLikeToPatterns(value_field, /*case_insensitive=*/ false); + const bool is_infix = isInfixPattern(like_pattern); + if (patterns.size() == 1 && (is_infix || affix_patterns_allowed)) { - const auto pattern_read_mode = candidate_for_exact_mode ? TextIndexDirectReadMode::Exact : getHintOrNoneMode(); + const auto is_exact = (is_infix || is_array_tokenizer) && candidate_for_exact_mode; + const auto pattern_read_mode = is_exact ? TextIndexDirectReadMode::Exact : direct_read_mode; out.function = RPNElement::FUNCTION_LIKE; out.text_search_queries.emplace_back( std::make_shared( - function_name, TextSearchMode::Any, pattern_read_mode, VectorWithMemoryTracking(), std::move(patterns))); + function_name, TextSearchMode::Any, pattern_read_mode, + VectorWithMemoryTracking(), std::move(patterns))); return true; } } @@ -1321,15 +1495,20 @@ bool MergeTreeIndexConditionText::traverseFunctionNode( if (has_postprocessor) return false; - auto patterns = stringLikeToPatterns(value_field, true); - if (patterns.size() == 1) + const auto & like_pattern = value_field.safeGet(); + auto patterns = stringLikeToPatterns(value_field, /*case_insensitive=*/ true); + const bool is_infix = isInfixPattern(like_pattern); + if (patterns.size() == 1 && (is_infix || affix_patterns_allowed)) { - const auto pattern_read_mode = candidate_for_exact_mode ? TextIndexDirectReadMode::Exact : getHintOrNoneMode(); + /// `getDirectReadMode` does not list `ilike`, which is served by the dictionary scan only. + const auto is_exact = (is_infix || is_array_tokenizer) && candidate_for_exact_mode; + const auto pattern_read_mode = is_exact ? TextIndexDirectReadMode::Exact : getHintOrNoneMode(); out.function = RPNElement::FUNCTION_LIKE; out.text_search_queries.emplace_back( std::make_shared( - function_name, TextSearchMode::Any, pattern_read_mode, VectorWithMemoryTracking(), std::move(patterns))); + function_name, TextSearchMode::Any, pattern_read_mode, + VectorWithMemoryTracking(), std::move(patterns))); return true; } return false; @@ -1466,9 +1645,13 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc /// It can be an arbitrary function that returns 0 for the default value of the map value type. /// It is true because `arrayElement` (and the equivalent subcolumn access) returns default value if key doesn't exist in the map, /// therefore we can use index to skip granules and use direct read as a hint for the original condition. + /// The result may be `Nullable` or `LowCardinality`, e.g. for a `Nullable` key: NULL reads as false below, as in WHERE. const auto * dag_node = function_node.getDAGNode(); - if (!dag_node || !dag_node->function_base || !dag_node->isDeterministic() || !WhichDataType(dag_node->result_type).isUInt8()) + if (!dag_node + || !dag_node->function_base + || !dag_node->isDeterministic() + || !WhichDataType(removeLowCardinalityAndNullable(dag_node->result_type)).isUInt8()) return false; auto subdag = ActionsDAG::cloneSubDAG({dag_node}, true); @@ -1508,10 +1691,14 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc if (map_argument->type != ActionsDAG::ActionType::INPUT || map_argument->result_name != required_column.name) return false; - if (const_key_argument->type != ActionsDAG::ActionType::COLUMN || !isStringOrFixedString(const_key_argument->result_type)) + /// A NULL key is declined: `arrayElement` returns NULL for it, not the default value. + Field key_field; + DataTypePtr key_type; + if (!RPNBuilderTreeNode(const_key_argument, function_node.getTreeContext()).tryGetConstant(key_field, key_type) + || key_field.getType() != Field::Types::String) return false; - key_const_value = std::string{const_key_argument->column->getDataAt(0)}; + key_const_value = key_field.safeGet(); } else { @@ -1567,7 +1754,7 @@ bool MergeTreeIndexConditionText::traverseMapElementKeyNode(const RPNBuilderFunc if (result_column->getBool(0)) return false; - auto tokens = stringToTokens(*key_const_value); + auto tokens = stringToTokens(std::string_view(*key_const_value)); out.function = RPNElement::FUNCTION_EQUALS; out.text_search_queries.emplace_back(std::make_shared("mapContainsKey", TextSearchMode::All, getHintOrNoneMode(), std::move(tokens))); return true; @@ -1601,7 +1788,7 @@ bool MergeTreeIndexConditionText::traverseMapElementValueNode(const RPNBuilderTr /// for functions like `func(arrayElement(m, 'const_key'), ...)`. /// If index can be used, than we can analyze the index as for scalar string column /// because `arrayElement(m, 'const_key')` projects Array(String) to String. - if (const_value.getType() != Field::Types::String || const_value.safeGet().empty()) + if (const_value.getType() != Field::Types::String || isMapValueDefault(const_value.safeGet(), header)) return false; return hasIndexForMapElementValue(index_column_node); @@ -1615,8 +1802,10 @@ bool MergeTreeIndexConditionText::traverseJSONSubcolumnKeyNode( /// Similar to traverseMapElementKeyNode but for JSON subcolumns. const auto * dag_node = function_node.getDAGNode(); - if (!dag_node || !dag_node->function_base || !dag_node->isDeterministic() - || !WhichDataType(removeNullable(dag_node->result_type)).isUInt8()) + if (!dag_node + || !dag_node->function_base + || !dag_node->isDeterministic() + || !WhichDataType(removeLowCardinalityAndNullable(dag_node->result_type)).isUInt8()) return false; auto subdag = ActionsDAG::cloneSubDAG({dag_node}, true); @@ -1681,10 +1870,18 @@ bool MergeTreeIndexConditionText::tryPrepareSetForTextSearch( { std::optional set_key_position; - auto has_index = [&](const RPNBuilderTreeNode & node) + /// `m['key']` answered by a `mapValues(m)` index: an absent key reads the value type's default. + bool has_index_for_map_element_value = false; + + auto has_index = [&](const RPNBuilderTreeNode & argument) { + const auto node = unwrapLosslessConversion(argument); + if (hasIndexForMapElementValue(node)) + { + has_index_for_map_element_value = true; + return true; + } return hasIndexForColumn(node.getColumnName()) - || hasIndexForMapElementValue(node) || tryMatchNodeToJSONIndex(node, header, "JSONAllValues"); }; @@ -1733,19 +1930,52 @@ bool MergeTreeIndexConditionText::tryPrepareSetForTextSearch( return false; const auto & set_column = *columns[*set_key_position]; - if (!WhichDataType(set_column.getDataType()).isStringOrFixedString()) + + /// With setting `transform_null_in = 1`, the IN set can be nullable. + const auto * set_column_nullable = typeid_cast(&set_column); + const auto & set_column_values = set_column_nullable ? set_column_nullable->getNestedColumn() : set_column; + + if (!WhichDataType(set_column_values.getDataType()).isStringOrFixedString()) return false; size_t total_row_count = prepared_set->getTotalRowCount(); + const bool set_is_fixed_string = WhichDataType(set_column_values.getDataType()).isFixedString(); + /// `IN` casts the value to the set's type, so a `FixedString` on either side drops trailing zero bytes like `equals`. + const bool has_fixed_string = set_is_fixed_string || indexed_fixed_string_size.has_value(); + const FixedStringNeedleContext context{ + .semantics = FixedStringPaddingSemantics::BothStripped, + .indexed_fixed_string_size = indexed_fixed_string_size, + .padding_never_in_terms = !has_preprocessor && tokenizerSplitsAtZeroByte(tokenizer->getType()), + }; + String normalized; for (size_t row = 0; row < total_row_count; ++row) { - auto ref = set_column.getDataAt(row); + /// The atom is an OR over the elements, and the index skips NULL rows when building a + /// granule, so a NULL element is a disjunct it cannot bind. Decline the atom. + if (set_column.isNullAt(row)) + { + out.text_search_queries.clear(); + return false; + } + + std::string_view element = set_column.getDataAt(row); + + if (has_fixed_string) + { + normalized.assign(element); + if (!tryNormalizeNeedlePadding(normalized, set_is_fixed_string, context)) + { + out.text_search_queries.clear(); + return false; + } + element = normalized; + } /// Reject the index usage when there is an empty string in the set. /// The condition with such a predicate will be always true on granule. /// See MergeTreeIndexGranuleText::hasAllQueryTokensOrEmpty. - if (ref.empty()) + if (element.empty() || (has_index_for_map_element_value && isMapValueDefault(element, header))) { out.text_search_queries.clear(); return false; @@ -1754,7 +1984,7 @@ bool MergeTreeIndexConditionText::tryPrepareSetForTextSearch( /// Apply preprocessor + tokenizer + postprocessor so set elements use the same /// tokens that were stored in the index. Skipping the postprocessor here would /// produce false negatives for postprocessors like lower(), stem(), etc. - VectorWithMemoryTracking tokens = stringToTokens(Field(String(ref))); + VectorWithMemoryTracking tokens = stringToTokens(element); /// An element that tokenizes to nothing cannot be proven present by the index. /// Bail out to keep the original predicate. diff --git a/src/Storages/MergeTree/MergeTreeIndexConditionText.h b/src/Storages/MergeTree/MergeTreeIndexConditionText.h index 442e21d6a59d..2573aebca1e0 100644 --- a/src/Storages/MergeTree/MergeTreeIndexConditionText.h +++ b/src/Storages/MergeTree/MergeTreeIndexConditionText.h @@ -163,7 +163,7 @@ class MergeTreeIndexConditionText final : public IMergeTreeIndexCondition, publi bool traverseFunctionNode( const RPNBuilderFunctionTreeNode & function_node, - const RPNBuilderTreeNode & index_column_node, + const RPNBuilderTreeNode & argument_node, DataTypePtr value_type, Field value_field, RPNElement & out) const; @@ -179,12 +179,14 @@ class MergeTreeIndexConditionText final : public IMergeTreeIndexCondition, publi bool hasIndexForMapElementValue(const RPNBuilderTreeNode & node) const; VectorWithMemoryTracking stringToTokens(const Field & field) const; + VectorWithMemoryTracking stringToTokens(std::string_view raw) const; VectorWithMemoryTracking substringToTokens(const Field & field, bool is_prefix, bool is_suffix) const; VectorWithMemoryTracking stringLikeToTokens(const Field & field) const; /// Builds the OR-list of token sets for a `match`-style regexp, folding the required substring /// into every alternative. Returns an empty list when the regexp imposes no token requirement. std::vector> regexpToTokensForQueries(const String & regexp_string) const; + /// Supports '%needle%', 'needle%' and '%needle'. See isInfixPattern for which of them is exact. std::vector stringLikeToPatterns(const Field & field, bool case_insensitive = false) const; bool tryPrepareSetForTextSearch(const RPNBuilderTreeNode & lhs, const RPNBuilderTreeNode & rhs, const String & function_name, RPNElement & out) const; @@ -197,6 +199,8 @@ class MergeTreeIndexConditionText final : public IMergeTreeIndexCondition, publi static bool requiresReadingAllTokens(const RPNElement & element); Block header; + /// N when the index is defined over a `FixedString(N)`, directly or as the array element type. + std::optional indexed_fixed_string_size; std::optional normalized_index_column_name; /// A private clone of the index tokenizer when it is stateful, so concurrent conditions do not /// share mutable parsing state; null otherwise. diff --git a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp index f58e8a1200c4..d61b342cbe7a 100644 --- a/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderTextIndex.cpp @@ -134,16 +134,16 @@ void MergeTreeReaderTextIndex::initializeFallbackReader(const IMergeTreeReader * /// - Pattern queries (LIKE): fallback when dictionary scan is abandoned. /// - Phrase queries (hasPhrase with Exact mode): fallback when estimated cardinality is too high /// and reading position data would be slower than evaluating directly. - bool has_fallback_candidates = condition_text->hasSearchPatterns() - || std::ranges::any_of( - search_queries, - [](const auto & search_query) - { - return search_query && search_query->getSearchMode() == TextSearchMode::Phrase - && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact; - }); + /// Only exact direct read needs it: a hint keeps the original predicate, so it can just be always true. + auto needs_fallback_for_query = [](const auto & search_query) + { + if (!search_query || search_query->getDirectReadMode() != TextIndexDirectReadMode::Exact) + return false; - if (!has_fallback_candidates) + return !search_query->getPatterns().empty() || search_query->getSearchMode() == TextSearchMode::Phrase; + }; + + if (std::ranges::none_of(search_queries, needs_fallback_for_query)) return; /// Build a fallback evaluation path. Compile each virtual column's default expression @@ -167,12 +167,7 @@ void MergeTreeReaderTextIndex::initializeFallbackReader(const IMergeTreeReader * { const auto & column = columns_to_read[i]; const auto & search_query = search_queries[i]; - if (!search_query) - continue; - - bool needs_fallback = !search_query->getPatterns().empty() - || (search_query->getSearchMode() == TextSearchMode::Phrase && search_query->getDirectReadMode() == TextIndexDirectReadMode::Exact); - if (!needs_fallback) + if (!needs_fallback_for_query(search_query)) continue; /// Compile the virtual column's default expression (the original search predicate). @@ -817,17 +812,8 @@ void MergeTreeReaderTextIndex::fillColumnLazy(IColumn & column, size_t column_id return; } - /// Convert postings to a sorted array and build a cursor from it. - auto key = TextIndexPostingsCache::hash(granule->getIndexIdForCaches(), columns_to_read[column_idx].name, static_cast(TextIndexPostingsCacheKind::Flat)); - - auto cell = condition_text->postingsCache()->getOrSet(key, [&] - { - auto flat = std::make_shared>(query_builder.postings->cardinality()); - query_builder.postings->toUint32Array(flat->data()); - return std::make_shared(std::move(flat)); - }); - - prebuilt_cursor = std::make_shared(std::get(cell->value)); + /// Build a cursor over the sorted array of postings, shared by all readers of the granule. + prebuilt_cursor = std::make_shared(query_builder.getFlatPostings()); cursors.push_back(prebuilt_cursor); } } diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e44eb4a2aaad..9c45cdd765f5 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -2830,6 +2830,16 @@ void MergeTreeSettings::applyChanges(const SettingsChanges & changes, ContextPtr impl->applyChanges(resolved_changes); } +void MergeTreeSettings::applyChangesLeavingDiskUnresolved(const SettingsChanges & changes) +{ + SettingsChanges changes_without_disk; + changes_without_disk.reserve(changes.size()); + for (const auto & change : changes) + if (change.name != "disk") + changes_without_disk.push_back(change); + impl->applyChanges(changes_without_disk); +} + void MergeTreeSettings::applyChange(const SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata) { auto resolved_change = change; diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 21206fca17a4..1a7910394b21 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -94,6 +94,13 @@ struct MergeTreeSettings SettingsChanges changesFrom(const MergeTreeSettings & base) const; void applyChanges(const SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata); void applyChange(const SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata); + /// For the computations that derive a value from the settings a statement would leave behind, and + /// read scalar settings only. Resolving the `disk` setting creates and registers the disk its + /// definition describes, and a computation that runs before the statement is known to be allowed + /// must not do that: a rejected `ALTER TABLE ... MODIFY SETTING disk = disk(...)` would leave the + /// disk, and the directory it created, behind. The settings update itself resolves it, see + /// `MergeTreeData::changeSettings`. + void applyChangesLeavingDiskUnresolved(const SettingsChanges & changes); VectorWithMemoryTracking getAllRegisteredNames() const; static std::vector getAllAliasNames(); std::string_view getDescription(std::string_view name) const; diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 87693b515ecc..167c6a53ef73 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,11 @@ namespace MergeTreeSetting extern const MergeTreeSettingsBool enable_index_granularity_compression; } +namespace FailPoints +{ + extern const char patch_part_index_write_empty[]; +} + MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeMutableDataPartPtr & data_part, MergeTreeSettingsPtr data_settings, @@ -380,9 +386,19 @@ MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDis /// throws `CORRUPTED_DATA` otherwise, including for empty covering parts. if (new_part->info.isPatch()) { + /// Writes an index without source parts, which is the corruption shape the load path + /// rejects: a patch part that holds rows but names no part it patches. Only for tests. + bool write_empty_index = false; + fiu_do_on(FailPoints::patch_part_index_write_empty, { write_empty_index = true; }); + write_hashed_file(PatchPartIndex::FILENAME, [&](auto & buffer) { - new_part->getPatchPartIndex().writeBinary(buffer); + const auto & patch_part_index = new_part->getPatchPartIndex(); + + if (write_empty_index) + patch_part_index.cloneEmpty().writeBinary(buffer); + else + patch_part_index.writeBinary(buffer); }); } } diff --git a/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp b/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp index 61bf1bb38a49..2ebcc2b6f0cb 100644 --- a/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp +++ b/src/Storages/MergeTree/PatchParts/PatchPartIndex.cpp @@ -271,6 +271,10 @@ PatchPartIndex PatchPartIndex::readBinary(ReadBuffer & in) } res.buildSourcePartsByVersion(); + + /// Consumes exactly the bytes of the index and nothing after them: the index is also embedded in + /// larger streams, so whether anything may follow it is for the caller to decide (see + /// `IMergeTreeDataPart::loadPatchPartIndex` for the file that holds nothing else). return res; } diff --git a/src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp b/src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp new file mode 100644 index 000000000000..92c3b967e9b8 --- /dev/null +++ b/src/Storages/MergeTree/PatchParts/tests/gtest_patch_part_index_read.cpp @@ -0,0 +1,82 @@ +#include + +#include +#include +#include +#include + +using namespace DB; + +/// A patch part carries the index of the parts it patches in `source_parts.dat`, and that file holds +/// nothing else. A zeroed block of the same size used to parse as a valid index: the first byte `0` is +/// the `V1` format version, the next eight zero bytes are `num_parts = 0`, and everything after them +/// was ignored. The load path now asserts that the file ends where the index ends, which relies on +/// `readBinary` consuming exactly the bytes of the index: the same parser reads the index out of larger +/// streams (the in-memory part data exchanged between replicas), where bytes do follow it. +TEST(PatchPartIndexRead, ConsumesExactlyTheIndex) +{ + PatchPartIndex index(MergeTreePatchPartsVersion::V1, ""); + index.addSourcePart("all_1_1_0", 2); + index.addSourcePart("all_2_2_0", 3); + + String written; + { + WriteBufferFromString out(written); + index.writeBinary(out); + } + + { + ReadBufferFromString in(written); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_FALSE(read_index.empty()); + EXPECT_EQ(read_index.getMinDataVersion("all_1_1_0"), 2); + EXPECT_EQ(read_index.getMaxDataVersion("all_2_2_0"), 3); + EXPECT_TRUE(in.eof()); + } + + /// The corruption shape from the issue: the file keeps its size, but its content is gone. The + /// parser stops after the nine bytes it understands, and what the loader does next is what turns + /// the rest of the block into a loud failure instead of an accepted empty index. + String zero_filled(written.size(), '\0'); + ASSERT_GT(zero_filled.size(), 9u); + { + ReadBufferFromString in(zero_filled); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_TRUE(read_index.empty()); + EXPECT_EQ(in.count(), 9u); + EXPECT_FALSE(in.eof()); + EXPECT_ANY_THROW(assertEOF(in)); + } + + /// Bytes after a well-formed index are left in the stream for the caller. + { + /// `ReadBufferFromString` only borrows the bytes, so the string has to outlive the buffer. + String with_trailing_bytes = written + String("tail"); + ReadBufferFromString in(with_trailing_bytes); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_FALSE(read_index.empty()); + EXPECT_EQ(in.count(), written.size()); + + String rest; + readStringUntilEOF(rest, in); + EXPECT_EQ(rest, "tail"); + } +} + +/// An index without source parts is still well-formed on its own - it is what an empty covering part +/// carries - so the parser accepts it and the load path is what rejects it for a part that holds rows. +TEST(PatchPartIndexRead, AcceptsAnEmptyIndex) +{ + PatchPartIndex index(MergeTreePatchPartsVersion::V1, ""); + + String written; + { + WriteBufferFromString out(written); + index.writeBinary(out); + } + + ReadBufferFromString in(written); + auto read_index = PatchPartIndex::readBinary(in); + EXPECT_TRUE(read_index.empty()); + EXPECT_TRUE(in.eof()); +} diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp index 47ed74cc8ff1..8580eedffbfc 100644 --- a/src/Storages/MergeTree/RPNBuilder.cpp +++ b/src/Storages/MergeTree/RPNBuilder.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -248,14 +249,14 @@ const Settings & RPNBuilderTreeContext::getSettings() const return query_context->getSettingsRef(); } -RPNBuilderTreeNode::RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, RPNBuilderTreeContext & tree_context_) +RPNBuilderTreeNode::RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, const RPNBuilderTreeContext & tree_context_) : dag_node(dag_node_) , tree_context(tree_context_) { chassert(dag_node); } -RPNBuilderTreeNode::RPNBuilderTreeNode(const IAST * ast_node_, RPNBuilderTreeContext & tree_context_) +RPNBuilderTreeNode::RPNBuilderTreeNode(const IAST * ast_node_, const RPNBuilderTreeContext & tree_context_) : ast_node(ast_node_) , tree_context(tree_context_) { @@ -371,6 +372,16 @@ ColumnWithTypeAndName RPNBuilderTreeNode::getConstantColumn() const return result; } +/// A `Field` is the plain value of a constant: `LowCardinality` is only an encoding of the column, +/// and a value that is not NULL has no `Nullable` type. +static DataTypePtr getTypeOfConstantValue(const Field & value, const DataTypePtr & type) +{ + auto value_type = removeLowCardinality(type); + if (!value.isNull()) + value_type = removeNullable(value_type); + return value_type; +} + bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & output_type) const { if (ast_node) @@ -389,12 +400,7 @@ bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & outp /// Simple literal output_value = literal->value; - output_type = block_with_constants.getByName(column_name).type; - - /// If constant is not Null, we can assume it's type is not Nullable as well. - if (!output_value.isNull()) - output_type = removeNullable(output_type); - + output_type = getTypeOfConstantValue(output_value, block_with_constants.getByName(column_name).type); return true; } if (block_with_constants.has(column_name) && isColumnConst(*block_with_constants.getByName(column_name).column)) @@ -402,11 +408,7 @@ bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & outp /// An expression which is dependent on constants only const auto & constant_column = block_with_constants.getByName(column_name); output_value = (*constant_column.column)[0]; - output_type = constant_column.type; - - if (!output_value.isNull()) - output_type = removeNullable(output_type); - + output_type = getTypeOfConstantValue(output_value, constant_column.type); return true; } } @@ -417,11 +419,7 @@ bool RPNBuilderTreeNode::tryGetConstant(Field & output_value, DataTypePtr & outp if (node_without_alias->column) { output_value = node_without_alias->column->getField(); - output_type = node_without_alias->result_type; - - if (!output_value.isNull()) - output_type = removeNullable(output_type); - + output_type = getTypeOfConstantValue(output_value, node_without_alias->result_type); return true; } } @@ -584,6 +582,77 @@ RPNBuilderTreeNode RPNBuilderFunctionTreeNode::getArgumentAt(size_t index) const return RPNBuilderTreeNode(dag_node->children[index], tree_context); } +namespace +{ + +/// Whether converting a value of type `from` to type `to` never changes it and never throws. +/// `Nullable` cannot be dropped, because it may throw on NULL. +bool isLosslessConversion(const DataTypePtr & from, const DataTypePtr & to) +{ + auto from_type = removeLowCardinality(from); + auto to_type = removeLowCardinality(to); + + if (to_type->isNullable()) + { + from_type = removeNullable(from_type); + to_type = removeNullable(to_type); + } + else if (from_type->isNullable()) + { + return false; + } + + if (from_type->equals(*to_type)) + return true; + + const auto * from_array = typeid_cast(from_type.get()); + const auto * to_array = typeid_cast(to_type.get()); + return from_array && to_array && isLosslessConversion(from_array->getNestedType(), to_array->getNestedType()); +} + +} + +bool isLosslessConversionFunction(const ActionsDAG::Node & node) +{ + if (node.type != ActionsDAG::ActionType::FUNCTION || !node.function_base) + return false; + + const auto function_name = node.function_base->getName(); + const size_t arguments_size = node.children.size(); + + const bool is_cast = (function_name == "CAST" || function_name == "_CAST") && arguments_size == 2; + const bool is_wrapper = (function_name == "toNullable" || function_name == "toLowCardinality") && arguments_size == 1; + + if (!is_cast && !is_wrapper) + return false; + + return isLosslessConversion(node.children.front()->result_type, node.result_type); +} + +RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node) +{ + if (!node.isFunction()) + return node; + + const auto function = node.toFunctionNode(); + const auto * function_dag_node = function.getDAGNode(); + + if (!function_dag_node || !isLosslessConversionFunction(*function_dag_node)) + return node; + + return unwrapLosslessConversion(function.getArgumentAt(0)); +} + +const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node) +{ + const auto * node_without_alias = getNodeWithoutAlias(node); + + if (!isLosslessConversionFunction(*node_without_alias)) + return node; + + return unwrapLosslessConversion(node_without_alias->children.front()); +} + template RPNBuilder::RPNBuilder( const ActionsDAG::Node * filter_actions_dag_node, diff --git a/src/Storages/MergeTree/RPNBuilder.h b/src/Storages/MergeTree/RPNBuilder.h index a785294769b2..04d502c5286e 100644 --- a/src/Storages/MergeTree/RPNBuilder.h +++ b/src/Storages/MergeTree/RPNBuilder.h @@ -76,10 +76,10 @@ class RPNBuilderTreeNode { public: /// Construct RPNBuilderTreeNode with non null dag node and tree context - explicit RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, RPNBuilderTreeContext & tree_context_); + explicit RPNBuilderTreeNode(const ActionsDAG::Node * dag_node_, const RPNBuilderTreeContext & tree_context_); /// Construct RPNBuilderTreeNode with non null ast node and tree context - explicit RPNBuilderTreeNode(const IAST * ast_node_, RPNBuilderTreeContext & tree_context_); + explicit RPNBuilderTreeNode(const IAST * ast_node_, const RPNBuilderTreeContext & tree_context_); /// Get AST node const IAST * getASTNode() const { return ast_node; } @@ -113,6 +113,7 @@ class RPNBuilderTreeNode /** Try get constant from node. If node is constant returns true, and constant value and constant type output parameters are set. * Otherwise false is returned. + * The output type is the type of the value: `LowCardinality` is removed, and `Nullable` is removed when the value is not NULL. */ bool tryGetConstant(Field & output_value, DataTypePtr & output_type) const; @@ -141,16 +142,10 @@ class RPNBuilderTreeNode return tree_context; } - /// Get tree context - RPNBuilderTreeContext & getTreeContext() - { - return tree_context; - } - protected: const IAST * ast_node = nullptr; const ActionsDAG::Node * dag_node = nullptr; - RPNBuilderTreeContext & tree_context; + const RPNBuilderTreeContext & tree_context; }; /** RPNBuilderFunctionTreeNode is wrapper around RPNBuilderTreeNode with function type. @@ -173,6 +168,15 @@ class RPNBuilderFunctionTreeNode : public RPNBuilderTreeNode RPNBuilderTreeNode getArgumentAt(size_t index) const; }; +/// Whether the node is `CAST`, `_CAST`, `toNullable` or `toLowCardinality` whose conversion never changes +/// the value and never throws: `LowCardinality` added or dropped and `Nullable` added, at any depth of `Array`. +bool isLosslessConversionFunction(const ActionsDAG::Node & node); + +/// Strips lossless conversions (see above) from the node. Indexes are analyzed on the expression under them. +/// The AST form carries no types and is returned as is. +RPNBuilderTreeNode unwrapLosslessConversion(const RPNBuilderTreeNode & node); +const ActionsDAG::Node * unwrapLosslessConversion(const ActionsDAG::Node * node); + /** RPN Builder build stack of reverse polish notation elements (RPNElements) required for index analysis. * * RPNBuilder client must provide RPNElement type that has following interface: diff --git a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp index 09786a9b73ba..72fad84b226d 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp +++ b/src/Storages/MergeTree/StorageFromMergeTreeProjection.cpp @@ -28,6 +28,7 @@ StorageFromMergeTreeProjection::StorageFromMergeTreeProjection( StorageID storage_id_, StoragePtr parent_storage_, StorageMetadataPtr parent_metadata_, ProjectionDescriptionRawPtr projection_) : IStorage(storage_id_) , parent_storage(std::move(parent_storage_)) + /// NOLINT(storage-cast): the table function resolves the source table before building this. , merge_tree(dynamic_cast(*parent_storage)) , parent_metadata(std::move(parent_metadata_)) , projection(projection_) diff --git a/src/Storages/MergeTree/TextIndexAnalyzer.cpp b/src/Storages/MergeTree/TextIndexAnalyzer.cpp index c1bbb05fd9d9..3b2f482d1bbe 100644 --- a/src/Storages/MergeTree/TextIndexAnalyzer.cpp +++ b/src/Storages/MergeTree/TextIndexAnalyzer.cpp @@ -156,6 +156,20 @@ void TextIndexAnalyzer::QueryBuilder::addPostings(const PostingList & token_post markFailed(); } +FlatPostingsPtr TextIndexAnalyzer::QueryBuilder::getFlatPostings() const +{ + chassert(postings); + + std::call_once(flat_postings_once, [&] + { + auto flat = std::make_shared>(postings->cardinality()); + postings->toUint32Array(flat->data()); + flat_postings = std::move(flat); + }); + + return flat_postings; +} + TextIndexAnalyzer::TextIndexAnalyzer(const MergeTreeIndexConditionText & condition_text) { global_search_mode = condition_text.getGlobalSearchMode(); @@ -318,6 +332,7 @@ void TextIndexAnalyzer::bypassPatternQueries() { auto & query_builder = query_builders.at(query_hash); query_builder.markBypassed(); + query_builder.is_analysis_incomplete = true; for (const auto & [query_token, _] : query_builder.tokens) queries_by_token[query_token].erase(query_hash); @@ -327,7 +342,7 @@ void TextIndexAnalyzer::bypassPatternQueries() double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_builder, size_t total_rows) const { const auto & query = *query_builder.query; - chassert(!query.getTokens().empty()); + chassert(!query.getTokens().empty() || !query.getPatterns().empty()); const double n = static_cast(total_rows); switch (query.getSearchMode()) @@ -367,6 +382,20 @@ double TextIndexAnalyzer::estimateQueryCardinality(const QueryBuilder & query_bu ? 1.0 - static_cast(query_builder.postings->cardinality()) / n : 1.0; + /// A pattern query declares no tokens, it owns the ones the dictionary scan matched. + if (query.getTokens().empty()) + { + for (const auto & [token, token_info] : query_builder.tokens) + { + if (hasReadPostings(token)) + continue; + + not_in_any *= (1.0 - static_cast(token_info->cardinality) / n); + } + + return n * (1.0 - not_in_any); + } + for (const auto & token : query.getTokens()) { auto it = query_builder.tokens.find(token); @@ -404,10 +433,8 @@ void TextIndexAnalyzer::analyzeCardinalitiesAndBypassHints(double selectivity_th if (query.getDirectReadMode() != TextIndexDirectReadMode::Hint) continue; - /// Pure-pattern queries have no declared tokens at parse time; their tokens are - /// discovered dynamically during dictionary scan. Skip the cardinality check in - /// that case — it would have no inputs to work with. - if (query.getTokens().empty()) + /// A pure-pattern query is estimated from the tokens the dictionary scan discovered. + if (query.getTokens().empty() && query_builder.tokens.empty()) continue; double estimated_cardinality = estimateQueryCardinality(query_builder, total_rows); diff --git a/src/Storages/MergeTree/TextIndexAnalyzer.h b/src/Storages/MergeTree/TextIndexAnalyzer.h index 03eba3c3958c..73979a99c515 100644 --- a/src/Storages/MergeTree/TextIndexAnalyzer.h +++ b/src/Storages/MergeTree/TextIndexAnalyzer.h @@ -1,7 +1,10 @@ #pragma once #include #include +#include #include +#include +#include namespace DB { @@ -42,6 +45,8 @@ class TextIndexAnalyzer bool is_failed = false; /// Query was discarded (low-selectivity hint, pattern bypass). bool is_bypassed = false; + /// The dictionary scan stopped early, so the matched tokens are incomplete and nothing can be pruned. + bool is_analysis_incomplete = false; /// Number of tokens whose posting list has already been folded into `postings`. size_t num_read_postings = 0; /// Declared tokens (`query->getTokens`) that may still contribute to an `Any` query. @@ -54,6 +59,14 @@ class TextIndexAnalyzer void addRowsRange(RowsRange token_rows_range); void addPostings(const PostingList & token_postings); bool needReadPostings() const { return num_read_postings < tokens.size(); } + + /// Sorted array of `postings`, built once and shared by all readers of the granule. + /// It is clipped to this query's readable rows, so it must never go to the server-wide postings cache. + FlatPostingsPtr getFlatPostings() const; + + private: + mutable std::once_flag flat_postings_once; + mutable FlatPostingsPtr flat_postings; }; explicit TextIndexAnalyzer(const MergeTreeIndexConditionText & condition_text); @@ -105,8 +118,8 @@ class TextIndexAnalyzer /* Fields built in the constructor from MergeTreeIndexConditionText. */ TextSearchMode global_search_mode; - /// One builder per parsed query, keyed by the query's stable hash. - absl::flat_hash_map query_builders; + /// One builder per parsed query, keyed by the query's stable hash. Node map because `QueryBuilder` is not movable. + absl::node_hash_map query_builders; /// Active queries that still depend on a given token. absl::flat_hash_map queries_by_token; /// Pattern queries grouped by their compiled regex; static for the analyzer's lifetime. diff --git a/src/Storages/MergeTree/TextIndexCache.h b/src/Storages/MergeTree/TextIndexCache.h index 7b814a9e11b8..a186932d67fd 100644 --- a/src/Storages/MergeTree/TextIndexCache.h +++ b/src/Storages/MergeTree/TextIndexCache.h @@ -127,13 +127,12 @@ enum class TextIndexPostingsCacheKind : UInt8 { Roaring = 0, Segment = 1, - Flat = 2, - Phrase = 3, /// phrase-search result, reusing the Flat (sorted doc-id) payload + Phrase = 3, /// phrase-search result, a sorted array of doc ids }; /// A single cell of TextIndexPostingsCache. It holds one of: /// - PostingListPtr: a decoded Roaring bitmap of one posting-list block; -/// - FlatPostingsPtr: a flattened sorted array of analyzer-folded postings (prebuilt or embedded cursor); +/// - FlatPostingsPtr: a sorted array of doc ids of a phrase-search result; /// - PostingListSegmentPtr: a decoded segment (payload + per-block index) of a compressed posting list (lazy cursor). /// Every payload is held by shared_ptr, so a consumer keeps its data alive by copying the inner pointer /// out of the cell — the data then outlives eviction of the (bounded) cache independently of the cell. diff --git a/src/Storages/MergeTree/WhatIfIndexEstimator.cpp b/src/Storages/MergeTree/WhatIfIndexEstimator.cpp index 5db904a0da9b..c5cc88c27ad5 100644 --- a/src/Storages/MergeTree/WhatIfIndexEstimator.cpp +++ b/src/Storages/MergeTree/WhatIfIndexEstimator.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -332,7 +333,7 @@ WhatIfResult estimateHypotheticalIndexes( { auto storage = tryResolveSingleTable(select_query, local_context); const auto & store = local_context->getHypotheticalObjectStore(); - if (const auto * mt = dynamic_cast(storage.get())) + if (const auto * mt = castStorage(storage, DeferredTable::Load).get()) { /// Empty table -> ReadNothing, report a zero baseline if (mt->getActivePartsCount() == 0) diff --git a/src/Storages/NATS/NATS_fwd.h b/src/Storages/NATS/NATS_fwd.h index f9b1043f7b7d..a471bffbbd1d 100644 --- a/src/Storages/NATS/NATS_fwd.h +++ b/src/Storages/NATS/NATS_fwd.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include namespace NATS @@ -26,7 +25,10 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - DB::maskURIPassword(&masked_value); + /// libnats takes the scheme as optional and ends the userinfo at the LAST '@' of the whole + /// value (`contrib/nats-io/src/url.c`, `natsUrl_Create`), so no URI authority bounds it. + if (masked_value.contains('@')) + masked_value = "[HIDDEN]"; return fmt::format("'{}'", masked_value); }} }; diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp index 5e2f139a9c76..7246b4b60e68 100644 --- a/src/Storages/NATS/StorageNATS.cpp +++ b/src/Storages/NATS/StorageNATS.cpp @@ -36,6 +36,8 @@ #include #include #include +#include +#include #include #include #include @@ -85,6 +87,7 @@ static const uint32_t QUEUE_SIZE = 100000; static const auto RESCHEDULE_MS = 500; static const auto MAX_THREAD_WORK_DURATION_MS = 60000; + namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -94,6 +97,57 @@ extern const int CANNOT_CONNECT_NATS; extern const int QUERY_NOT_ALLOWED; } +namespace +{ + +/// Checks a NATS address against the remote host filter and returns the address rebuilt from its parsed +/// parts - the string to hand to libnats in place of the original value. The remote host filter must see +/// exactly the host and port libnats will dial, so the value is not passed on as it was written: the +/// rebuilt address is `[scheme://][credentials@]host:port` with an explicit port, a form libnats +/// re-parses to the same host and port. +/// +/// libnats reads a URL of the form `[scheme://][user[:password]@]host[:port]` as a C string, splits the +/// credentials at the last `@`, substitutes `localhost` for an empty host, allows a `/path` after the +/// port, and connects to port 4222 when none is given (`natsUrl_Create`). An address which such a +/// re-parse could read differently - a NUL, a `/`, an empty host, a character outside printable ASCII - +/// is rejected instead of repaired. Throws `UNACCEPTABLE_URL` for a host the filter does not allow and +/// `BAD_ARGUMENTS` for an address it cannot parse safely. +String validateNATSAddress(const String & address, const RemoteHostFilter & remote_host_filter) +{ + if (address.contains('\0')) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "NATS address must not contain NUL characters"); + + String host_and_port = address; + + String scheme; + if (const auto scheme_end = host_and_port.find("://"); scheme_end != String::npos) + { + scheme = host_and_port.substr(0, scheme_end + strlen("://")); + host_and_port = host_and_port.substr(scheme_end + strlen("://")); + + for (const char c : scheme.substr(0, scheme_end)) + if (!isAlphaASCII(c)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid scheme in NATS address '{}'", address); + } + + String credentials; + if (const auto credentials_end = host_and_port.rfind('@'); credentials_end != String::npos) + { + credentials = host_and_port.substr(0, credentials_end + 1); + host_and_port = host_and_port.substr(credentials_end + 1); + + /// The credentials are kept verbatim (a password may contain almost anything, including + /// non-ASCII), only ASCII control characters (which include DEL) are rejected. + for (const char c : credentials) + if (isASCII(c) && !isPrintableASCII(c)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unexpected character in the credentials of NATS address '{}'", address); + } + + return scheme + credentials + remote_host_filter.checkAndGetCanonicalHostAndPort(host_and_port, 4222, "NATS address"); +} + +} + StorageNATS::StorageNATS( const StorageID & table_id_, @@ -154,6 +208,13 @@ StorageNATS::StorageNATS( .reconnect_wait = static_cast((*nats_settings)[NATSSetting::nats_reconnect_wait].value), .secure = (*nats_settings)[NATSSetting::nats_secure].value}; + const auto & remote_host_filter = context_->getRemoteHostFilter(); + if (!configuration.url.empty()) + configuration.url = validateNATSAddress(configuration.url, remote_host_filter); + for (auto & server : configuration.servers) + if (!server.empty()) + server = validateNATSAddress(server, remote_host_filter); + StorageInMemoryMetadata storage_metadata; storage_metadata.setColumns(columns_); storage_metadata.setComment(comment); diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp index 47bbc604ad4b..1bc42de2adec 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp @@ -38,6 +38,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; + extern const int DATALAKE_DATABASE_ERROR; } namespace FailPoints @@ -615,12 +616,12 @@ ReadFromFormatInfo DeltaLakeMetadataDeltaKernel::prepareReadingFromFormat( SinkToStoragePtr DeltaLakeMetadataDeltaKernel::write( SharedHeader sample_block, - const StorageID & /* table_id */, + const StorageID & table_id, ObjectStoragePtr object_storage_, StorageObjectStorageConfigurationPtr configuration, const std::optional & format_settings, ContextPtr context, - std::shared_ptr /* catalog */) + std::shared_ptr catalog) { if (!context->getSettingsRef()[Setting::allow_experimental_delta_lake_writes]) { @@ -643,14 +644,30 @@ SinkToStoragePtr DeltaLakeMetadataDeltaKernel::write( "Writing to DeltaLake tables with column mapping enabled is not supported"); } - auto delta_transaction = std::make_shared(kernel_helper); + auto write_object_storage = object_storage_; + auto write_kernel_helper = kernel_helper; + if (catalog && object_storage_->hasCredentialsRefreshCallback()) + { + if (auto write_credentials_callback = catalog->getWriteCredentialsConfigurationCallback(table_id)) + { + write_object_storage = configuration->createObjectStorage(context, /* is_readonly */ false, write_credentials_callback); + if (!write_object_storage->tryRefreshCredentialsViaCallback()) + throw Exception( + ErrorCodes::DATALAKE_DATABASE_ERROR, + "The catalog did not vend storage credentials for writing into table {}", + table_id.getNameForLogs()); + write_kernel_helper = DB::getKernelHelper(configuration, write_object_storage); + } + } + + auto delta_transaction = std::make_shared(write_kernel_helper); delta_transaction->create(partition_columns, snapshot->getTableSchema()); if (partition_columns.empty()) { return std::make_shared( delta_transaction, - object_storage_, + write_object_storage, context, sample_block, format_settings, @@ -661,7 +678,7 @@ SinkToStoragePtr DeltaLakeMetadataDeltaKernel::write( return std::make_shared( delta_transaction, partition_columns, - object_storage_, + write_object_storage, context, sample_block, format_settings, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp index beb714c61a23..bb21135a0aab 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Compaction.cpp @@ -407,6 +407,7 @@ static void writeDataFiles( } output_format->flush(); output_format->finalize(); + data_file->manifest_list->statistics.addColumnSizesOnDisk(output_format->getColumnSizesOnDisk(), *sample_block); write_buffer->finalize(); auto file_bytes = write_buffer->count(); if (file_bytes == 0 && !data_file->patched_path.empty()) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h index b58d769a373d..ab5dfd3d0e7b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Constant.h @@ -129,6 +129,7 @@ DEFINE_ICEBERG_FIELD_ALIAS(partition_transform, transform); DEFINE_ICEBERG_FIELD_ALIAS(partition_name, name); DEFINE_ICEBERG_FIELD_ALIAS(default_spec_id, default-spec-id); DEFINE_ICEBERG_FIELD_ALIAS(partition_spec, partition-spec); +DEFINE_ICEBERG_FIELD_ALIAS(manifest_partition_spec_id, partition-spec-id); DEFINE_ICEBERG_FIELD_ALIAS(partition_specs, partition-specs); DEFINE_ICEBERG_FIELD_ALIAS(spec_id, spec-id); DEFINE_ICEBERG_FIELD_ALIAS(added_records, added-records); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp index 4e6879496faf..95e3293aa664 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.cpp @@ -3,10 +3,16 @@ #include #include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + #if USE_AVRO DataFileStatistics::DataFileStatistics(Poco::JSON::Array::Ptr schema_) @@ -33,9 +39,8 @@ void DataFileStatistics::update(const Chunk & chunk) if (!chunk.hasRows()) return; size_t num_columns = chunk.getNumColumns(); - if (column_sizes.empty()) + if (null_counts.empty()) { - column_sizes.resize(num_columns, 0); null_counts.resize(num_columns, 0); for (size_t i = 0; i < num_columns; ++i) { @@ -48,7 +53,6 @@ void DataFileStatistics::update(const Chunk & chunk) for (size_t i = 0; i < num_columns; ++i) { const auto & col = chunk.getColumns()[i]; - column_sizes[i] += col->byteSize(); if (const auto * nullable_col = checkAndGetColumn(col.get())) { for (UInt8 v : nullable_col->getNullMapData()) @@ -58,23 +62,61 @@ void DataFileStatistics::update(const Chunk & chunk) } } -void DataFileStatistics::merge(const DataFileStatistics & other) +void DataFileStatistics::addColumnSizesOnDisk(const std::unordered_map & sizes_by_column_name, const Block & sample_block) { - if (other.column_sizes.empty()) + if (sizes_by_column_name.empty()) return; + if (sample_block.columns() != field_ids.size()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "Iceberg data file has {} columns while its schema has {} fields", + sample_block.columns(), + field_ids.size()); + if (column_sizes.empty()) + column_sizes.resize(field_ids.size(), 0); + + for (size_t i = 0; i < field_ids.size(); ++i) + { + const auto & column_name = sample_block.getByPosition(i).name; + auto it = sizes_by_column_name.find(column_name); + if (it == sizes_by_column_name.end()) + throw Exception( + ErrorCodes::LOGICAL_ERROR, "Written data file does not report the on-disk size of column {}", column_name); + column_sizes[i] += static_cast(it->second); + } +} + +void DataFileStatistics::merge(const DataFileStatistics & other) +{ + if (!other.column_sizes.empty()) + { + if (column_sizes.empty()) + { + column_sizes = other.column_sizes; + } + else + { + chassert(column_sizes.size() == other.column_sizes.size()); + for (size_t i = 0; i < column_sizes.size(); ++i) + column_sizes[i] += other.column_sizes[i]; + } + } + + if (other.null_counts.empty()) + return; + + if (null_counts.empty()) { - column_sizes = other.column_sizes; null_counts = other.null_counts; ranges = other.ranges; return; } - chassert(column_sizes.size() == other.column_sizes.size()); - for (size_t i = 0; i < column_sizes.size(); ++i) + chassert(null_counts.size() == other.null_counts.size()); + for (size_t i = 0; i < null_counts.size(); ++i) { - column_sizes[i] += other.column_sizes[i]; null_counts[i] += other.null_counts[i]; ranges[i] = uniteRanges(ranges[i], other.ranges[i]); } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h index 680b283e0388..6352f2df9ad7 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/DataFileStatistics.h @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -25,6 +26,7 @@ class DataFileStatistics explicit DataFileStatistics(Poco::JSON::Array::Ptr schema_); void update(const Chunk & chunk); + void addColumnSizesOnDisk(const std::unordered_map & sizes_by_column_name, const Block & sample_block); void merge(const DataFileStatistics & other); std::vector> getColumnSizes() const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 2af675f6cf10..b47dec44d1a0 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -184,7 +184,7 @@ Iceberg::PersistentTableComponents IcebergMetadata::initializePersistentTableCom LoggerPtr log) { const auto [metadata_version, metadata_file_path, compression_method] - = getLatestOrExplicitMetadataFileAndVersion(object_storage, configuration->getPathForRead().path, configuration->getDataLakeSettings(), cache_ptr, context_, log.get(), std::nullopt, CompressionMethod::None, true); + = getLatestOrExplicitMetadataFileAndVersion(object_storage, configuration->getPathForRead().path, configuration->getDataLakeSettings(), cache_ptr, context_, log.get(), std::nullopt, CompressionMethod::None, /* force_fetch_latest_metadata */ false); LOG_DEBUG(log, "Latest metadata file path is {}, version {}", metadata_file_path, metadata_version); auto metadata_object = getMetadataJSONObject(metadata_file_path, object_storage, cache_ptr, context_, log, compression_method, std::nullopt); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp index a83d8b5f3278..666f5037f58e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergWrites.cpp @@ -296,21 +296,22 @@ std::vector dumpFieldToBytes(const Field & field, DataTypePtr type) } } -bool canWriteStatistics( +/// Retains only the bounds that can be serialized. A field left out is simply absent from the +/// manifest bounds map, which readers treat as "bound unknown" for that column. +std::vector> filterWritableStatistics( const std::vector> & statistics, const std::unordered_map & field_id_to_column_index, SharedHeader sample_block) { - if (statistics.empty()) - return false; - + std::vector> writable; + writable.reserve(statistics.size()); for (const auto & [field_id, stat] : statistics) { auto type = sample_block->getDataTypes()[field_id_to_column_index.at(field_id)]; - if (!canDumpIcebergStats(stat, type)) - return false; + if (canDumpIcebergStats(stat, type)) + writable.emplace_back(field_id, stat); } - return true; + return writable; } } @@ -510,10 +511,11 @@ void generateManifestFile( /// original id-carrying JSON as the avro.schema header so external readers can plan a scan. writer.setMetadata(Iceberg::f_avro_schema, schema_representation); writer.setMetadata(Iceberg::f_schema, json_representation); + writer.setMetadata(Iceberg::f_schema_id, std::to_string(schema_object_to_write->getValue(Iceberg::f_schema_id))); writer.setMetadata(Iceberg::f_format_version, std::to_string(version)); writer.setMetadata(Iceberg::f_partition_spec, stringifyJSON(partition_spec->getArray(Iceberg::f_fields))); - writer.setMetadata(Iceberg::f_partition_spec_id, std::to_string(partition_spec_id)); + writer.setMetadata(Iceberg::f_manifest_partition_spec_id, std::to_string(partition_spec_id)); writer.setMetadata(Iceberg::f_format_version, std::to_string(version)); for (size_t file_idx = 0; file_idx < data_file_names.size(); ++file_idx) { @@ -596,10 +598,11 @@ void generateManifestFile( } else if (effective_statistics) { - auto statistics = effective_statistics->getColumnSizes(); - set_fields(statistics, Iceberg::f_column_sizes, [](size_t, size_t value) { return static_cast(value); }); + auto column_sizes = effective_statistics->getColumnSizes(); + if (!column_sizes.empty()) + set_fields(column_sizes, Iceberg::f_column_sizes, [](size_t, size_t value) { return static_cast(value); }); - statistics = effective_statistics->getNullCounts(); + auto statistics = effective_statistics->getNullCounts(); set_fields(statistics, Iceberg::f_null_value_counts, [](size_t, size_t value) { return static_cast(value); }); std::unordered_map field_id_to_column_index; @@ -610,13 +613,15 @@ void generateManifestFile( auto dump_fields = [&](size_t field_id, Field value) { return dumpFieldToBytes(value, sample_block->getDataTypes()[field_id_to_column_index.at(field_id)]); }; - auto lower_statistics = effective_statistics->getLowerBounds(); - if (canWriteStatistics(lower_statistics, field_id_to_column_index, sample_block)) + auto lower_statistics + = filterWritableStatistics(effective_statistics->getLowerBounds(), field_id_to_column_index, sample_block); + if (!lower_statistics.empty()) { set_fields(lower_statistics, Iceberg::f_lower_bounds, dump_fields); } - auto upper_statistics = effective_statistics->getUpperBounds(); - if (canWriteStatistics(upper_statistics, field_id_to_column_index, sample_block)) + auto upper_statistics + = filterWritableStatistics(effective_statistics->getUpperBounds(), field_id_to_column_index, sample_block); + if (!upper_statistics.empty()) { set_fields(upper_statistics, Iceberg::f_upper_bounds, dump_fields); } @@ -928,32 +933,66 @@ void generateManifestList( forEachAvroEntry(resolved_manifest_list_path, object_storage, context, "IcebergWrites", [&](const avro::GenericDatum & datum) { + if (datum.type() != avro::AVRO_RECORD) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} contains an entry with Avro type {}, but a record is required", + resolved_manifest_list_path, + static_cast(datum.type())); + const avro::GenericRecord & old_entry = datum.value(); + + auto validate_field_type = [&](const String & field_name, avro::Type expected_type) -> const avro::GenericDatum & + { + if (!old_entry.hasField(field_name)) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} entry is missing required field '{}'", + resolved_manifest_list_path, + field_name); + + const avro::GenericDatum & field = old_entry.field(field_name); + if (field.type() != expected_type) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} field '{}' has Avro type {}, but type {} is required", + resolved_manifest_list_path, + field_name, + static_cast(field.type()), + static_cast(expected_type)); + + return field; + }; + + const avro::GenericDatum & old_manifest_path = validate_field_type(Iceberg::f_manifest_path, avro::AVRO_STRING); + /// When a path filter is supplied, copy only the matching entries. if (!carry_forward_manifest_paths.empty() - && !carry_forward_manifest_paths.contains(old_entry.field(Iceberg::f_manifest_path).value())) + && !carry_forward_manifest_paths.contains(old_manifest_path.value())) return; avro::GenericDatum new_datum(schema.root()); avro::GenericRecord & new_entry = new_datum.value(); - new_entry.field(f_manifest_path) = old_entry.field(Iceberg::f_manifest_path); - new_entry.field(f_manifest_length) = old_entry.field(Iceberg::f_manifest_length); - new_entry.field(f_partition_spec_id) = old_entry.field(Iceberg::f_partition_spec_id); + + auto copy_required_field = [&](const String & field_name, avro::Type expected_type) + { + new_entry.field(field_name) = validate_field_type(field_name, expected_type); + }; + + new_entry.field(f_manifest_path) = old_manifest_path; + copy_required_field(Iceberg::f_manifest_length, avro::AVRO_LONG); + copy_required_field(Iceberg::f_partition_spec_id, avro::AVRO_INT); /// iceberg-spark changed `f_added_snapshot_id` from 'null, long' to 'long' (apache/iceberg#11626); rewrite with the new schema in case we read the old type. if (old_entry.hasField(Iceberg::f_added_snapshot_id)) { const avro::GenericDatum & old_added_snapshot_id_entry = old_entry.field(Iceberg::f_added_snapshot_id); - if (old_added_snapshot_id_entry.isUnion()) - { - if (old_added_snapshot_id_entry.unionBranch() == 0) /// it means add_snapshot_id is null - { - /// This only happens when we read data written by a old version of iceberg, which violates the spec of iceberg. - throw Exception( - ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Manifest list {} has null value for field '{}', but it is required", - resolved_manifest_list_path, - Iceberg::f_added_snapshot_id); - } - } + if (old_added_snapshot_id_entry.type() != avro::AVRO_LONG) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest list {} field '{}' has Avro type {}, but a non-null long is required", + resolved_manifest_list_path, + Iceberg::f_added_snapshot_id, + static_cast(old_added_snapshot_id_entry.type())); + new_entry.field(f_added_snapshot_id) = old_added_snapshot_id_entry.value(); } else diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp index 9c0d2db914de..23a0000af455 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.cpp @@ -9,6 +9,7 @@ #include +#include #include #include @@ -17,6 +18,7 @@ #include #include +#include #include #include #include @@ -39,6 +41,11 @@ namespace DB::ErrorCodes extern const int BAD_ARGUMENTS; } +namespace DB::Setting +{ + extern const SettingsBool iceberg_tolerate_conflicting_manifest_schemas; +} + namespace ProfileEvents { extern const Event IcebergPartitionPrunedFiles; @@ -315,11 +322,38 @@ std::shared_ptr ManifestFileIterator::create( path_to_manifest_file_, f_schema); - Poco::Dynamic::Var json = parser.parse(*schema_json_string); - const Poco::JSON::Object::Ptr & schema_object = json.extract(); - Int32 manifest_schema_id = schema_object->getValue(f_schema_id); + const bool tolerate_conflicting_manifest_schemas = context_->getSettingsRef()[Setting::iceberg_tolerate_conflicting_manifest_schemas]; - schema_processor.addIcebergTableSchema(schema_object); + std::optional header_schema_id; + if (auto schema_id_string = manifest_file_deserializer_->tryGetAvroMetadataValue(f_schema_id)) + header_schema_id = parse(*schema_id_string); + + Int32 manifest_schema_id = 0; + if (header_schema_id.has_value() && tolerate_conflicting_manifest_schemas + && schema_processor.isSchemaRegisteredFromMetadata(*header_schema_id)) + { + manifest_schema_id = *header_schema_id; + } + else + { + Poco::Dynamic::Var json = parser.parse(*schema_json_string); + const Poco::JSON::Object::Ptr & schema_object = json.extract(); + Int32 embedded_schema_id = schema_object->getValue(f_schema_id); + if (header_schema_id.has_value() && *header_schema_id != embedded_schema_id) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Manifest file '{}' has header key '{}' = {} that differs from the '{}' = {} of its '{}' header", + path_to_manifest_file_, + f_schema_id, + *header_schema_id, + f_schema_id, + embedded_schema_id, + f_schema); + manifest_schema_id = embedded_schema_id; + + schema_processor.addIcebergTableSchema( + schema_object, IcebergSchemaProcessor::SchemaSource::ManifestFile, tolerate_conflicting_manifest_schemas); + } PartitionSpecification partition_spec_vec; for (size_t i = 0; i != partition_specification->size(); ++i) @@ -464,13 +498,14 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) /// those manifests still carry the original snapshot_id. The manifest file's own Avro header /// records the correct schema_id for the data files it describes, so falling back to /// manifest_schema_id is safe and correct in this case. - LOG_DEBUG( - getLogger("ManifestFileIterator"), - "Manifest file '{}' has entry with snapshot_id '{}' whose snapshot metadata is not present " - "(snapshot may have been expired by the catalog). Falling back to manifest schema_id {}.", - path_to_manifest_file, - resolved_snapshot_id, - manifest_schema_id); + if (!logged_missing_snapshot_metadata.exchange(true, std::memory_order_relaxed)) + LOG_TEST( + getLogger("ManifestFileIterator"), + "Manifest file '{}' has entry with snapshot_id '{}' whose snapshot metadata is not present " + "(snapshot may have been expired by the catalog). Falling back to manifest schema_id {}.", + path_to_manifest_file, + resolved_snapshot_id, + manifest_schema_id); } const auto resolved_schema_id = schema_id_opt.has_value() ? *schema_id_opt : manifest_schema_id; @@ -500,20 +535,18 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) PruningReturnStatus pruning_status = PruningReturnStatus::NOT_PRUNED; if (filter_dag) { + const ManifestFilesPruner * current_pruner = getOrCreatePruner(entry->resolved_schema_id); + /// Compute per-column hyperrectangles for DATA files std::unordered_map hyperrectangles; if (parsed_entry->content_type == FileContentType::DATA) { - for (const auto & [column_id, bounds] : parsed_entry->value_bounds) + for (const auto & [column_id, column_type] : current_pruner->getMinMaxColumnTypes()) { - auto field_characteristics = schema_processor_ptr->tryGetFieldCharacteristics(resolved_schema_id, column_id); - /// If we don't have column characteristics, bounds don't have any sense. - /// This happens if the subfield is inside map or array, because we don't support - /// name generation for such subfields (we support names of nested subfields in structs only). - if (!field_characteristics) + auto bounds_it = parsed_entry->value_bounds.find(column_id); + if (bounds_it == parsed_entry->value_bounds.end()) continue; - - const auto & name_and_type = *field_characteristics; + const auto & bounds = bounds_it->second; String left_str; String right_str; @@ -521,13 +554,13 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) if (!bounds.first.tryGet(left_str) || !bounds.second.tryGet(right_str)) continue; - if (const auto type_id = name_and_type.type->getTypeId(); + if (const auto type_id = column_type->getTypeId(); type_id == DB::TypeIndex::Tuple || type_id == DB::TypeIndex::Map || type_id == DB::TypeIndex::Array || type_id == DB::TypeIndex::Variant) continue; - auto left = deserializeFieldFromBinaryRepr(left_str, name_and_type.type, true); - auto right = deserializeFieldFromBinaryRepr(right_str, name_and_type.type, false); + auto left = deserializeFieldFromBinaryRepr(left_str, column_type, true); + auto right = deserializeFieldFromBinaryRepr(right_str, column_type, false); if (!left || !right) { /// Pruning is skipped either way, but at scale 38 a bound that only loses its widened @@ -548,12 +581,12 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) /// declared expose that inversion, which is why they are read again here. std::optional declared_left = left; std::optional declared_right = right; - if (DB::WhichDataType(DB::removeNullable(name_and_type.type)).isDecimal()) + if (DB::WhichDataType(DB::removeNullable(column_type)).isDecimal()) { declared_left = deserializeFieldFromBinaryRepr( - left_str, name_and_type.type, true, /*compensate_rounding=*/false); + left_str, column_type, true, /*compensate_rounding=*/false); declared_right = deserializeFieldFromBinaryRepr( - right_str, name_and_type.type, false, /*compensate_rounding=*/false); + right_str, column_type, false, /*compensate_rounding=*/false); } /// A pair inverted as declared means the manifest's statistics are untrustworthy, so no @@ -575,7 +608,6 @@ ProcessedManifestFileEntryPtr ManifestFileIterator::processRow(size_t row_index) } } - const ManifestFilesPruner * current_pruner = getOrCreatePruner(entry->resolved_schema_id); pruning_status = current_pruner->canBePruned(entry, hyperrectangles); } insertRowToLogTable( diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h index 40bb9b4c45c1..d6e90f78590e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFileIterator.h @@ -160,6 +160,7 @@ class ManifestFileIterator : public boost::noncopyable std::atomic current_row_index{0}; std::atomic fully_initialized{false}; std::atomic active_fetchers{0}; + std::atomic logged_missing_snapshot_metadata{false}; /// Cached results accumulated during iteration mutable SharedMutex files_mutex; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp index 011cc96f16a0..ab8e95a358b6 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.cpp @@ -6,6 +6,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -134,6 +137,7 @@ ManifestFilesPruner::ManifestFilesPruner( if (!name_and_type.has_value()) continue; + min_max_column_types.emplace(used_column_id, name_and_type->type); name_and_type->name = DB::backQuote(DB::toString(used_column_id)); ExpressionActionsPtr expression @@ -187,12 +191,185 @@ Field decodePartitionDecimalByType(const String & bytes, const IDataType & type) } +namespace +{ + +enum class PartitionTransformKind : uint8_t +{ + Day, + Month, + Year, + Hour, + NotInvertible, +}; + +PartitionTransformKind parsePartitionTransformKind(const String & transform_name_src) +{ + const String transform_name = Poco::toLower(transform_name_src); + + if (transform_name == "day" || transform_name == "days" || transform_name == "date" || transform_name == "dates") + return PartitionTransformKind::Day; + if (transform_name == "month" || transform_name == "months") + return PartitionTransformKind::Month; + if (transform_name == "year" || transform_name == "years") + return PartitionTransformKind::Year; + if (transform_name == "hour" || transform_name == "hours") + return PartitionTransformKind::Hour; + return PartitionTransformKind::NotInvertible; +} + +struct Interval +{ + Int64 first; + Int64 past_last; +}; + +std::optional unitInterval(Int64 value) +{ + Int64 past_last = 0; + if (common::addOverflow(value, Int64{1}, past_last)) + return {}; + return Interval{value, past_last}; +} + +std::optional refineInterval(std::optional interval, Int64 factor) +{ + Int64 first = 0; + Int64 past_last = 0; + if (!interval || common::mulOverflow(interval->first, factor, first) || common::mulOverflow(interval->past_last, factor, past_last)) + return {}; + return Interval{first, past_last}; +} + +std::optional closedRange(std::optional interval, std::optional decimal_scale) +{ + Int64 last = 0; + if (!interval || common::subOverflow(interval->past_last, Int64{1}, last)) + return {}; + + if (decimal_scale) + return Range( + DecimalField(interval->first, *decimal_scale), true, DecimalField(last, *decimal_scale), true); + return Range(interval->first, true, last, true); +} + +std::optional dayIntervalOfMonthNum(Int64 month) +{ + auto months = unitInterval(month); + if (!months) + return {}; + + const auto & utc = DateLUT::instance("UTC"); + const auto epoch = ExtendedDayNum(0); + const auto first = utc.addMonths(epoch, months->first); + const auto past_last = utc.addMonths(epoch, months->past_last); + if (utc.toMonthNumSinceEpoch(first) != months->first || utc.toMonthNumSinceEpoch(past_last) != months->past_last) + return {}; + + return Interval{Int64{first}, Int64{past_last}}; +} + +std::optional dayIntervalOfYearNum(Int64 year) +{ + auto years = unitInterval(year); + if (!years) + return {}; + + const auto & utc = DateLUT::instance("UTC"); + const auto epoch = ExtendedDayNum(0); + const auto first = utc.addYears(epoch, years->first); + const auto past_last = utc.addYears(epoch, years->past_last); + if (utc.toYearSinceEpoch(first) != years->first || utc.toYearSinceEpoch(past_last) != years->past_last) + return {}; + + return Interval{Int64{first}, Int64{past_last}}; +} + +std::optional dayIntervalOfPartitionValue(PartitionTransformKind kind, Int64 value) +{ + switch (kind) + { + case PartitionTransformKind::Day: + return unitInterval(value); + case PartitionTransformKind::Month: + return dayIntervalOfMonthNum(value); + case PartitionTransformKind::Year: + return dayIntervalOfYearNum(value); + case PartitionTransformKind::Hour: + case PartitionTransformKind::NotInvertible: + return {}; + } + UNREACHABLE(); +} + +std::optional secondIntervalOfPartitionValue(PartitionTransformKind kind, Int64 value) +{ + static constexpr Int64 seconds_per_hour = 3600; + static constexpr Int64 seconds_per_day = 86400; + + switch (kind) + { + case PartitionTransformKind::Hour: + return refineInterval(unitInterval(value), seconds_per_hour); + case PartitionTransformKind::Day: + case PartitionTransformKind::Month: + case PartitionTransformKind::Year: + return refineInterval(dayIntervalOfPartitionValue(kind, value), seconds_per_day); + case PartitionTransformKind::NotInvertible: + return {}; + } + UNREACHABLE(); +} + +std::optional partitionValueAsInt64(const Field & partition_value) +{ + if (partition_value.getType() == Field::Types::Int64) + return partition_value.safeGet(); + + if (partition_value.getType() == Field::Types::UInt64) + { + const UInt64 value = partition_value.safeGet(); + if (value <= static_cast(std::numeric_limits::max())) + return static_cast(value); + } + + return {}; +} + +std::optional rangeOfPartitionValue(const String & transform_name, const Field & partition_value, const IDataType & source_type) +{ + const auto value = partitionValueAsInt64(partition_value); + if (!value) + return {}; + + const PartitionTransformKind kind = parsePartitionTransformKind(transform_name); + const WhichDataType which(source_type); + + if (which.isDateOrDate32()) + return closedRange(dayIntervalOfPartitionValue(kind, *value), std::nullopt); + + if (which.isDateTime()) + return closedRange(secondIntervalOfPartitionValue(kind, *value), std::nullopt); + + if (which.isDateTime64()) + { + const UInt32 scale = getDecimalScale(source_type); + return closedRange( + refineInterval(secondIntervalOfPartitionValue(kind, *value), DecimalUtils::scaleMultiplier(scale)), scale); + } + + return {}; +} + +} + PruningReturnStatus ManifestFilesPruner::canBePruned( const ProcessedManifestFileEntryPtr & entry, const std::unordered_map & entry_hyperrectangles) const { + const auto & partition_value = entry->parsed_entry->partition_key_value; + if (partition_key_condition.has_value()) { - const auto & partition_value = entry->parsed_entry->partition_key_value; std::vector index_value(partition_value.begin(), partition_value.end()); for (size_t i = 0; i < index_value.size(); ++i) { @@ -218,23 +395,39 @@ PruningReturnStatus ManifestFilesPruner::canBePruned( for (const auto & [column_id, key_condition] : min_max_key_conditions) { - std::optional name_and_type = schema_processor.tryGetFieldCharacteristics(initial_schema_id, column_id); - /// There is no such column in this manifest file - if (!name_and_type.has_value()) - { - continue; - } - - auto rect_it = entry_hyperrectangles.find(column_id); - if (rect_it == entry_hyperrectangles.end()) + auto type_it = min_max_column_types.find(column_id); + if (type_it == min_max_column_types.end()) continue; + const auto & column_type = type_it->second; auto info_it = entry->parsed_entry->columns_infos.find(column_id); bool has_no_nulls = info_it != entry->parsed_entry->columns_infos.end() && info_it->second.nulls_count.has_value() && *info_it->second.nulls_count == 0; - if (has_no_nulls && !key_condition.mayBeTrueInRange(1, &rect_it->second.left, &rect_it->second.right, {name_and_type->type})) + const DataTypes data_types{column_type}; + + if (entry->common_partition_specification) + { + for (const auto & partition_field : *entry->common_partition_specification) + { + if (partition_field.source_id != column_id || partition_field.tuple_index < 0 + || static_cast(partition_field.tuple_index) >= partition_value.size()) + continue; + + auto range = rangeOfPartitionValue( + partition_field.transform_name, + partition_value[partition_field.tuple_index], + *removeNullable(column_type)); + + if (range && !key_condition.mayBeTrueInRange(1, &range->left, &range->right, data_types)) + return PruningReturnStatus::PARTITION_PRUNED; + } + } + + auto rect_it = entry_hyperrectangles.find(column_id); + if (has_no_nulls && rect_it != entry_hyperrectangles.end() + && !key_condition.mayBeTrueInRange(1, &rect_it->second.left, &rect_it->second.right, data_types)) { return PruningReturnStatus::MIN_MAX_INDEX_PRUNED; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h index 78c136167a88..e9aa14f97e57 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/ManifestFilesPruning.h @@ -43,6 +43,7 @@ class ManifestFilesPruner std::optional partition_key_condition; std::unordered_map min_max_key_conditions; + std::unordered_map min_max_column_types; /// NOTE: tricky part to support RENAME column. /// Takes ActionDAG representation of user's WHERE expression and /// rename columns to the their origina numeric ID's in iceberg @@ -58,6 +59,8 @@ class ManifestFilesPruner DB::ContextPtr context); PruningReturnStatus canBePruned(const ProcessedManifestFileEntryPtr & entry, const std::unordered_map & entry_hyperrectangles) const; + + const std::unordered_map & getMinMaxColumnTypes() const { return min_max_column_types; } }; } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp index 06f1ca6fcec6..1935421c3701 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp @@ -85,6 +85,12 @@ void MultipleFileWriter::finalize() { output_format->flush(); output_format->finalize(); + + auto column_sizes_on_disk = output_format->getColumnSizesOnDisk(); + if (current_file_stats) + current_file_stats->addColumnSizesOnDisk(column_sizes_on_disk, *sample_block); + stats.addColumnSizesOnDisk(column_sizes_on_disk, *sample_block); + buffer->finalize(); UInt64 file_bytes = buffer->count(); total_bytes += file_bytes; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 3bb6890d7f48..6ffc50d493b8 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -267,6 +267,8 @@ static std::optional writeDataFiles( { delete_data_writers[partition_key]->flush(); delete_data_writers[partition_key]->finalize(); + delete_data_statistics.at(partition_key).addColumnSizesOnDisk( + delete_data_writers[partition_key]->getColumnSizesOnDisk(), getPositionDeleteFileSampleBlock()); delete_data_write_buffers[partition_key]->finalize(); { auto delete_bytes = delete_data_write_buffers[partition_key]->count(); @@ -289,6 +291,7 @@ static std::optional writeDataFiles( PullingPipelineExecutor executor(pipeline); auto header = interpreter->getUpdatedHeader(); + auto update_sample_block = getNonVirtualColumns(header, /*remove_low_cardinality=*/ true); Block block; while (executor.pull(block)) @@ -338,6 +341,8 @@ static std::optional writeDataFiles( { update_data_writers[partition_key]->flush(); update_data_writers[partition_key]->finalize(); + update_data_statistics.at(partition_key).addColumnSizesOnDisk( + update_data_writers[partition_key]->getColumnSizesOnDisk(), update_sample_block); update_data_write_buffers[partition_key]->finalize(); { auto update_bytes = update_data_write_buffers[partition_key]->count(); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index c154a025ab3a..eba2434a0b42 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -393,10 +393,18 @@ namespace Iceberg std::string IcebergSchemaProcessor::default_link{}; -void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr) +void IcebergSchemaProcessor::dropCachedSchema(Int32 schema_id) { - std::lock_guard lock(mutex); + iceberg_table_schemas_by_ids.erase(schema_id); + clickhouse_table_schemas_by_ids.erase(schema_id); + std::erase_if(transform_dags_by_ids, [schema_id](const auto & item) { return item.first.first == schema_id || item.first.second == schema_id; }); + std::erase_if(clickhouse_types_by_source_ids, [schema_id](const auto & item) { return item.first.first == schema_id; }); + std::erase_if(clickhouse_ids_by_source_names, [schema_id](const auto & item) { return item.first.first == schema_id; }); +} +void IcebergSchemaProcessor::addIcebergTableSchema( + Poco::JSON::Object::Ptr schema_ptr, SchemaSource source, bool tolerate_conflicting_manifest_schemas) +{ Int32 schema_id = schema_ptr->getValue(f_schema_id); /// Databricks UniForm writes a degenerate placeholder schema (e.g. {"schema-id":0,"fields":[]}) @@ -404,54 +412,121 @@ void IcebergSchemaProcessor::addIcebergTableSchema(Poco::JSON::Object::Ptr schem if (!schema_ptr->isArray(f_fields) || schema_ptr->getArray(f_fields)->size() == 0) return; - current_schema_id = schema_id; - if (iceberg_table_schemas_by_ids.contains(schema_id)) + std::unordered_map type_mapping; + if (allow_geo_parser) { - chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); - std::unordered_map type_mapping; - if (allow_geo_parser) + type_mapping[f_geography] = f_binary; + type_mapping[f_geometry] = f_binary; + } + + Poco::JSON::Object::Ptr registered_schema; + { + SharedLockGuard lock(mutex); + auto it = iceberg_table_schemas_by_ids.find(schema_id); + if (it != iceberg_table_schemas_by_ids.end()) { - type_mapping[f_geography] = f_binary; - type_mapping[f_geometry] = f_binary; + const bool registered_from_manifest = manifest_sourced_schema_ids.contains(schema_id); + if (source == SchemaSource::ManifestFile && tolerate_conflicting_manifest_schemas && !registered_from_manifest) + return; + if (source == SchemaSource::ManifestFile || !registered_from_manifest) + registered_schema = it->second; } - /// A schema-id is immutable per the Iceberg spec: re-binding it to different fields is malformed metadata. - if (!schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping)) - throw Exception( - ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Iceberg schema with schema-id {} is bound to two different schemas across metadata versions", - schema_id); } - else + if (registered_schema && schemasAreIdentical(*registered_schema, *schema_ptr, type_mapping)) + return; + + std::lock_guard lock(mutex); + + if (iceberg_table_schemas_by_ids.contains(schema_id)) { - auto fields = schema_ptr->get(f_fields).extract(); - /// A field name is required per the Iceberg spec, and an empty column name is not representable in ClickHouse. - for (size_t i = 0; i != fields->size(); ++i) + chassert(clickhouse_table_schemas_by_ids.contains(schema_id)); + if (schemasAreIdentical(*iceberg_table_schemas_by_ids.at(schema_id), *schema_ptr, type_mapping)) + { + /// An identical metadata.json copy confirms a copy that was registered from a manifest header. + if (source == SchemaSource::Metadata) + manifest_sourced_schema_ids.erase(schema_id); + return; + } + + /// The 'schema' key in a manifest file header is only a copy of the table schema at the + /// time the manifest was written; metadata.json is the authoritative source. Broken writers + /// have been observed storing degraded copies in manifest headers under an already-used + /// schema-id (e.g. AWS S3 Tables maintenance jobs writing `timestamp` instead of + /// `timestamptz`, or a schema containing only the partition source columns). Other engines + /// (Spark, Trino, PyIceberg, DuckDB) resolve schemas from metadata.json and ignore such + /// divergent copies that came from a manifest. + const bool registered_from_manifest = manifest_sourced_schema_ids.contains(schema_id); + if (registered_from_manifest) { - auto field = fields->getObject(static_cast(i)); - if (field->getValue(f_name).empty()) + if (source == SchemaSource::Metadata) + { + /// A read registers the metadata.json schemas before it walks any manifest, but the + /// maintenance entrypoints (`remove_orphan_files`, `expire_snapshots`, manifest + /// compaction, mutation validation) can reach a manifest header first on an empty + /// processor. A schema that came from a manifest is never authoritative, so the + /// metadata.json copy replaces it, along with everything that was derived from it. + LOG_WARNING( + getLogger("IcebergSchemaProcessor"), + "Schema-id {} was registered from a manifest file header and differs from the schema " + "metadata.json binds to that id; replacing the schema that came from the manifest", + schema_id); + dropCachedSchema(schema_id); + manifest_sourced_schema_ids.erase(schema_id); + } + else + { + /// Two manifest headers disagree on a schema-id that metadata.json has not defined: + /// there is no authoritative copy to decide which one the data was written with. throw Exception( ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, - "Iceberg schema with schema-id {} has a field with id {} whose name is empty", - schema_id, - field->getValue(f_id)); + "Iceberg schema with schema-id {} is bound to two different schemas by manifest file headers, " + "and metadata.json does not define it", + schema_id); + } } - - auto clickhouse_schema = std::make_shared(); - String current_full_name{}; - for (size_t i = 0; i != fields->size(); ++i) + else { - auto field = fields->getObject(static_cast(i)); - auto name = field->getValue(f_name); - bool required = field->getValue(f_required); - current_full_name = name; - auto type = getFieldType(field, f_type, required, current_full_name, true); - clickhouse_schema->push_back(NameAndTypePair{name, type}); - clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; - clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + if (source == SchemaSource::ManifestFile && tolerate_conflicting_manifest_schemas) + return; + /// A schema-id is immutable per the Iceberg spec: re-binding it to different fields is malformed metadata. + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg schema with schema-id {} is bound to two different schemas across metadata versions", + schema_id); } - clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; - iceberg_table_schemas_by_ids[schema_id] = schema_ptr; } + + auto fields = schema_ptr->get(f_fields).extract(); + /// A field name is required per the Iceberg spec, and an empty column name is not representable in ClickHouse. + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + if (field->getValue(f_name).empty()) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Iceberg schema with schema-id {} has a field with id {} whose name is empty", + schema_id, + field->getValue(f_id)); + } + + current_schema_id = schema_id; + auto clickhouse_schema = std::make_shared(); + String current_full_name{}; + for (size_t i = 0; i != fields->size(); ++i) + { + auto field = fields->getObject(static_cast(i)); + auto name = field->getValue(f_name); + bool required = field->getValue(f_required); + current_full_name = name; + auto type = getFieldType(field, f_type, required, current_full_name, true); + clickhouse_schema->push_back(NameAndTypePair{name, type}); + clickhouse_types_by_source_ids[{schema_id, field->getValue(f_id)}] = NameAndTypePair{current_full_name, type}; + clickhouse_ids_by_source_names[{schema_id, current_full_name}] = field->getValue(f_id); + } + clickhouse_table_schemas_by_ids[schema_id] = clickhouse_schema; + iceberg_table_schemas_by_ids[schema_id] = schema_ptr; + if (source == SchemaSource::ManifestFile) + manifest_sourced_schema_ids.insert(schema_id); current_schema_id = std::nullopt; } @@ -735,7 +810,15 @@ std::shared_ptr IcebergSchemaProcessor::getSchemaTransformationDag( /// a whitespace-only difference is the same type and needs only a rename, not a cast. if (canonicalizeTypeSpacing(old_type) == canonicalizeTypeSpacing(new_type)) { - if (old_json->getValue(f_name) != name) + /// Nullability is carried by the separate `required` key, so equal type strings + /// can still resolve to different types. Only relaxing required to optional is + /// legal evolution; the reverse keeps the plain passthrough. + const bool old_required = old_json->getValue(f_required); + if (old_required && !required && !old_node->result_type->equals(*type)) + { + node = &dag->addCast(*old_node, type, name, nullptr); + } + else if (old_json->getValue(f_name) != name) { node = &dag->addAlias(*old_node, name); } @@ -859,6 +942,13 @@ bool IcebergSchemaProcessor::hasClickHouseTableSchemaById(Int32 id) const return clickhouse_table_schemas_by_ids.contains(id); } +bool IcebergSchemaProcessor::isSchemaRegisteredFromMetadata(Int32 id) const +{ + SharedLockGuard lock(mutex); + + return iceberg_table_schemas_by_ids.contains(id) && !manifest_sourced_schema_ids.contains(id); +} + std::unordered_map IcebergSchemaProcessor::traverseSchema(Poco::JSON::Array::Ptr schema) { std::unordered_map result; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index a072f5ebc127..cb186791710e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -88,9 +88,23 @@ class IcebergSchemaProcessor using Node = ActionsDAG::Node; public: + /// Where a schema copy being registered comes from. metadata.json is the authoritative source; + /// the 'schema' key of a manifest file header is only a snapshot of the table schema at the time + /// the manifest was written. A schema that came from a manifest is never authoritative: it is + /// replaced by the metadata.json copy of the same schema-id whenever that one is registered, and + /// it may be ignored if it conflicts with an already registered metadata.json copy. + enum class SchemaSource + { + Metadata, + ManifestFile, + }; + explicit IcebergSchemaProcessor(bool allow_geo_parser_ = false) : allow_geo_parser(allow_geo_parser_) {} - void addIcebergTableSchema(Poco::JSON::Object::Ptr schema_ptr); + void addIcebergTableSchema( + Poco::JSON::Object::Ptr schema_ptr, + SchemaSource source = SchemaSource::Metadata, + bool tolerate_conflicting_manifest_schemas = false); std::shared_ptr getClickHouseTableSchemaById(Int32 id); std::shared_ptr getSchemaTransformationDagByIds(Int32 old_id, Int32 new_id); NameAndTypePair getFieldCharacteristics(Int32 schema_version, Int32 source_id) const; @@ -99,6 +113,7 @@ class IcebergSchemaProcessor std::optional tryGetColumnIDByName(Int32 schema_id, const std::string & name) const; Poco::JSON::Object::Ptr getIcebergTableSchemaById(Int32 id) const; bool hasClickHouseTableSchemaById(Int32 id) const; + bool isSchemaRegisteredFromMetadata(Int32 id) const; static DataTypePtr getSimpleType(const String & type_name, bool allow_geo_parser = true); @@ -125,6 +140,16 @@ class IcebergSchemaProcessor mutable std::map, Int32> clickhouse_ids_by_source_names TSA_GUARDED_BY(mutex); std::optional current_schema_id TSA_GUARDED_BY(mutex) = 0; std::unordered_map schema_id_by_snapshot TSA_GUARDED_BY(mutex); + /// Schema-ids whose registered copy came from a manifest file header and has not been confirmed + /// by an identical metadata.json copy yet. Such a copy is replaced when metadata.json binds the + /// schema-id to a different schema, and two manifest headers disagreeing on such a schema-id is + /// an error, because no authoritative copy is left to decide between them. + std::unordered_set manifest_sourced_schema_ids TSA_GUARDED_BY(mutex); + + /// Forget the schema registered for `schema_id` together with everything derived from it: the + /// per-field lookups and the cached schema transformation DAGs in either direction. They are + /// keyed by schema-id and never rebuilt once populated, so a stale entry would keep answering. + void dropCachedSchema(Int32 schema_id) TSA_REQUIRES(mutex); NamesAndTypesList getSchemaType(const Poco::JSON::Object::Ptr & schema); DataTypePtr getComplexTypeFromObject(const Poco::JSON::Object::Ptr & type, String & current_full_name, bool is_subfield_of_root); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp index 497f559d7c95..b6a29bd096fe 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Utils.cpp @@ -1520,7 +1520,22 @@ KeyDescription getSortingKeyDescriptionFromMetadata(Poco::JSON::Object::Ptr meta auto column_name = source_id_to_column_name[source_id]; int direction = field->getValue(f_direction) == "asc" ? 1 : -1; auto iceberg_transform_name = field->getValue(f_transform); - auto clickhouse_transform_name = parseTransformAndArgument(iceberg_transform_name); + std::optional clickhouse_transform_name; + try + { + clickhouse_transform_name = parseTransformAndArgument(iceberg_transform_name); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::BAD_ARGUMENTS) + throw; + } + if (!clickhouse_transform_name.has_value()) + { + /// An unknown or malformed transform is not a reason to reject the table: an Iceberg + /// sort order is only an optimization hint, so drop it and read/write the table as unsorted. + return KeyDescription{}; + } /// Quote the column name so identifiers with special characters (e.g. `@timestamp`) /// produce a parseable ORDER BY clause. auto quoted_column_name = backQuoteIfNeed(column_name); @@ -1610,6 +1625,13 @@ void forEachAvroEntry( auto reader_base = std::make_unique(std::move(input_stream), MAX_AVRO_SCHEMA_DEPTH); avro::DataFileReader reader(std::move(reader_base)); + if (reader.readerSchema().root()->type() != avro::AVRO_RECORD) + throw Exception( + ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION, + "Avro file {} has root schema type {}, but Iceberg manifest-list entries must be records", + filename, + static_cast(reader.readerSchema().root()->type())); + avro::GenericDatum datum(reader.readerSchema()); while (reader.read(datum)) callback(datum); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp index 504f0f8fc7e9..b79d3ba94033 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/tests/gtest_iceberg_schema_processor.cpp @@ -7,6 +7,11 @@ #include #include +namespace DB::ErrorCodes +{ + extern const int ICEBERG_SPECIFICATION_VIOLATION; +} + using namespace DB::Iceberg; namespace @@ -185,6 +190,145 @@ TEST(IcebergSchemaProcessor, RebindingSchemaIdToDifferentTypeStillRejected) EXPECT_THROW(processor.addIcebergTableSchema(second), DB::Exception); } +/// The manifest header 'schema' key is only a copy of the table schema at write time; metadata.json +/// is authoritative. Broken writers (observed with AWS S3 Tables maintenance jobs) store degraded +/// copies in manifest headers under an already-used schema-id, e.g. `timestamp` instead of +/// `timestamptz`. With toleration enabled (the default of `iceberg_tolerate_conflicting_manifest_schemas`), +/// the divergent manifest copy must be ignored and the metadata.json schema kept. +TEST(IcebergSchemaProcessor, ConflictingManifestSchemaToleratedWhenEnabled) +{ + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_metadata); + EXPECT_NO_THROW(processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + + /// The metadata.json copy must win: the field keeps the timestamptz type. + auto schema = processor.getClickHouseTableSchemaById(0); + ASSERT_EQ(schema->size(), 1u); + EXPECT_EQ(schema->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + +/// With toleration disabled (`compatibility` below 26.9), the same conflict must still fail. +TEST(IcebergSchemaProcessor, ConflictingManifestSchemaRejectedWhenDisabled) +{ + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_metadata); + EXPECT_THROW( + processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + DB::Exception); +} + +/// Toleration only applies to manifest header copies: two conflicting metadata.json definitions of +/// the same schema-id are genuine catalog corruption and must always be rejected. +TEST(IcebergSchemaProcessor, ConflictingMetadataSchemaAlwaysRejected) +{ + auto first = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto second = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(first); + EXPECT_THROW( + processor.addIcebergTableSchema( + second, IcebergSchemaProcessor::SchemaSource::Metadata, /*tolerate_conflicting_manifest_schemas=*/true), + DB::Exception); +} + +/// A manifest header carrying a schema-id NOT registered from metadata.json (e.g. an expired schema +/// still referenced by an old manifest) must register normally regardless of the toleration flag. +TEST(IcebergSchemaProcessor, ManifestSchemaWithNewIdRegistersNormally) +{ + auto from_manifest = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + EXPECT_NO_THROW(processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + EXPECT_TRUE(processor.hasClickHouseTableSchemaById(5)); +} + +/// The maintenance entrypoints (`remove_orphan_files`, `expire_snapshots`, manifest compaction) can +/// register a schema that came from a manifest before metadata.json is read. Such a schema is never +/// authoritative: the metadata.json copy of the same schema-id must replace it, together with the +/// per-field lookups and the cached transformation DAGs derived from it. +TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsReplacedByMetadata) +{ + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + auto other = parseSchema(R"json({"schema-id":1,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"},{"id":2,"name":"v","required":false,"type":"int"}]})json"); + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + processor.addIcebergTableSchema(other); + + /// Populate everything derived from the schema that came from the manifest. + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "Nullable(DateTime64(6))"); + auto stale_dag = processor.getSchemaTransformationDagByIds(0, 1); + ASSERT_NE(stale_dag, nullptr); + + EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata)); + + auto schema = processor.getClickHouseTableSchemaById(0); + ASSERT_EQ(schema->size(), 1u); + EXPECT_EQ(schema->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); + EXPECT_EQ(processor.getFieldCharacteristics(0, 1).type->getName(), "Nullable(DateTime64(6, 'UTC'))"); + EXPECT_NE(processor.getSchemaTransformationDagByIds(0, 1).get(), stale_dag.get()); + + /// Once metadata.json has settled the schema-id, a later divergent schema from a manifest is handled like + /// any header conflicting with a known metadata.json copy: ignored when tolerated, rejected otherwise. + EXPECT_NO_THROW(processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + EXPECT_THROW( + processor.addIcebergTableSchema( + from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + DB::Exception); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + +/// An identical metadata.json copy confirms a schema that came from a manifest and was registered before it; the schema-id then +/// behaves as if it had been registered from metadata.json in the first place. +TEST(IcebergSchemaProcessor, ManifestHeaderRegisteredFirstIsConfirmedByIdenticalMetadata) +{ + auto from_manifest = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto from_metadata = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto degraded = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(from_manifest, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + EXPECT_NO_THROW(processor.addIcebergTableSchema(from_metadata)); + + EXPECT_NO_THROW(processor.addIcebergTableSchema( + degraded, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true)); + EXPECT_THROW( + processor.addIcebergTableSchema( + degraded, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/false), + DB::Exception); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + +/// Two manifest headers that disagree on a schema-id metadata.json has not defined leave no +/// authoritative copy to decide between them, so this is a specification violation even when +/// conflicts with metadata.json are tolerated. Other schema-ids are unaffected. +TEST(IcebergSchemaProcessor, ConflictingManifestHeadersWithoutMetadataSchemaRejected) +{ + auto first = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamp"}]})json"); + auto second = parseSchema(R"json({"schema-id":5,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + auto other = parseSchema(R"json({"schema-id":0,"fields":[{"id":1,"name":"ts","required":false,"type":"timestamptz"}]})json"); + IcebergSchemaProcessor processor; + processor.addIcebergTableSchema(other); + processor.addIcebergTableSchema(first, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + try + { + processor.addIcebergTableSchema(second, IcebergSchemaProcessor::SchemaSource::ManifestFile, /*tolerate_conflicting_manifest_schemas=*/true); + FAIL() << "expected ICEBERG_SPECIFICATION_VIOLATION"; + } + catch (const DB::Exception & e) + { + EXPECT_EQ(e.code(), DB::ErrorCodes::ICEBERG_SPECIFICATION_VIOLATION); + } + EXPECT_TRUE(processor.hasClickHouseTableSchemaById(0)); + EXPECT_EQ(processor.getClickHouseTableSchemaById(0)->front().type->getName(), "Nullable(DateTime64(6, 'UTC'))"); +} + /// A renamed field bound to the same schema-id must still be rejected (issue #107316). TEST(IcebergSchemaProcessor, RebindingSchemaIdToRenamedFieldStillRejected) { diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp index 50484b2e20bd..08fac3e0d656 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonClient.cpp @@ -298,6 +298,7 @@ PaimonSnapshot PaimonTableClient::getSnapshot(const std::pair & s std::pair, size_t> PaimonTableClient::getManifestMeta(String manifest_list_path, bool disable_filesystem_cache) { /// read manifest list file + Paimon::checkPathIsRelativeToTable(manifest_list_path, "manifest list"); auto context = getContext(); RelativePathWithMetadata relative_path(std::filesystem::path(table_location) / PAIMON_MANIFEST_DIR / manifest_list_path); auto read_settings = getPaimonMetadataReadSettings(disable_filesystem_cache); @@ -320,6 +321,7 @@ std::pair, size_t> PaimonTableClient::getMan PaimonManifest PaimonTableClient::getDataManifest(String manifest_path, const PaimonTableSchema & table_schema, const String & partition_default_name, bool disable_filesystem_cache) { + Paimon::checkPathIsRelativeToTable(manifest_path, "manifest"); String manifest_file_name(manifest_path.begin() + manifest_path.find_last_of('/') + 1, manifest_path.end()); if (manifest_file_name.starts_with("index-manifest-")) return {}; diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp index b97affbdeff5..89e441c7ea7c 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonMetadata.cpp @@ -896,6 +896,7 @@ Strings PaimonMetadata::collectDataFilesFromManifests( auto manifest = getManifest(meta.file_name, snapshot_state->schema_id); for (const auto & entry : manifest->entries) { + Paimon::checkPathIsRelativeToTable(entry.file.file_name, "data file"); String file_path = (std::filesystem::path(persistent_components.table_path) / entry.file.bucket_path / entry.file.file_name); diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp index bda89875692e..88ab26fb305f 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/PaimonStreamState.cpp @@ -10,6 +10,7 @@ #include #include #include +#include namespace DB { @@ -20,6 +21,11 @@ extern const int LOGICAL_ERROR; extern const int REPLICA_IS_ALREADY_ACTIVE; } +namespace FailPoints +{ +extern const char paimon_incremental_read_pause_before_is_active_remove[]; +} + PaimonStreamState::PaimonStreamState( zkutil::ZooKeeperPtr keeper_, const String & keeper_path_, @@ -162,14 +168,19 @@ bool PaimonStreamState::activate() { /// Stale node from our previous session — safe to reclaim. /// Use versioned delete (CAS) to guard against TOCTOU races. + FailPointInjection::pauseFailPoint( + FailPoints::paimon_incremental_read_pause_before_is_active_remove); auto remove_code = keeper->tryRemove(is_active_path, stat.version); - if (remove_code != Coordination::Error::ZOK) + /// ZNONODE is this removal's postcondition, not a failure: Keeper reaps the + /// ephemeral itself once the session that created it expires. + if (remove_code != Coordination::Error::ZOK && remove_code != Coordination::Error::ZNONODE) { LOG_WARNING(log, "Failed to remove stale is_active node at {} (code: {}). " "Will retry on next attempt.", is_active_path.string(), remove_code); return false; } - LOG_INFO(log, "Removed stale is_active node from previous session at {}", is_active_path.string()); + LOG_INFO(log, "Cleared stale is_active node from previous session at {} ({})", + is_active_path.string(), remove_code); } else { diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp index a3885f3c55f6..bc1fef9cbf51 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -29,6 +31,7 @@ namespace ErrorCodes extern const int CANNOT_PRINT_FLOAT_OR_DOUBLE_NUMBER; extern const int BAD_ARGUMENTS; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +extern const int PATH_ACCESS_DENIED; } } namespace Paimon @@ -371,6 +374,17 @@ String getBucketPath(const String & partition, Int32 bucket, const PaimonTableSc return bucket_path; } +void checkPathIsRelativeToTable(const String & path, std::string_view kind) +{ + const std::filesystem::path fs_path(path); + if (fs_path.has_root_path() || std::ranges::any_of(fs_path, [](const auto & component) { return component == ".."; })) + throw Exception( + ErrorCodes::PATH_ACCESS_DENIED, + "Paimon {} path `{}` must be relative to the table directory and must not contain `..`", + kind, + path); +} + String concatPath(std::initializer_list paths) { if (paths.size() == 1) diff --git a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h index b01d8d33a9eb..fb1171b109be 100644 --- a/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h +++ b/src/Storages/ObjectStorage/DataLakes/Paimon/Utils.h @@ -73,6 +73,7 @@ class PathEscape DB::Row getPartitionFields(const String & partition, const PaimonTableSchema & table_schema); String getBucketPath(const String & partition, Int32 bucket, const PaimonTableSchema & table_schema, const String & partition_default_name); String concatPath(std::initializer_list paths); +void checkPathIsRelativeToTable(const String & path, std::string_view kind); template void getValueFromJSON(T & t, const Poco::JSON::Object::Ptr & json, const String & key) diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp index 735827eb8222..806825224e3c 100644 --- a/src/Storages/ObjectStorage/S3/Configuration.cpp +++ b/src/Storages/ObjectStorage/S3/Configuration.cpp @@ -179,15 +179,16 @@ ObjectStoragePtr StorageS3Configuration::createObjectStorage(ContextPtr context, url, *s3_settings, context, /* for_disk_s3 */ false, /*opt_disk_name*/ {}, /*refresh_credentials_callback*/ std::nullopt, is_loading_from_existing_metadata, force_anonymous_load_fallback); - auto client_refresher = [refresh_credentials_callback, this, context_ = Context::createCopy(context)] () -> std::unique_ptr + S3ObjectStorage::S3CredentialsRefreshCallback client_refresher; + if (refresh_credentials_callback) { - if (!refresh_credentials_callback) - return nullptr; - auto new_client = getClient( - url, *s3_settings, context_, /* for_disk_s3 */ false, /*opt_disk_name*/ {}, refresh_credentials_callback, - is_loading_from_existing_metadata, force_anonymous_load_fallback); - return new_client; - }; + client_refresher = [refresh_credentials_callback, this, context_ = Context::createCopy(context)] () -> std::unique_ptr + { + return getClient( + url, *s3_settings, context_, /* for_disk_s3 */ false, /*opt_disk_name*/ {}, refresh_credentials_callback, + is_loading_from_existing_metadata, force_anonymous_load_fallback); + }; + } return std::make_shared( std::move(client), std::make_unique(*s3_settings), diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 42f4fcaa6d45..99db81265c8c 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -208,6 +209,7 @@ namespace Setting extern const SettingsUInt64 s3_path_filter_limit; extern const SettingsBool use_parquet_metadata_cache; extern const SettingsBool s3_validate_etag_on_read; + extern const SettingsBool azure_validate_etag_on_read; } static void logIcebergFileStats(const ObjectInfoPtr & object_info, const LoggerPtr & log) @@ -793,6 +795,11 @@ Chunk StorageObjectStorageSource::generate() const auto column_pos = read_from_format_info.source_header.getPositionByName(name_and_type.name); auto partition_column = name_and_type.type->createColumnConst(chunk.getNumRows(), value)->convertToFullColumnIfConst(); + /// The `_delta_log` type differs from the declared one when the columns were + /// specified rather than inferred, and the block follows the declared schema. + const auto & declared_type = read_from_format_info.source_header.getByPosition(column_pos).type; + if (!name_and_type.type->equals(*declared_type)) + partition_column = castColumn({partition_column, name_and_type.type, name_and_type.name}, declared_type); /// This column is filled with default value now, remove it. chunk.erase(column_pos); /// Add correct values. @@ -1573,16 +1580,24 @@ std::unique_ptr createReadBuffer( /// 2. object etag suggests a cache key in case we use filesystem cache /// 3. object etag as a cache key for parquet metadata caching /// 4. object etag to detect a concurrent in-place overwrite during the read + /// Whether the read is pinned to the generation of the object seen at listing time. Each backend + /// that supports it has its own setting, because they are documented per backend and a user may + /// want to opt out of the check for one store but not the other. + bool validate_etag_on_read = false; + if (object_storage->getType() == ObjectStorageType::S3) + validate_etag_on_read = settings[Setting::s3_validate_etag_on_read]; + else if (object_storage->getType() == ObjectStorageType::Azure) + validate_etag_on_read = settings[Setting::azure_validate_etag_on_read]; + if (!object_info.metadata) { object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); } - else if (!object_info.metadata->is_fetched && settings[Setting::s3_validate_etag_on_read] - && object_storage->getType() == ObjectStorageType::S3) + else if (!object_info.metadata->is_fetched && validate_etag_on_read) { - /// Refresh the s3Cluster skip_object_metadata placeholder to obtain its size + ETag for read-time - /// validation (it carries no tags, so the with_tags=false HEAD drops nothing). A real fetch that - /// merely lacks an ETag (e.g. GCS) has is_fetched=true and is left as-is - no extra HEAD. + /// Refresh the cluster function's skip_object_metadata placeholder to obtain its size + ETag for + /// read-time validation (it carries no tags, so the with_tags=false HEAD drops nothing). A real fetch + /// that merely lacks an ETag (e.g. GCS) has is_fetched=true and is left as-is - no extra HEAD. object_info.metadata = object_storage->getObjectMetadata(object_info, /*with_tags=*/ false); } @@ -1648,13 +1663,18 @@ std::unique_ptr createReadBuffer( /// filename to `readWithDistributedCache` (it ends up in `getFileName()` and in /// `system.distributed_cache_log.filename`). Use the object path so the DC log /// shows a useful name rather than an empty string. - const auto stored_object_size = is_size_known ? object_size : StoredObject::UnknownSize; + /// The size is used by the object storage as the right bound of the read, so it must come from a + /// real listing or HEAD: the skip_object_metadata placeholder is default-constructed, and its + /// `size_bytes == 0` would otherwise read every non-empty object as empty. + const auto stored_object_size = is_size_known && object_info.metadata->is_fetched + ? object_size + : StoredObject::UnknownSize; StoredObject stored_object(object_info.getPath(), object_info.getPath(), stored_object_size, object_info.read_source_index); /// Pin the read to the object generation seen here (etag from the LIST/HEAD): a GET with a - /// different ETag means an in-place overwrite, reported as S3_OBJECT_CHANGED_DURING_READ - /// instead of torn cross-generation data. - if (settings[Setting::s3_validate_etag_on_read] && object_info.metadata.has_value()) + /// different ETag means an in-place overwrite, reported as S3_OBJECT_CHANGED_DURING_READ or + /// AZURE_OBJECT_CHANGED_DURING_READ instead of torn cross-generation data. + if (validate_etag_on_read && object_info.metadata.has_value()) stored_object.etag = object_info.metadata->etag; pipeline.setSource(object_storage, StoredObjects{stored_object}, modified_read_settings); diff --git a/src/Storages/RabbitMQ/RabbitMQ_fwd.h b/src/Storages/RabbitMQ/RabbitMQ_fwd.h index af4a3c6fd2bb..6dc017ae1e25 100644 --- a/src/Storages/RabbitMQ/RabbitMQ_fwd.h +++ b/src/Storages/RabbitMQ/RabbitMQ_fwd.h @@ -1,7 +1,6 @@ #pragma once #include #include -#include #include namespace RabbitMQ @@ -21,7 +20,10 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - DB::maskURIPassword(&masked_value); + /// AMQP-CPP ends the login at the FIRST '@' after the scheme, unbounded by the `/?#` that closes + /// an RFC 3986 authority (`contrib/AMQP-CPP/include/amqpcpp/address.h`), so no URI masker bounds it. + if (masked_value.contains('@')) + masked_value = "[HIDDEN]"; return fmt::format("'{}'", masked_value); }} }; diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp index d19fd8ed616c..61eb507f3036 100644 --- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp +++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include @@ -61,7 +62,7 @@ void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr con for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { StoragePtr table = iterator->table(); - RocksDBStoragePtr rocksdb_table = table ? std::dynamic_pointer_cast(table) : nullptr; + RocksDBStoragePtr rocksdb_table = table ? castStorage(table, DeferredTable::Skip) : nullptr; if (!rocksdb_table) continue; diff --git a/src/Storages/Statistics/StatisticsBasic.cpp b/src/Storages/Statistics/StatisticsBasic.cpp index 3c9f88b061a8..8e6a5254a2a9 100644 --- a/src/Storages/Statistics/StatisticsBasic.cpp +++ b/src/Storages/Statistics/StatisticsBasic.cpp @@ -152,8 +152,8 @@ void StatisticsBasic::serialize(WriteBuffer & buf) mask |= BasicFeatureMask::NumericMinMax; if (tracks_string) mask |= BasicFeatureMask::StringLengthSum; - - mask |= BasicFeatureMask::DefaultCount; + if (has_default_count) + mask |= BasicFeatureMask::DefaultCount; writeIntBinary(mask, buf); if (tracks_numeric) @@ -163,8 +163,8 @@ void StatisticsBasic::serialize(WriteBuffer & buf) } if (tracks_string) writeIntBinary(string_total_bytes, buf); - - writeIntBinary(default_count, buf); + if (has_default_count) + writeIntBinary(default_count, buf); } void StatisticsBasic::deserialize(ReadBuffer & buf, StatisticsFileVersion /*version*/) diff --git a/src/Storages/Statistics/tests/gtest_stats.cpp b/src/Storages/Statistics/tests/gtest_stats.cpp index 321f879a39a1..94daafd96dce 100644 --- a/src/Storages/Statistics/tests/gtest_stats.cpp +++ b/src/Storages/Statistics/tests/gtest_stats.cpp @@ -762,6 +762,91 @@ TEST(Statistics, BasicDefaultCountRoundTrip) auto eq0 = restored->estimateEqual(Field(Int64(0))); ASSERT_TRUE(eq0.has_value()); EXPECT_DOUBLE_EQ(*eq0, 4.0); + + /// `basic` statistics written by 26.6 and 26.7 store the default-value count only for `Nullable` columns, as + /// the NULL count (bit 2 of the feature mask, then called `NullCount`). For other columns it is absent, so it + /// is unknown, and it must stay unknown when the loaded statistics are written again, as a mutation does for + /// the columns it does not change. Check each layout these versions wrote. + constexpr UInt8 numeric_min_max = 1u << 0; + constexpr UInt8 string_length_sum = 1u << 1; + constexpr UInt8 null_count = 1u << 2; + + auto load_legacy_and_rewrite = [](const DataTypePtr & type, UInt8 feature_mask, std::function write_features) + { + /// `StatisticsBasic::serialize` of 26.7. + String payload; + { + WriteBufferFromString buf(payload); + writeIntBinary(static_cast(1000), buf); /// row_count + writeIntBinary(feature_mask, buf); + write_features(buf); + buf.finalize(); + } + + /// `ColumnStatistics::serialize` of 26.7 (`V4`) with the single `Basic` statistic. + String file; + { + WriteBufferFromString buf(file); + writeIntBinary(static_cast(4), buf); /// StatisticsFileVersion::V4 + writeIntBinary(static_cast(1ULL << static_cast(StatisticsType::Basic)), buf); + writeStringBinary(type->getName(), buf); + writeIntBinary(static_cast(1000), buf); /// rows + writeIntBinary(static_cast(payload.size()), buf); + buf.write(payload.data(), payload.size()); + buf.finalize(); + } + + ReadBufferFromString file_rb(file); + auto loaded = ColumnStatistics::deserialize(file_rb, type); + WriteBufferFromOwnString rewritten_wb; + loaded->serialize(rewritten_wb); + ReadBufferFromString rewritten_rb(rewritten_wb.str()); + return ColumnStatistics::deserialize(rewritten_rb, type); + }; + + /// A numeric column: min and max. + { + auto rewritten = load_legacy_and_rewrite(data_type, numeric_min_max, [](WriteBuffer & buf) + { + writeFieldBinary(Field(Int64(0)), buf); + writeFieldBinary(Field(Int64(999)), buf); + }); + auto estimate = rewritten->getEstimate(); + EXPECT_FALSE(estimate.estimated_default_count.has_value()); + /// Without the count there is no estimate for `x = 0`, rather than an estimate of zero rows. + EXPECT_FALSE(rewritten->estimateEqual(Field(Int64(0))).has_value()); + ASSERT_TRUE(estimate.estimated_min.has_value()); + ASSERT_TRUE(estimate.estimated_max.has_value()); + EXPECT_EQ(*estimate.estimated_min, Field(Int64(0))); + EXPECT_EQ(*estimate.estimated_max, Field(Int64(999))); + } + + /// A `String` column: the total byte length of the values. + { + auto string_type = DataTypeFactory::instance().get("String"); + auto rewritten = load_legacy_and_rewrite(string_type, string_length_sum, [](WriteBuffer & buf) + { + writeIntBinary(static_cast(2890), buf); + }); + EXPECT_FALSE(rewritten->getEstimate().estimated_default_count.has_value()); + EXPECT_FALSE(rewritten->estimateEqual(Field(String(""))).has_value()); + const auto & basic = assert_cast(*rewritten->getStats().at(StatisticsType::Basic)); + EXPECT_EQ(basic.getStringTotalBytes(), 2890u); + } + + /// A `Nullable` column: min, max and the NULL count, which is the default-value count of a `Nullable` column. + { + auto nullable_type = std::make_shared(std::make_shared()); + auto rewritten = load_legacy_and_rewrite(nullable_type, numeric_min_max | null_count, [](WriteBuffer & buf) + { + writeFieldBinary(Field(Int64(1)), buf); + writeFieldBinary(Field(Int64(999)), buf); + writeIntBinary(static_cast(250), buf); + }); + EXPECT_TRUE(rewritten->hasNullCount()); + EXPECT_EQ(rewritten->getNullCount(), 250u); + EXPECT_EQ(rewritten->estimateDefaults(), 250u); + } } TEST(Statistics, BasicDefaultCountArray) diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index fa0262c17b82..3fa188702e06 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -35,6 +35,10 @@ class StorageAlias final : public IStorage, WithContext bool readsFromOtherTables() const override { return true; } + /// An `Alias` has no data of its own, so a bulk `TRUNCATE ALL TABLES` must skip it. + /// Only the bulk paths consult this; an explicit `TRUNCATE TABLE ` still truncates the target. + bool supportsTruncate() const override { return false; } + /// Get the target storage this alias points to StoragePtr getTargetTable(std::optional access_check = std::nullopt) const; StoragePtr tryGetTargetTable() const { return DatabaseCatalog::instance().tryGetTable(StorageID(target_database, target_table), getContext()); } diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index 08922daedfd6..cd6eaf90454b 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -37,6 +37,7 @@ namespace DB { namespace Setting { + extern const SettingsBool allow_executable_tables; extern const SettingsBool allow_experimental_analyzer; extern const SettingsSeconds max_execution_time; } @@ -58,6 +59,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int UNSUPPORTED_METHOD; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SUPPORT_IS_DISABLED; } namespace @@ -163,6 +165,12 @@ void StorageExecutable::readImpl( size_t max_block_size, size_t /*threads*/) { + if (!context->getSettingsRef()[Setting::allow_executable_tables]) + throw Exception( + ErrorCodes::SUPPORT_IS_DISABLED, + "The `executable` table function and the `Executable` and `ExecutablePool` table " + "engines are disabled. Set `allow_executable_tables` setting to enable them"); + auto & script_name = settings->script_name; auto user_scripts_path = context->getUserScriptsPath(); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index f2afa34d2958..2c4cb0a58661 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -3270,7 +3271,7 @@ void StorageMergeTree::replacePartitionFrom(const StoragePtr & source_table, con /// Clang's thread-safety analyzer, which cannot track mutex ownership across `std::lock`. void StorageMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr local_context) TSA_NO_THREAD_SAFETY_ANALYSIS { - auto dest_table_storage = std::dynamic_pointer_cast(dest_table); + auto dest_table_storage = std::dynamic_pointer_cast(resolveStorageProxyLoading(dest_table)); if (!dest_table_storage) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table {} supports movePartitionToTable only for MergeTree family of table engines. Got {}", diff --git a/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp b/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp index b83639f01468..969ba92a36d3 100644 --- a/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp +++ b/src/Storages/StorageMergeTreeAnalyzeIndexes.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -128,7 +129,7 @@ class MergeTreeAnalyzeIndexSource final : public ISource, WithContext auto reader_settings = MergeTreeReaderSettings::createForQuery(context, *table_settings, query_info); const auto metadata_snapshot = storage->getInMemoryMetadataPtr(context, false); - const auto * merge_tree_data = dynamic_cast(storage.get()); + const auto * merge_tree_data = castStorage(storage, DeferredTable::Load).get(); if (!merge_tree_data) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeAnalyzeIndexes expected MergeTree table, got: {}", storage->getName()); @@ -305,11 +306,11 @@ StorageMergeTreeAnalyzeIndexes::StorageMergeTreeAnalyzeIndexes( const ASTPtr & predicate_, const OptionalVectorSearchParameters & vector_search_parameters_) : StorageWithCommonVirtualColumns(table_id_) - , source_table(source_table_) + , source_table(resolveStorageProxyLoading(source_table_)) , predicate(predicate_) , vector_search_parameters(vector_search_parameters_) { - const auto * merge_tree_data = dynamic_cast(source_table.get()); + const auto * merge_tree_data = castStorage(source_table, DeferredTable::Load).get(); if (!merge_tree_data) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeAnalyzeIndexes expected MergeTree table, got: {}", source_table->getName()); diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp index a02ea2d0473f..6dc20b3c09af 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.cpp +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -313,7 +314,7 @@ void StorageMergeTreeCodecBlockCounts::checkSourceTableAccess(const StoragePtr & context->checkAccess(AccessType::SELECT, source_table->getStorageID(), source_metadata->getColumns().getNamesOfPhysical()); } -StoragePtr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context) +std::shared_ptr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context) { /// `SHOW TABLES` is the privilege that governs whether the table's existence may be learned, and it is implied /// by a grant on any single column of it, so this only adds a tier below the `SELECT` check on every column @@ -324,11 +325,13 @@ StoragePtr StorageMergeTreeCodecBlockCounts::resolveSourceTable(const StorageID auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); checkSourceTableAccess(source_table, context); - if (!dynamic_cast(source_table.get())) + /// Reading the parts is what the function is for, so a not yet loaded source table is loaded here. + auto merge_tree = castStorage(source_table, DeferredTable::Load); + if (!merge_tree) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeCodecBlockCounts expected MergeTree table, got: {}", source_table->getName()); - return source_table; + return merge_tree; } void StorageMergeTreeCodecBlockCounts::read( @@ -344,15 +347,11 @@ void StorageMergeTreeCodecBlockCounts::read( storage_snapshot->check(column_names); /// Under the reader's context, see the constructor. - auto source_table = resolveSourceTable(source_table_id, context); - const auto source_storage_id = source_table->getStorageID(); - - /// A cast to a base class, so not `assert_cast`, which asserts the exact type and would reject every - /// `MergeTree` table. `resolveSourceTable` has already rejected a source table that is not a `MergeTree`. - const auto & merge_tree = dynamic_cast(*source_table); + auto merge_tree = resolveSourceTable(source_table_id, context); + const auto source_storage_id = merge_tree->getStorageID(); /// `system.parts_columns` lists patch parts, so this function does too. - auto data_parts = merge_tree.getDataPartsVectorForInternalUsage( + auto data_parts = merge_tree->getDataPartsVectorForInternalUsage( {MergeTreeData::DataPartState::Active}, {MergeTreeData::DataPartKind::Regular, MergeTreeData::DataPartKind::Patch}); std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); @@ -374,7 +373,7 @@ void StorageMergeTreeCodecBlockCounts::read( } /// The parts reference the source table's MergeTreeData without owning it. - query_plan.addStorageHolder(source_table); + query_plan.addStorageHolder(merge_tree); query_plan.addStep( std::make_unique( diff --git a/src/Storages/StorageMergeTreeCodecBlockCounts.h b/src/Storages/StorageMergeTreeCodecBlockCounts.h index e8852112a0ae..4faa519d942e 100644 --- a/src/Storages/StorageMergeTreeCodecBlockCounts.h +++ b/src/Storages/StorageMergeTreeCodecBlockCounts.h @@ -6,6 +6,8 @@ namespace DB { +class MergeTreeData; + /// Backing storage for `mergeTreeCodecBlockCounts(database, table)`. One row per (part, column, substream). /// Counts compressed blocks per codec by reading each stream's `.bin` header. Selecting `part_name`/`column`/`substream` is metadata-only. class StorageMergeTreeCodecBlockCounts final : public IStorage @@ -27,7 +29,7 @@ class StorageMergeTreeCodecBlockCounts final : public IStorage /// `SHOW TABLES` on the name, before the catalog is consulted, so that an inaccessible table and a missing one /// answer alike; then `SELECT` on every column, before the engine is examined, so that a user without it cannot /// learn the engine from the `BAD_ARGUMENTS` that rejects a table that is not a `MergeTree`. - static StoragePtr resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context); + static std::shared_ptr resolveSourceTable(const StorageID & source_table_id, const ContextPtr & context); void read( QueryPlan & query_plan, diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp index 54566034cc77..3c463efbe935 100644 --- a/src/Storages/StorageMergeTreeIndex.cpp +++ b/src/Storages/StorageMergeTreeIndex.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -294,7 +295,7 @@ StorageMergeTreeIndex::StorageMergeTreeIndex( , with_marks(with_marks_) , with_minmax(with_minmax_) { - const auto * merge_tree = dynamic_cast(source_table.get()); + const auto * merge_tree = castStorage(source_table, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeIndex expected MergeTree table, got: {}", source_table->getName()); diff --git a/src/Storages/StorageMergeTreeTextIndex.cpp b/src/Storages/StorageMergeTreeTextIndex.cpp index 2ebf5b4a57ce..4ca65f7c9cc1 100644 --- a/src/Storages/StorageMergeTreeTextIndex.cpp +++ b/src/Storages/StorageMergeTreeTextIndex.cpp @@ -1,4 +1,5 @@ +#include #include #include @@ -362,6 +363,7 @@ void ReadFromMergeTreeTextIndex::applyFilters(ActionDAGNodes added_filter_nodes) void ReadFromMergeTreeTextIndex::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) { /// Taken at read time: the storage outlives the query in a table created from the function before that was forbidden. + /// NOLINT(storage-cast): the table function resolves the source table before building this. auto data_parts = dynamic_cast(*storage->source_table).getDataPartsVectorForInternalUsage(); std::erase_if(data_parts, [](const MergeTreeData::DataPartPtr & part) { return part->isEmpty(); }); auto filtered_parts = VirtualColumnUtils::filterDataPartsWithExpression(data_parts, virtual_columns_filter); @@ -410,7 +412,7 @@ StorageMergeTreeTextIndex::StorageMergeTreeTextIndex( , source_table(source_table_) , text_index(std::move(text_index_)) { - if (!dynamic_cast(source_table.get())) + if (!castStorage(source_table, DeferredTable::Load)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table->getName()); StorageInMemoryMetadata storage_metadata; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index bccc4ffeae5e..2dc249080a2c 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -16,6 +16,14 @@ class StorageProxy : public IStorage virtual StoragePtr getNested() const = 0; + /// The wrapped storage if it already exists, or null. Never creates it, so an observer + /// iterating every table cannot trigger a load. + virtual StoragePtr tryGetNested() const { return nullptr; } + + /// Whether the proxy only defers the creation of the storage and adds no behaviour of its own, + /// so that an operation may run on the wrapped storage directly. + virtual bool isLazyStandIn() const { return false; } + String getName() const override { return "Proxy"; } bool isRemote() const override { return getNested()->isRemote(); } @@ -194,5 +202,57 @@ class StorageProxy : public IStorage }; +/// The storage an operation should run on: the lazy stand-in is replaced by the storage it wraps +/// once that exists, while the other proxies add behaviour of their own and are kept. +inline StoragePtr resolveStorageProxy(const StoragePtr & storage) +{ + const auto * proxy = dynamic_cast(storage.get()); + if (!proxy || !proxy->isLazyStandIn()) + return storage; + auto nested = proxy->tryGetNested(); + return nested ? nested : storage; +} + +/// Same, but creates the wrapped storage when it does not exist yet. For operations that name a +/// table explicitly, where loading it is the expected cost of the operation. +inline StoragePtr resolveStorageProxyLoading(const StoragePtr & storage) +{ + const auto * proxy = dynamic_cast(storage.get()); + return proxy && proxy->isLazyStandIn() ? proxy->getNested() : storage; +} + +/// Proxies stack: a lazily loaded `URL` table is a `StorageTableProxy` over a `StorageURLSchemeDispatch` +/// over the real storage. The bound only guards against a cycle. +constexpr size_t max_storage_proxy_depth = 16; + +/// What a cast does with a table that is not loaded yet. +enum class DeferredTable : uint8_t +{ + /// Load it. For an operation that names the table, where loading is its expected cost. + Load, + /// Leave it unloaded, so the cast yields null and the caller skips it. For an observer that + /// walks every table and must not turn a listing into a load. + Skip, +}; + +/// The single way to cast a catalog pointer to a concrete engine type. A lazily loaded table is +/// reached through `StorageTableProxy`, so a direct cast fails even once the table is loaded. +template +std::shared_ptr castStorage(const StoragePtr & storage, DeferredTable deferred_table) +{ + /// The type test looks through every layer, whichever wrappers sit on top of the engine. + StoragePtr resolved = storage; + for (size_t depth = 0; depth < max_storage_proxy_depth && resolved; ++depth) + { + const auto * proxy = dynamic_cast(resolved.get()); + if (!proxy) + break; + auto nested = deferred_table == DeferredTable::Load ? proxy->getNested() : proxy->tryGetNested(); + if (!nested) + break; + resolved = nested; + } + return std::dynamic_pointer_cast(resolved); +} } diff --git a/src/Storages/StorageQueryRunner.cpp b/src/Storages/StorageQueryRunner.cpp index e10d6f5d681d..a6c2191919f5 100644 --- a/src/Storages/StorageQueryRunner.cpp +++ b/src/Storages/StorageQueryRunner.cpp @@ -466,7 +466,7 @@ class QueryRunnerDispatcher : WithContext } else { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "The `QueryRunner` engine does not support this query: {}", job.query); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "The `QueryRunner` engine does not support this query"); } } io.onFinish(); @@ -598,6 +598,8 @@ class QueryRunnerDispatcher : WithContext const auto event_time = std::chrono::system_clock::now(); + const String query_for_logging = formatQueryForLogging(job.query, settings); + query_log->add([&](QueryLogElement & element) { element.type = type; @@ -606,7 +608,7 @@ class QueryRunnerDispatcher : WithContext element.query_start_time = timeInSeconds(query_start_time); element.query_start_time_microseconds = timeInMicroseconds(query_start_time); element.query_duration_ms = duration_ms; - element.query = job.query; + element.query = query_for_logging; element.current_database = job.database; element.log_comment = settings[Setting::log_comment]; element.client_info = job_context->getClientInfo(); diff --git a/src/Storages/StorageRedis.cpp b/src/Storages/StorageRedis.cpp index b29f3c83332b..04906d787887 100644 --- a/src/Storages/StorageRedis.cpp +++ b/src/Storages/StorageRedis.cpp @@ -123,8 +123,13 @@ class RedisDataSource final : public ISource MutableColumns columns = sample_block.cloneEmptyColumns(); RedisArray values = storage.multiGet(scan_keys); - for (size_t i = 0; i < scan_keys.size() && !values.get(i).isNull(); i++) + for (size_t i = 0; i < scan_keys.size(); ++i) { + /// MGET answers by position, and a scanned key can hold another Redis type or expire + /// before the MGET runs, so a nil marks one absent value, not the end of the batch. + if (values.get(i).isNull()) + continue; + fillColumns(scan_keys.get(i).value(), values.get(i).value(), primary_key_pos, sample_block, columns @@ -461,7 +466,7 @@ Chunk StorageRedis::getBySerializedKeys(const RedisArray & keys, PaddedPODArray< "StorageRedis::getBySerializedKeys: null_map size {} does not match keys size {}", null_map->size(), keys.size()); - for (size_t i = 0; i < values.size(); ++i) + for (size_t i = 0; i < keys.size(); ++i) { if (null_map && !(*null_map)[i]) { @@ -517,7 +522,18 @@ RedisArray StorageRedis::multiGet(const RedisArray & keys) const for (size_t i = 0; i < keys.size(); ++i) cmd_mget.add(keys.get(i)); - return connection->client->execute(cmd_mget); + RedisArray values = connection->client->execute(cmd_mget); + + /// Callers pair the reply with the request by position, into arrays sized from `keys`. + if (values.isNull() || values.size() != keys.size()) + throw Exception( + ErrorCodes::INTERNAL_REDIS_ERROR, + "Redis table {} returned {} values for MGET of {} keys", + getStorageID().getFullNameNotQuoted(), + values.isNull() ? 0 : values.size(), + keys.size()); + + return values; } void StorageRedis::multiSet(const RedisArray & data) const diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 1af170cec475..5637e4431ebe 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -75,6 +75,7 @@ #include #include #include +#include #include #include #include @@ -3207,7 +3208,8 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) auto clone_data_parts_from_source_table = [&] () -> size_t { - source_table = DatabaseCatalog::instance().tryGetTable(source_table_id, getContext()); + /// Leaving this proxied would make the checks below read the source as not replicated. + source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().tryGetTable(source_table_id, getContext())); if (!source_table) { LOG_DEBUG(log, "Can't use {} as source table for REPLACE PARTITION command. It does not exist.", source_table_id.getNameForLogs()); @@ -3339,7 +3341,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) /// However, it's quite dangerous, because part may appear in source table. /// So we enqueue it for check only if no replicas of source table have part either. bool need_check = true; - if (auto * replicated_src_table = typeid_cast(source_table.get())) + if (auto * replicated_src_table = castStorage(source_table, DeferredTable::Load).get()) { String src_replica = replicated_src_table->findReplicaHavingPart(part_desc->src_part_name, false); if (!src_replica.empty()) @@ -3403,7 +3405,7 @@ bool StorageReplicatedMergeTree::executeReplaceRange(LogEntry & entry) throw Exception(ErrorCodes::UNFINISHED, "Checksums of {} is suddenly changed", part_desc->src_table_part->name); /// Don't do hardlinks in case of zero-copy at any side (defensive programming) - bool source_zero_copy_enabled = (*dynamic_cast(source_table.get())->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; + bool source_zero_copy_enabled = (*castStorage(source_table, DeferredTable::Load)->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; bool our_zero_copy_enabled = (*storage_settings_ptr)[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; IDataPartStorage::ClonePartParams clone_params @@ -9184,7 +9186,9 @@ void StorageReplicatedMergeTree::replacePartitionFrom( if (replace) throw DB::Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Only support DROP/DETACH/ATTACH PARTITION ALL currently"); + /// Patch parts cannot be copied to another table. Partitions with unapplied patches are rejected by `replacePartitionFromImpl`. partitions = src_data.getAllPartitionIds(); + std::erase_if(partitions, isPatchPartitionId); } else { @@ -9201,7 +9205,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom( const auto zookeeper = getZooKeeper(); const bool zero_copy_enabled = (*storage_settings_ptr)[MergeTreeSetting::allow_remote_fs_zero_copy_replication] - || (*dynamic_cast(source_table.get())->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; + || (*src_data.getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; using Entry = std::unique_ptr; std::vector entries(partitions.size()); @@ -9529,7 +9533,7 @@ std::unique_ptr StorageReplicatedMergeTree::rep void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_table, const ASTPtr & partition, ContextPtr query_context) { auto component_guard = Coordination::setCurrentComponent("StorageReplicatedMergeTree::movePartitionToTable"); - auto dest_table_storage = std::dynamic_pointer_cast(dest_table); + auto dest_table_storage = castStorage(dest_table, DeferredTable::Load); if (!dest_table_storage) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Table {} supports movePartitionToTable only for ReplicatedMergeTree family of table engines. " @@ -9654,7 +9658,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta /// Don't do hardlinks in case of zero-copy at any side (defensive programming) bool zero_copy_enabled = (*storage_settings_ptr)[MergeTreeSetting::allow_remote_fs_zero_copy_replication] - || (*dynamic_cast(dest_table.get())->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; + || (*dest_table_storage->getSettings())[MergeTreeSetting::allow_remote_fs_zero_copy_replication]; IDataPartStorage::ClonePartParams clone_params { diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index aef8d35af603..a5964106745e 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -184,14 +184,14 @@ class SQLiteSink final : public SinkToStorage WriteBufferFromOwnString sqlbuf; sqlbuf << "INSERT INTO "; - sqlbuf << doubleQuoteString(remote_table_name); + sqlbuf << doubleQuoteStringSQLite(remote_table_name); sqlbuf << " ("; for (auto it = block.begin(); it != block.end(); ++it) { if (it != block.begin()) sqlbuf << ", "; - sqlbuf << quoteString(it->name); + sqlbuf << doubleQuoteStringSQLite(it->name); } sqlbuf << ") VALUES "; diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h index 70697cfcdaca..443333d82584 100644 --- a/src/Storages/StorageTableFunction.h +++ b/src/Storages/StorageTableFunction.h @@ -35,6 +35,12 @@ class StorageTableFunctionProxy final : public StorageProxy setInMemoryMetadata(cached_metadata); } + StoragePtr tryGetNested() const override + { + std::lock_guard lock{nested_mutex}; + return nested; + } + StoragePtr getNestedImpl() const { std::lock_guard lock{nested_mutex}; diff --git a/src/Storages/StorageTableProxy.h b/src/Storages/StorageTableProxy.h index 26f69e01992a..11f5c0b1130a 100644 --- a/src/Storages/StorageTableProxy.h +++ b/src/Storages/StorageTableProxy.h @@ -52,6 +52,14 @@ class StorageTableProxy final : public StorageProxy return IStorage::getInMemoryMetadataPtr(context_, bypass_metadata_cache); } + StoragePtr tryGetNested() const override + { + std::lock_guard lock{nested_mutex}; + return nested; + } + + bool isLazyStandIn() const override { return true; } + StoragePtr getNested() const override { std::lock_guard lock{nested_mutex}; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 7f3e7f6dca7a..cd59a8c92aa7 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1882,8 +1882,9 @@ String StorageURL::resolveURLBase(const String & url, const String & base, const } auto scheme_end = base.find("://"); + /// Not echoed back: the value can carry a credential, and password masking anchors on the `://` it lacks. if (scheme_end == String::npos) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "The `{}` setting must contain a scheme (e.g. https://), got: {}", base_setting_name, base); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "The `{}` setting must contain a scheme (e.g. https://)", base_setting_name); /// Find the boundary of the path component in the base URL (before '?' or '#'). auto authority_start = scheme_end + 3; /// skip "://" @@ -2285,6 +2286,7 @@ class StorageURLSchemeDispatch final : public StorageProxy } StoragePtr getNested() const override { return nested; } + StoragePtr tryGetNested() const override { return nested; } /// The table was created with `ENGINE = URL(...)`; report it as such for consistency with /// `SHOW CREATE TABLE` and `system.tables`, even though reads/writes go to the delegate. String getName() const override { return "URL"; } @@ -2536,6 +2538,7 @@ static StoragePtr tryDispatchURLEngineByScheme(const StorageFactory::Arguments & /// `format = auto` that would force re-inference (and external I/O) on every `ATTACH`/restart. if (const auto * file = typeid_cast(delegate_storage.get())) resolved_format = file->getFormatName(); + /// NOLINT(storage-cast): the delegate is created right here, it never comes from the catalog. else if (const auto * object_storage = typeid_cast(delegate_storage.get())) resolved_format = object_storage->getFormatName(); else diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index b98dfce1db65..1d27a13b7d6e 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -771,6 +771,9 @@ ContextPtr StorageView::getViewSubqueryContext(ContextPtr context, const Storage view_settings[Setting::max_result_bytes] = 0; view_settings[Setting::extremes] = false; view_context->setSettings(view_settings); + /// The inlined view body is the inner query of the view, just like in `getViewContext`: + /// e.g. it must read the table itself, not the inserted block of a materialized view. + view_context->setIsViewInnerQuery(true); return view_context; } diff --git a/src/Storages/System/StatusRequestsPool.cpp b/src/Storages/System/StatusRequestsPool.cpp index f4d657612dd8..44402fc10eda 100644 --- a/src/Storages/System/StatusRequestsPool.cpp +++ b/src/Storages/System/StatusRequestsPool.cpp @@ -2,17 +2,21 @@ #include #include +#include namespace DB { StoragePtr resolveStatusRequestTable(const StorageID & storage_id) { + /// The status is read by casting to the engine, so a lazily loaded table has to be resolved to + /// the storage behind its proxy. A table with no storage yet has no status to report. /// Resolution by UUID does not depend on the current table name, so it survives renames. if (storage_id.hasUUID()) - return DatabaseCatalog::instance().tryGetByUUID(storage_id.uuid).second; + return resolveStorageProxy(DatabaseCatalog::instance().tryGetByUUID(storage_id.uuid).second); - return DatabaseCatalog::instance().tryGetTable(storage_id, Context::getGlobalContextInstance()); + return resolveStorageProxy( + DatabaseCatalog::instance().tryGetTable(storage_id, Context::getGlobalContextInstance())); } DatabasePtr resolveStatusRequestDatabase(const String & database_name) diff --git a/src/Storages/System/StorageSystemCertificates.cpp b/src/Storages/System/StorageSystemCertificates.cpp index 3b8acceea62a..2ae44bb8a5f3 100644 --- a/src/Storages/System/StorageSystemCertificates.cpp +++ b/src/Storages/System/StorageSystemCertificates.cpp @@ -15,6 +15,7 @@ #include #include #include + #include #endif #include @@ -140,12 +141,19 @@ void StorageSystemCertificates::fillData([[maybe_unused]] MutableColumns & res_c } }; + /// The CA certificates may have been reloaded since the context was created, `CertificateReloader` knows the current ones. + auto current_ca_paths = [](const std::string & prefix, Poco::Net::Context::Ptr ssl_context) + { + if (auto reloaded_ca_paths = CertificateReloader::instance().getCAPaths(prefix)) + return *reloaded_ca_paths; + return ssl_context->getCAPaths(); + }; + const auto & config = Context::getGlobalContextInstance()->getConfigRef(); try { - const auto & ca_paths = Poco::Net::SSLManager::instance().defaultServerContext()->getCAPaths(); - process_ca_paths(ca_paths, ""); + process_ca_paths(current_ca_paths(Poco::Net::SSLManager::CFG_SERVER_PREFIX, Poco::Net::SSLManager::instance().defaultServerContext()), ""); } catch (const Poco::Net::SSLException &) { @@ -162,10 +170,7 @@ void StorageSystemCertificates::fillData([[maybe_unused]] MutableColumns & res_c continue; if (auto ctx = Poco::Net::SSLManager::instance().getCustomServerContext(prefix)) - { - const auto & ca_paths = ctx->getCAPaths(); - process_ca_paths(ca_paths, protocol_name); - } + process_ca_paths(current_ca_paths(prefix, ctx), protocol_name); } #endif } diff --git a/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp b/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp index 40f8da6ac1e9..3d9578979cf1 100644 --- a/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp +++ b/src/Storages/System/StorageSystemDashboardsFilesystemCache.cpp @@ -382,9 +382,10 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_FilesystemCacheReserveAttempts) AS Attempts, - avg(ProfileEvent_FilesystemCacheFailedReserveAttempts) AS Failed, + avg(ProfileEvent_FilesystemCacheFailedReserveAttempts - ProfileEvent_FilesystemCacheReserveAheadRetries) AS Failed, avg(ProfileEvent_FilesystemCacheFailToReserveSpaceBecauseOfLockContention) AS SkippedOnLockContention, - avg(ProfileEvent_FilesystemCacheFailToReserveSpaceBecauseOfCacheResize) AS SkippedOnCacheResize + avg(ProfileEvent_FilesystemCacheFailToReserveSpaceBecauseOfCacheResize) AS SkippedOnCacheResize, + avg(ProfileEvent_FilesystemCacheReserveAheadRetries) AS ReserveAheadRetries FROM merge('system', '^metric_log') WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to GROUP BY t diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp index 6e008205e2b4..7b45899dd5e7 100644 --- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp +++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -42,7 +43,7 @@ StoragesDroppedInfoStream::StoragesDroppedInfoStream(std::optional f String database_name = storage->getStorageID().getDatabaseName(); String table_name = storage->getStorageID().getTableName(); String engine_name = storage->getName(); - if (!dynamic_cast(storage.get())) + if (!castStorage(storage, DeferredTable::Skip)) continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name)) diff --git a/src/Storages/System/StorageSystemGraphite.cpp b/src/Storages/System/StorageSystemGraphite.cpp index 0bc4f2156621..0d079dd1fb23 100644 --- a/src/Storages/System/StorageSystemGraphite.cpp +++ b/src/Storages/System/StorageSystemGraphite.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -52,11 +53,7 @@ static StorageSystemGraphite::Configs getConfigs(ContextPtr context) for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); - if (!table) - continue; - - const MergeTreeData * table_data = dynamic_cast(table.get()); + auto table_data = castStorage(iterator->table(), DeferredTable::Skip); if (!table_data) continue; diff --git a/src/Storages/System/StorageSystemIcebergFiles.cpp b/src/Storages/System/StorageSystemIcebergFiles.cpp index c428bd784e6f..c1ce1d7d99f4 100644 --- a/src/Storages/System/StorageSystemIcebergFiles.cpp +++ b/src/Storages/System/StorageSystemIcebergFiles.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include @@ -221,7 +222,7 @@ class SystemIcebergFilesSource : public ISource if (!lock) return false; - auto * object_storage_table = dynamic_cast(storage.get()); + auto object_storage_table = castStorage(storage, DeferredTable::Skip); if (!object_storage_table || !object_storage_table->isIcebergStorage()) return false; diff --git a/src/Storages/System/StorageSystemIcebergHistory.cpp b/src/Storages/System/StorageSystemIcebergHistory.cpp index e9cd3b66eb28..f51cfadafc44 100644 --- a/src/Storages/System/StorageSystemIcebergHistory.cpp +++ b/src/Storages/System/StorageSystemIcebergHistory.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -202,7 +203,7 @@ void StorageSystemIcebergHistory::fillData( // Table was dropped while acquiring the lock, skipping table continue; - if (auto * object_storage_table = dynamic_cast(storage.get())) + if (auto * object_storage_table = castStorage(storage, DeferredTable::Skip).get()) { add_history_record(database_name, table_name, object_storage_table); } diff --git a/src/Storages/System/StorageSystemKafkaConsumers.cpp b/src/Storages/System/StorageSystemKafkaConsumers.cpp index 2dfdc0304e31..04731fcb31df 100644 --- a/src/Storages/System/StorageSystemKafkaConsumers.cpp +++ b/src/Storages/System/StorageSystemKafkaConsumers.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -276,9 +277,9 @@ void StorageSystemKafkaConsumers::fillData(MutableColumns & res_columns, Context for (auto it = db.second->getTablesIterator(context); it->isValid(); it->next()) { StoragePtr storage = it->table(); - if (auto * kafka_table = dynamic_cast(storage.get())) + if (auto kafka_table = castStorage(storage, DeferredTable::Skip)) handle_table(it, *kafka_table); - else if (auto * kafka_2_table = dynamic_cast(storage.get())) + else if (auto kafka_2_table = castStorage(storage, DeferredTable::Skip)) handle_table(it, *kafka_2_table); } } diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 08b9ac0360d1..095673f0f43c 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -88,13 +89,10 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - if (!dynamic_cast(table.get())) - continue; - if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; @@ -142,6 +140,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c std::vector statuses; { const IStorage * storage = merge_tree_tables[database][table].get(); + /// NOLINT(storage-cast): `merge_tree_tables` is filled with already resolved storages. if (const auto * merge_tree = dynamic_cast(storage)) statuses = merge_tree->getMutationsStatus(); } diff --git a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp index 0894963bc019..ec3f099ba390 100644 --- a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp +++ b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB @@ -66,7 +67,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { StoragePtr storage = iterator->table(); - if (auto * queue_table = dynamic_cast(storage.get())) + if (auto queue_table = castStorage(storage, DeferredTable::Skip)) { add_table(iterator, *queue_table); } diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp index b1aa1c8a263d..47ee9ea2873a 100644 --- a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp +++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -70,11 +71,9 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - if (!dynamic_cast(table.get())) - continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; replicated_tables[db.first][iterator->name()] = table; @@ -119,6 +118,7 @@ void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, String database = (*col_database_to_filter)[i].safeGet(); String table = (*col_table_to_filter)[i].safeGet(); + /// NOLINT(storage-cast): `replicated_tables` is filled with already resolved storages. auto moves = dynamic_cast(*replicated_tables[database][table]).getPartMovesBetweenShardsEntries(); for (auto & entry : moves) diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 38d2ced83ce8..6cc120847304 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -161,7 +162,7 @@ StoragesInfoStream::StoragesInfoStream(std::optional filter_by_datab for (auto iterator = database->getTablesIterator(context); iterator->isValid(); iterator->next()) { String table_name = iterator->name(); - StoragePtr storage = iterator->table(); + auto storage = castStorage(iterator->table(), DeferredTable::Skip); if (!storage) continue; @@ -175,9 +176,6 @@ StoragesInfoStream::StoragesInfoStream(std::optional filter_by_datab storage_uuid = hash.get128(); } - if (!dynamic_cast(storage.get())) - continue; - if (check_access_for_tables_in_db && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name)) continue; diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h index fb6581878023..382211b67864 100644 --- a/src/Storages/System/StorageSystemPartsBase.h +++ b/src/Storages/System/StorageSystemPartsBase.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -75,7 +76,7 @@ class StoragesInfoStreamBase info.engine = info.storage->getName(); - info.data = dynamic_cast(info.storage.get()); + info.data = castStorage(info.storage, DeferredTable::Skip).get(); if (!info.data) throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown engine {}", info.engine); diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp index 0161fbc3dcac..73e10da25be4 100644 --- a/src/Storages/System/StorageSystemReplicas.cpp +++ b/src/Storages/System/StorageSystemReplicas.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -193,12 +194,9 @@ void StorageSystemReplicas::readImpl( const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first); for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - - if (!dynamic_cast(table.get())) - continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; replicated_tables[db.first][iterator->name()] = table; diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp index 27e1660c4607..ad26b8e612e7 100644 --- a/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -89,11 +90,9 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { - const auto & table = iterator->table(); + auto table = castStorage(iterator->table(), DeferredTable::Skip); if (!table) continue; - if (!dynamic_cast(table.get())) - continue; if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name())) continue; replicated_tables[db.first][iterator->name()] = table; @@ -141,6 +140,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, Conte String database = (*col_database_to_filter)[i].safeGet(); String table = (*col_table_to_filter)[i].safeGet(); + /// NOLINT(storage-cast): `replicated_tables` is filled with already resolved storages. dynamic_cast(*replicated_tables[database][table]).getQueue(queue, replica_name); for (size_t j = 0, queue_size = queue.size(); j < queue_size; ++j) diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp index 4e2b8faedca9..cf8a79439f11 100644 --- a/src/Storages/System/StorageSystemTables.cpp +++ b/src/Storages/System/StorageSystemTables.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -981,7 +982,7 @@ class TablesBlockSource final : public ISource ++res_index; } - auto table_merge_tree = std::dynamic_pointer_cast(table); + auto table_merge_tree = castStorage(table, DeferredTable::Skip); if (columns_mask[src_index++]) { if (table_merge_tree) diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index 3c06a812616b..086002a38a3e 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -715,7 +715,8 @@ std::vector checkActionsDAGForAggregateFunctions( { auto offsets = ColumnArray::ColumnOffsets::create(); offsets->getData().push_back(1); - candidates.push_back(ColumnArray::create(element->cloneResized(1), std::move(offsets))); + candidates.push_back( + ColumnArray::create(element->convertToFullColumnIfConst()->cloneResized(1), std::move(offsets))); } } else diff --git a/src/Storages/Utils.cpp b/src/Storages/Utils.cpp index 7143ca6cf76d..d5922374d39c 100644 --- a/src/Storages/Utils.cpp +++ b/src/Storages/Utils.cpp @@ -25,6 +25,7 @@ namespace DB { return {CurrentMetrics::AttachedDictionary}; } + /// NOLINT(storage-cast): runs under the database lock, and attach and detach must count a proxy alike. if (typeid_cast(storage.get()) != nullptr) { return {CurrentMetrics::AttachedTable, CurrentMetrics::AttachedReplicatedTable}; diff --git a/src/TableFunctions/TableFunctionMergeTreeIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeIndex.cpp index f6b5de4fe455..4d5b07f6ea73 100644 --- a/src/TableFunctions/TableFunctionMergeTreeIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeIndex.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -150,10 +151,10 @@ static NameSet getAllPossibleStreamNames( ColumnsDescription TableFunctionMergeTreeIndex::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto metadata_snapshot = source_table->getInMemoryMetadataPtr(context, false); - const auto * merge_tree = dynamic_cast(source_table.get()); + const auto * merge_tree = castStorage(source_table, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function mergeTreeIndex expected MergeTree table, got: {}", source_table->getName()); @@ -204,7 +205,7 @@ StoragePtr TableFunctionMergeTreeIndex::executeImpl( ColumnsDescription /*cached_columns*/, bool is_insert_query) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto columns = getActualTableStructure(context, is_insert_query); StorageID storage_id(getDatabaseName(), table_name); diff --git a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp index 29f2a251d9e1..fbd90bd9464d 100644 --- a/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp +++ b/src/TableFunctions/TableFunctionMergeTreeTextIndex.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -117,7 +118,7 @@ std::pair TableFunctionMergeTreeTextIndex::resolv /// Otherwise the errors below would reveal the engine and the indexes of a table the user cannot see. context->checkAccess(AccessType::SHOW_TABLES, source_database, source_table); - auto source_table_ptr = DatabaseCatalog::instance().getTable(StorageID{source_database, source_table}, context); + auto source_table_ptr = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(StorageID{source_database, source_table}, context)); auto metadata_snapshot = source_table_ptr->getInMemoryMetadataPtr(context, false); const auto & index_desc = metadata_snapshot->getSecondaryIndices().getByName(source_index_name); @@ -127,7 +128,7 @@ std::pair TableFunctionMergeTreeTextIndex::resolv "Got index '{}' of type '{}', expected 'text'", source_index_name, index_desc.type); - const auto * merge_tree = dynamic_cast(source_table_ptr.get()); + const auto * merge_tree = castStorage(source_table_ptr, DeferredTable::Load).get(); if (!merge_tree) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Storage MergeTreeTextIndex expected MergeTree table, got: {}", source_table_ptr->getName()); diff --git a/src/TableFunctions/TableFunctionProjection.cpp b/src/TableFunctions/TableFunctionProjection.cpp index 7b6f252b6b8b..bcf88e1dc970 100644 --- a/src/TableFunctions/TableFunctionProjection.cpp +++ b/src/TableFunctions/TableFunctionProjection.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -65,7 +66,7 @@ void TableFunctionMergeTreeProjection::parseArguments(const ASTPtr & ast_functio ColumnsDescription TableFunctionMergeTreeProjection::getActualTableStructure(ContextPtr context, bool /*is_insert_query*/) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto metadata_snapshot = source_table->getInMemoryMetadataPtr(context, false); if (!metadata_snapshot->getProjections().has(projection_name)) @@ -85,7 +86,7 @@ StoragePtr TableFunctionMergeTreeProjection::executeImpl( ColumnsDescription /*cached_columns*/, bool /* is_insert_query */) const { - auto source_table = DatabaseCatalog::instance().getTable(source_table_id, context); + auto source_table = resolveStorageProxyLoading(DatabaseCatalog::instance().getTable(source_table_id, context)); auto metadata_snapshot = source_table->getInMemoryMetadataPtr(context, false); ProjectionDescriptionRawPtr projection = &metadata_snapshot->getProjections().get(projection_name); diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index b58a0b842505..1e95ac17dee1 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -267,15 +267,25 @@ StoragePtr TableFunctionURL::executeImpl( /// reports the delegate's engine name and access URI, so the outer check (or the caller that /// explicitly disabled it and took over) has already covered exactly the delegate's source. if (delegate) + { + /// The query text still names `url`, while the delegate is a different backend. If the delegate + /// created its `*Cluster` storage for `parallel_replicas_for_cluster_engines`, the forwarded query + /// would be rewritten from the surface AST name into `urlCluster(...)` - a function that rejects + /// every non-HTTP scheme - and with the argument grammar of the delegate rather than of `url`. + /// Scheme dispatch is therefore resolved on this node: the delegate builds its plain storage. + ContextMutablePtr delegate_context = Context::createCopy(context); + delegate_context->setSetting("parallel_replicas_for_cluster_engines", false); + return delegate->execute( ast_function, - context, + delegate_context, table_name, std::move(cached_columns), /*use_global_context=*/false, is_insert_query, /*check_create_temporary_table=*/false, /*check_source_access=*/false); + } /// Stored columns accompany a table definition rather than an ad-hoc query, so creation and /// replay must resolve to the same storage. @@ -554,6 +564,8 @@ SELECT * FROM url('s3://clickhouse-public-datasets/hits_compatible/hits.csv'); Scheme dispatch is not yet wired through [`urlCluster`](/reference/functions/table-functions/urlCluster): a non-`http(s)` scheme passed to `urlCluster` is rejected with an error. Use the corresponding cluster function (`s3Cluster`, `azureBlobStorageCluster`, `hdfsCluster`, …) for those backends instead. +For the same reason, a dispatched `url` call is read on the node that received the query: the [parallel_replicas_for_cluster_engines](/reference/settings/session-settings/parallel-replicas#parallel_replicas_for_cluster_engines) fan-out is not applied to it. Use the corresponding cluster function directly when you want the read distributed across replicas. + ## Globs in URL {#globs-in-url} Patterns in `{ }` are used to generate a set of shards or to specify failover addresses. Supported pattern types and examples see in the description of the [remote](/reference/functions/table-functions/remote#globs-in-addresses) function. diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 5c20520e2fb7..1c7e063b7f5d 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -2153,8 +2153,8 @@ def test_api_call_quota_ignores_subquery_settings(started_cluster): ) outer_wins = int(get_profile_events(qid)["api_calls"]) - # The quota is set only in the subquery; the outer query leaves it at the default (far - # above 64). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - + # The quota is set only in the subquery; the outer query leaves it at the default (0 - + # no limit). The subquery cap is ignored, so all 64 rows run rather than stopping at 5 - # a quota set only in a subquery has no effect. qid = unique_query_id("quota_levels_subquery_only") instance.query( diff --git a/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py b/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py new file mode 100644 index 000000000000..d998b1690edf --- /dev/null +++ b/tests/integration/test_arrowflight_interface/test_session_options_settings_profile.py @@ -0,0 +1,76 @@ +# coding: utf-8 + +import pytest +import random +import string + +from .flight_sql_client import FlightSQLClient, SetSessionOptionsResult + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=[ + "configs/flight_port.xml", + ], +) + +PROFILE_NAME = "profile_arrowflight_session_options_constraints" + + +def get_client(): + session_id = ''.join(random.choices(string.ascii_letters + string.digits, k=16)) + return FlightSQLClient( + host=node.ip_address, + port=8888, + insecure=True, + disable_server_verification=True, + metadata={'x-clickhouse-session-id': session_id}, + features={'metadata-reflection': 'true'}, + ) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + node.wait_until_port_is_ready(8888, timeout=10) + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def settings_profile(): + node.query(f"DROP SETTINGS PROFILE IF EXISTS {PROFILE_NAME}") + node.query( + f"CREATE SETTINGS PROFILE {PROFILE_NAME} SETTINGS max_execution_time = 10 CONST" + ) + try: + yield PROFILE_NAME + finally: + node.query(f"DROP SETTINGS PROFILE IF EXISTS {PROFILE_NAME}") + + +def _query_scalar(client, query): + flight_info = client.execute(query) + table = client.do_get(flight_info.endpoints[0].ticket).read_all() + return table.column(0)[0].as_py() + + +def test_profile_constraints_apply_within_same_request(): + """A profile set in a SetSessionOptions request constrains the other options of the same request.""" + client = get_client() + + result = client.set_session_options( + {"profile": PROFILE_NAME, "max_execution_time": "999"} + ) + + assert "profile" not in result.errors + # A constraint violation is neither a parse nor an unknown-setting error, so it maps to UNSPECIFIED. + assert ( + result.errors["max_execution_time"].value + == SetSessionOptionsResult.UNSPECIFIED + ) + assert float(_query_scalar(client, "SELECT getSetting('max_execution_time')")) == 10 diff --git a/tests/integration/test_async_insert_pool_saturation/__init__.py b/tests/integration/test_async_insert_pool_saturation/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_async_insert_pool_saturation/configs/async_insert.xml b/tests/integration/test_async_insert_pool_saturation/configs/async_insert.xml new file mode 100644 index 000000000000..287c26322722 --- /dev/null +++ b/tests/integration/test_async_insert_pool_saturation/configs/async_insert.xml @@ -0,0 +1,11 @@ + + 1 + true + + + tcp + true + 9010 + + + diff --git a/tests/integration/test_async_insert_pool_saturation/configs/no_flush_on_shutdown.xml b/tests/integration/test_async_insert_pool_saturation/configs/no_flush_on_shutdown.xml new file mode 100644 index 000000000000..40503bcac555 --- /dev/null +++ b/tests/integration/test_async_insert_pool_saturation/configs/no_flush_on_shutdown.xml @@ -0,0 +1,3 @@ + + false + diff --git a/tests/integration/test_async_insert_pool_saturation/test.py b/tests/integration/test_async_insert_pool_saturation/test.py new file mode 100644 index 000000000000..8d43a92d6612 --- /dev/null +++ b/tests/integration/test_async_insert_pool_saturation/test.py @@ -0,0 +1,260 @@ +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from uuid import uuid4 + +import pytest + +from helpers.client import Client +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", main_configs=["configs/async_insert.xml"], stay_alive=True +) +node_no_flush = cluster.add_instance( + "node_no_flush", + main_configs=["configs/async_insert.xml", "configs/no_flush_on_shutdown.xml"], + stay_alive=True, +) + +FAILPOINT = "async_insert_flush_pause_in_executor" +ADMISSION_FAILPOINT = "async_insert_pause_before_schedule" +SETTINGS = { + "async_insert": 1, + "wait_for_async_insert": 0, + "async_insert_max_data_size": 2, + "async_insert_use_adaptive_busy_timeout": 0, + "async_insert_busy_timeout_min_ms": 3600000, + "async_insert_busy_timeout_max_ms": 3600000, +} + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield + finally: + cluster.shutdown() + + +def metric(name): + return int(node.query(f"SELECT value FROM system.metrics WHERE metric = '{name}'")) + + +@pytest.fixture +def tables(): + for table in ("saturated", "unrelated"): + node.query(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY x") + try: + yield + finally: + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE", timeout=30) + for table in ("saturated", "unrelated"): + node.query(f"DROP TABLE {table}") + + +@pytest.mark.parametrize("wait_for_async_insert", [0, 1]) +@pytest.mark.parametrize( + "flush_query", + ["SYSTEM FLUSH ASYNC INSERT QUEUE", "SYSTEM FLUSH ASYNC INSERT QUEUE unrelated"], +) +def test_saturated_pool_does_not_lock_queue_shard( + tables, wait_for_async_insert, flush_query +): + # One worker also means one queue shard: both tables necessarily share its mutex. + waiter_id = f"pool_waiter_{uuid4().hex}" + flush_id = f"explicit_flush_{uuid4().hex}" + with ThreadPoolExecutor(max_workers=2) as executor: + try: + node.query(f"SYSTEM ENABLE FAILPOINT {FAILPOINT}") + + # Exactly two bytes reach the size threshold without the oversized-input + # fallback. HTTP keeps parsing in the server, where the failpoint runs. + node.http_query( + "INSERT INTO saturated FORMAT TSV", + data="1\n", + params=SETTINGS, + timeout=30, + ) + node.query(f"SYSTEM WAIT FAILPOINT {FAILPOINT} PAUSE", timeout=30) + assert metric("AsynchronousInsertThreadsScheduled") == 1 + + waiter = executor.submit( + node.http_query, + "INSERT INTO saturated FORMAT TSV", + data="2\n", + params={ + **SETTINGS, + "wait_for_async_insert": wait_for_async_insert, + "query_id": waiter_id, + }, + timeout=120, + ) + # This is emitted with the shard lock held, just before the blocking + # submission. Waiting for it prevents the unrelated insert racing ahead. + node.wait_for_log_line( + rf"\{{{waiter_id}\}}.*Scheduling async insert processing job because enough bytes accumulated", + timeout=30, + ) + + # This insert only buffers. On the original code it times out waiting + # for the mutex held by the producer trying to submit the second flush. + assert ( + node.http_query( + "INSERT INTO unrelated FORMAT TSV", + data="3\n", + params={**SETTINGS, "async_insert_max_data_size": 1024}, + timeout=30, + ) + == "" + ) + assert not waiter.done() + assert metric("AsynchronousInsertThreadsScheduled") == 1 + assert metric("PendingAsyncInsert") == 3 + + flush = executor.submit( + node.query, flush_query, query_id=flush_id, timeout=120 + ) + node.wait_for_log_line( + rf"\{{{flush_id}\}}.*Requested to flush asynchronous insert queue", + timeout=30, + ) + assert not flush.done() + + node.query(f"SYSTEM DISABLE FAILPOINT {FAILPOINT}") + assert waiter.result(timeout=30) == "" + assert flush.result(timeout=30) == "" + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + + assert node.query("SELECT x FROM saturated ORDER BY x") == "1\n2\n" + assert node.query("SELECT x FROM unrelated") == "3\n" + assert metric("PendingAsyncInsert") == 0 + assert metric("AsynchronousInsertThreadsScheduled") == 0 + finally: + # Release the worker even when the regression times out, before joining + # HTTP requests that need it to make progress. + node.query(f"SYSTEM DISABLE FAILPOINT {FAILPOINT}") + + +@pytest.mark.parametrize("trigger", ["size", "deadline"]) +@pytest.mark.parametrize("bad_data", [False, True]) +def test_forced_flush_waits_for_batch_before_pool_admission(tables, trigger, bad_data): + settings = {**SETTINGS, "async_insert_max_data_size": 4} + if trigger == "deadline": + settings["async_insert_busy_timeout_min_ms"] = 1000 + settings["async_insert_busy_timeout_max_ms"] = 1000 + + flush_id = f"drain_handoff_{uuid4().hex}" + with ThreadPoolExecutor(max_workers=2) as executor: + try: + node.query(f"SYSTEM ENABLE FAILPOINT {ADMISSION_FAILPOINT}") + + # The first request is acknowledged while still buffered. A subsequent + # producer or the deadline worker removes its batch from the shard queue. + node.http_query( + "INSERT INTO saturated FORMAT TSV", + data="x\n" if bad_data else "1\n", + params=settings, + timeout=30, + ) + producer = None + if trigger == "size": + producer = executor.submit( + node.http_query, + "INSERT INTO saturated FORMAT TSV", + data="x\n" if bad_data else "2\n", + params=settings, + timeout=120, + ) + node.query(f"SYSTEM WAIT FAILPOINT {ADMISSION_FAILPOINT} PAUSE", timeout=30) + assert metric("AsynchronousInsertThreadsScheduled") == 0 + + flush = executor.submit( + node.query, + "SYSTEM FLUSH ASYNC INSERT QUEUE", + query_id=flush_id, + timeout=120, + ) + node.wait_for_log_line( + rf"\{{{flush_id}\}}.*Will wait for finishing of 0 flushing jobs", + timeout=30, + ) + # An empty pool must not let the flush overlook the acknowledged row + # in the batch paused just before admission. + with pytest.raises(TimeoutError): + flush.result(timeout=1) + + # Waiting for that batch must also leave the shard mutex available. + node.http_query( + "INSERT INTO unrelated FORMAT TSV", + data="3\n", + params={**SETTINGS, "async_insert_max_data_size": 1024}, + timeout=30, + ) + + node.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}") + if producer is not None: + assert producer.result(timeout=30) == "" + assert flush.result(timeout=30) == "" + expected = "" if bad_data else "1\n2\n" if trigger == "size" else "1\n" + assert node.query("SELECT x FROM saturated ORDER BY x") == expected + node.query("SYSTEM FLUSH ASYNC INSERT QUEUE") + assert node.query("SELECT x FROM unrelated") == "3\n" + assert metric("PendingAsyncInsert") == 0 + finally: + node.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}") + + +@pytest.mark.parametrize("flush_on_shutdown", [False, True]) +def test_shutdown_handles_batch_before_pool_admission(flush_on_shutdown): + shutdown_node = node if flush_on_shutdown else node_no_flush + introspection = Client(shutdown_node.ip_address, 9010, command=cluster.client_bin_path) + table = f"shutdown_pending_{uuid4().hex}" + shutdown_node.query(f"CREATE TABLE {table} (x UInt64) ENGINE = MergeTree ORDER BY x") + + try: + with ThreadPoolExecutor(max_workers=1) as executor: + stop = None + try: + shutdown_node.query(f"SYSTEM ENABLE FAILPOINT {ADMISSION_FAILPOINT}") + # A deadline-triggered batch contains an acknowledged insert and has no + # producer connection left for server shutdown to wait on. + assert ( + shutdown_node.http_query( + f"INSERT INTO {table} FORMAT TSV", + data="1\n", + params={ + **SETTINGS, + "async_insert_max_data_size": 1024, + "async_insert_busy_timeout_min_ms": 1000, + "async_insert_busy_timeout_max_ms": 1000, + }, + timeout=30, + ) + == "" + ) + shutdown_node.query(f"SYSTEM WAIT FAILPOINT {ADMISSION_FAILPOINT} PAUSE", timeout=30) + assert shutdown_node.query("SELECT value FROM system.metrics WHERE metric = 'AsynchronousInsertThreadsScheduled'") == "0\n" + + stop = executor.submit(shutdown_node.stop_clickhouse, stop_wait_sec=60, kill=False) + shutdown_node.wait_for_log_line("Shutting down the asynchronous insertion queue", timeout=30) + assert not stop.done() + # Normal listeners have closed. The introspection listener remains + # available until after the async queue and database catalog shut down. + introspection.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}", timeout=30) + stop.result(timeout=90) + assert shutdown_node.contains_in_log("Asynchronous insertion queue finished", from_host=True) + assert shutdown_node.contains_in_log("Background threads finished", from_host=True) + finally: + if stop is None: + shutdown_node.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}") + elif not stop.done(): + introspection.query(f"SYSTEM DISABLE FAILPOINT {ADMISSION_FAILPOINT}", timeout=30) + stop.result(timeout=90) + if stop is not None: + shutdown_node.start_clickhouse() + + expected = "1\n" if flush_on_shutdown else "0\n" + assert shutdown_node.query(f"SELECT count() FROM {table}") == expected + finally: + shutdown_node.query(f"DROP TABLE {table}") diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py b/tests/integration/test_auth_method_valid_until_stateful_protocols/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml b/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml new file mode 100644 index 000000000000..65eafc59c3f8 --- /dev/null +++ b/tests/integration/test_auth_method_valid_until_stateful_protocols/configs/protocols.xml @@ -0,0 +1,4 @@ + + 9001 + 5433 + diff --git a/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py b/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py new file mode 100644 index 000000000000..1faf74afbbeb --- /dev/null +++ b/tests/integration/test_auth_method_valid_until_stateful_protocols/test.py @@ -0,0 +1,112 @@ +import time + +import psycopg2 +import pymysql.connections +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# The authentication method's `VALID UNTIL` must be enforced per query, not only at login. +# Stateful protocols (MySQL, PostgreSQL) authenticate once at connection startup and then run +# every later command through `Session::makeQueryContext`, so without a per-query re-check a +# credential that expires after login would keep working for the lifetime of the connection. +# The check lives in `Session::makeQueryContextImpl`, so every protocol shares it. +node = cluster.add_instance("node", main_configs=["configs/protocols.xml"]) + +MYSQL_PORT = 9001 +POSTGRES_PORT = 5433 + +# Lifetime of the expiring credential, measured from user creation. The first query on each +# connection runs within a fraction of a second of creation (well inside the lifetime); the second +# query runs after sleeping past the expiry. +EXPIRING_LIFETIME_S = 6 + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def create_users(): + node.query("DROP USER IF EXISTS u_expiring, u_lasting") + expiry = node.query(f"SELECT toString(now() + INTERVAL {EXPIRING_LIFETIME_S} SECOND)").strip() + # Plaintext-stored passwords so both the MySQL and PostgreSQL frontends can verify them. + node.query(f"CREATE USER u_expiring IDENTIFIED WITH plaintext_password BY 'pw' VALID UNTIL '{expiry}'") + node.query("CREATE USER u_lasting IDENTIFIED WITH plaintext_password BY 'pw' VALID UNTIL '2999-01-01 00:00:00'") + node.query("GRANT SELECT ON system.one TO u_expiring, u_lasting") + return expiry + + +def sleep_past(expiry): + while int(node.query(f"SELECT now() > toDateTime('{expiry}')").strip()) != 1: + time.sleep(0.5) + # The check is `now > valid_until` with second precision, so cross the boundary decisively. + time.sleep(1.5) + + +def test_mysql_connection_stops_working_after_expiry(started_cluster): + expiry = create_users() + host = started_cluster.get_instance_ip("node") + + def connect(user): + return pymysql.connections.Connection(host=host, user=user, password="pw", database="default", port=MYSQL_PORT) + + expiring = connect("u_expiring") + lasting = connect("u_lasting") + + # Both credentials are valid at connection time and for the first query. + for client in (expiring, lasting): + cursor = client.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == ((1,),) + + sleep_past(expiry) + + # The same, still-open connection must stop working once the method has expired ... + with pytest.raises(pymysql.err.MySQLError, match="expired"): + expiring.cursor().execute("SELECT 1") + + # ... while a connection under a non-expired method keeps working. + cursor = lasting.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == ((1,),) + + lasting.close() + + +def test_postgresql_connection_stops_working_after_expiry(started_cluster): + expiry = create_users() + host = started_cluster.get_instance_ip("node") + + def connect(user): + conn = psycopg2.connect(host=host, user=user, password="pw", dbname="default", port=POSTGRES_PORT) + conn.autocommit = True + return conn + + expiring = connect("u_expiring") + lasting = connect("u_lasting") + + for client in (expiring, lasting): + cursor = client.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == [(1,)] + + sleep_past(expiry) + + # The PostgreSQL handler sends an ErrorResponse and then terminates the connection on an expired + # credential, and psycopg2 may surface either the message or the connection loss - both are + # fail-close; the point is that the query must not succeed. + with pytest.raises(psycopg2.Error, match="expired|closed the connection"): + expiring.cursor().execute("SELECT 1") + + cursor = lasting.cursor() + cursor.execute("SELECT 1") + assert cursor.fetchall() == [(1,)] + + lasting.close() diff --git a/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml new file mode 100644 index 000000000000..294b4bc7dcb4 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/configs/remote_url_allow_hosts.xml @@ -0,0 +1,5 @@ + + + minio1:9001 + + diff --git a/tests/integration/test_backup_restore_s3/test_remote_host_filter.py b/tests/integration/test_backup_restore_s3/test_remote_host_filter.py new file mode 100644 index 000000000000..977d02c84e73 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/test_remote_host_filter.py @@ -0,0 +1,56 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/remote_url_allow_hosts.xml"], + with_minio=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + node.query( + "CREATE TABLE t (id UInt64, s String) ENGINE = MergeTree ORDER BY id" + ) + node.query("INSERT INTO t VALUES (1, 'a'), (2, 'b')") + yield cluster + finally: + cluster.shutdown() + + +def test_backup_to_allowed_host(): + name = uuid.uuid4().hex + destination = f"S3('http://minio1:9001/root/data/backups/{name}', 'minio', '{minio_secret_key}')" + node.query(f"BACKUP TABLE t TO {destination}") + node.query(f"RESTORE TABLE t AS t_{name} FROM {destination}") + assert node.query(f"SELECT count() FROM t_{name}") == "2\n" + node.query(f"DROP TABLE t_{name} SYNC") + + +@pytest.mark.parametrize( + "url", + [ + "http://minio1:9002/root/data/backups/x", + "http://resolver:8080/root/data/backups/x", + "http://127.0.0.1:9/root/data/backups/x", + ], +) +def test_backup_to_disallowed_host(url): + destination = f"S3('{url}', 'minio', '{minio_secret_key}')" + settings = "SETTINGS backup_restore_s3_retry_attempts = 0" + + error = node.query_and_get_error(f"BACKUP TABLE t TO {destination} {settings}") + assert "UNACCEPTABLE_URL" in error, error + + error = node.query_and_get_error( + f"RESTORE TABLE t AS t_restored FROM {destination} {settings}" + ) + assert "UNACCEPTABLE_URL" in error, error diff --git a/tests/integration/test_backup_restore_s3_role_arn/test.py b/tests/integration/test_backup_restore_s3_role_arn/test.py index a4ed1c43ca7d..8c4126518e29 100644 --- a/tests/integration/test_backup_restore_s3_role_arn/test.py +++ b/tests/integration/test_backup_restore_s3_role_arn/test.py @@ -9,9 +9,11 @@ from helpers.cluster import ClickHouseCluster from helpers.mock_servers import start_mock_servers -# `role_arn` and `role_session_name` are not secrets -- assuming the role still needs the server's -# own identity and a matching trust policy -- so they survive into the `` locator and -# let every hop of a chain reopen its base. The mock STS accepts the role and session name below. +# `role_arn` and `role_session_name` survive into the `` locator and let every hop of a +# chain reopen its base: assuming the role still needs the server's own identity and a matching trust +# policy. The session name is masked in logged query text, because a trust policy can pin it, but the +# stored locator keeps it on purpose (see `BackupInfo.cpp`). The mock STS accepts the role and +# session name below. # # Metadata written by a version that stripped the identifiers names no credentials and carries no # marker to reconstruct them from, so its base backup opens unauthenticated. Such a chain must still diff --git a/tests/integration/test_backup_source_grants/configs/backups.xml b/tests/integration/test_backup_source_grants/configs/backups.xml index 2d99cb1d1ef9..acf77f033a2f 100644 --- a/tests/integration/test_backup_source_grants/configs/backups.xml +++ b/tests/integration/test_backup_source_grants/configs/backups.xml @@ -1,5 +1,14 @@ + + + + local + /var/lib/clickhouse/disks/backup_disk/ + + + /var/lib/clickhouse/backups + backup_disk diff --git a/tests/integration/test_backup_source_grants/test.py b/tests/integration/test_backup_source_grants/test.py index 42b44da713dc..ff8efd1037bd 100644 --- a/tests/integration/test_backup_source_grants/test.py +++ b/tests/integration/test_backup_source_grants/test.py @@ -256,9 +256,8 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d # the check is skipped here and the restricted user reaches the embedded locator unchecked. # # The outer locator is File and IS granted, so a denial naming READ ON S3 can only come from the - # embedded S3 locator. `BACKUP DATABASE` serializes the locator as a string that - # `BackupInfo::fromAST` rejects before any check, so the function form is written into the - # manifest directly - the attacker-controlled-manifest shape this authorization exists for. + # embedded S3 locator, which is why the manifest is rewritten to hold one - the + # attacker-controlled-manifest shape this authorization exists for. # Every locator here is credential-free (File locally, a 1-argument S3 URL in the manifest): the # later definition-mismatch error logs both definitions, and a masked credential in that line # trips the tests-only `throw_on_match` masking rule and aborts the server. @@ -289,7 +288,7 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d assert "READ ON S3" in error, error # With the grant, authorization passes: the restore proceeds past CHECKING_ACCESS_RIGHTS and - # fails later on the pre-existing string-vs-function definition mismatch instead. + # fails later on the mismatch between the pre-existing definition and the rewritten one instead. node.query(f"GRANT READ ON S3 TO {USER}") error = node.query_and_get_error( "RESTORE DATABASE dbembedded ON CLUSTER one_shard FROM File('outer10')", user=USER @@ -300,41 +299,103 @@ def test_restore_on_cluster_authorizes_an_embedded_definition_over_an_existing_d node.query("DROP DATABASE dbembedded SYNC") -def test_restore_on_cluster_of_a_real_backup_engine_manifest_is_unchanged(started_cluster): - # The manifest is NOT crafted here, which is the point: `BACKUP DATABASE` serializes the inner - # locator as a string, and authorizing it would parse it and reject it with BAD_ARGUMENTS before - # any access decision. Only this shape can catch that, so it is a separate case from the crafted - # one - which asserts the security property but cannot see this class. +def test_restore_on_cluster_refuses_a_quoted_embedded_locator_over_an_existing_database( + started_cluster, +): + # The pre-fix carrier: metadata an older server rewrote holds the locator as a string literal, and + # such a manifest can end up inside a backup. On this path the creation-time refusal never runs - + # the target database exists, so `CREATE DATABASE IF NOT EXISTS` returns before `DatabaseFactory`, + # and `allow_different_database_def = 1` waives the definition mismatch that would stop it next. + # The preflight is therefore the only place that can see the quoted locator, and it must refuse it + # rather than wave it through unauthorized: the function form of the very same manifest is denied + # for a missing `READ ON S3` in the case above. + # + # The refusal must not print the locator, which is why the assertion below looks for the host that + # a quoted locator could carry credentials next to. + node.query("DROP DATABASE IF EXISTS dbquoted SYNC") + node.query("BACKUP DATABASE d67785 TO File('inner13') FORMAT Null") + node.query("CREATE DATABASE dbquoted ENGINE = Backup('d67785', File('inner13'))") + node.query("BACKUP DATABASE dbquoted TO File('outer13') FORMAT Null") + manifest = ( + "CREATE DATABASE dbquoted ENGINE = Backup('d67785', " + "'S3(\\'http://minio1:9001/root/data/denied/b13\\')')" + ) + node.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/backups/outer13/metadata/dbquoted.sql <<'MANIFEST'\n" + f"{manifest}\nMANIFEST", + ], + user="root", + ) + + assert node.query("SELECT count() FROM system.databases WHERE name = 'dbquoted'") == "1\n" + node.query(f"GRANT READ ON FILE TO {USER}") + + error = node.query_and_get_error( + "RESTORE DATABASE dbquoted ON CLUSTER one_shard FROM File('outer13') " + "SETTINGS allow_different_database_def = 1", + user=USER, + ) + assert "BAD_ARGUMENTS" in error, error + assert "Expected function as the backup destination" in error, error + assert "minio1" not in error, error + + node.query("DROP DATABASE dbquoted SYNC") + + +def test_restore_on_cluster_of_a_real_backup_engine_manifest_authorizes_the_inner_locator( + started_cluster, +): + # The manifest is NOT crafted here, which is the point: `BACKUP DATABASE` writes the inner locator + # as the function it is, so a real manifest carries a locator that authorization can decode, and + # the embedded one is checked exactly as a rewritten one is. Only this shape can see that, so it is + # a separate case from the rewritten one - which asserts the security property against a locator + # the definition never held. The outer locator is a Disk backup, which needs READ ON DISK, so a + # denial naming READ ON FILE can only come from the inner one. node.query("DROP DATABASE IF EXISTS dbreal SYNC") node.query("BACKUP DATABASE d67785 TO File('inner11') FORMAT Null") node.query("CREATE DATABASE dbreal ENGINE = Backup('d67785', File('inner11'))") - node.query("BACKUP DATABASE dbreal TO File('outer11') FORMAT Null") + node.query("BACKUP DATABASE dbreal TO Disk('backup_disk', 'outer11') FORMAT Null") manifest = node.exec_in_container( - ["bash", "-c", "cat /var/lib/clickhouse/backups/outer11/metadata/dbreal.sql"], + [ + "bash", + "-c", + "cat /var/lib/clickhouse/disks/backup_disk/outer11/metadata/dbreal.sql", + ], user="root", ) - # The locator really is the string form: an ASTLiteral, not an ASTFunction. - assert "Backup('d67785', 'File(\\'inner11\\')')" in manifest, manifest + # The locator really is the function form: an ASTFunction, not an ASTLiteral holding its text. + assert "Backup('d67785', File('inner11'))" in manifest, manifest - # Only the outer locator's own grant; the string-form inner one must not be authorized at all. - node.query(f"GRANT READ ON FILE TO {USER}") + # Only the outer locator's own grant, so the embedded File locator is the one that is missing. + node.query(f"GRANT READ ON DISK TO {USER}") + + error = node.query_and_get_error( + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11')", + user=USER, + ) + assert "ACCESS_DENIED" in error, error + assert "READ ON FILE" in error, error - # Target exists, so nothing is created and the restore fully succeeds. Authorizing the string - # form turns this into `Code: 36` out of CHECKING_ACCESS_RIGHTS. + # Target exists and its definition matches the manifest, so nothing is created and the restore + # fully succeeds. + node.query(f"GRANT READ ON FILE TO {USER}") node.query( - "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM File('outer11') FORMAT Null", user=USER + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11') FORMAT Null", + user=USER, ) - # Target absent, so creation runs and rejects the string form - as it does without this feature. - # `While creating database` is what pins the failure to the creation stage rather than the - # access-check one, which is where the same code would report it. + # Target absent, so creation runs: the locator parses, and the created database reads the tables of + # the inner backup. node.query("DROP DATABASE dbreal SYNC") - error = node.query_and_get_error( - "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM File('outer11')", user=USER + node.query( + "RESTORE DATABASE dbreal ON CLUSTER one_shard FROM Disk('backup_disk', 'outer11') FORMAT Null", + user=USER, ) - assert "ACCESS_DENIED" not in error, error - assert "BAD_ARGUMENTS" in error, error - assert "While creating database" in error, error + assert node.query("SELECT x FROM dbreal.secrets") == "42\n" + node.query("DROP DATABASE dbreal SYNC") def test_explicit_base_backup_locator_is_authorized_on_the_initiator(started_cluster): diff --git a/tests/integration/test_database_backup/test.py b/tests/integration/test_database_backup/test.py index 8316d798e14f..40aa406a050e 100644 --- a/tests/integration/test_database_backup/test.py +++ b/tests/integration/test_database_backup/test.py @@ -12,6 +12,17 @@ with_minio=True, ) +# `test_database_backup_metadata_with_quoted_locator_loads_on_restart` rewrites a database metadata file +# in place, and such a file only exists when the metadata lives on the local disk, so that test runs on an +# instance which keeps the local database disk. +instance_local_metadata = cluster.add_instance( + "instance_local_metadata", + main_configs=["configs/backups.xml"], + stay_alive=True, + with_minio=True, + with_remote_database_disk=False, +) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -271,3 +282,94 @@ def test_database_backup_unavailable_but_server_starts(backup_destination): instance.query("DROP DATABASE IF EXISTS test_database_backup SYNC") instance.query("DROP DATABASE IF EXISTS test_database SYNC") cleanup_backup_files(instance) + + +def test_database_backup_metadata_with_quoted_locator_loads_on_restart(): + # Regression test for https://github.com/ClickHouse/ClickHouse/issues/118349 + # An older server regenerated the definition of a `Backup` database with the locator quoted into a + # string literal, and `ALTER DATABASE ... MODIFY COMMENT` wrote that back into `metadata/.sql`. + # The next start replays the stored full `ATTACH DATABASE ... ENGINE = Backup(...)` statement, which + # is neither the short `ATTACH` nor a force-restore load, so it has to accept that form on its own. + # + # The metadata file is rewritten in place here, so this runs on the instance whose metadata is a file + # on the local disk rather than an object on a remote database disk. + instance = instance_local_metadata + + cleanup_backup_files(instance) + + instance.query( + """ + DROP DATABASE IF EXISTS test_database SYNC; + DROP DATABASE IF EXISTS test_database_backup SYNC; + + CREATE DATABASE test_database; + + CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + INSERT INTO test_database.test_table VALUES (0, 'test_database.test_table'); + + BACKUP DATABASE test_database TO File('test_database_backup_file'); + CREATE DATABASE test_database_backup ENGINE = Backup('test_database', File('test_database_backup_file')); + """ + ) + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + + # The metadata file exactly as a pre-fix server left it after a comment change. + instance.stop_clickhouse() + metadata = ( + "ATTACH DATABASE test_database_backup\n" + "ENGINE = Backup('test_database', 'File(\\'test_database_backup_file\\')')\n" + "COMMENT 'written by an older server'\n" + ) + instance.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/metadata/test_database_backup.sql <<'SQL'\n" + + metadata + + "SQL\n", + ], + user="root", + ) + assert "'File(\\'test_database_backup_file\\')'" in instance.exec_in_container( + ["cat", "/var/lib/clickhouse/metadata/test_database_backup.sql"] + ) + instance.start_clickhouse() + + # The server started and the database loaded with its tables and comment. + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by an older server\n" + ) + # The definition is regenerated with the locator as the function it is. + assert ( + "ENGINE = Backup('test_database', File('test_database_backup_file'))" + in instance.query("SHOW CREATE DATABASE test_database_backup FORMAT TSVRaw") + ) + + # A comment change on this server writes the function form, and that survives a restart too. + instance.query( + "ALTER DATABASE test_database_backup MODIFY COMMENT 'written by this server'" + ) + instance.restart_clickhouse() + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by this server\n" + ) + + instance.query("DROP DATABASE test_database_backup SYNC") + instance.query("DROP DATABASE test_database SYNC") diff --git a/tests/integration/test_database_glue/test.py b/tests/integration/test_database_glue/test.py index 4ca41b7ac988..145a84b0810a 100644 --- a/tests/integration/test_database_glue/test.py +++ b/tests/integration/test_database_glue/test.py @@ -1371,11 +1371,104 @@ def test_sts_smoke(started_cluster): result = node.query(f"SELECT sum(value) FROM {db_name_success}.`{root_namespace}.{table_name}`") assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + # `aws_role_session_name` can act as a shared secret (the trust policy can pin it, which is what the + # STS mock does), so every display surface hides it while `aws_role_arn` stays visible. This database + # carries no other secret, so `[HIDDEN]` can only come from the session name. + show_create = node.query(f"SHOW CREATE DATABASE {db_name_success}") + assert "miniorole" not in show_create + assert "arn::role" in show_create + assert "[HIDDEN]" in show_create + + engine_full = node.query( + f"SELECT engine_full FROM system.databases WHERE name = '{db_name_success}'" + ) + assert "miniorole" not in engine_full + assert "[HIDDEN]" in engine_full + + node.query("SYSTEM FLUSH LOGS system.query_log") + logged_create = node.query( + f"SELECT arrayStringConcat(groupArray(query), '\\n') FROM system.query_log " + f"WHERE query_kind = 'Create' AND type = 'QueryFinish' AND query LIKE '%{db_name_success}%'" + ) + assert "[HIDDEN]" in logged_create + assert "miniorole" not in logged_create + # Cleanup node.query(f"DROP DATABASE IF EXISTS {db_name_fail} SYNC") node.query(f"DROP DATABASE IF EXISTS {db_name_success} SYNC") +def test_sts_backup_restore_keeps_role_session_name(started_cluster): + """A backup archives the database definition with the real `aws_role_session_name`, although every + display surface shows `[HIDDEN]`. The STS mock grants the role only for the session name `miniorole`, + so the restored catalog can read its table only if the archived value was the real one.""" + node = started_cluster.instances["node1"] + + test_ref = f"test_sts_backup_{uuid.uuid4()}" + table_name = f"{test_ref}_table" + root_namespace = f"{test_ref}_namespace" + + catalog = load_catalog_impl(started_cluster) + catalog.create_namespace(root_namespace) + + schema = Schema( + NestedField(field_id=1, name="id", field_type=StringType(), required=False), + NestedField(field_id=2, name="value", field_type=DoubleType(), required=False), + ) + table = create_table(catalog, root_namespace, table_name, schema, PartitionSpec(), DEFAULT_SORT_ORDER, dir=table_name) + table.append( + pa.Table.from_pylist( + [ + {"id": "row1", "value": 10.0}, + {"id": "row2", "value": 20.0}, + {"id": "row3", "value": 30.0}, + ] + ) + ) + + db_name = f"db_backup_{test_ref.replace('-', '_')}" + restored_db_name = f"{db_name}_restored" + create_clickhouse_glue_database( + started_cluster, + node, + db_name, + additional_settings={ + "aws_role_arn": "arn::role", + "aws_role_session_name": "miniorole", + }, + query_settings={"s3_allow_server_credentials_in_user_queries": 1}, + with_credentials=False, + ) + result = node.query(f"SELECT sum(value) FROM {db_name}.`{root_namespace}.{table_name}`") + assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + + # A DataLakeCatalog database owns no tables, so the backup holds just the database definition. The + # default server config allows File backups under its `backups` directory. + backup = f"File('{db_name}')" + node.query(f"BACKUP DATABASE {db_name} TO {backup}") + node.query(f"DROP DATABASE IF EXISTS {restored_db_name} SYNC") + node.query( + f"RESTORE DATABASE {db_name} AS {restored_db_name} FROM {backup}", + settings={ + "allow_database_glue_catalog": 1, + "s3_allow_server_credentials_in_user_queries": 1, + }, + ) + + # Readable only with the real session name: the STS mock rejects any other value. + result = node.query(f"SELECT sum(value) FROM {restored_db_name}.`{root_namespace}.{table_name}`") + assert result.strip() == "60", f"Expected sum to be 60 but got: {result}" + + # The archived value came back, and the display of the restored definition is masked like any other. + show_create = node.query(f"SHOW CREATE DATABASE {restored_db_name}") + assert "miniorole" not in show_create + assert "arn::role" in show_create + assert "[HIDDEN]" in show_create + + node.query(f"DROP DATABASE IF EXISTS {restored_db_name} SYNC") + node.query(f"DROP DATABASE IF EXISTS {db_name} SYNC") + + def test_sts_smoke_no_opt_in(started_cluster): """A Glue DataLakeCatalog with aws_role_arn set, no explicit aws_access_key_id/aws_secret_access_key, and no s3_allow_server_credentials_in_user_queries opt-in. GlueCatalog routes through the same diff --git a/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml new file mode 100644 index 000000000000..9d5a3fe658f3 --- /dev/null +++ b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml @@ -0,0 +1,30 @@ + + + dict_local_infile + + + test + mysql80 + 3306 + root + ClickHouse_MySQL_P@ssw0rd + dict_local_infile_table
+ 1 + + + + + + + + id + + + value + String + + + + 0 + + diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index d2bd9de40401..b7be44a3ef57 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -19,6 +19,7 @@ "configs/dictionaries/mysql_dict_compression.xml", "configs/dictionaries/mysql_dict_compression_wire.xml", "configs/dictionaries/mysql_dict_no_compression_wire.xml", + "configs/dictionaries/mysql_dict_local_infile.xml", ] CONFIG_FILES = [ "configs/remote_servers.xml", @@ -1032,3 +1033,74 @@ def reload(name): mysql_connection, "DROP TABLE IF EXISTS test.inherit_pool_test;" ) mysql_connection.close() + + +def test_enable_local_infile_xml_dict(started_cluster): + """`enable_local_infile` is rejected in a dictionary created with a DDL query, but a dictionary + defined in a server configuration file is written by an operator and keeps working.""" + mysql_connection = get_mysql_conn(started_cluster) + + try: + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE test.dict_local_infile_table (id INT NOT NULL, value TEXT, PRIMARY KEY(id));", + ) + execute_mysql_query( + mysql_connection, + "INSERT INTO test.dict_local_infile_table VALUES (1, 'local_infile');", + ) + + # The dictionary is lazily loaded, so nothing has touched the source yet; reload it now that + # its table exists. + for _ in range(10): + try: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile") + break + except Exception: + time.sleep(0.5) + + # Loading the dictionary means the source was instantiated and connected with the option on, + # which is what the guard must not prevent for this route. + value = instance.query( + "SELECT dictGet('dict_local_infile', 'value', toUInt64(1))" + ).strip() + last_exception = instance.query( + "SELECT last_exception FROM system.dictionaries WHERE name = 'dict_local_infile'" + ).strip() + assert value == "local_infile", ( + " was rejected in the XML dict config: " + f"{last_exception!r}" + ) + + # The same option from a DDL query is rejected, at the source and at a replica alike. The + # endpoint here is the working MySQL server, so a rejection cannot be a connection failure + # in disguise. + credentials = f"USER 'root' PASSWORD '{mysql_pass}' DB 'test' TABLE 'dict_local_infile_table'" + for source in ( + f"HOST 'mysql80' PORT 3306 {credentials} ENABLE_LOCAL_INFILE 1", + f"{credentials} REPLICA(PRIORITY 1 HOST 'mysql80' PORT 3306 ENABLE_LOCAL_INFILE 1)", + ): + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + instance.query( + f""" + CREATE DICTIONARY dict_local_infile_ddl (id UInt64, value String) + PRIMARY KEY id + SOURCE(MYSQL({source})) + LAYOUT(FLAT()) + LIFETIME(0) + """ + ) + with pytest.raises(Exception) as exc: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile_ddl") + assert "cannot be enabled in a dictionary created with a DDL query" in str( + exc.value + ), f"Unexpected error for {source!r}: {exc.value}" + finally: + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + mysql_connection.close() diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 915eaef3ad15..9aa9080bf737 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -611,6 +611,53 @@ def test_external_table(): ) +def test_settings_profile_constraints(): + profile = "grpc_profile_constraints" + query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") + query( + f"CREATE SETTINGS PROFILE {profile} SETTINGS max_execution_time = 10 CONST, max_result_rows = 12345, format_csv_delimiter = '|'" + ) + try: + # `settings` is a protobuf map with no order: the profile goes first, so its constraints bind the + # other settings and they override the values it sets. + e = query_and_get_error( + "SELECT 1", settings={"profile": profile, "max_execution_time": "999"} + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT getSetting('max_result_rows')", + settings={"profile": profile, "max_result_rows": "7"}, + ) + == "7\n" + ) + + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + + def ext(settings): + return clickhouse_grpc_pb2.ExternalTable( + name="ext1", columns=columns, data=b"1;Alex\n", format="CSV", settings=settings + ) + + e = query_and_get_error( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "max_execution_time": "999"})], + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "format_csv_delimiter": ";"})], + ) + == "1\tAlex\n" + ) + finally: + query(f"DROP SETTINGS PROFILE {profile}") + + def test_external_table_streaming(): columns = [ clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), diff --git a/tests/integration/test_keeper_session/test.py b/tests/integration/test_keeper_session/test.py index 07062f42f5e6..213e1f3b22bc 100644 --- a/tests/integration/test_keeper_session/test.py +++ b/tests/integration/test_keeper_session/test.py @@ -36,6 +36,9 @@ reply_header_struct = struct.Struct("!iqi") stat_struct = struct.Struct("!qqqqiiiqiiq") +# Protocol version Keeper sends when it rejects a connection. +KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT = 42 + @pytest.fixture(scope="module") def started_cluster(): @@ -86,7 +89,9 @@ def read_buffer(bytes, offset): return bytes[index : index + length], offset -def handshake(node_name=node1.name, session_timeout=1000, session_id=0): +def handshake( + node_name=node1.name, session_timeout=1000, session_id=0, full_reply=False +): client = None try: client = get_keeper_socket(node_name) @@ -130,6 +135,12 @@ def handshake(node_name=node1.name, session_timeout=1000, session_id=0): read_only = False print("negotiated_timeout - session_id", negotiated_timeout, session_id) + if full_reply: + (reply_length,) = int_struct.unpack_from(data, 0) + rest = data[int_struct.size + reply_length :] + while chunk := client.recv(1_000): + rest += chunk + return proto_version, negotiated_timeout, session_id, rest return negotiated_timeout, session_id finally: if client is not None: @@ -149,6 +160,35 @@ def test_session_timeout(started_cluster): assert negotiated_timeout == 10000 +def test_handshake_to_continue_session_is_expired(started_cluster): + wait_nodes() + negotiated_timeout, session_id = handshake( + node1.name, session_timeout=8000, session_id=0 + ) + assert negotiated_timeout == 8000 and session_id > 0 + # Keeper cannot restore a session, so a request to continue one must not be answered with a new session. + # The expired reply is the last thing the server sends before it closes the connection. + assert handshake( + node1.name, session_timeout=8000, session_id=session_id, full_reply=True + ) == (0, 0, 0, b"") + + try: + node2.stop_clickhouse() + node3.stop_clickhouse() + keeper_utils.wait_until_quorum_lost(cluster, node1) + # A rejected client gets no session id that it would send back as the session to continue. + assert handshake( + node1.name, session_timeout=8000, session_id=0, full_reply=True + ) == (KEEPER_PROTOCOL_VERSION_CONNECTION_REJECT, 8000, 0, b"") + assert handshake( + node1.name, session_timeout=8000, session_id=session_id, full_reply=True + ) == (0, 0, 0, b"") + finally: + node2.start_clickhouse() + node3.start_clickhouse() + wait_nodes() + + def test_session_close_shutdown(started_cluster): wait_nodes() diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index d16d8f5623f7..24b95f076493 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -396,8 +396,8 @@ def generate_create_table_numbered(tail): generate_create_table_numbered(f"(`x` int) ENGINE = AzureQueue('{azure_storage_account_url}', 'cont', '*', '{azure_account_name}', '[HIDDEN]', 'CSV', 'none') SETTINGS mode = 'unordered'"), generate_create_table_numbered(f"(`x` int) ENGINE = AzureBlobStorage('{masked_sas_conn_string}', 'exampledatasets', 'example.csv')"), generate_create_table_numbered("(`x` int) ENGINE = S3('https://my-s3-endpoint/bucket/data.csv', 'myaccess', '[HIDDEN]', 'CSV')"), - generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@'"), - generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://schema_user:[HIDDEN]@domain.com'"), + generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://[HIDDEN]@'"), + generate_create_table_numbered("(`x` int) ENGINE = Kafka SETTINGS kafka_broker_list = '127.0.0.1', kafka_topic_list = 'topic', kafka_group_name = 'group', kafka_format = 'JSONEachRow', kafka_security_protocol = 'sasl_ssl', kafka_sasl_mechanism = 'PLAIN', kafka_sasl_username = 'user', kafka_sasl_password = '[HIDDEN]', format_avro_schema_registry_url = 'http://[HIDDEN]@domain.com'"), generate_create_table_numbered("(`x` int) ENGINE = S3('http://minio1:9001/root/data/test5.csv.gz', 'CSV', access_key_id = 'minio', secret_access_key = '[HIDDEN]', compression_method = 'gzip')"), generate_create_table_numbered("(`x` int) ENGINE = ArrowFlight('arrowflight1:5006', 'dataset', 'arrowflight_user', '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = ArrowFlight(named_collection_1, host = 'arrowflight1', port = 5006, dataset = 'dataset', username = 'arrowflight_user', password = '[HIDDEN]')"), @@ -405,12 +405,12 @@ def generate_create_table_numbered(tail): generate_create_table_numbered("(`x` int) ENGINE = Redis('localhost', 0, '[HIDDEN]') PRIMARY KEY x"), generate_create_table_numbered("(`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = JDBC('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = ODBC('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = JDBC('[HIDDEN]', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = ODBC('[HIDDEN]', 'mydb', 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), + generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = 'mydb', external_table = 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')"), - generate_create_table_numbered("(`x` int) ENGINE = ODBC(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb', external_database = 'mydb', external_table = 'mytable')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', external_table = '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = JDBC(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]', external_table = '[HIDDEN]')"), generate_create_table_numbered("(`x` int) ENGINE = NATS SETTINGS nats_url = 'localhost:4222', nats_subjects = 'subject', nats_format = 'JSONEachRow', nats_token = '[HIDDEN]'"), @@ -656,12 +656,12 @@ def make_test_case(i): "CREATE TABLE tablefunc49 (`x` int) AS redis('localhost', 'key', 'key Int64', 0, '[HIDDEN]')", "CREATE TABLE tablefunc50 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", "CREATE TABLE tablefunc51 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", - "CREATE TABLE tablefunc52 (`x` int) AS jdbc('jdbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", - "CREATE TABLE tablefunc53 (`x` int) AS odbc('odbc://user:[HIDDEN]@localhost:5432/mydb', 'mydb', 'mytable')", + "CREATE TABLE tablefunc52 (`x` int) AS jdbc('[HIDDEN]', 'mydb', 'mytable')", + "CREATE TABLE tablefunc53 (`x` int) AS odbc('[HIDDEN]', 'mydb', 'mytable')", "CREATE TABLE tablefunc54 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", "CREATE TABLE tablefunc55 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", - "CREATE TABLE tablefunc56 (`x` int) AS jdbc(named_collection_1, datasource = 'jdbc://user:[HIDDEN]@localhost:5432/mydb')", - "CREATE TABLE tablefunc57 (`x` int) AS odbc(named_collection_1, connection_settings = 'odbc://user:[HIDDEN]@localhost:5432/mydb')", + "CREATE TABLE tablefunc56 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]')", + "CREATE TABLE tablefunc57 (`x` int) AS odbc(named_collection_1, connection_settings = '[HIDDEN]')", "CREATE TABLE tablefunc58 (`x` int) AS jdbc(named_collection_1, datasource = '[HIDDEN]', connection_settings = '[HIDDEN]')", "CREATE TABLE tablefunc59 (`x` int) AS jdbc(named_collection_1, connection_settings = '[HIDDEN]', external_database = '[HIDDEN]', datasource = '[HIDDEN]')", "CREATE TABLE tablefunc60 (`x` int) AS deltaLakeS3('http://minio1:9001/root/data/test11.csv.gz', 'minio', '[HIDDEN]')", @@ -1058,6 +1058,62 @@ def test_database_backup_engine_s3(): node.query("DROP DATABASE IF EXISTS backup_db_s3_2") +def test_database_backup_engine_azure_display_surfaces(): + """A live `Backup` database over an `AzureBlobStorage` locator: `account_key` must not reach + `SHOW CREATE DATABASE` or `system.databases.engine_full`. Only `S3` locators are reconstructed + argument by argument, so an Azure one keeps its engine name and arity and hides every argument.""" + azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + azure_account_name = "devstoreaccount1" + azure_account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + + # A backup destination is write-once and the container is emptied only when the cluster starts, + # so give every run its own blob path: a repeat inside one session hits BACKUP_ALREADY_EXISTS. + blob = "db_backup_azure_surfaces_" + "".join( + random.choice(string.ascii_lowercase) for _ in range(8) + ) + locator = ( + f"AzureBlobStorage('{azure_storage_account_url}', '{cluster.azurite_container}', " + f"'{blob}', '{azure_account_name}', '{azure_account_key}')" + ) + masked_locator = ( + "AzureBlobStorage('[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]')" + ) + surfaces = [ + "SHOW CREATE DATABASE backup_db_azure_view", + "SELECT engine_full FROM system.databases WHERE name = 'backup_db_azure_view'", + ] + + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("CREATE DATABASE backup_db_azure_src") + node.query( + "CREATE TABLE backup_db_azure_src.t (x int) ENGINE = MergeTree ORDER BY x" + ) + node.query("INSERT INTO backup_db_azure_src.t SELECT * FROM numbers(10)") + node.query(f"BACKUP DATABASE backup_db_azure_src TO {locator} FORMAT Null") + node.query( + f"CREATE DATABASE backup_db_azure_view ENGINE = Backup('backup_db_azure_src', {locator})" + ) + + # TSVRaw so the locator is compared as written rather than through TSV escaping. + for surface in surfaces: + shown = node.query(f"{surface} FORMAT TSVRaw") + assert masked_locator in shown, shown + assert azure_account_key not in shown, shown + + # The key is in the locator, so its absence above is masking rather than an argument that the + # regenerated definition never carried. + for surface in surfaces: + shown = node.query(f"{surface} {show_secrets}=1 FORMAT TSVRaw") + assert azure_account_key in shown, shown + + # The masked surfaces belong to a working database, not to one that failed to open. + assert node.query("SELECT count() FROM backup_db_azure_view.t") == "10\n" + + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + + def test_backup_table_azure_named_collection(): """Test that secrets in Azure named collection backups are masked in system.backups and logs.""" azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] diff --git a/tests/integration/test_mq_remote_host_filter/__init__.py b/tests/integration/test_mq_remote_host_filter/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml b/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml new file mode 100644 index 000000000000..054acd185f73 --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml @@ -0,0 +1,8 @@ + + + localhost:19092 + localhost:14222 + allowed-any-port + .*\.allowed\.example\.com + + diff --git a/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml b/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml new file mode 100644 index 000000000000..e6b7a5f8f93f --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml @@ -0,0 +1,5 @@ + + + localhost:9999 + + diff --git a/tests/integration/test_mq_remote_host_filter/test.py b/tests/integration/test_mq_remote_host_filter/test.py new file mode 100644 index 000000000000..7d3e4b1afebe --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/test.py @@ -0,0 +1,193 @@ +""" +Tests that the `Kafka` and `NATS` table engines respect `remote_url_allow_hosts`. + +Both engines used to hand the user-supplied broker addresses (`kafka_broker_list`, +`nats_url`, `nats_server_list`) to the client library without consulting +`RemoteHostFilter`, so `CREATE TABLE` opened outbound TCP connections to hosts the +operator had explicitly forbidden. `RabbitMQ`, the sibling engine, rejects the same +addresses at `CREATE` with `UNACCEPTABLE_URL`. + +The addresses are parsed, validated element-wise, and rebuilt before they reach the +client library, so an entry whose re-parse by the library could disagree with the +validated form (a path, an empty host, a NUL, ...) is rejected with `BAD_ARGUMENTS` +instead of being repaired. + +No broker runs in this test: a forbidden or malformed address must be rejected before +any connection attempt, and an allowed one must get past the filter (and then fail to +connect, for the engine which connects at DDL time). + +Each case is a triplet: the engine input, the expected error code (`None` means the +table must be created), and an optional string the error message must contain. +""" + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/allowed_hosts.xml"], +) +# The section of the server configuration (and named collections, loaded by the same code) +# can override the broker list librdkafka receives, behind the back of the validated +# `kafka_broker_list` setting. This instance carries such an override to a forbidden host. +node_kafka_override = cluster.add_instance( + "node_kafka_override", + main_configs=["configs/allowed_hosts.xml", "configs/kafka_bootstrap_override.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# (kafka_broker_list value, expected error code or None for success, required error message part) +KAFKA_CASES = [ + pytest.param("localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="forbidden_broker"), + # One allowed broker must not smuggle a forbidden one past the filter. + pytest.param("localhost:19092,localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="element_wise"), + # librdkafka accepts `scheme://host:port` entries; the filter must see host:port. + pytest.param("PLAINTEXT://localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="scheme_prefix"), + # A bare host must be checked with librdkafka's default port 9092. + pytest.param("localhost", "UNACCEPTABLE_URL", "localhost:9092", id="default_port"), + # A bare `` allowlist entry allows the host on any port, as for other engines. + pytest.param("allowed-any-port:9095", None, None, id="host_allowed_for_any_port"), + pytest.param("localhost:19092", None, None, id="allowed_broker"), + # librdkafka cuts an entry at the first `/` after `scheme://`. Without the strict parse, the + # filter would see the host `evil.com/.allowed.example.com`, the `host_regexp` allowlist entry + # would fully match it, and librdkafka would dial `evil.com`. + pytest.param( + "PLAINTEXT://evil.com/.allowed.example.com", "BAD_ARGUMENTS", "Unexpected character '/'", + id="path_does_not_bypass_host_regexp", + ), + # librdkafka splits the broker list on spaces too. Without doing the same, the filter would see + # the single host `evil.com .allowed.example.com`, the `host_regexp` allowlist entry would fully + # match it, and librdkafka would dial `evil.com`. + pytest.param("evil.com .allowed.example.com", "UNACCEPTABLE_URL", "evil.com:9092", id="space_separator"), + # librdkafka substitutes `localhost` for an empty host. + pytest.param(":9092", "BAD_ARGUMENTS", "Empty host", id="empty_host"), + # The broker list reaches librdkafka as a C string, so a NUL must not truncate it after the check. + pytest.param("evil.com\\0.allowed.example.com", "BAD_ARGUMENTS", "NUL", id="nul"), + # The validated entries are rebuilt and rejoined; the original text never reaches librdkafka. + pytest.param("localhost:19092 , localhost:19092", None, None, id="canonicalized"), + # An allowed host by the `host_regexp` allowlist entry. + pytest.param("sub.allowed.example.com", None, None, id="host_regexp_allowed"), +] + +# (NATS settings, expected error code or None for success, required error message part) +NATS_CASES = [ + pytest.param("nats_url = 'localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", id="forbidden_url"), + # libnats accepts `nats://user:password@host:port`; the filter must see host:port. + pytest.param( + "nats_url = 'nats://user:password@localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", + id="scheme_and_credentials", + ), + # One allowed server must not smuggle a forbidden one past the filter. + pytest.param( + "nats_server_list = 'localhost:14222,localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", + id="server_list_element_wise", + ), + # A bare host must be checked with libnats' default port 4222. + pytest.param("nats_url = 'localhost'", "UNACCEPTABLE_URL", "localhost:4222", id="default_port"), + # An allowed address must get past the filter and reach the connection attempt. `NATS` connects + # at `CREATE` time and nothing listens on the allowed port, so the statement fails - but with a + # connection error, not `UNACCEPTABLE_URL`. + pytest.param( + "nats_url = 'localhost:14222', nats_startup_connect_tries = 1", "CANNOT_CONNECT_NATS", None, + id="allowed_url_passes_filter", + ), + # libnats allows a `/path` after the port, so a path must not reach the filter as part of the host. + pytest.param( + "nats_url = 'nats://evil.com:4222/x.allowed.example.com'", "BAD_ARGUMENTS", "Unexpected character '/'", + id="path_does_not_bypass_host_regexp", + ), + # libnats substitutes `localhost` for an empty host. + pytest.param("nats_url = 'nats://:4222'", "BAD_ARGUMENTS", "Empty host", id="empty_host"), + # The URL reaches libnats as a C string, so a NUL must not truncate it after the check. + pytest.param("nats_url = 'evil.com\\0.allowed.example.com'", "BAD_ARGUMENTS", "NUL", id="nul"), + # The credentials must survive the rebuild of the validated address. + pytest.param( + "nats_url = 'nats://user:password@localhost:14222', nats_startup_connect_tries = 1", + "CANNOT_CONNECT_NATS", None, + id="credentials_preserved", + ), +] + + +def check(query, expected_error, message_part): + if expected_error is None: + node.query(query) + return + + error = node.query_and_get_error(query) + assert expected_error in error, error + if message_part is not None: + assert message_part in error, error + + +@pytest.mark.parametrize("broker_list, expected_error, message_part", KAFKA_CASES) +def test_kafka(started_cluster, broker_list, expected_error, message_part): + node.query("DROP TABLE IF EXISTS kafka_filtered SYNC") + check( + f""" + CREATE TABLE kafka_filtered (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = '{broker_list}', + kafka_topic_list = 'topic', + kafka_group_name = 'group', + kafka_format = 'JSONEachRow' + """, + expected_error, + message_part, + ) + + +def test_kafka_config_override_is_validated(started_cluster): + """A broker list supplied by the server configuration must be validated too. + + `getConsumerConfiguration` seeds `metadata.broker.list` from the validated + `kafka_broker_list`, but the `` section of the server configuration (or a named + collection) is merged afterwards and can override it - here through the + `bootstrap.servers` alias. The merged value is validated again when the consumer is + created, so the `CREATE` (which sees only the allowed setting) succeeds and the read + fails on the forbidden override. + """ + node_kafka_override.query( + """ + CREATE TABLE kafka_override (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'localhost:19092', + kafka_topic_list = 'topic', + kafka_group_name = 'group', + kafka_format = 'JSONEachRow' + """ + ) + error = node_kafka_override.query_and_get_error( + "SELECT * FROM kafka_override LIMIT 1" + " SETTINGS stream_like_engine_allow_direct_select = 1" + ) + assert "UNACCEPTABLE_URL" in error, error + assert "localhost:9999" in error, error + node_kafka_override.query("DROP TABLE kafka_override SYNC") + + +@pytest.mark.parametrize("settings, expected_error, message_part", NATS_CASES) +def test_nats(started_cluster, settings, expected_error, message_part): + node.query("DROP TABLE IF EXISTS nats_filtered SYNC") + check( + f""" + CREATE TABLE nats_filtered (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS {settings}, + nats_subjects = 'subject', + nats_format = 'JSONEachRow' + """, + expected_error, + message_part, + ) diff --git a/tests/integration/test_paimon_incremental_read/test.py b/tests/integration/test_paimon_incremental_read/test.py index 4e256a4a1de6..2ceb6c298730 100644 --- a/tests/integration/test_paimon_incremental_read/test.py +++ b/tests/integration/test_paimon_incremental_read/test.py @@ -21,6 +21,7 @@ CH_MV_PAIMON_TABLE = "paimon_mv_source" CH_MV_MERGETREE_TABLE = "paimon_mv_dest" CH_MV_NAME = "paimon_refresh_mv" +CH_TABLE_NAME_ACTIVATE_RECLAIM = "paimon_inc_read_activate_reclaim" cluster = ClickHouseCluster(__file__) node = cluster.add_instance( @@ -425,3 +426,105 @@ def test_paimon_to_mergetree_via_refresh_mv(started_cluster): node.query(f"DROP VIEW IF EXISTS {CH_MV_NAME} SYNC;") node.query(f"DROP TABLE IF EXISTS {CH_MV_MERGETREE_TABLE} SYNC;") node.query(f"DROP TABLE IF EXISTS {CH_MV_PAIMON_TABLE} SYNC;") + + +def test_paimon_incremental_read_activate_tolerates_reaped_is_active(started_cluster): + """A read after a Keeper session loss reclaims the `is_active` marker its own + previous session left behind, and must survive Keeper reaping that ephemeral + concurrently: the marker is visible to the reclaim path's `tryGet` and already + gone by its versioned `tryRemove`, which then reports ZNONODE. The marker here + is fabricated because it reproduces exactly that observable state, whereas + waiting for the real reap to land inside a two-round-trip window would make the + test nondeterministic.""" + writer_container_id = cluster.get_instance_docker_id("paimon-incremental-writer") + + warehouse_name = "warehouse_activate_reclaim" + warehouse_uri = f"file://{USER_FILES_PATH}/{warehouse_name}/" + warehouse_dir = f"{USER_FILES_PATH}/{warehouse_name}" + table_path = f"{USER_FILES_PATH}/{warehouse_name}/test.db/test_table" + # Unique per run: committed_snapshot persists in Keeper, so a rerun against + # the same cluster must not inherit an earlier run's watermark. + keeper_path = f"/clickhouse/paimon_activate_reclaim_{uuid.uuid4().hex}" + is_active_path = f"{keeper_path}/replicas/r1/is_active" + failpoint = "paimon_incremental_read_pause_before_is_active_remove" + + _clean_warehouse(writer_container_id, warehouse_dir) + + # Warm-up commit (snapshot 1), consumed to establish the baseline. + _run_writer(writer_container_id, warehouse_uri=warehouse_uri, start_id=0, rows_per_commit=1, commit_times=1) + _create_clickhouse_table_for_paimon_incremental_read( + CH_TABLE_NAME_ACTIVATE_RECLAIM, table_path, keeper_path=keeper_path + ) + count_query = f"SELECT count() FROM {CH_TABLE_NAME_ACTIVATE_RECLAIM}" + _wait_until_query_result(count_query, "1\n", database="default") + _wait_until_query_result(count_query, "0\n", database="default") + + zk = cluster.get_kazoo_client("zoo1") + reader = None + reader_result = {} + try: + # Read the server's own marker instead of hardcoding its payload format. + identifier = zk.get(is_active_path)[0] + + # Snapshot 2: the batch the post-reconnect read must deliver. + _run_writer(writer_container_id, warehouse_uri=warehouse_uri, start_id=1, rows_per_commit=10, commit_times=1) + + session_query = ( + "SELECT client_id FROM system.zookeeper_connection WHERE name = 'default'" + ) + old_session = node.query(session_query) + # Finalizes the shared session, so the Keeper handle latched in + # PaimonStreamState stays expired and the next read takes the + # needsNewKeeper() branch that calls activate(). + node.query("SYSTEM RECONNECT ZOOKEEPER") + assert node.query(session_query) != old_session, ( + f"the Keeper session was not replaced (still {old_session!r})" + ) + + deadline = time.monotonic() + 60 + while zk.exists(is_active_path) is not None: + assert time.monotonic() < deadline, "the old session's is_active was never reaped" + time.sleep(0.5) + + # Persistent, not ephemeral: activate() never inspects stat.ephemeralOwner, + # so persistence is invisible to the code under test while keeping a second + # Keeper session out of the test's failure modes. + zk.create(is_active_path, identifier) + + node.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + reader = threading.Thread( + target=lambda: reader_result.update( + zip(("out", "err"), node.query_and_get_answer_with_error(count_query)) + ) + ) + reader.start() + + # Returns only once a thread has parked at the failpoint. Since the + # failpoint sits inside the identifier-matched branch, that proves the read + # entered the reclaim path with a marker it recognises as its own. + node.query(f"SYSTEM WAIT FAILPOINT {failpoint} PAUSE", timeout=60) + assert reader.is_alive(), ( + f"the reader returned before parking at the failpoint: {reader_result!r}" + ) + + # The reap lands inside the read's tryGet/tryRemove window. + zk.delete(is_active_path) + node.query(f"SYSTEM NOTIFY FAILPOINT {failpoint}") + + reader.join(timeout=120) + assert not reader.is_alive(), "the reader thread never finished" + assert not reader_result.get("err"), f"the read failed: {reader_result!r}" + # Not just "did not throw": the pending snapshot must still be delivered. + assert reader_result.get("out") == "10\n", ( + f"the pending snapshot was not delivered: {reader_result!r}" + ) + assert zk.get(f"{keeper_path}/committed_snapshot")[0] == b"2" + _wait_until_query_result(count_query, "0\n", database="default") + finally: + # The failpoint is process-global: an early failure must leave neither it + # armed nor the reader thread parked on it. + node.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + zk.stop() + if reader is not None: + reader.join(timeout=60) + node.query(f"DROP TABLE IF EXISTS {CH_TABLE_NAME_ACTIVATE_RECLAIM} SYNC;") diff --git a/tests/integration/test_query_runner/test.py b/tests/integration/test_query_runner/test.py index 9b2a055705ff..49036e56c360 100644 --- a/tests/integration/test_query_runner/test.py +++ b/tests/integration/test_query_runner/test.py @@ -134,6 +134,22 @@ def test_cluster_query_log(): ) +def test_cluster_query_log_hides_secrets(): + node_query_runner.query(runner_ddl("query String, settings Map(String, String)", "synchronous", "cluster")) + node_query_runner.query( + "INSERT INTO runner VALUES (" + "'SELECT * FROM url(''http://cluster_user:cluster_secret@node_cluster:8123/?query=SELECT+1'', ''LineAsString'', ''s String'')', " + "{'log_comment': 'qr_cluster_log_secret'})" + ) + node_query_runner.query("SYSTEM FLUSH LOGS query_log") + assert_eq_with_retry( + node_query_runner, + "SELECT type, query LIKE '%[HIDDEN]%', query LIKE '%cluster_secret%' FROM system.query_log " + "WHERE log_comment = 'qr_cluster_log_secret' AND is_internal ORDER BY type", + "QueryStart\t1\t0\nQueryFinish\t1\t0", + ) + + def test_cluster_insert_requires_remote(): node_query_runner.query(runner_ddl("query String", "synchronous", "cluster")) node_query_runner.query("CREATE USER no_remote_user") diff --git a/tests/integration/test_reload_ca_certificate/__init__.py b/tests/integration/test_reload_ca_certificate/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/integration/test_reload_ca_certificate/certs/ca.crt b/tests/integration/test_reload_ca_certificate/certs/ca.crt new file mode 100644 index 000000000000..b836c066c43e --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIURxZvEwx6gcq/IbNHsPts6W5c7P0wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDEwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCQMESmjuEmo6A25FY5Bt11iOrs +ySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5Uei/SsY0kLTiz +PZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJtwVf9E13eKGw7 +qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAgLoLUeVC/HpNQ +dVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4bJV53nOJ+1Jw +IEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTYdzicaA9JAgMB +AAGjUzBRMB0GA1UdDgQWBBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAfBgNVHSMEGDAW +gBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBiopVu+5LqgH4FwKh6YLvWEWdtw5gWs1Eda27cN2b1nTnJ+Xfn +UlwAdaS1lzGBkgUDYfbL0OJo9wx/yg5MuakE2kNO1QALb4r5mpGqTBMYT1XVzcGx +6LCOweHBI4a/ff4s1rj1xRR+PIPIHnCd5mKEgscKMvCmR5kUzkkmjRPFnBJFLHH9 +Z88L4J5NBwF8wJsFCv9td8aeeeTc9rg19zKOn6vkR5AczeXVAeBDBxAtcmX2YRMt +ytIWGmKfW3OE8kxsZdJJn4b+9jE5pLbTEL/So0YuNELRgcpK+9MGApOKjcc3DJg0 +LhsYev318QV4SjluwR1T/jgRvAlFbwk8+10L +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca1.crt b/tests/integration/test_reload_ca_certificate/certs/ca1.crt new file mode 100644 index 000000000000..b836c066c43e --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca1.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIURxZvEwx6gcq/IbNHsPts6W5c7P0wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDEwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCQMESmjuEmo6A25FY5Bt11iOrs +ySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5Uei/SsY0kLTiz +PZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJtwVf9E13eKGw7 +qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAgLoLUeVC/HpNQ +dVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4bJV53nOJ+1Jw +IEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTYdzicaA9JAgMB +AAGjUzBRMB0GA1UdDgQWBBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAfBgNVHSMEGDAW +gBSJsd7zDPOJkBExjUxJ8bKVSg2pHTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQBiopVu+5LqgH4FwKh6YLvWEWdtw5gWs1Eda27cN2b1nTnJ+Xfn +UlwAdaS1lzGBkgUDYfbL0OJo9wx/yg5MuakE2kNO1QALb4r5mpGqTBMYT1XVzcGx +6LCOweHBI4a/ff4s1rj1xRR+PIPIHnCd5mKEgscKMvCmR5kUzkkmjRPFnBJFLHH9 +Z88L4J5NBwF8wJsFCv9td8aeeeTc9rg19zKOn6vkR5AczeXVAeBDBxAtcmX2YRMt +ytIWGmKfW3OE8kxsZdJJn4b+9jE5pLbTEL/So0YuNELRgcpK+9MGApOKjcc3DJg0 +LhsYev318QV4SjluwR1T/jgRvAlFbwk8+10L +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca1.key b/tests/integration/test_reload_ca_certificate/certs/ca1.key new file mode 100644 index 000000000000..562b4debf1ac --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCQMESmjuEmo6A2 +5FY5Bt11iOrsySvgaWC7DkFaRRy/EZOvQg/HiGXG/oRtt1BsKLYdxRrvtxgJ9H5U +ei/SsY0kLTizPZFsEomesNwxV1Y8dyeKf9EDSKCjCnrlJwaTmpJ2FpjJRQHf5YJt +wVf9E13eKGw7qGoCxGGUx14kU7ftdfpEtCC8HRjktWlkXt16nCAHInXx8WSi6SAg +LoLUeVC/HpNQdVWyPHUhYyFppkYnYONIAJa2UaXB4OlaE+d9/edJeKWGc46oIFK4 +bJV53nOJ+1JwIEf13ax0IcKc1I0q1hJrsnB68Wowi6NR33THHbPNCvU4h/+1qcTY +dzicaA9JAgMBAAECggEABbduL1gkknXkBcJw2IRec8OaBHQs2cdfwwbGM0b1xEL/ +2Dp9+j7ngcJTO9or1XVElCKoMNxmgaAzK9RSJfqtV3j2AaWKyOO6wk3YKwTKXy2+ +WvS6k7RgTwUnjm3TykwTa1xgZ2wQQ8EmC3CCCtScMPu+CnGgw1Y9GVXmqaQf5jQ1 +xAoxP8/POBLjco8PAFWtai+wK9X1OVxQ6gr5Tw3TwAIHrYGsTY19g5c5IM4Y2k34 +QoaSGHkmSFOrBZs2DaWCaJ08Iz3zWUBP0VV2/H0szdIIrkv/Tqw2K6peRbJB8mYN +TID/BpbAadbEllzfn31O3Si0urHNZMrJI5IS2IChEQKBgQDHXrc6AUQQArbEKCxB +dq6B9+tLukaiMctxLHNXhY2NYlJxtASxkmYJewmU/d16lMyo3S+IPkEueo0Q1u3Z +k/MotSiis5kkL2EJWJjHq4H/ktw6k2a0vRd4DtcoysJ2qkCcgubSGYqSE+YcI1KC +MyOoKfb2IffCucyYAzntKmh+2QKBgQC5JQH9+CdyPlcWTJsXMztgDwyhzBixSqIl +Ovfrzzi7QSIX9csccZoXWkOfDEUJp2CZImErASO06+RM2ETQ1n2art4eWxX3UubD +pLXfr1m8NL/LJb56k9sWT+YwLXm4JeHeaErK6xTQCDNxH6szXUXQdXUpnbrrjg0S +6xVG0tCt8QKBgBI+U4vmQ8EnTmwitPIElzFja0+Rqxb6cYBYrfFLUkmmvp6S9378 +Q4QIkzbkCBlIdnXZT5krATHsmu34jOlFBZIrCZ3hy1ipUTrWtZxH0Gx/ltFxXYua +ZgRhb0TXUPYk3Ca2P8Ln/WsikQLwJIOvhErGFEgvkYlrERKz8OAH6mn5AoGAXQxd +YO9bm8365KkhdNp5p8BIf/RcIJY6wW1OdkPR5kJIyTPtnWD2qW/i9kcrVzu4j524 +qe1Lrby0I265vx9dRuVFmon6ky8l7QOVqFKvTahRD97rSR2QCTmknWfteYAIcUeG +906ISjkk6WCaIRlqYeb2ODEeZQ4iQfTF369J03ECgYA3NPk8j5ma59w/TYUPgZWN +aFlEKWLO8xOal9kWhx5dpXi1YCv3YoQztrfYus5b2nj7iDshK5cK6TzZaxPahqxA +rW9ExDjN2q89AvqP9vKPDJZ8Rk7+vTlLSNkFSe9LvvtfnU6IHdNcj+t/CI2823GC +Myj/NocqikSkeRCcq4SGCA== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca2.crt b/tests/integration/test_reload_ca_certificate/certs/ca2.crt new file mode 100644 index 000000000000..eff5f7fd97bc --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca2.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRzCCAi+gAwIBAgIUdDLEtjezpK/A2pIn/0YjbO3JhyAwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMjAeFw0yNjA4MzExNzI4MTVaFw0zNjA4MjgxNzI4MTVaMDMxGDAWBgNV +BAoMD0NsaWNrSG91c2UgVGVzdDEXMBUGA1UEAwwOVGVzdCBSb290IENBIDIwggEi +MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1bXrQUqqds9kXelFc7J+IrXsE +UdFZtxktFglh9eFrffqr0/+vthrcnqZ/OZrLmZV5PSW+gSNuaAVArbgGwYWW1lqL +0eRfQOktX/NcjoYlWUYlK7lWReFQj2CTCUrP3ojrdo/Oe0/0LsV3kctfUagcNxMX +oV5NAwHd9dDf9aL1ulljc+RO4ym8BlRqSApShkyH3xklkDLBneHOUJncwrWcC12p +1kV3fvt9iBURGK/MYAhhBJGnBeHM+OHLcyVZaUhqXcZ4+fFA2dEA5yGvPs2Gaf/1 +aAE/FsU/zLZkHmI8gslCoBrLtc10dgiUne07Hphu14wplVrSCWJImFMNN31fAgMB +AAGjUzBRMB0GA1UdDgQWBBRQyTcy9OASBuFcqS92wJ9LmtdwejAfBgNVHSMEGDAW +gBRQyTcy9OASBuFcqS92wJ9LmtdwejAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4IBAQCnd9ONRHb484+BSpRjfWpuDwGgjdqW9aglEFHOVsUestWe8rSl +9lZBveng/4zIqpIKZpOK04shPwP28lKwPim7EFp9iMSd7lnvSy0vHUhYoVzDtK/V ++QoYcrS/LgijOd+ohaszn2hzWWenNdWkgzSfjzp0WgiGg6j2R5pM9fmX/Wh0KtaC +XF23cM8eLaA5CyVCGRmH637qircr9wLbulld14GNBa9hTqcsp12TE+UX5ULwrGdO +PfiFeC8dmtiNW1p1dhdas52kB92e8EWU1wjFMZACyWpY6x7fxmdLexys5TqKkSn+ +JODwownsXCO4RGHTgQUuZNKFcyfbQGUbF3Ig +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/ca2.key b/tests/integration/test_reload_ca_certificate/certs/ca2.key new file mode 100644 index 000000000000..14928a74fcad --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/ca2.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1bXrQUqqds9kX +elFc7J+IrXsEUdFZtxktFglh9eFrffqr0/+vthrcnqZ/OZrLmZV5PSW+gSNuaAVA +rbgGwYWW1lqL0eRfQOktX/NcjoYlWUYlK7lWReFQj2CTCUrP3ojrdo/Oe0/0LsV3 +kctfUagcNxMXoV5NAwHd9dDf9aL1ulljc+RO4ym8BlRqSApShkyH3xklkDLBneHO +UJncwrWcC12p1kV3fvt9iBURGK/MYAhhBJGnBeHM+OHLcyVZaUhqXcZ4+fFA2dEA +5yGvPs2Gaf/1aAE/FsU/zLZkHmI8gslCoBrLtc10dgiUne07Hphu14wplVrSCWJI +mFMNN31fAgMBAAECggEADGEHnovqwk88LSLrxvMKQPF9UaJ3g1nqk4NLvphf+iKi +u53/yzBvYsWccf/Z37HUsMIK/5WgzO63X7NlZBNI7ILRs3VDJ3tZWFQ8rwBhMiPb +lREHrgBJ9beNYlr29M9So6ZM/Pdi/DPuG1DNAe2jZ9Fw3GLmo/XkPGMtDlWcpIwm +DpJDpFDCPfpi0g0z5gIyqVQQW3m9dE8WFvBPUYoDALmKfj8xvPuw1ezEsSkUj06N +YD+/PTOvpQyDhJGVVS9fZyQvdfCszTPMYsNx7e3FVsDdrXRwCFr4gbzxkjQhcwoq +VeZc+1k9R6V17uY3lPL5hfUpLBz8CxfitBzHJjH7QQKBgQDhiAXlNGn0+ggKg3r0 +H2wJ9E6ecyhUVOXoS9f65b+V6sqPSEHWk0jKPJqMXZV6WY4lQHq90WU3hxx2sNiX +jIP6AZdTUVfqJ7gAvBIn14kkLrirmPYx3MURFYoPm+B4LIHcvMlnMtY7BzO3eiv9 +1GDZYDl0mNcx/m5ipYxYLIH34QKBgQDN8CKzju8R1Ogp7m7bbjxSv3PCEzV0jHrs +jQBTSjb60d+Lz/STI6N0H7BMkwvNewnMwQU1xZ8DgGsDXQa4/6x2UiDGHvsCDFTF +HHsXzQCHP8n10vaU4xeCFqyj9/KYElqS8WOn/faH+c1N246+ATTMRDiU3lhI64Bo +iXKmuQcdPwKBgHKYPwabf0su0G8nJ45reOYF8Pyp3tAa40cJYpDltFdkmc/8ExgI +dm/sI0s3MgCdCJD9FmDkyN1SFbBpY2R9zYF21YFMT7N2wxP8e+0qo1BzPPpUGqRz +XN61ZxVPSttFIica9est9ZTAsBKGTVwIUb2iGw+XqaCJe2U8YPdchh2BAoGBAJ+L +fAbqJHMHJDpgG4hqlddxtafUo+RAdXdQIcFlTMTy1aKGoK9hu99aMYaRoWI3ATed +DoFDMldPJRj8+BlZEu6z3+o91C8ZCI+Q6hhdXRxrIfcN0rU0XmENWgDKNir0hTE0 +TAW5LkbYE+NOxv6TBql97OwAehs8QEY8vhNGY6mXAoGASwKe9byhQKXk0X6Q31g+ +5qaDc+fAqirKnPkqexw+1pUv/j6pK0QFBmPAT46Iwt/znAXqXCfSCNwIpR2UW78n +2J9UQR076SUat71MvcbPhesoUhLw4ZyFQYZ1Lo3piokhzjI0zwlmMusDpUdOjdB6 +JJcZmm30hrRMYNncfobFvXc= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert1.crt b/tests/integration/test_reload_ca_certificate/certs/cert1.crt new file mode 100644 index 000000000000..3c2faa29525a --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert1.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUB2rETXTp2XN0iWipA9fQo9alsGQwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAN7xjZT2K9ocTmXgCKTgRU70rTvDrW9lpupOAASUm+pwWjtwtYs6AyI1eOlM +5ecZlMHC5zhgngxHlQ1MKqMmvFeMJmPZlpG4xPiwXQXxoFvbQ9T6nRvcnxhB2R/p +wSxBvVwwX2Tf7h+f03XwVndC3CkPwdmwR3VsnVld7WPsLaTemUDKefZ6yTR0OVYZ +fN86bw5Ao9ZzjyH079fAvL2BAQ27I64WJz2480dJX9xgp9wzo91oYyy/ox6cA+ls +lPxUIvV/Se9mV0NhtkkdEgC0DidXyKZM79PsDTxnQUYiqF8p6YGv+bDbvBgyaoTJ +uJQ5MyEDKEl6PcTWq06mHWGF7iMCAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCLJtr15jan+ipoJ4MHQ8hHuj5SmMB8G +A1UdIwQYMBaAFImx3vMM84mQETGNTEnxspVKDakdMA0GCSqGSIb3DQEBCwUAA4IB +AQAhoPNnfpwuqjZClvtMNGZz4He2g09XO7RgE6uFBPG4DKZyJoRwNkrUrdqpkG13 +2D1lV8IUUzgRBqzP4JmVjmm6Vb7106V6epaZXQPar9PaM9pS/whf6G+5lXTVbn0G +skjMAuDeqZLIPh+547hvdI3WguJ+Lt+iqtlpVpqjPqySgG21H7CFMEZ7pKz7bKgc +Q7qzlvVNJc/XDE8nkuxfdBDZODQD3HEQXOVVGx/1x82HXy/EsTS6nBZnO9gOyj5H +gd/+uN2S5oIuxs1a1b1b54MuWZaFz9OGaqGr3CAjSZO1imIfgT4Ftfoa0FAFrSdO +pXpfpZLT5iOYwG+WxcYu1fW5 +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert1.key b/tests/integration/test_reload_ca_certificate/certs/cert1.key new file mode 100644 index 000000000000..61143a040b98 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert1.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe8Y2U9ivaHE5l +4Aik4EVO9K07w61vZabqTgAElJvqcFo7cLWLOgMiNXjpTOXnGZTBwuc4YJ4MR5UN +TCqjJrxXjCZj2ZaRuMT4sF0F8aBb20PU+p0b3J8YQdkf6cEsQb1cMF9k3+4fn9N1 +8FZ3QtwpD8HZsEd1bJ1ZXe1j7C2k3plAynn2esk0dDlWGXzfOm8OQKPWc48h9O/X +wLy9gQENuyOuFic9uPNHSV/cYKfcM6PdaGMsv6MenAPpbJT8VCL1f0nvZldDYbZJ +HRIAtA4nV8imTO/T7A08Z0FGIqhfKemBr/mw27wYMmqEybiUOTMhAyhJej3E1qtO +ph1hhe4jAgMBAAECggEAOTrp3OTqseVVTLqbjXOS5ydRNwfOxEtkcz5Nq99YPPDV +gO+4csKUHlp6rO0UEWSUNr8pKuRGfiF8BjtYsKQXciPkkPpAuCylx69CWe3CfAIH +4irpXMcgQhJZQeN4Nruzd/Bk9Ji1YIHfPyXQlHHh4VqNqSui1GZq6A+ACogM2Ybe +Qu2tEKGYC7LIYATfcnWMF8QpdF+FDCQmjL8yZTz6WhCDTk+h58y/lDfFwzhCKEwx +Y3g69S16lgIRlzxqoXU9UR31ieGsG+op6hmgrdO57chTUF/+g3qS7fGoFN6g6gsm +GeanWfM2nHpM4A73+TF7aI8xf4IHIgkXYa/KEXXh4QKBgQDz+Kj7OEQvv2NIs+lY +RkYTl0ljof8IzFPp8qZEo9z0PpD/Nl9sphfoV/ZYj79Ijnb8bkPcK2aqn2f9Ortx +I28XtzZog/p4XVSLw0o6hRRENsZEpsxYebaGp6kighZM9JHPm67T4neQ5PcR0r4g +Kz6DpoN/bM+uiWiULIMBKeHlwwKBgQDp73x77kvFXGb/Xc67qjfT7H8gPdBcAajd +SGeQHP+AbvsxZaEp0hMSXHYoJRq4eSsA5h8BOHDjM39LZ4uvlcphQDpuCHC/cbUI +bt5PmuddY/YTDkvEi/WIZi9+HSuAZ+RnVgbZowuTV0/t776Fj8R/MefDuWSuiDvF +Bt2z1XZwIQKBgBGZxtcY4BJxxD/ietsbdsLDD1BYx4Vi+ErQbp5VFAOq39sJmSjF +csQYVHVfKXWakYr0iYDAwM9eYKosKomm/MTBOvOfUdqNISRUGm7OWv/w06zwO53G +ahycy97pc6JpontPx/URSX7yhcCLa5v2grQMtz/iIbl9wEWwUGMtGlbxAoGANQD3 +IplWf6w1Bg06JxklNxYxo5t91yrlGOYr2OJJHc+HiKSvRGt9uL5MY0Is8Lk7fiOl +yMACC+iCIhKe+rSkuy4zTvUInsfjrbp5Em5Vl7pradvmXO0dP79vaVKwpZJklOlP ++gXQPJ0e1hlpAJgXfH5RNe6OmmDxse2hU/q8sCECgYBmK1MbPoX0SbuzPmAT6iu8 +4spHZp/AsiWOAnXU42VgEJCRfH+086CvZdf8N7Q6cJDfxofO95syDIMkC/7bkTGT +WT+R+1s0ouZFBtcX+6uPf9eR0/5W3SCzV1YkQtw6iXhoFociRdXDGO8JON80+x8e +BWtbo3oEMlwc6/rNAELLbQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert2.crt b/tests/integration/test_reload_ca_certificate/certs/cert2.crt new file mode 100644 index 000000000000..4fb702bf143b --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert2.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUIcogGF2yo/W8Z+f1vXaSVWnpDO4wDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMjAeFw0yNjA4MzExNzI4MTVaFw0zNjA4MjgxNzI4MTVaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAJKHwEUY5CUuRs7G9Zv+slTng5op18vSYWpj9m1/VagGhkRLlp6FGqf0+pnb +m1A0T0sXKVeuKFgJWqkTTJ5BuP2IOnS+HsfJ2oc0g2vRpO8v0mtZBdT8KA/W6fur +bxQeUOAoEIGca1r0ha/nRAdwvptFRIiIbDgIe9NXR7DQVBWe7Q+DxpzDhjR3/EvZ +W1/jjvTyg/WRhnM3WhrxfeBrHm6o1d8SsDR1t2FTe+7tiBh4ItI1ZWQUSpPhs9qf +LGe5jDx0+9fy4bj8isyMfGvnT+Ll5QFQn34DZrhQ+CAnU6AukZw2Bc3tIfLqtx28 +dh1VUDl0j/BRdbXerEzla1AVCb8CAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCjHe0KJ1rhLUhZfsMAtv5KSBYC0MB8G +A1UdIwQYMBaAFFDJNzL04BIG4VypL3bAn0ua13B6MA0GCSqGSIb3DQEBCwUAA4IB +AQAbU63n0EM1JWgCDaHYTo9FvnCmnmc1Ge2Nnz5lCwwom5kzB7GPjHljsx3L93OG +ETPbFD1On4OZI9ynURh+9mGlTAE3RiCHT+UjnHnQlKpn0mEjNihXGGM9iMIe4a/q +iVM2To3ayfjuMviLqgpRk48941rEbSSlnaZu+9bXG87QfRdRww2brRF1YOh3IvdT +lawdGc4Rlw/Hh9XjEBJ4u9dtRneoScKI//dzkTKuY0Jj3LeWRq8quX4+DnJZaBlx +vhU9YHstiNIooT1WGlHfNErOB7rsjWbAexWAdIo4gJVTM8caQtpaYPbOBxvCMaMX +/GD8THgD62RZxT80PWXdr4RW +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/cert2.key b/tests/integration/test_reload_ca_certificate/certs/cert2.key new file mode 100644 index 000000000000..beeebe86de1f --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/cert2.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCSh8BFGOQlLkbO +xvWb/rJU54OaKdfL0mFqY/Ztf1WoBoZES5aehRqn9PqZ25tQNE9LFylXrihYCVqp +E0yeQbj9iDp0vh7HydqHNINr0aTvL9JrWQXU/CgP1un7q28UHlDgKBCBnGta9IWv +50QHcL6bRUSIiGw4CHvTV0ew0FQVnu0Pg8acw4Y0d/xL2Vtf44708oP1kYZzN1oa +8X3gax5uqNXfErA0dbdhU3vu7YgYeCLSNWVkFEqT4bPanyxnuYw8dPvX8uG4/IrM +jHxr50/i5eUBUJ9+A2a4UPggJ1OgLpGcNgXN7SHy6rcdvHYdVVA5dI/wUXW13qxM +5WtQFQm/AgMBAAECggEAF9DYR21vbhBH8kV0gl69Zb8H7O9zw2KAcoDTJY072FFQ +fbAwHQ1dku071UmpbuSZpgbv3Brn3sCNIOGMo0TjstLIoX83KdodL7rcHhLsdPMi +XE/opUCN/bO8wwAO+pz0yyy9MLvCLSiExAguzpYcvMg02+DEf2pNlJY2hj6jXqyq +PKQXqTdtclrTfsjLBDJfqysZA5TrEV8tglU4uYr7fD2q3iFJ90VsBOKZaNs5jEsC +TJfXYWOf5OngwUD5nVCWHfsltJyysbhHUPQJcsk4G2zzQdX9ASS6wMc67rBuz25B +EMzR0ZP2eGAYqhbmYa6oVr0kE1GnukxLkQqtHOeAuQKBgQDEKEu4CRri/fw+RXOB +S33jdEPFP5hzPhx/41bKQzvZ/VkVRH7SLxMGIQjgkd1j7Klx7o42iUJPUYtlN8Gc +0+HxmnUjAh81F2fAq+C3rqRo2D0KQT3izHPGKxfmyPhPXVKrL8FVXGWMv1wvSHpJ +HQ3M4KaD9GMKmnswwyPArM0OBwKBgQC/O5/bqJf+ltCVeE4q77VxsHNcrJfxwgn6 +1I6ZcoAUPPyywOAG4WLaiH41I3CPydFYLfDk6AnbO9JQybvnu69OPa+At++Dggm6 +MSAYvYigQ4RsZuc+kzaV9//zhWIVa5kcfljyo1L49cLf2uBz3JWziEqYsl4TmCtp +bs8oWME4iQKBgAJU5EmEujAWisgGtU/FIPLyL9gJYHuGMnqGrkJrOCvoKgXpsYQ4 +EQbSn7NjqHkGmCEFj+UwDny44GpMll2R2y6vAlNvNAXCiHYu1NX6GnQwldEoY17t +xTaGzprsqp7u4gus3qRwG7jnkWXye5mg4cgcp34MCp1Wpr42o5cntqxDAoGAKP89 +XDgercPjX8f06huNyJvNf5a41GmG/jFHiPoVH0Gb4y6aWJ9FNBiDBh1c6laX/NGM +jWZ5hniitBMrp5iDEsECuRO103mzYClb+jHX8pPG9f5xoOaqkyghxTFZP8JbhtJH +e20sQpddeeRQrkYiCeU0KNxEcuryk53f54RvmBECgYAO8Yo/NA6Cz7FIidmb4tL5 +n3frL1fdA7bY74ltjMJp4dCmvlezbKG2xdSKqDlEvfoWgRrZEBtjH/i5esZhzH/H +6z0rK/TmoLpW5HLmBJ5Fqmj5lWOvmtNVh63QcW/hTFOGRSzuESzTD1qafMjtAAvZ +HBvkHzbEbZ6tVlKpNnNZzg== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh b/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh new file mode 100755 index 000000000000..c487b3941426 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/generate_certs.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Generates the certificates for test_reload_ca_certificate: +# ca1.crt / ca1.key - first root CA +# ca2.crt / ca2.key - second, unrelated root CA +# cert1.crt / cert1.key - leaf certificate issued by ca1 +# cert2.crt / cert2.key - leaf certificate issued by ca2 +# Both leaf certificates are valid for all host names used in the test and carry no extended key usage, +# so they can be used as server, client and Keeper (Raft) certificates. +# ca.crt, node.crt, node.key - copies of ca1.crt, cert1.crt, cert1.key: the initial content of the files +# that the configs point to and that the test overwrites to rotate certificates. +set -e +cd "$(dirname "${BASH_SOURCE[0]}")" + +DAYS=3650 + +cat > leaf.cnf << 'EOC' +[req] +distinguished_name = dn +prompt = no +[dn] +CN = clickhouse-test +[v3_leaf] +basicConstraints = CA:FALSE +keyUsage = digitalSignature, keyEncipherment +subjectAltName = DNS:localhost, DNS:node, DNS:node1, DNS:node2, DNS:node3, IP:127.0.0.1 +EOC + +for i in 1 2; do + openssl req -x509 -newkey rsa:2048 -nodes -batch -sha256 -days $DAYS \ + -subj "/O=ClickHouse Test/CN=Test Root CA $i" -keyout ca$i.key -out ca$i.crt + + openssl req -newkey rsa:2048 -nodes -batch -config leaf.cnf -keyout cert$i.key -out cert$i.csr + openssl x509 -req -in cert$i.csr -CA ca$i.crt -CAkey ca$i.key -CAcreateserial -sha256 -days $DAYS \ + -extfile leaf.cnf -extensions v3_leaf -out cert$i.crt + rm -f cert$i.csr ca$i.srl +done +rm -f leaf.cnf + +cp ca1.crt ca.crt +cp cert1.crt node.crt +cp cert1.key node.key diff --git a/tests/integration/test_reload_ca_certificate/certs/node.crt b/tests/integration/test_reload_ca_certificate/certs/node.crt new file mode 100644 index 000000000000..3c2faa29525a --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/node.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDbjCCAlagAwIBAgIUB2rETXTp2XN0iWipA9fQo9alsGQwDQYJKoZIhvcNAQEL +BQAwMzEYMBYGA1UECgwPQ2xpY2tIb3VzZSBUZXN0MRcwFQYDVQQDDA5UZXN0IFJv +b3QgQ0EgMTAeFw0yNjA4MzExNzI4MTRaFw0zNjA4MjgxNzI4MTRaMBoxGDAWBgNV +BAMMD2NsaWNraG91c2UtdGVzdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBAN7xjZT2K9ocTmXgCKTgRU70rTvDrW9lpupOAASUm+pwWjtwtYs6AyI1eOlM +5ecZlMHC5zhgngxHlQ1MKqMmvFeMJmPZlpG4xPiwXQXxoFvbQ9T6nRvcnxhB2R/p +wSxBvVwwX2Tf7h+f03XwVndC3CkPwdmwR3VsnVld7WPsLaTemUDKefZ6yTR0OVYZ +fN86bw5Ao9ZzjyH079fAvL2BAQ27I64WJz2480dJX9xgp9wzo91oYyy/ox6cA+ls +lPxUIvV/Se9mV0NhtkkdEgC0DidXyKZM79PsDTxnQUYiqF8p6YGv+bDbvBgyaoTJ +uJQ5MyEDKEl6PcTWq06mHWGF7iMCAwEAAaOBkjCBjzAJBgNVHRMEAjAAMAsGA1Ud +DwQEAwIFoDA1BgNVHREELjAsgglsb2NhbGhvc3SCBG5vZGWCBW5vZGUxggVub2Rl +MoIFbm9kZTOHBH8AAAEwHQYDVR0OBBYEFCLJtr15jan+ipoJ4MHQ8hHuj5SmMB8G +A1UdIwQYMBaAFImx3vMM84mQETGNTEnxspVKDakdMA0GCSqGSIb3DQEBCwUAA4IB +AQAhoPNnfpwuqjZClvtMNGZz4He2g09XO7RgE6uFBPG4DKZyJoRwNkrUrdqpkG13 +2D1lV8IUUzgRBqzP4JmVjmm6Vb7106V6epaZXQPar9PaM9pS/whf6G+5lXTVbn0G +skjMAuDeqZLIPh+547hvdI3WguJ+Lt+iqtlpVpqjPqySgG21H7CFMEZ7pKz7bKgc +Q7qzlvVNJc/XDE8nkuxfdBDZODQD3HEQXOVVGx/1x82HXy/EsTS6nBZnO9gOyj5H +gd/+uN2S5oIuxs1a1b1b54MuWZaFz9OGaqGr3CAjSZO1imIfgT4Ftfoa0FAFrSdO +pXpfpZLT5iOYwG+WxcYu1fW5 +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_ca_certificate/certs/node.key b/tests/integration/test_reload_ca_certificate/certs/node.key new file mode 100644 index 000000000000..61143a040b98 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/certs/node.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDe8Y2U9ivaHE5l +4Aik4EVO9K07w61vZabqTgAElJvqcFo7cLWLOgMiNXjpTOXnGZTBwuc4YJ4MR5UN +TCqjJrxXjCZj2ZaRuMT4sF0F8aBb20PU+p0b3J8YQdkf6cEsQb1cMF9k3+4fn9N1 +8FZ3QtwpD8HZsEd1bJ1ZXe1j7C2k3plAynn2esk0dDlWGXzfOm8OQKPWc48h9O/X +wLy9gQENuyOuFic9uPNHSV/cYKfcM6PdaGMsv6MenAPpbJT8VCL1f0nvZldDYbZJ +HRIAtA4nV8imTO/T7A08Z0FGIqhfKemBr/mw27wYMmqEybiUOTMhAyhJej3E1qtO +ph1hhe4jAgMBAAECggEAOTrp3OTqseVVTLqbjXOS5ydRNwfOxEtkcz5Nq99YPPDV +gO+4csKUHlp6rO0UEWSUNr8pKuRGfiF8BjtYsKQXciPkkPpAuCylx69CWe3CfAIH +4irpXMcgQhJZQeN4Nruzd/Bk9Ji1YIHfPyXQlHHh4VqNqSui1GZq6A+ACogM2Ybe +Qu2tEKGYC7LIYATfcnWMF8QpdF+FDCQmjL8yZTz6WhCDTk+h58y/lDfFwzhCKEwx +Y3g69S16lgIRlzxqoXU9UR31ieGsG+op6hmgrdO57chTUF/+g3qS7fGoFN6g6gsm +GeanWfM2nHpM4A73+TF7aI8xf4IHIgkXYa/KEXXh4QKBgQDz+Kj7OEQvv2NIs+lY +RkYTl0ljof8IzFPp8qZEo9z0PpD/Nl9sphfoV/ZYj79Ijnb8bkPcK2aqn2f9Ortx +I28XtzZog/p4XVSLw0o6hRRENsZEpsxYebaGp6kighZM9JHPm67T4neQ5PcR0r4g +Kz6DpoN/bM+uiWiULIMBKeHlwwKBgQDp73x77kvFXGb/Xc67qjfT7H8gPdBcAajd +SGeQHP+AbvsxZaEp0hMSXHYoJRq4eSsA5h8BOHDjM39LZ4uvlcphQDpuCHC/cbUI +bt5PmuddY/YTDkvEi/WIZi9+HSuAZ+RnVgbZowuTV0/t776Fj8R/MefDuWSuiDvF +Bt2z1XZwIQKBgBGZxtcY4BJxxD/ietsbdsLDD1BYx4Vi+ErQbp5VFAOq39sJmSjF +csQYVHVfKXWakYr0iYDAwM9eYKosKomm/MTBOvOfUdqNISRUGm7OWv/w06zwO53G +ahycy97pc6JpontPx/URSX7yhcCLa5v2grQMtz/iIbl9wEWwUGMtGlbxAoGANQD3 +IplWf6w1Bg06JxklNxYxo5t91yrlGOYr2OJJHc+HiKSvRGt9uL5MY0Is8Lk7fiOl +yMACC+iCIhKe+rSkuy4zTvUInsfjrbp5Em5Vl7pradvmXO0dP79vaVKwpZJklOlP ++gXQPJ0e1hlpAJgXfH5RNe6OmmDxse2hU/q8sCECgYBmK1MbPoX0SbuzPmAT6iu8 +4spHZp/AsiWOAnXU42VgEJCRfH+086CvZdf8N7Q6cJDfxofO95syDIMkC/7bkTGT +WT+R+1s0ouZFBtcX+6uPf9eR0/5W3SCzV1YkQtw6iXhoFociRdXDGO8JON80+x8e +BWtbo3oEMlwc6/rNAELLbQ== +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper1.xml b/tests/integration/test_reload_ca_certificate/configs/keeper1.xml new file mode 100644 index 000000000000..427a4b8f6872 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper1.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper2.xml b/tests/integration/test_reload_ca_certificate/configs/keeper2.xml new file mode 100644 index 000000000000..5dac68a6af56 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper2.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 2 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/keeper3.xml b/tests/integration/test_reload_ca_certificate/configs/keeper3.xml new file mode 100644 index 000000000000..28a155e92631 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/keeper3.xml @@ -0,0 +1,36 @@ + + 0.0.0.0 + 0.0.0.0 + + 0 + 9181 + 3 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + trace + + + + true + + 1 + node1 + 9234 + + + 2 + node2 + 9234 + + + 3 + node3 + 9234 + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/ssl.xml b/tests/integration/test_reload_ca_certificate/configs/ssl.xml new file mode 100644 index 000000000000..3b93b0a7552a --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/ssl.xml @@ -0,0 +1,31 @@ + + 8443 + + + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + false + + relaxed + false + sslv2,sslv3 + true + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + false + relaxed + false + sslv2,sslv3 + true + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml b/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml new file mode 100644 index 000000000000..6a1077ce40b9 --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/configs/ssl_with_default_cas.xml @@ -0,0 +1,29 @@ + + 8443 + + + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + + relaxed + false + sslv2,sslv3 + true + + + /etc/clickhouse-server/config.d/node.crt + /etc/clickhouse-server/config.d/node.key + /etc/clickhouse-server/config.d/ca.crt + relaxed + false + sslv2,sslv3 + true + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_reload_ca_certificate/test.py b/tests/integration/test_reload_ca_certificate/test.py new file mode 100644 index 000000000000..473a2e850e2c --- /dev/null +++ b/tests/integration/test_reload_ca_certificate/test.py @@ -0,0 +1,275 @@ +""" +Hot reload of the trusted CA certificates (`openSSL.*.caConfig`). + +The certificates in certs/ are produced by certs/generate_certs.sh: +two unrelated root CAs (ca1, ca2) and one leaf certificate issued by each of them (cert1, cert2). +Every instance starts with ca.crt = ca1 as the trusted CA and node.crt/node.key = cert1 as its own certificate, +and the tests overwrite these files in the container to rotate them without restarting anything. +""" + +import time +import uuid + +import pytest + +import helpers.keeper_utils as ku +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +CONFIG_DIR = "/etc/clickhouse-server/config.d" +CERT_FILES = [ + "certs/ca.crt", + "certs/ca1.crt", + "certs/ca2.crt", + "certs/node.crt", + "certs/node.key", + "certs/cert1.crt", + "certs/cert1.key", + "certs/cert2.crt", + "certs/cert2.key", +] + +# Serves HTTPS and acts as a TLS client towards itself. +node = cluster.add_instance("node", main_configs=["configs/ssl.xml"] + CERT_FILES) + +# Does not set `loadDefaultCAFile`, and OpenSSL's default CA file is ca2 for this instance. +node_with_default_cas = cluster.add_instance( + "node_with_default_cas", + main_configs=["configs/ssl_with_default_cas.xml"] + CERT_FILES, + env_variables={"SSL_CERT_FILE": f"{CONFIG_DIR}/ca2.crt"}, +) + +# Three nodes with embedded Keeper talking Raft over TLS. `loadDefaultCAFile` is not set for them: Keeper assumes `false` for +# the Raft connections then, unlike everything else, and their CA certificates have to be reloaded all the same. +keeper_nodes = [ + cluster.add_instance(f"node{i}", main_configs=[f"configs/keeper{i}.xml", "configs/ssl_with_default_cas.xml"] + CERT_FILES) for i in (1, 2, 3) +] + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def set_trusted_cas(instance, *cas): + """Overwrite ca.crt (the configured `caConfig`) with the given CA certificates.""" + sources = " ".join(f"{CONFIG_DIR}/{ca}.crt" for ca in cas) + instance.exec_in_container(["bash", "-c", f"cat {sources} > {CONFIG_DIR}/ca.crt.tmp && mv {CONFIG_DIR}/ca.crt.tmp {CONFIG_DIR}/ca.crt"]) + + +def set_own_certificate(instance, cert): + """Overwrite node.crt/node.key (the configured `certificateFile`/`privateKeyFile`) with the given leaf certificate.""" + instance.exec_in_container( + [ + "bash", + "-c", + f"cp {CONFIG_DIR}/{cert}.crt {CONFIG_DIR}/node.crt.tmp && mv {CONFIG_DIR}/node.crt.tmp {CONFIG_DIR}/node.crt && " + f"cp {CONFIG_DIR}/{cert}.key {CONFIG_DIR}/node.key.tmp && mv {CONFIG_DIR}/node.key.tmp {CONFIG_DIR}/node.key", + ] + ) + + +@pytest.fixture(autouse=True) +def restore_certificates(started_cluster): + yield + for instance in [node, node_with_default_cas] + keeper_nodes: + set_trusted_cas(instance, "ca1") + set_own_certificate(instance, "cert1") + instance.query("SYSTEM RELOAD CONFIG") + for instance in keeper_nodes: + kill_raft_connections(instance) + ku.wait_nodes(cluster, keeper_nodes) + + +def https_request_with_client_certificate(cert, instance=node): + """Query the HTTPS port of `instance` presenting the given client certificate. Returns the response, or None if the TLS handshake failed.""" + result = instance.exec_in_container( + [ + "bash", + "-c", + f"curl --silent --show-error --cacert {CONFIG_DIR}/ca1.crt --cert {CONFIG_DIR}/{cert}.crt --key {CONFIG_DIR}/{cert}.key " + f"'https://localhost:8443/?query=SELECT%201' 2>&1 || echo CURL_FAILED", + ] + ) + return None if "CURL_FAILED" in result else result + + +def assert_eventually(predicate, description, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.5) + assert predicate(), description + + +def test_server_reloads_ca(started_cluster): + """The CAs used to verify client certificates follow the content of `caConfig` without a restart or an explicit reload.""" + assert https_request_with_client_certificate("cert1") == "1\n" + assert https_request_with_client_certificate("cert2") is None + + # Trust both CAs: the file change alone triggers the reload. + set_trusted_cas(node, "ca1", "ca2") + assert_eventually(lambda: https_request_with_client_certificate("cert2") == "1\n", "cert2 is accepted after ca2 was added") + assert https_request_with_client_certificate("cert1") == "1\n" + + # Drop the old CA: certificates issued by it are not accepted anymore. + set_trusted_cas(node, "ca2") + assert_eventually(lambda: https_request_with_client_certificate("cert1") is None, "cert1 is rejected after ca1 was removed") + assert https_request_with_client_certificate("cert2") == "1\n" + + assert node.contains_in_log("Reloaded CA certificates") + + +def test_client_reloads_ca(started_cluster): + """The CAs used to verify server certificates of outgoing connections follow the content of `caConfig`.""" + # `node` connects to its own HTTPS port, which serves cert1. `Connection: close` rules out reusing a pooled connection. + query = "SELECT * FROM url('https://localhost:8443/?query=SELECT%201', 'TSV', 'x UInt8', headers('Connection'='close'))" + + set_trusted_cas(node, "ca2") + node.query("SYSTEM RELOAD CONFIG") + error = node.query_and_get_error(query) + assert "certificate verify failed" in error, error + + set_trusted_cas(node, "ca1") + node.query("SYSTEM RELOAD CONFIG") + assert node.query(query) == "1\n" + + +def test_default_cas_are_kept(started_cluster): + """With `loadDefaultCAFile` left at its default, both the CAs from `caConfig` and the default ones are trusted, before and after a reload.""" + # cert1 is trusted through `caConfig` (ca.crt = ca1), cert2 through the default CA file (SSL_CERT_FILE = ca2). + assert https_request_with_client_certificate("cert1", node_with_default_cas) == "1\n" + assert https_request_with_client_certificate("cert2", node_with_default_cas) == "1\n" + + reloads = int(node_with_default_cas.count_in_log("Reloaded CA certificates").strip()) + set_trusted_cas(node_with_default_cas, "ca1") # same content, new modification time + node_with_default_cas.query("SYSTEM RELOAD CONFIG") + assert int(node_with_default_cas.count_in_log("Reloaded CA certificates").strip()) > reloads + + assert https_request_with_client_certificate("cert1", node_with_default_cas) == "1\n" + assert https_request_with_client_certificate("cert2", node_with_default_cas) == "1\n" + + +def test_system_certificates_follows_reload(started_cluster): + """`system.certificates` shows the CA certificates that are currently used, also after `caConfig` is changed to another file.""" + query = "SELECT path, subject LIKE '%Test Root CA {}%' FROM system.certificates WHERE NOT default" + assert node.query(query.format(1)) == f"{CONFIG_DIR}/ca.crt\t1\n" + + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca.crt", f"{CONFIG_DIR}/ca2.crt") + try: + node.query("SYSTEM RELOAD CONFIG") + assert node.query(query.format(2)) == f"{CONFIG_DIR}/ca2.crt\t1\n" + finally: + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca2.crt", f"{CONFIG_DIR}/ca.crt") + node.query("SYSTEM RELOAD CONFIG") + + +def test_ca_directory(started_cluster): + """`caConfig` can be a directory with certificates named by their subject hash. Replacing a certificate in it is noticed too.""" + ca_dir = f"{CONFIG_DIR}/ca_dir" + node.exec_in_container(["bash", "-c", f"mkdir -p {ca_dir} && cp {CONFIG_DIR}/ca1.crt {ca_dir}/$(openssl x509 -noout -subject_hash -in {CONFIG_DIR}/ca1.crt).0"]) + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", f"{CONFIG_DIR}/ca.crt", ca_dir) + try: + node.query("SYSTEM RELOAD CONFIG") + assert https_request_with_client_certificate("cert1") == "1\n" + assert https_request_with_client_certificate("cert2") is None + + # Overwrite the only file in place: the set of file names in the directory does not change. + node.exec_in_container(["bash", "-c", f"cat {CONFIG_DIR}/ca2.crt > {ca_dir}/$(openssl x509 -noout -subject_hash -in {CONFIG_DIR}/ca1.crt).0"]) + node.query("SYSTEM RELOAD CONFIG") + assert https_request_with_client_certificate("cert1") is None + finally: + node.replace_in_config(f"{CONFIG_DIR}/ssl.xml", ca_dir, f"{CONFIG_DIR}/ca.crt") + node.exec_in_container(["rm", "-rf", ca_dir]) + node.query("SYSTEM RELOAD CONFIG") + + +def kill_raft_connections(instance): + instance.exec_in_container( + ["bash", "-c", "ss --kill -tn state established '( dport = :9234 or sport = :9234 )' > /dev/null"], nothrow=True + ) + + +def raft_port_accepts_client_certificate(instance, target, cert): + """Whether the Raft port of `target` completes a TLS handshake with a client that presents `cert`.""" + # With TLS 1.2 the server verifies the client certificate before the handshake completes on the client side, + # so a rejected certificate reliably shows up as a failed handshake in s_client. + result = instance.exec_in_container( + [ + "bash", + "-c", + f"openssl s_client -brief -tls1_2 -connect {target.name}:9234 -cert {CONFIG_DIR}/{cert}.crt -key {CONFIG_DIR}/{cert}.key " + f"&1 || true", + ] + ) + return "CONNECTION ESTABLISHED" in result + + +def raft_port_certificate_issuer(instance, target): + return instance.exec_in_container( + [ + "bash", + "-c", + f"openssl s_client -connect {target.name}:9234 /dev/null | openssl x509 -noout -issuer 2>/dev/null || true", + ] + ).strip() + + +def check_keeper_cluster_works(path): + connections = [] + try: + for instance in keeper_nodes: + connections.append(ku.get_fake_zk(cluster, instance.name)) + connections[0].create(path, b"data") + for connection in connections: + connection.sync(path) + assert connection.get(path)[0] == b"data" + finally: + for connection in connections: + connection.stop() + connection.close() + + +def reload_and_reconnect_raft(): + for instance in keeper_nodes: + instance.query("SYSTEM RELOAD CONFIG") + for instance in keeper_nodes: + kill_raft_connections(instance) + ku.wait_nodes(cluster, keeper_nodes) + + +def test_keeper_raft_reloads_ca(started_cluster): + """Rotate the CA of the Raft connections between Keeper nodes without restarting them.""" + run = uuid.uuid4().hex + ku.wait_nodes(cluster, keeper_nodes) + check_keeper_cluster_works(f"/before_{run}") + assert "Test Root CA 1" in raft_port_certificate_issuer(keeper_nodes[0], keeper_nodes[1]) + + # 1. Trust the new CA in addition to the old one. + for instance in keeper_nodes: + set_trusted_cas(instance, "ca1", "ca2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/both_cas_{run}") + + # 2. Switch the nodes to certificates issued by the new CA. + for instance in keeper_nodes: + set_own_certificate(instance, "cert2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/new_certs_{run}") + for target in keeper_nodes[1:]: + assert "Test Root CA 2" in raft_port_certificate_issuer(keeper_nodes[0], target) + + # 3. Stop trusting the old CA. + for instance in keeper_nodes: + set_trusted_cas(instance, "ca2") + reload_and_reconnect_raft() + check_keeper_cluster_works(f"/new_ca_{run}") + + assert raft_port_accepts_client_certificate(keeper_nodes[0], keeper_nodes[1], "cert2") + assert not raft_port_accepts_client_certificate(keeper_nodes[0], keeper_nodes[1], "cert1") diff --git a/tests/integration/test_s3_cluster/test.py b/tests/integration/test_s3_cluster/test.py index 30d5a6190217..76b8f0df2881 100644 --- a/tests/integration/test_s3_cluster/test.py +++ b/tests/integration/test_s3_cluster/test.py @@ -2,23 +2,21 @@ import logging import os import shutil -import uuid +import time +from email.errors import HeaderParseError import pytest from helpers.cluster import ClickHouseCluster -from helpers.config_cluster import minio_access_key, minio_secret_key +from helpers.config_cluster import minio_secret_key from helpers.mock_servers import start_mock_servers from helpers.test_tools import TSV -from helpers.utility import random_string logging.getLogger().setLevel(logging.INFO) logging.getLogger().addHandler(logging.StreamHandler()) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) - -# Base CSV files committed to git. Immutable on disk, safe to read concurrently. -S3_BASE_FILES = [ +S3_DATA = [ "data/clickhouse/part1.csv", "data/clickhouse/part123.csv", "data/database/part2.csv", @@ -26,48 +24,14 @@ ] -def _generated_host_dir(): - """Per-xdist-worker host directory for the dynamically generated CSV files. - - Concurrent xdist workers running this module (especially under - `--dist=each`, used by the targeted/flaky integration jobs) all execute - this fixture in parallel. Their MinIO containers are isolated by - `ClickHouseCluster` (it appends `PYTEST_XDIST_WORKER` to `project_name`), - but the host filesystem path is not. With a shared host directory, one - worker's teardown (`shutil.rmtree(...)`) deletes files while another - worker is still uploading them, producing setup-phase - `FileNotFoundError` on `data/generated/file_*.csv` and failing every - test in the module via fixture error. - - The MinIO bucket key for tests stays `data/generated/file_N.csv` (each - worker uploads to its own MinIO instance); only the host write/read path - is scoped per worker. - """ - worker_id = os.environ.get("PYTEST_XDIST_WORKER", "") - suffix = f"_{worker_id}" if worker_id else "" - return os.path.join(SCRIPT_DIR, f"data/generated{suffix}") - - def create_buckets_s3(cluster): minio = cluster.minio_client - host_dir = _generated_host_dir() - os.makedirs(host_dir, exist_ok=True) - - files_to_upload = [] # list of (bucket_object_key, host_file_path) - - # Base files: committed in git, read from the shared SCRIPT_DIR location. - for relative_path in S3_BASE_FILES: - files_to_upload.append( - (relative_path, os.path.join(SCRIPT_DIR, relative_path)) - ) - - # Generated files: written into a per-worker host directory but uploaded - # to MinIO under the canonical `data/generated/` prefix that tests query. for file_number in range(100): - bucket_object_key = f"data/generated/file_{file_number}.csv" - host_path = os.path.join(host_dir, f"file_{file_number}.csv") - with open(host_path, "w+", encoding="utf-8") as f: + file_name = f"data/generated/file_{file_number}.csv" + os.makedirs(os.path.join(SCRIPT_DIR, "data/generated/"), exist_ok=True) + S3_DATA.append(file_name) + with open(os.path.join(SCRIPT_DIR, file_name), "w+", encoding="utf-8") as f: # a String, b UInt64 data = [] @@ -79,13 +43,12 @@ def create_buckets_s3(cluster): writer = csv.writer(f) writer.writerows(data) - files_to_upload.append((bucket_object_key, host_path)) - for bucket_object_key, host_path in files_to_upload: + for file in S3_DATA: minio.fput_object( bucket_name=cluster.minio_bucket, - object_name=bucket_object_key, - file_path=host_path, + object_name=file, + file_path=os.path.join(SCRIPT_DIR, file), ) for obj in minio.list_objects(cluster.minio_bucket, recursive=True): print(obj.object_name) @@ -104,7 +67,6 @@ def run_s3_mocks(started_cluster): @pytest.fixture(scope="module") def started_cluster(): - cluster = None try: cluster = ClickHouseCluster(__file__) cluster.add_instance( @@ -140,10 +102,8 @@ def started_cluster(): yield cluster finally: - # Only this worker's directory; never touches other workers' data. - shutil.rmtree(_generated_host_dir(), ignore_errors=True) - if cluster is not None: - cluster.shutdown() + shutil.rmtree(os.path.join(SCRIPT_DIR, "data/generated/")) + cluster.shutdown() def test_select_all(started_cluster): @@ -549,218 +509,3 @@ def test_cluster_default_expression(started_cluster): ) assert result == expected_result - - -@pytest.mark.parametrize("allow_experimental_analyzer", [0, 1]) -@pytest.mark.parametrize("use_partition_strategy", [False, True]) -def test_hive_partitioning(started_cluster, allow_experimental_analyzer, use_partition_strategy): - node = started_cluster.instances["s0_0_0"] - - data_path = f"root/data/hive_{allow_experimental_analyzer}/{random_string(6)}" - - if use_partition_strategy: - node.query( - f""" - INSERT - INTO FUNCTION s3( - 'http://minio1:9001/{data_path}', - 'minio', - '{minio_secret_key}', - format='Parquet', - partition_strategy='hive', - structure='key Int32, value Int32') - PARTITION BY key - SELECT number+1 as key, number+1 as value - FROM system.numbers - LIMIT 4 - """ - ) - else: - for i in range(1, 5): - node.query( - f""" - INSERT - INTO FUNCTION s3('http://minio1:9001/{data_path}/key={i}/data.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - SELECT {i}, {i} - """ - ) - - # With use_partition_strategy=1 and use_hive_partitioning=0 we have no 'key' column - if not use_partition_strategy: - query_id_full = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3('http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 0, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_full, - ) - result = int(result) - assert result == 2 - - query_id_optimized = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3('http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 1, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_optimized, - ) - result = int(result) - assert result == 2 - - if not use_partition_strategy: - query_id_cluster_full = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3Cluster(cluster_simple, 'http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 0, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_cluster_full, - ) - result = int(result) - assert result == 2 - - query_id_cluster_optimized = str(uuid.uuid4()) - result = node.query( - f""" - SELECT count() - FROM s3Cluster(cluster_simple, 'http://minio1:9001/{data_path}/key=**.parquet', 'minio', '{minio_secret_key}', 'Parquet', 'key Int32, value Int32') - WHERE key <= 2 - FORMAT TSV - SETTINGS enable_filesystem_cache = 0, use_query_cache = 0, use_cache_for_count_from_files = 0, use_hive_partitioning = 1, allow_experimental_analyzer={allow_experimental_analyzer} - """, - query_id=query_id_cluster_optimized, - ) - result = int(result) - assert result == 2 - - node.query("SYSTEM FLUSH LOGS ON CLUSTER 'cluster_simple'") - - event = "EngineFileLikeReadFiles" - - if not use_partition_strategy: - full_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_full}' - FORMAT TSV - """ - ) - full_traffic = int(full_traffic) - assert full_traffic == 4 - - optimized_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_optimized}' - FORMAT TSV - """ - ) - optimized_traffic = int(optimized_traffic) - assert optimized_traffic == 2 - - if not use_partition_strategy: - cluster_full_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_cluster_full}' - FORMAT TSV - """ - ) - cluster_full_traffic = int(cluster_full_traffic) - assert cluster_full_traffic == full_traffic - - cluster_optimized_traffic = node.query( - f""" - SELECT sum(ProfileEvents['{event}']) - FROM clusterAllReplicas(cluster_simple, system.query_log) - WHERE type='QueryFinish' AND initial_query_id='{query_id_cluster_optimized}' - FORMAT TSV - """ - ) - cluster_optimized_traffic = int(cluster_optimized_traffic) - assert cluster_optimized_traffic == optimized_traffic - -def test_iceberg_s3_cluster_read_task_failpoint(started_cluster): - """INSERT INTO SELECT FROM icebergS3Cluster fails and does not hang. - Reproduces the scenario from https://github.com/ClickHouse/ClickHouse/issues/98165 - """ - node = started_cluster.instances["s0_0_0"] - run_id = uuid.uuid4().hex[:8] - iceberg_table = f"iceberg_src_{run_id}" - dst_table = f"local_dst_{run_id}" - iceberg_url = f"http://minio1:9001/root/{iceberg_table}/" - - # Create and populate the source Iceberg table before enabling the failpoint. - node.query( - f""" - CREATE TABLE {iceberg_table} (id UInt64, data String) - ENGINE = IcebergS3('{iceberg_url}', '{minio_access_key}', '{minio_secret_key}') - """ - ) - node.query( - f"INSERT INTO {iceberg_table} SELECT number AS id, randomString(10) AS data FROM numbers(50)", - settings={"allow_insert_into_iceberg": 1}, - ) - - # Local destination table — we only care that the query fails, not the data. - node.query( - f""" - CREATE TABLE {dst_table} (id UInt64, data String) - ENGINE = MergeTree() ORDER BY id - """ - ) - - all_node_names = ["s0_0_0", "s0_0_1", "s0_1_0"] - try: - for name in all_node_names: - started_cluster.instances[name].query( - "SYSTEM ENABLE FAILPOINT storage_cluster_read_sleep" - ) - - # The failpoint sleeps 10s uninterruptibly before max_execution_time can - # fire, so the server's legitimate abort lands well after 5s. The client - # timeout must outlast that abort (plus sanitizer/runner slowdown), not - # the 5s execution limit, otherwise a slow runner raises a false - # "Client timed out!". - _, error = node.query_and_get_answer_with_error( - f""" - INSERT INTO {dst_table} - SELECT * FROM icebergS3Cluster( - 'cluster_simple', - '{iceberg_url}', - '{minio_access_key}', '{minio_secret_key}') - SETTINGS max_execution_time = 5 - """, - timeout=120, - ) - - assert error, ( - "Expected a timeout error but the query succeeded. " - "The `storage_cluster_read_sleep` failpoint should have caused the query " - "to hang inside ReadTaskIterator::next() on every worker, " - "and `max_execution_time` should have aborted it." - ) - assert "Timeout" in error or "timeout" in error.lower(), ( - f"Expected a timeout-related error, got: {error}" - ) - finally: - for name in all_node_names: - started_cluster.instances[name].query( - "SYSTEM DISABLE FAILPOINT storage_cluster_read_sleep" - ) - node.query(f"DROP TABLE IF EXISTS {dst_table}") - node.query(f"DROP TABLE IF EXISTS {iceberg_table}") diff --git a/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml b/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml new file mode 100644 index 000000000000..f8e015f0b761 --- /dev/null +++ b/tests/integration/test_s3_style_link/configs/config.d/parallel_replicas.xml @@ -0,0 +1,23 @@ + + + + + + + http://minio1:9001/root/ + minio + ClickHouse_Minio_P@ssw0rd + + + + + + + node9000 + node9000 + node9000 + + + + diff --git a/tests/integration/test_s3_style_link/test.py b/tests/integration/test_s3_style_link/test.py index da7e64676d17..353c3a0764bf 100644 --- a/tests/integration/test_s3_style_link/test.py +++ b/tests/integration/test_s3_style_link/test.py @@ -10,6 +10,7 @@ "node", main_configs=[ "configs/config.d/minio.xml", + "configs/config.d/parallel_replicas.xml", ], user_configs=[ "configs/users.d/users.xml", @@ -139,3 +140,55 @@ def test_s3_question_mark_wildcards(started_cluster): assert result_s3_scheme == result_http_scheme assert result_s3_scheme.startswith('20\t') assert "['a1','a2']" in result_s3_scheme or "['a2','a1']" in result_s3_scheme + + +def test_url_s3_scheme_with_parallel_replicas(started_cluster): + """ + `url('s3://...')` is delegated to the `s3` backend, but the query text still names `url`. + The cluster fan-out of `parallel_replicas_for_cluster_engines` rewrites the forwarded query + from that surface name, so it used to send `urlCluster('s3://...')` - a shape `urlCluster` + rejects - both for a plain `SELECT` and for the distributed `INSERT ... SELECT`. + """ + node.query( + f""" + INSERT INTO FUNCTION s3 + ( + 'minio://data/parallel_replicas_url.csv', 'minio', '{minio_secret_key}', + 'CSV', 'a UInt32' + ) SETTINGS s3_truncate_on_insert=1 + SELECT number FROM numbers(10); + """ + ) + + parallel_replicas_settings = """ + SETTINGS cluster_for_parallel_replicas = 'parallel_replicas', + enable_parallel_replicas = 1, + max_parallel_replicas = 3, + parallel_replicas_for_cluster_engines = 1 + """ + + assert ( + node.query( + f""" + SELECT count() FROM url('s3://data/parallel_replicas_url.csv', 'CSV', 'a UInt32') + {parallel_replicas_settings} + """ + ) + == "10\n" + ) + + node.query("DROP TABLE IF EXISTS url_s3_parallel_replicas SYNC") + node.query( + "CREATE TABLE url_s3_parallel_replicas (a UInt32) ENGINE = MergeTree ORDER BY a" + ) + node.query( + f""" + INSERT INTO url_s3_parallel_replicas + SELECT * FROM url('s3://data/parallel_replicas_url.csv', 'CSV', 'a UInt32') + {parallel_replicas_settings}, parallel_distributed_insert_select = 2 + """ + ) + + # The rows must be inserted exactly once, not once per replica. + assert node.query("SELECT count() FROM url_s3_parallel_replicas") == "10\n" + node.query("DROP TABLE url_s3_parallel_replicas SYNC") diff --git a/tests/integration/test_ssh/test.py b/tests/integration/test_ssh/test.py index 5fa570bfe6c0..350aa0023645 100644 --- a/tests/integration/test_ssh/test.py +++ b/tests/integration/test_ssh/test.py @@ -1,6 +1,7 @@ import os import re import socket +import struct import subprocess import time @@ -495,3 +496,117 @@ def test_interactive_tab_completion_respects_session_user(started_cluster): instance.query("DROP USER IF EXISTS completer") instance.query("DROP TABLE IF EXISTS default.visible_completion_target") instance.query("DROP TABLE IF EXISTS default.hidden_completion_target") + + +def test_interactive_session_torn_down_with_a_dead_pty(started_cluster): + """Losing the pty while the embedded client is shutting down must not kill the server. + + `ReplxxLineReader::~ReplxxLineReader` writes an escape sequence to the + terminal to reset cursor blinking when overwrite mode was ever enabled. + `Replxx::print` throws `std::runtime_error("write failed")` when that write + does not go through, and a destructor is implicitly `noexcept`, so the + exception used to `std::terminate` the whole server process. + + Reproduce it the way a real disconnect does: turn overwrite mode on with + the `Insert` key, then drop the TCP connection with a RST so that the + server's side of the pty is gone by the time the line reader is destroyed. + """ + # The daemon watchdog restarts the server after `std::terminate`, so "the + # server answers queries again" is not evidence of anything, and neither is + # a growing `uptime()`: on a fresh cluster the uptime before the disconnect + # is only a few seconds, so a restarted server reaches a larger value well + # within the sampling window below. Pin the process identity instead: the + # watchdog restarts the server by forking a new child, so the set of + # `clickhouse-server` pids in the container changes and cannot recover. + # The pattern is anchored at argv0 so that the shell running `pgrep` (whose + # own command line contains the pattern) does not match itself. + def server_pids(): + return instance.exec_in_container( + ["bash", "-c", "pgrep -f '^[^ ]*clickhouse(-| )server' | sort -n"], + nothrow=True, + ).split() + + pids_before = server_pids() + assert pids_before, "no `clickhouse-server` process in the container" + uptime_before = float(instance.query("SELECT uptime()").strip()) + + pkey = paramiko.Ed25519Key.from_private_key_file(f"{SCRIPT_DIR}/keys/lucy_ed25519") + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=instance.ip_address, + port=9022, + username="lucy", + pkey=pkey, + timeout=30, + ) + try: + channel = client.invoke_shell(term="xterm", width=80, height=24) + channel.settimeout(20) + output = _read_channel_until(channel, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + # `Insert` toggles overwrite mode, which is what makes the destructor + # print the "reset cursor blinking" sequence in the first place. Wait + # for the raw `\033[5 q` ("blinking cursor") escape the key handler + # prints: it is the only observable proof that the server really + # consumed the key and that `overwrite_mode` became true. Without it + # the destructor writes nothing and the test would pass even unfixed. + channel.sendall("\x1b[2~") + raw = b"" + deadline = time.time() + 10 + while time.time() < deadline and b"\x1b[5 q" not in raw: + if channel.recv_ready(): + raw += channel.recv(65536) + else: + time.sleep(0.05) + assert ( + b"\x1b[5 q" in raw + ), f"overwrite mode was not enabled, the destructor would write nothing: {raw!r}" + + # Abort the connection with a RST instead of a graceful shutdown, so + # writes on the server side fail rather than being silently discarded. + sock = client.get_transport().sock + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + sock.close() + finally: + client.close() + + # The session teardown is asynchronous, so keep sampling for a while. A + # sample that fails only means the teardown is still in flight, but the + # *last* sample of the window must succeed and must still show the very + # same process: the server that survived the disconnect, not a fresh one. + last_failure = None + last_uptime = None + deadline = time.time() + 30 + while time.time() < deadline: + try: + uptime = float(instance.query("SELECT uptime()").strip()) + except Exception as e: # down or restarting — the next sample decides + last_failure = e + last_uptime = None + time.sleep(0.5) + continue + last_uptime = uptime + pids = server_pids() + assert pids == pids_before, ( + "the server process was replaced after the SSH disconnect, i.e. it " + "died while tearing the session down and was started again " + f"(pids {pids} != {pids_before})" + ) + assert uptime >= uptime_before, ( + "the server restarted after the SSH disconnect, i.e. it died while " + f"tearing the session down (uptime {uptime} < {uptime_before})" + ) + time.sleep(0.5) + + assert last_uptime is not None, ( + "the server did not answer a query at the end of the 30 s window after " + f"the SSH disconnect, i.e. it died while tearing the session down: {last_failure}" + ) + + # `from_host=True` also greps the rotated logs, so a restart cannot hide the + # fatal line by rotating it out of the active `clickhouse-server.log`. + assert not instance.contains_in_log( + "std::terminate", from_host=True + ), "the server called `std::terminate` while tearing down the SSH session" diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem new file mode 100644 index 000000000000..82687debfcde --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFPzCCAyegAwIBAgIUb8W/doRsWnXgJSqDgDz8Yb+cLVwwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBhMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRow +GAYDVQQDDBFjbGllbnQxAC5ldmlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBAOMBJ2yqK72eew557aeGID0RXFzVs6lLTN4/71C4/309gAcIoqsw +Phulwzj3IfH7v2PPntsROxGtRCVS5LWarXUTEviEaiWUp1MwoFlrhn2Dm78hIgYC +7NrdtI48tb9t9y6eTio99WdIiXGY2KNlctfnTyCxKeanZi7JZUmb9VW95gh/5dl3 +sajZPjUDrFxqUQbzsSzbgElFkjvOzN5xEoUM1qbjHcyML7aKweXJKhSswdi4K3oA +v/FbFleJQgoRPB43JDqc5YjpPARKJlt5VZqpkgiKFAm0R+o4TYeCBd0kZFtL+7Y3 +TO27ZdQiOFKR/DINPCdA3/nxtQ7GJ17g/nwXLQRKUDxXain7jo+87OHghMA8WfwQ +lMSkQ+sqJdxA/VunQEHNCljw64BHeuU6CqQDU9eEnFNq9+3Qb0INiMRn6EGj20LK +5+q/fJqGm/N8BcuoYHUymbtojAjnJmCIWwvTzo7OYaG84nRwOPkZZvcSOrzn2CbT +zhmXHiSnP7ZAL/r24ZW5oikjw6Ie40iqQ2MyknTv3Ko1LWIKkjAbMcE6fo3SlzMA +fVsm9wAyVa/UyjO1TcnPNNta1TdqSgdRsEPDvR/IMPQYJ7Be0H0SZLj2/v8ZmHmZ +KVfGKdWnFBQKiTSHVEBHBdXADcXi2grYjnOBKnQ7H2UQK9jfGMu4FPYlAgMBAAEw +DQYJKoZIhvcNAQELBQADggIBAF4piZQxvyxwfhQxmNbyc8EtUo5qBIWE026e5Hsl +4QX3glKhAo+xLEQbKAImsMdXXYNdGaBJDTEZRbGSvyEgVa8e4YF0FP+CkFDcM0oF +k7YUgQHkZvlR/lsd5idMM7JJ1WXnux6jay9hpF9GEi2QMB2QkwuRwBHkgtjEun8C +qdyvApSecY6BXQllAJyoalSVR4ZzaMGVgSGL99RKyn320icevO6WDOVocEiohJTK +KOwf8q9G6riKWsRbNNPhkt9QwZzC0/RuKV6r25QR7H+GSjmllcBnbyqW3+DGGuJL +iMtKrCmx4xHedvDXRxi8ZxsfIrWVI7f5hgcWCXFWFMeqkU9nD3TGJrAuamBBQMZn +aC3q9AyVudroJs/1ojrSD92+H3qSzhqoLZ4bd531ShPasQ/EHIiOeN794j3Rxo0d +ituD2LSboe2Zr4TNhi7ou/dSfDXPEwJSb5V1Xgp9OyhQCKnFa+RkbQVtME5dVBMR +oO4HMTqaCgYSzCKhrxIO/vid2p6NQp7ZI8V2hAhUR4aBwnWFFspmuWaXpZqXxE6+ +a46rTA4HLpQ7TKwbi4fS2zWSh64nCHmv72GOHQ86dlXDNvfqLo8vyz79chKPo+31 +fsTZJks9wBzeSsq0tqAWflN/hQQCZJT1IpIc5VwU45FLSraKBGYRpCP4FnciXUuJ +RiyF +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem new file mode 100644 index 000000000000..452e63223383 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA4wEnbKorvZ57Dnntp4YgPRFcXNWzqUtM3j/vULj/fT2ABwii +qzA+G6XDOPch8fu/Y8+e2xE7Ea1EJVLktZqtdRMS+IRqJZSnUzCgWWuGfYObvyEi +BgLs2t20jjy1v233Lp5OKj31Z0iJcZjYo2Vy1+dPILEp5qdmLsllSZv1Vb3mCH/l +2XexqNk+NQOsXGpRBvOxLNuASUWSO87M3nEShQzWpuMdzIwvtorB5ckqFKzB2Lgr +egC/8VsWV4lCChE8HjckOpzliOk8BEomW3lVmqmSCIoUCbRH6jhNh4IF3SRkW0v7 +tjdM7btl1CI4UpH8Mg08J0Df+fG1DsYnXuD+fBctBEpQPFdqKfuOj7zs4eCEwDxZ +/BCUxKRD6yol3ED9W6dAQc0KWPDrgEd65ToKpANT14ScU2r37dBvQg2IxGfoQaPb +Qsrn6r98moab83wFy6hgdTKZu2iMCOcmYIhbC9POjs5hobzidHA4+Rlm9xI6vOfY +JtPOGZceJKc/tkAv+vbhlbmiKSPDoh7jSKpDYzKSdO/cqjUtYgqSMBsxwTp+jdKX +MwB9Wyb3ADJVr9TKM7VNyc8021rVN2pKB1GwQ8O9H8gw9BgnsF7QfRJkuPb+/xmY +eZkpV8Yp1acUFAqJNIdUQEcF1cANxeLaCtiOc4EqdDsfZRAr2N8Yy7gU9iUCAwEA +AQKCAgAhsUz9M2q866tUaY4iXxAk5ZFBvtQ6NztzAfv5YrlFCPOaLP1xOX6rvYLd +eczCLP0/Xr8IwSTKdF7uHUC7Svhzgn++fsTUdse5Bg7JXDdregphUr/JMvZIa+n9 +xzlV7t07wEerZw59Txuwo8uFFlchKtX97QhiPDa4kLAlve4AKmiHRW3DmnY2MKcA +2Qk5nuwPg0Go/fFWBBX20N8WevjmjOfFKxDZXctCeujZkW3bVg2K0ZcJC4McWx/U +oUI52qIDfWgKrQ6NL3T20VyibL64bx0GLccDQ7dS0RrXbISF9Kin5lNiEbuAzjTd +yLB3DO+TgY8L58n2lUSsKzFdGI7yFjJTr6yzmKIayS11c00TCBQ0w/l+oCWOhrvM +zn/k3mrORcRA1aqX5mcqAdzs6hYFIyFbKXNH8hlE7JnUUbcvR76WbzMl6IxPxMT+ +ikbTIQnG+T2PMgmaxVbe9YhyHT2YbMHF7igocz5mI0PHsdNaB1IBSV1G2Knq8QJi +rbYUdCShTJksrNIXghyjbCPx5VIo05fx8NleCuNND18WPs32tI2rceLlF3KSk8GF +moCnPJFTtEgk5IvzWavOGO+abkU1A9xGOhNHCMZVxOca/ZGlhHSRBCuhagpLPx9e +QprxfFVIFG8jNrN0/Mhgv3ivKmbYZuv03wdcWDtKCdMrxkah9wKCAQEA8Xp+V4zk +Ta4nqkjL4TTv+uczszGt/Kb5OYZyb1EzUseabRyOFkb2iMUf+Ol/92UMBlh/FzTa +RFHE1lRed45l8Jhgsbaq9r8gmJZ1WZM2IUByUI++z+USlEtG8O+8wlniyTn9Fc+n +y+wkZwSeJSLWZAWIzeygJs/ceKG+4TpnLZnC2rQqOoKGc+0Dimc1E5HV3mOkhwe1 +CtiYOUvgNf0rEV7E5D7smjpAeiJ8xPhYRg0XXLztN+9rQA+6AvwpbI3VnyAKbPUQ +/J+ZVg0cVMH7pTDXMj0jB2wevrIa7xj2Pn4w/jR8i/tNjFauMMvnofr6RW6KqYk/ +opXhfSE1Pk9w6wKCAQEA8KfWOQEmKlC0KJrN1/D1ZchNZf57pNFtjyWdjLUah05k +a0QLUe1yme7lTw59tsFIfrd/ozeiWKSvhum9/mUNvwBgPjnqp73pmrEEu1bgQ8Ib +cCm55GAmBWlNMO52iD+EPdsYvj+KgAWxu6W8B6XsyjoVLBYqHwNzitZlW9D1PAbY +KWBppmntVAhj7oNvk40uWEFIuTU0o0RIrvcYmX795sr2PY69ozOdiY6UziR4oRAs +DhJ+0a8Sjct0/RMq+sOpBONrpOBHqJzoCX+/alOjbJ8FYIMuc6OU2cNtcxjL3tcZ +cJKInsMl7bMYhi0SIEnttuAgh6mzD4Iox1WX3hbxLwKCAQEAySWPHmCnQCnEsqzG +38aX7DkwsKC+XEm+KnPa2O0bwmWwNhlmJhpgfBcwBciDJtHODW8kFnGZKvWt8BcB +RbehJKPZT89oY/dbMJ+MCtx8Z4BmeML0X/ph2pNF+abJZl83cQVT0xpRnKUMwZ9w +GGEHkvOlPFtSIGJfNUEOXlCm565ASKtwzaIyW1hf7acA6Fc/fmsj/rKl1O4NBxU0 +I+TKPkLh4Xqk3eeOE+6sXeq8pUV8Y2ygcUqG3Sr8eEYSP3F6M32eEZql1rkfIjXz +loqrkrO3XgrdJe6jQZfcpbP2UqINL4MLCBOCSldd7Bm9zgjg9nsZGBXSox0UYoYJ +8uh7lwKCAQA+I3gi+/L41iHOojooWeVjRJcHkPAoHJNndNT7cf/JlCpFsCokG2WN +7at0AE/hkoK/hW4FnOXkcZGJCm2udDVabiRrrNS0P0tEUBTisonxtPsUuRFwsIrg +ttHhopEkmRHyTtJSvWFrsQy1YRPt/Z/oj5rL6WUy4NdCsB032fqYZ0QFWwmsmIlZ +O1liSrnSpY+j5id6+wv+ZDFITDEj4TB6GUn/lw3MSBWTYSd3Gt+y5tQZRhlM1yG5 +TyGD/yEH4uGPi5FN30NhfDJF0aCBOdtmvqDKzNR/s2tJ1zY5k9uATJYbBRsVs1nl +yGq6qoSVpcEliTWdEepURM12utkd1VqtAoIBAFLriLD2AF0+ayheiDFgsbRo78V3 +dygUPBZi7b6nPWv3gugnwvWrTFaCAndACxNZPWNwEcING2Utjp8PTvPTzad9yibl +3xqNSKR9gaqmDYPEY1DwYGCpHRc969PIYcyK6z3dICj1ySEo0RwTa+Ab0OHeRWzS +ROnmmy9a9Vpx7KyMZcmOGhSXL3q7nr2imI+qZ4tGiEJts85d2fXAVhGeBOagzy5n +uD00wYzBc8rZ9u+d7ZOyQetgoJBnhyQNzW7IBCN+Z3EWW5lPlHQJpTXjHCdXgiZU +28pX/4J/mIcaURA0xhexuuLqTEjLlFoAnjNjWCCXv0u5uP8Ba35g4BSAz0o= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem new file mode 100644 index 000000000000..25c2687c0edf --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIUTgWFBpS4cJ/+Fgg5jAfOubkwokowDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBmMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMR8w +HQYDVQQDDBZldmlsAC5jb3JwLmV4YW1wbGUuY29tMIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8x +qMXeoHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp +/X6eDAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1Ey +G5smL6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMU +vVfRZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NA +iZbZphA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43 +hAa0vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60S +rEQJ30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBB +fg+iti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRA +N13RUMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zC +SAb88JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMC +AwEAAaMlMCMwIQYDVR0RBBowGIIWZXZpbAAuY29ycC5leGFtcGxlLmNvbTANBgkq +hkiG9w0BAQsFAAOCAgEAFuyPgncpYKGYffQPoOWR2yTn6L56XxvQcFNrchyKkwAn +zsBQFLOX4T4z1FuIQTwwKiSRwz+hwuIfcK2++I1yyxgYzw8Cxw7uqpvRi6AgN9pk +LEsXbSOvqaQyFEukVFenbkIejBVexF74gk2LKJYnT7MRVgs/k8C5Lqy5lKpB/2Gk +7xnzVe6mLhdZrMRbtCmt+Pr2m5l+XMGucnoaNCK87zSIGEt8fCYNB//vQ61GeANv +hRYiWgdWcOCnl2ip+EWbq0+ZK2JvJ8Ih2d0S+Qtbhek8XQsuCoa5P3YjLyAfzhpk +/CZb+VbOH41qZ2uWi05xVE3IRbJBlRDnX/bXC4Nx2ojzOnWAqzmrJffsKal/04oP +BcA6dOzm+CvT7AlzImlRI/6ZEC6DlrXSkvm5yRNwdUhTZ5ZjQnSyZBF4lNY3u9Sa +G1ipxyR9wRcA7uac8cFv/MBAGP5godwi3WPvqC/7noGDbZqDxcsZOeVFdP+NFtFg +dilITLxlgaQimN2nxhPvAAz6EcJ2owFgJJk6J3f6ICq8gdbJzFq2uVJrKBBL35St +/aOpflDRhQkMdJwrCuI/+4NBcXG5fQywjH8H+9dvbUONIlttbwE2zZJgXtY0gb+A +kY8fmy5pWBYnYAB8lFdAZxW0Sx6ehkmLA/ExtH/7jz/GwnwS12/of6wSvQHj8so= +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem new file mode 100644 index 000000000000..10d1b89a2701 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8xqMXe +oHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp/X6e +DAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1EyG5sm +L6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMUvVfR +ZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NAiZbZ +phA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43hAa0 +vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60SrEQJ +30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBBfg+i +ti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRAN13R +UMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zCSAb8 +8JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMCAwEA +AQKCAgAEaIj8Y6VR/EQNyxFgQ7nRQC3VrU1jUM7CgttRbb4wEtFdGr3DojH9awnF +qGEac+2mp3XAtorZnu7oSEFdpH0F64kZro2OeuOAaUoVps/wHkNffOPT17AOxJCT +3OwBNmOho7F6cW1ipV+6U6hX4yK0sTBpdrr5iO9Uz6R35NIkehGIq93b/YCgwmXE +u5xFp1pSkfoaIwjskwE8Mx/Eh9mwi5OMVq6kvVBdvbp++4pmTnQL0UPKAOb/PIIA +ih+v80GniCXk//tzhBow2ISqQE4MKabt+REE5JNnjjA4wDf6C3Hh7lmYwX0VAi/Z +PrnVlzCvcBQEObhxFqMdIY+C9awzGrf0iJ1UGZms5IrcKwvEL7VsrQqO2COEv42K +KtbVNpRMYG5n+VwtylChmh2g6oX4wMLbYIykDfXZVv/FIjP3vgPxs19t/CRVUEer +x6GztwhAmKvp4WzerVH1xJ7QKcaIuy4cjhNLVUQCcjPbFbtHsHzvuScXHfppCM67 +6rpNQ5DrQ0Wj0a1kCal/57aKQQdD37k4aTTEoCbDJkbD10SoaljJYElHleB26xCV +AwbyXz6gwJ80CxbXS1uco3c86keDekrGV5f73XX53MZouRR90Ii/TJvvdGutXFkX +mpAIaOZlfeDSFMJoFaSpOj3UaY4RBwcXXDB69QpFMDe2ND0gwQKCAQEA6jw1G0dP +HwDYfQSMzTQkhle7hHqxPoyuooBe4Vl6MJ64S22tgvRLE+FuLfubR9tMBYYr/Ub4 +CmSBE/XMbNGpH31yct5I/8rc9OnRg0bcCw2dz/Q7b84LKas6fkBFaf1mMlPvH2zi +nwtS3Bw3Q6DAky1ra6rqjiZDtDCN+vQOh0zXVdOyVjApDBbngX/x8hVhFLs3SuwS +jm3P8YloEQ6aQ0wCjvsemJs+IpDlH7cHwYlgxBREqQ1wOKdo8IZ/MTTvlPOXW04F +pP995LCvtVacjhfOLaWdHTs442yuFvFBoPWFEvx2CGYTQe+dk8zmzLFdYVNpEvbV +dh/m1hMrA3lUdwKCAQEAxF1d+QuDanP39u+16u4+u61p3gzYv2f9m+bjViiYMv6N +kSDzdXrKidD1I+iEgIy1QjRfdHlQMCIpP8D4zuWCrNY57QKdV4bRKs0RSgpOXPv+ +FGeVf9taPIH/lUH8FUJ1gqfoLX+6W2RbZxcNPOSOf13o6jTjBbB6oVetljNjsnDS +us7CJe3+MDk20pO2+E49Q0TuyeALDPRSciSvJrVCVmoPs6rpu22z5zMCxd6+WztL +YBWfFM38QpIMUg1miz7hU7m7lQHlYE9/p9RSI5ePD9mgepPQN+bQg0fGQaD0tzad +MJr/BI0OvHLdwFQNLya66nWK7s85aD/UjDRO/A6kRQKCAQB1zGes21ToM6WsYeBp +xsJjqbWNb6K54UhmQwb0b+pqjzgB/xuW00L6sZGWoIW8QoZd9Ncknk9Z8qeToTb4 +twxF4PHw4Od3dM9ggEK0sasyB9wI3DwUA1xLzWgyXCJMpnqB7wJAHKNv9uLp/Wqx +oSOYIOx4DlG9wXKlKRIOVjUESFm3OSrj+355LP+qeez0oVnccjbhgA3pAULlpwPm +KCDenVhgDdyaROCfw5znMUY+R9eZZNQO7Mo2Q8Mby5gl6AhhMYw6B+gAzdjDbTRA +j1lWgJRZEoQMUl9OyLZYpWYrC66sGLlHigY/T8FAtniQEtbyfl9GgUpjCLIvkR49 +tgQLAoIBAF/WefD9D4y7QQDCifU5hmCvCIaZmogAxyR6EeaRNYdd+dYlUO27mnKd +C6gU6eabxjOjwBrmwp5bbepx0n2YQqj8fZURu51mbVwIbjHGyexUCPQIgky+0FHL +2OQOKmxt3VCBhq3+MwQ7/OhZtdpMasf7G5yDZ3H1akSouE4gkr4alp8aHmPIvlDm ++7zW32xdM0VLtYfN01blP//5q4qm2NO4PCWieyVBK5bhrK7KQfng/K7Onq/WwRH0 +mhLJ+4xmii8E7WqSXFMfOdy9ocFBTU+dFdf9oJhIDOil9Ts+xXFONHXukBy2g8Sy +A0zFORIUQxH/gGmBtjENRj2PoiUfOEkCggEBAMiBLM8L+k7ovGd7KVoeSgk7FjsV +5gq0GDwXGov9HEyeY/mmmuS5mfxZUYvRgzbpQutqy88MvbMsCS7VDtT9m0IFs8zR +qXIUQmOrYJuI37BrA16K5EDTj6wY/7DKRSM4guNqQjYnJ2bYBrXCZLWFUF/OVl3w +VF5reN7BCMOTXGc4aAeJzcmvhGU7M3WnGwL7yGkiX8BCDXo+ddLuf1SyDUttfrmo +Kbb4h4LAe2ATtRO0F9SfEetdmHUwO6iRJGQNQQWO7pWOjUStSry3Z8AM0uVDBRDG +Ol3DmKQlxbFAhLD62KkcWt30eNnNMlYaq8huKY4Y2juyi5ZQ31sDij4worA= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh index 19354a5a3e36..b8bbd34567a2 100755 --- a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -60,3 +60,76 @@ openssl x509 -req -days 36525 -in client_far_future-req.pem -CA ca-cert.pem -CAk # 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" + +# 7. Generate a CA-signed certificate whose CN carries an embedded NUL byte ("client1\0.evil.com"), +# to test that server-side CN extraction does not truncate at the NUL. User 'john' is configured with +# client1, so a server that truncated the CN at the NUL would extract +# "client1" and wrongly authenticate this certificate as 'john'; the full CN must be preserved so the +# match fails. openssl's CLI cannot place a NUL inside a -subj field, so we build this certificate +# with the 'cryptography' library instead. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "client1\x00.evil.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .sign(ca_key, hashes.SHA256())) +with open("client13-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client13-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY + +# 8. Generate a CA-signed certificate whose CN and DNS SAN both carry an embedded NUL byte +# ("evil\0.corp.example.com"), to test that wildcard matching rejects it. Users 'wildcard_cn' and +# 'wildcard_dns' are configured with '*.corp.example.com' and 'DNS:*.corp.example.com': without an +# explicit check, '*' would match the single "label" "evil\0" and authenticate this certificate. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "evil\x00.corp.example.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("evil\x00.corp.example.com")]), critical=False) + .sign(ca_key, hashes.SHA256())) +with open("client14-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client14-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 977bc82ea276..118172905b9a 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -147,6 +147,18 @@ def test_native_fallback_to_password(): assert "AUTHENTICATION_FAILED" in str(err.value) +def test_native_cn_nul_byte_no_bypass(): + # Authentication bypass: client13's CN is "client1\0.evil.com" and user 'john' is configured with + # client1. If server-side CN extraction truncated at the embedded NUL + # byte, the CN would collapse to "client1" and the certificate would authenticate as user 'john'. + # The full CN must be preserved, so the match must fail. + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="client13" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + + def get_ssl_context(cert_name): context = WrapSSLContextWithSNI(SSL_HOST, ssl.PROTOCOL_TLS_CLIENT) context.load_verify_locations(cafile=f"{SCRIPT_DIR}/certs/ca-cert.pem") @@ -221,6 +233,14 @@ def test_https_wrong_cert(): ) +def test_https_cn_nul_byte_no_bypass(): + # Same bypass as test_native_cn_nul_byte_no_bypass, over the HTTPS interface: client13's CN + # "client1\0.evil.com" must not be truncated to "client1" and authenticate as user 'john'. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="client13") + assert "403" in str(err.value) + + def test_https_non_ssl_auth(): # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" @@ -546,6 +566,21 @@ def test_x509_cn_wildcard_single_label(): assert "403" in str(err.value) +def test_x509_wildcard_nul_byte_no_bypass(): + # Authentication bypass: client14's CN and DNS SAN are both "evil\0.corp.example.com". Users + # 'wildcard_cn' and 'wildcard_dns' are configured with '*.corp.example.com' and + # 'DNS:*.corp.example.com'. The '*' must not match the span "evil\0", so both must fail. + for user in ["wildcard_cn", "wildcard_dns"]: + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user=user, cert_name="client14" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user=user, cert_name="client14") + assert "403" in str(err.value) + + def test_x509_uri_san_wildcard_dot_in_segment(): # Non-regression: '.' separates labels for DNS/CN but is NOT a separator for URI SANs, # whose separator is '/'. A wildcard URI path segment may legitimately contain dots, so diff --git a/tests/integration/test_storage_azure_blob_storage/test_cluster.py b/tests/integration/test_storage_azure_blob_storage/test_cluster.py index 602af40f7c66..e7341ad6d62c 100644 --- a/tests/integration/test_storage_azure_blob_storage/test_cluster.py +++ b/tests/integration/test_storage_azure_blob_storage/test_cluster.py @@ -82,6 +82,47 @@ def test_select_all(cluster): assert TSV(pure_azure) == TSV(distributed_azure) +def test_select_all_split_by_buckets(cluster): + """ + With `cluster_table_function_split_granularity = 'bucket'` the initiator opens every Parquet file + itself to split it into row-group buckets, using the object info of the cluster iterator, which + skips the metadata probe. That placeholder must not be taken as a known size of zero: the Azure + read is bounded by the size recorded in the object info, so the file would be read as empty. + """ + node = cluster.instances["node_0"] + storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + azure_query( + node, + f"INSERT INTO TABLE FUNCTION azureBlobStorage('{storage_account_url}', 'cont', 'test_cluster_split_by_buckets.parquet', 'devstoreaccount1'," + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'Parquet', 'auto', 'key UInt64, data String') " + f"SELECT number, toString(number) FROM numbers(100000)", + settings={ + "azure_truncate_on_insert": 1, + "output_format_parquet_row_group_size": 1000, + }, + ) + + pure_azure = azure_query( + node, + f"SELECT count(), sum(key), sum(length(data)) from azureBlobStorage('{storage_account_url}', 'cont', 'test_cluster_split_by_buckets.parquet', 'devstoreaccount1'," + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'Parquet')", + ) + assert TSV(pure_azure) == TSV("100000\t4999950000\t488890") + + for validate_etag_on_read in (0, 1): + distributed_azure = azure_query( + node, + f"SELECT count(), sum(key), sum(length(data)) from azureBlobStorageCluster('simple_cluster', '{storage_account_url}', 'cont', 'test_cluster_split_by_buckets.parquet', 'devstoreaccount1'," + f"'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==', 'Parquet')", + settings={ + "cluster_table_function_split_granularity": "bucket", + "cluster_table_function_buckets_batch_size": 10, + "azure_validate_etag_on_read": validate_etag_on_read, + }, + ) + assert TSV(pure_azure) == TSV(distributed_azure) + + def test_count(cluster): node = cluster.instances["node_0"] port = cluster.env_variables["AZURITE_PORT"] diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index cc8f7034012f..efc207fa0d18 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -5358,24 +5358,16 @@ def test_early_return_limit(started_cluster, use_delta_kernel): assert first_check_hits > 0 or queue_check_hits > 0 - assert 1 == int(instance.query( - f"SELECT count() FROM system.text_log WHERE query_id = '{query_id}' AND message LIKE '%List batch size is 1/1, shutdown: true%'" - )) - # Early return should scan significantly fewer files # With s3_list_object_keys_size=1, queue pauses frequently forcing shutdown checks # Should stop very early after consuming just a few files assert scanned_files < full_scan_files, \ f"Early return should scan fewer files: {scanned_files} >= {full_scan_files}" - # 3 because: - # we have async reader creation with 2 existing readers at a moment of time, - # each calls next() and consumes 2 files from the scan. - # It takes 1 file for the query to stop because of LIMIT 1. - # But because scan is also asynchronous and continues once batch limit is not reached, - # we get +1 scanned file. - assert scanned_files == 3, \ - f"Early return should scan 3 files with LIMIT 1, but scanned {scanned_files}" + # At most 3: two async readers each consume one file, plus one the scan produces before + # it observes shutdown. Fewer is legal when shutdown lands earlier. + assert scanned_files <= 3, \ + f"Early return should scan at most 3 files with LIMIT 1, but scanned {scanned_files}" def test_struct_dotted_field_names(started_cluster): diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py index 2cc7d839778a..63ac6063258f 100644 --- a/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py @@ -1,11 +1,22 @@ +import glob +import os + +import avro.datafile +import avro.io +import pyarrow.parquet as pq import pytest from helpers.iceberg_utils import ( check_validity_and_get_prunned_files_general, create_iceberg_table, - get_uuid_str + default_download_directory, + get_last_snapshot, + get_uuid_str, + unescape_path, ) +TABLE_ROOT = "/var/lib/clickhouse/user_files/iceberg_data/default" + @pytest.mark.parametrize("format_version", [1, 2]) @pytest.mark.parametrize("storage_type", ["s3", "azure", "local"]) @@ -133,4 +144,102 @@ def check_validity_and_get_prunned_files(select_expression): f"SELECT * FROM {TABLE_NAME} WHERE number <= 5 ORDER BY ALL" ) == 3 - ) \ No newline at end of file + ) + + +def _read_avro(path): + with open(path, "rb") as f: + return list(avro.datafile.DataFileReader(f, avro.io.DatumReader())) + + +def _data_files_of_last_snapshot(table_path): + """The `data_file` record of every manifest entry of the newest snapshot.""" + snapshot_id = get_last_snapshot(table_path) + manifest_lists = glob.glob(f"{table_path}/metadata/snap-{snapshot_id}-*.avro") + assert len(manifest_lists) == 1, manifest_lists + + data_files = [] + for list_entry in _read_avro(manifest_lists[0]): + manifest = os.path.join( + table_path, "metadata", os.path.basename(unescape_path(list_entry["manifest_path"])) + ) + data_files.extend(entry["data_file"] for entry in _read_avro(manifest)) + return data_files + + +def _column_sizes_from_parquet(path): + """Per-column compressed size as the Parquet footer of one data file reports it, keyed by the + Iceberg field id the column carries.""" + parquet_file = pq.ParquetFile(path) + field_ids = [ + int(field.metadata[b"PARQUET:field_id"]) for field in parquet_file.schema_arrow + ] + + sizes = dict.fromkeys(field_ids, 0) + metadata = parquet_file.metadata + for row_group_index in range(metadata.num_row_groups): + row_group = metadata.row_group(row_group_index) + assert row_group.num_columns == len(field_ids) + for column_index, field_id in enumerate(field_ids): + sizes[field_id] += row_group.column(column_index).total_compressed_size + return sizes + + +@pytest.mark.parametrize("format_version", [1, 2]) +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_writes_column_sizes_are_on_disk_sizes( + started_cluster_iceberg_no_spark, format_version, storage_type +): + """`data_file.column_sizes` is the size the column occupies inside the data file, so every + entry must repeat what the Parquet footer of the file it names says. An in-memory size passes + neither check below: `s` holds 1000000 bytes of one repeated character, which compresses to a + fraction of the file, and the sum over a file cannot exceed the file itself.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + TABLE_NAME = ( + "test_writes_column_sizes_are_on_disk_sizes_" + + storage_type + + "_" + + get_uuid_str() + ) + + create_iceberg_table( + storage_type, + instance, + TABLE_NAME, + started_cluster_iceberg_no_spark, + "(id Int32, s String)", + format_version, + order_by="id", + ) + # Blocks of 1000 rows that the insert pipeline does not squash, so the writer rolls over at + # exactly 4000 rows and the commit holds three data files of known sizes. + instance.query( + f"INSERT INTO {TABLE_NAME} SELECT number, repeat('x', 100) FROM numbers(10000)", + settings={ + "iceberg_insert_max_rows_in_data_file": 4000, + "max_block_size": 1000, + "max_insert_block_size": 1000, + "min_insert_block_size_rows": 0, + "min_insert_block_size_bytes": 0, + "max_insert_threads": 1, + }, + ) + + table_path = f"{TABLE_ROOT}/{TABLE_NAME}/" + default_download_directory( + started_cluster_iceberg_no_spark, storage_type, table_path, table_path + ) + + data_files = _data_files_of_last_snapshot(table_path) + assert len(data_files) == 3 + + for data_file in data_files: + data_path = os.path.join( + table_path, "data", os.path.basename(unescape_path(data_file["file_path"])) + ) + written = {pair["key"]: pair["value"] for pair in data_file["column_sizes"]} + + assert written == _column_sizes_from_parquet(data_path) + assert sum(written.values()) <= data_file["file_size_in_bytes"] + + assert instance.query(f"SELECT count(), uniqExact(s) FROM {TABLE_NAME}") == "10000\t1\n" diff --git a/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py new file mode 100644 index 000000000000..51bb446b5f31 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py @@ -0,0 +1,268 @@ +import pytest + +from helpers.iceberg_utils import ( + check_validity_and_get_prunned_files_general, + execute_spark_query_general, + get_creation_expression, + get_uuid_str, +) + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_with_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + # A filter that wraps the partition source column in a monotonic function must still be able to + # use the partition value, which is what https://github.com/ClickHouse/ClickHouse/issues/103433 + # reported for `toDate`. + assert check_validity_and_get_prunned_files(select("toDate(ts_day) = toDate('2024-01-20')")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfDay(ts_day) = toDateTime64('2024-01-20 00:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMMDD(ts_day) = 20240120")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) IN (toDate('2024-01-20'), toDate('2024-02-20'))")) == 2 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) > toDate('2024-02-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(ts_day) = 2025")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) = toDateTime64('2024-01-20 10:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_hour) = toDate('2024-01-21')")) == 2 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) = toDate('2024-02-01')")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMM(ts_month) = 202503")) == 4 + assert check_validity_and_get_prunned_files(select("toYear(d_year) = 2026")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfYear(d_year) = toDate('2028-01-01')")) == 4 + + # A constant that is not aligned to the transform's granularity: 10:30 is inside the 10:00 hour, + # so the file of that hour must survive, and the same for a day and a month boundary. + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) < toDateTime64('2024-01-20 10:30:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) >= toDateTime64('2024-01-20 10:30:00', 6)")) == 1 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) < toDate('2024-01-20') + INTERVAL 12 HOUR")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) > toDate('2024-02-10')")) == 2 + + # A single day is one weekday, so the partition value answers this too: only 2024-01-20 is a Saturday. + assert check_validity_and_get_prunned_files(select("toDayOfWeek(ts_day) = 6")) == 3 + + # The partition value of a `day` transform says nothing about the hour of the day, so a filter on + # it must not prune anything. + assert check_validity_and_get_prunned_files(select("toHour(ts_day) = 10")) == 0 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # The transforms count from 1970, so these partition values are negative. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1970-01-01', 2), + (TIMESTAMP '1970-01-01 00:30:00', DATE '1971-06-01', 3), + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 3 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) < toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 3 + assert check_validity_and_get_prunned_files(select("toYear(d) >= 1971")) == 2 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch_separate_commits(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_separate_commits_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # One manifest file per commit, so the partition summaries of the first manifest are negative on + # both of their sides, while the ones of the second manifest are positive on both. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1969-06-01', 2); + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 3); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 1 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_without_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_without_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day < '2024-01-21 00:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day >= '2024-02-01 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day BETWEEN '2024-01-21 00:00:00' AND '2024-02-21 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day IN ('2024-01-20 10:00:00', '2024-02-20 10:00:00')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_hour = '2024-01-20 11:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_hour > '2024-01-21 09:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_month = '2024-02-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_month < '2024-03-01 00:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("d_year = '2026-01-20'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year < '2025-01-01'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year >= '2026-01-01'")) == 2 + assert check_validity_and_get_prunned_files(select("d_year IN ('2024-01-20', '2026-01-20')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 12:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year = '2026-06-01'")) == 3 diff --git a/tests/integration/test_storage_kafka/test_batch_fast.py b/tests/integration/test_storage_kafka/test_batch_fast.py index c36d42a7b0df..f55249b5a7d6 100644 --- a/tests/integration/test_storage_kafka/test_batch_fast.py +++ b/tests/integration/test_storage_kafka/test_batch_fast.py @@ -1931,6 +1931,57 @@ def test_kafka_producer_consumer_separate_settings( assert property_in_log in kafka_producer_applied_properties +@pytest.mark.parametrize( + "create_query_generator", + [ + k.generate_old_create_table_query, + k.generate_new_create_table_query, + ], +) +def test_kafka_password_not_logged(kafka_cluster, create_query_generator): + suffix = k.random_string(6) + kafka_table = f"kafka_{suffix}" + username = f"kafka_user_{suffix}" + password = f"secret_kafka_password_{suffix}" + + instance.rotate_logs() + instance.query( + create_query_generator( + kafka_table, + "key UInt64", + topic_list="password_not_logged", + consumer_group="test", + settings={ + "kafka_sasl_username": username, + "kafka_sasl_password": password, + }, + ) + ) + + # Create an mv to initialize the librdkafka consumers + instance.query(f"CREATE MATERIALIZED VIEW test.{kafka_table}_view ENGINE=MergeTree ORDER BY tuple() AS SELECT * FROM test.{kafka_table}") + instance.wait_for_log_line(f"{kafka_table}.*Created #0 consumer") + instance.query(f"DROP TABLE test.{kafka_table}_view") + instance.query(f"INSERT INTO test.{kafka_table} VALUES (1)") + + assert instance.contains_in_log(f"{kafka_table}.*Kafka producer created") + + # The property-logging loops ran for both the consumer and the producer, + # but they hid the values of the sensitive properties. `sasl.username` is + # hidden because librdkafka marks it with the _RK_SENSITIVE flag, not + # because of the name, so it validates the generated blacklist. + for client_type in ["Consumer", "Producer"]: + for property_name in ["sasl.username", "sasl.password"]: + assert instance.contains_in_log( + f"{kafka_table}.*{client_type} set property {property_name}:\\[HIDDEN\\]" + ) + # The username still appears in the logged CREATE TABLE text (only + # kafka_sasl_password is masked there), so check only the password value. + assert not instance.contains_in_log(password) + + instance.query(f"DROP TABLE test.{kafka_table}") + + @pytest.mark.parametrize( "create_query_generator, log_line", [ diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py index 252fc892b9e4..9f02e5fbe90e 100644 --- a/tests/integration/test_storage_rabbitmq/test.py +++ b/tests/integration/test_storage_rabbitmq/test.py @@ -3602,7 +3602,7 @@ def test_hiding_credentials(rabbitmq_cluster, db, unique): instance.query("SYSTEM FLUSH LOGS") message = instance.query(f"SELECT message FROM system.text_log WHERE message ILIKE '%CREATE TABLE {db}.{table_name}%'") assert "rabbitmq_password = \\'[HIDDEN]\\'" in message - assert "rabbitmq_address = \\'amqp://root:[HIDDEN]@rabbitmq1:5672/\\'" in message + assert "rabbitmq_address = \\'[HIDDEN]\\'" in message def test_rabbitmq_default_mode_nack_on_parse_error(rabbitmq_cluster, db, unique): diff --git a/tests/integration/test_storage_redis/fake_redis.py b/tests/integration/test_storage_redis/fake_redis.py new file mode 100644 index 000000000000..240744332982 --- /dev/null +++ b/tests/integration/test_storage_redis/fake_redis.py @@ -0,0 +1,128 @@ +""" +A minimal RESP server that answers MGET with a reply of the test's choosing. + +`StorageRedis` pairs the elements of an MGET reply with the requested keys by position, so both +the element count and the position of a nil decide what the engine does. Usage: + + fake_redis.py [ ] + +Every MGET is answered with `max(0, len(keys) + delta)` elements: the requested keys echoed back +as their own values, followed by RESP nils. `` and `` are comma-separated +key names; when they are given, SCAN answers with those keys and MGET answers nil for the names +in ``, so a full scan meets a fixed valid/nil sequence rather than Redis's own key +order. Anything else is answered `+OK`. +""" + +import socket +import sys +import threading + + +def read_exactly(stream, size): + data = stream.read(size) + if len(data) != size: + raise EOFError + return data + + +def read_line(stream): + line = stream.readline() + if not line: + raise EOFError + return line.rstrip(b"\r\n") + + +def read_command(stream): + """Read one RESP array. + + Argument payloads are read by declared length, never up to a newline: the keys on the wire + are ClickHouse's serializeBinary output and may contain \\r, \\n or non-UTF-8 bytes. + """ + header = read_line(stream) + if not header.startswith(b"*"): + return None + args = [] + for _ in range(int(header[1:])): + arg_header = read_line(stream) + if not arg_header.startswith(b"$"): + raise EOFError + args.append(read_exactly(stream, int(arg_header[1:]))) + read_exactly(stream, 2) # trailing CRLF + return args + + +def serialize_string(name): + """A String shorter than 128 bytes as ClickHouse serializes it: one length byte, then bytes. + + The engine deserializes the keys SCAN reports and the values MGET returns, so both have to + arrive in that form. See serialize_binary_for_string in test.py. + """ + return bytes([len(name)]) + name.encode() + + +def mget_reply(keys, delta, nil_keys): + count = max(0, len(keys) + delta) + out = [b"*%d\r\n" % count] + for i in range(count): + if i < len(keys) and keys[i] not in nil_keys: + out.append(b"$%d\r\n%s\r\n" % (len(keys[i]), keys[i])) + else: + out.append(b"$-1\r\n") + return b"".join(out) + + +def scan_reply(scan_keys): + """Cursor 0 with every key, so the engine reads the whole keyspace in one batch.""" + out = [b"*2\r\n$1\r\n0\r\n", b"*%d\r\n" % len(scan_keys)] + for key in scan_keys: + out.append(b"$%d\r\n%s\r\n" % (len(key), key)) + return b"".join(out) + + +def handle(conn, delta, scan_keys, nil_keys): + try: + with conn.makefile("rb") as stream: + while True: + args = read_command(stream) + if not args: + break + command = args[0].upper() + if command == b"MGET": + conn.sendall(mget_reply(args[1:], delta, nil_keys)) + elif command == b"SCAN" and scan_keys: + conn.sendall(scan_reply(scan_keys)) + else: + conn.sendall(b"+OK\r\n") + except (EOFError, OSError, ValueError): + pass + finally: + conn.close() + + +def parse_names(argument): + return [serialize_string(name) for name in argument.split(",") if name] + + +def main(): + port = int(sys.argv[1]) + delta = int(sys.argv[2]) + scan_keys = parse_names(sys.argv[3]) if len(sys.argv) > 3 else [] + nil_keys = parse_names(sys.argv[4]) if len(sys.argv) > 4 else [] + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", port)) + # The engine holds a connection pool and a direct join reads on several pipeline threads, so + # connections have to be served concurrently. + server.listen(64) + print(f"listening on {port}, MGET reply length = requested + {delta}", flush=True) + + while True: + conn, _ = server.accept() + threading.Thread( + target=handle, args=(conn, delta, scan_keys, nil_keys), daemon=True + ).start() + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_storage_redis/test.py b/tests/integration/test_storage_redis/test.py index af2892125fce..8bf1f96526a7 100644 --- a/tests/integration/test_storage_redis/test.py +++ b/tests/integration/test_storage_redis/test.py @@ -1,5 +1,6 @@ ## sudo -H pip install redis import json +import os import struct import sys @@ -8,7 +9,9 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, wait_condition + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) @@ -529,3 +532,161 @@ def check_query(query, read_type, keys_count, rows_read): plan = node.query("EXPLAIN actions=1, optimize=0 SELECT * FROM test_get_keys") assert 'ReadType: FullScan' in plan + + +# Ports of the three fake Redis endpoints, inside the ClickHouse container. +FAKE_REDIS_PORT_OK = 16379 +FAKE_REDIS_PORT_LONG = 16380 +FAKE_REDIS_PORT_SHORT = 16381 +FAKE_REDIS_PORT_SCAN = 16382 + + +def start_fake_redis(port, delta, keys=""): + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /fake_redis.py {port} {delta} {keys}" + f" > /var/log/clickhouse-server/fake_redis_{port}.log 2>&1", + ], + detach=True, + user="root", + ) + wait_condition( + lambda: node.exec_in_container( + ["bash", "-c", f"exec 3<>/dev/tcp/127.0.0.1/{port} && echo OK"], + nothrow=True, + ), + lambda r: "OK" in r, + max_attempts=40, + delay=0.5, + ) + + +def test_malformed_mget_reply(started_cluster): + """An MGET reply whose element count differs from the request must be rejected. + + The result loop of StorageRedis was bounded by the reply length while the null map it + indexes is sized from the request, so an over-long reply read and wrote past the end of the + null map. A short reply produced fewer rows than keys, which breaks the row-per-key contract + that IKeyValueEntity::getByKeys promises to a direct join. + """ + tables = ("redis_fake_ok", "redis_fake_long", "redis_fake_short", "t_fake_left") + for table in tables: + drop_table(table) + + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_OK, 0) + start_fake_redis(FAKE_REDIS_PORT_LONG, 1) + start_fake_redis(FAKE_REDIS_PORT_SHORT, -1) + + node.query( + f""" + CREATE TABLE redis_fake_ok (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_OK}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_long (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_LONG}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_short (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SHORT}') PRIMARY KEY (key); + + CREATE TABLE t_fake_left (k String) ENGINE = TinyLog; + INSERT INTO t_fake_left VALUES ('a'), ('b'); + """ + ) + + def direct_join(table): + return node.query( + f"SELECT key, value FROM (SELECT k AS key FROM t_fake_left) AS t " + f"INNER JOIN {table} USING (key) ORDER BY key " + f"SETTINGS join_algorithm = 'direct' FORMAT TSV" + ) + + # An endpoint that answers with one element per key still works, so a mock that never + # listens or frames RESP wrongly reddens here instead of green-washing the arms below. + assert TSV.toMat(direct_join("redis_fake_ok")) == [["a", "a"], ["b", "b"]] + + with pytest.raises(QueryRuntimeException) as long_join: + direct_join("redis_fake_long") + assert "INTERNAL_REDIS_ERROR" in str(long_join.value) + assert "for MGET of" in str(long_join.value) + + with pytest.raises(QueryRuntimeException) as short_join: + direct_join("redis_fake_short") + assert "INTERNAL_REDIS_ERROR" in str(short_join.value) + + # The other caller of the reply reads it with no null map at all. + with pytest.raises(QueryRuntimeException) as long_in: + node.query("SELECT * FROM redis_fake_long WHERE key IN ('a', 'b')") + assert "INTERNAL_REDIS_ERROR" in str(long_in.value) + + # A zero-element reply is a null array in Poco, not an empty one, so it is the isNull() + # term of the guard that rejects it. One key against the delta = -1 endpoint produces it. + with pytest.raises(QueryRuntimeException) as zero_in: + node.query("SELECT * FROM redis_fake_short WHERE key IN ('a')") + assert "INTERNAL_REDIS_ERROR" in str(zero_in.value) + assert "returned 0 values" in str(zero_in.value) + + for table in tables: + drop_table(table) + + +def test_full_scan_skips_missing_values(started_cluster): + """A full scan must skip the keys MGET answers with nil, not stop at the first one. + + MGET answers by position, and a key SCAN listed can hold a non-string type or expire + before the MGET runs, so a nil marks one absent value and not the end of the batch. + """ + address = get_address_for_ch() + table = "test_full_scan_missing" + fake_table = "redis_fake_scan" + + client = get_redis_connection(db_id=4) + client.flushdb() + drop_table(table) + drop_table(fake_table) + + # Redis alone decides in what order SCAN reports keys, so the mock pins the one thing this + # test is about: a nil arriving before a key that still has a value. + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_SCAN, 0, "a,b,c b") + + node.query( + f""" + CREATE TABLE {fake_table} (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SCAN}') PRIMARY KEY (key); + """ + ) + rows = node.query(f"SELECT key, value FROM {fake_table} ORDER BY key FORMAT TSV") + assert TSV.toMat(rows) == [["a", "a"], ["c", "c"]] + + # The same thing on a real Redis, which answers nil for any key that holds another type. + node.query( + f""" + CREATE TABLE {table}(k String, v String) + Engine=Redis('{address}', 4, 'clickhouse') PRIMARY KEY (k); + + INSERT INTO {table} SELECT toString(number), toString(number) FROM numbers(16); + """ + ) + + # Control: every row is readable before the keys below exist, so a fixture that writes + # nothing reddens here instead of leaving the assertion after it vacuous. + assert int(node.query(f"SELECT uniqExact(k) FROM {table}")) == 16 + + for i in range(16): + client.rpush(f"list_{i}", "x") + + # SCAN can report a key twice while Redis rehashes and no read path dedupes the rows, so the + # oracle here is the key set: a lost key is the defect, a repeated one is not. + keys = node.query(f"SELECT DISTINCT k FROM {table} ORDER BY toUInt32(k) FORMAT TSV") + assert TSV.toMat(keys) == [[str(i)] for i in range(16)] + + client.flushdb() + drop_table(table) + drop_table(fake_table) diff --git a/tests/integration/test_storage_s3_queue/test_sts_smoke.py b/tests/integration/test_storage_s3_queue/test_sts_smoke.py index 761606c336af..5b02ffebace7 100644 --- a/tests/integration/test_storage_s3_queue/test_sts_smoke.py +++ b/tests/integration/test_storage_s3_queue/test_sts_smoke.py @@ -111,14 +111,14 @@ def get_count(node, table_name): assert get_count(node, dst_table_name) == 10 assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) node.restart_clickhouse() assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) @@ -181,7 +181,7 @@ def test_s3_queue_extra_credentials_backup(started_cluster): # are equally invalid for MinIO, so a restored table that silently dropped # the clause would fail with the same error. Pin the round trip explicitly. assert ( - "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'miniorole\\')" + "extra_credentials(\\'role_arn\\' = \\'arn::role\\', \\'role_session_name\\' = \\'[HIDDEN]\\')" in node.query(f"SHOW CREATE TABLE {table_name}") ) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj index 160599290aae..cc8e3f678a10 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/counter.clj @@ -34,9 +34,12 @@ :type :ok :value (long (count (zk-list conn root-path)))) (catch Exception _ (assoc op :type :info, :error :connect-error))) - :final-read (chu/exec-with-retries 30 (fn [] (assoc op - :type :ok - :value (long (count (zk-list conn root-path)))))) + :final-read (chu/exec-with-retries 30 (fn [] + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (assoc op + :type :ok + :value (long (count (zk-list conn root-path)))))))) :add (try (do (zk-multi-create-many-seq-nodes conn (concat-path root-path "seq-") (:value op) :with-acl (:with-auth test)) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj index d26849ee127e..be4b3f1481a6 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/queue.clj @@ -45,8 +45,10 @@ :drain ; drain via delete is to long, just list all nodes (chu/exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))) + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (zk-sync conn) + (assoc op :type :ok :value (into #{} (map #(str %1) (zk-list conn root-path)))))))))) (teardown! [_ test]) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj index 87b4f91b5863..afe25f28a809 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/set.clj @@ -25,10 +25,12 @@ (invoke! [this test op] (case (:f op) :read (chu/exec-with-retries 30 (fn [] - (zk-sync conn) - (assoc op - :type :ok - :value (read-string (:data (zk-get-str conn k)))))) + (with-fresh-conn nodename (:with-auth test) + (fn [conn] + (zk-sync conn) + (assoc op + :type :ok + :value (read-string (:data (zk-get-str conn k)))))))) :add (try (do (zk-add-to-set conn k (:value op)) diff --git a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj index 9947973e51ae..cfdf7d1a829a 100644 --- a/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj +++ b/tests/jepsen.clickhouse/src/jepsen/clickhouse/keeper/utils.clj @@ -32,6 +32,16 @@ conn) conn))) +(defn with-fresh-conn + "Calls (f conn) on a new connection to node and closes it. Keeper cannot continue a session, + so a worker's connection that dropped after its last operation is expired." + [node with-auth f] + (let [conn (zk-connect node 9181 30000 with-auth)] + (try + (f conn) + (finally + (zk/close conn))))) + (defn zk-create-range [conn n & {:keys [with-acl] :or {with-acl false}}] (dorun (map (fn [v] (zk/create-all conn v diff --git a/tests/performance/aggregation_single_level_result_fanout.xml b/tests/performance/aggregation_single_level_result_fanout.xml new file mode 100644 index 000000000000..e7a9f16cc9f3 --- /dev/null +++ b/tests/performance/aggregation_single_level_result_fanout.xml @@ -0,0 +1,111 @@ + + + + 8 + 0 + 0 + + + CREATE TABLE single_level_fanout_t1 (oid UInt64, cid Int32, cr DateTime, h String, s1 Int32) ENGINE = MergeTree ORDER BY oid + CREATE TABLE single_level_fanout_t2 (oid UInt64) ENGINE = MergeTree ORDER BY oid + CREATE TABLE single_level_fanout_t3 (cid Int32, h String, ref DateTime, s2 Float64) ENGINE = MergeTree ORDER BY (cid, h, ref) + + + INSERT INTO single_level_fanout_t1 + SELECT + intDiv(number, 5), + toInt32(intDiv(number, 5) % 4), + toDateTime('2026-09-08 00:00:00') + toIntervalMinute(intDiv(number, 5) % 1440) + toIntervalSecond(number % 5), + toString(intDiv(number, 5) % 200), + toInt32(number % 300) + FROM numbers(300000) + + INSERT INTO single_level_fanout_t2 SELECT number FROM numbers(60000) + + INSERT INTO single_level_fanout_t3 + SELECT + toInt32(number % 4), + toString(intDiv(number, 4) % 200), + toDateTime('2026-09-08 00:00:00') + toIntervalMinute(intDiv(number, 800)), + toFloat64(number % 5) / 2 + FROM numbers(1152000) + + + + + join_algorithm + + hash + parallel_hash + + + + + = e.t0 - toIntervalMinute(3) AND p.ref <= e.t0 + toIntervalMinute(1) + GROUP BY e.oid, e.cid, e.h, e.s + ) + GROUP BY cid + FORMAT Null + SETTINGS join_algorithm = '{join_algorithm}' + ]]> + + + = e.t0 - toIntervalMinute(3) AND p.ref <= e.t0 + toIntervalMinute(1) + GROUP BY e.oid, e.cid, e.h, e.s + ) + GROUP BY cid + FORMAT Null + SETTINGS enable_parallel_single_level_merge = 1, enable_adaptive_aggregator = 1 + ]]> + + DROP TABLE IF EXISTS single_level_fanout_t1 + DROP TABLE IF EXISTS single_level_fanout_t2 + DROP TABLE IF EXISTS single_level_fanout_t3 + diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index b3fc24d31665..af586967907b 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -1,4 +1,6 @@ + CREATE TABLE array_join_wide_chain (A Array(String), B Array(String), C Array(String)) ENGINE = MergeTree ORDER BY tuple() + INSERT INTO array_join_wide_chain SELECT arrayMap(i -> concat('a', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('b', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('c', toString(number), '_', toString(i)), range(100)) FROM numbers(100000) @@ -10,4 +12,10 @@ SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 with 'clickhouse' as str select arrayJoin(range(number % 10)), materialize(str) from numbers(10000000) + + + SELECT a, b, c FROM array_join_wide_chain ARRAY JOIN A AS a ARRAY JOIN B AS b ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + SELECT a, b, c FROM array_join_wide_chain LEFT ARRAY JOIN A AS a LEFT ARRAY JOIN B AS b LEFT ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + + DROP TABLE IF EXISTS array_join_wide_chain diff --git a/tests/performance/hash_join_low_cardinality_many_matches.xml b/tests/performance/hash_join_low_cardinality_many_matches.xml new file mode 100644 index 000000000000..8c6d62517602 --- /dev/null +++ b/tests/performance/hash_join_low_cardinality_many_matches.xml @@ -0,0 +1,11 @@ + + + + 8 + 0 + + + SELECT count() FROM (SELECT number % 100 AS k FROM numbers(4000)) AS a INNER JOIN (SELECT intDiv(number, 4000) AS k, toLowCardinality(toString(number % 7)) AS lc FROM numbers(400000)) AS b ON a.k = b.k WHERE NOT ignore(b.lc) SETTINGS join_algorithm = 'hash' + SELECT count() FROM (SELECT number % 100 AS k FROM numbers(4000)) AS a INNER JOIN (SELECT intDiv(number, 4000) AS k, toLowCardinality(toString(number % 7)) AS lc FROM numbers(400000)) AS b ON a.k = b.k WHERE NOT ignore(b.lc) SETTINGS join_algorithm = 'parallel_hash' + diff --git a/tests/performance/map_lowcardinality_key_lookup.xml b/tests/performance/map_lowcardinality_key_lookup.xml new file mode 100644 index 000000000000..ee61624b8bf3 --- /dev/null +++ b/tests/performance/map_lowcardinality_key_lookup.xml @@ -0,0 +1,43 @@ + + + + + CREATE TABLE map_lc_keys (id UInt64, m Map(LowCardinality(String), String)) + ENGINE = MergeTree ORDER BY id + + + + INSERT INTO map_lc_keys + SELECT number, mapFromArrays( + arrayMap(x -> concat('k', leftPad(toString(x), 2, '0')), range(20)), + arrayMap(x -> concat('v', toString(cityHash64(number, x) % 1000)), range(20))) + FROM numbers(1500000) + + + SELECT sum(ignore(m['k00'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['k09'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['k19'])) FROM map_lc_keys SETTINGS max_threads = 1 + SELECT sum(ignore(m['absent'])) FROM map_lc_keys SETTINGS max_threads = 1 + + DROP TABLE map_lc_keys + + + + + CREATE TABLE map_uuid_keys (id UInt64, m Map(UUID, DateTime64(3))) + ENGINE = MergeTree ORDER BY id + + + + INSERT INTO map_uuid_keys + SELECT number, mapFromArrays( + arrayMap(x -> reinterpretAsUUID(toFixedString(leftPad(toString(x), 16, '0'), 16)), range(20)), + arrayMap(x -> toDateTime64(number + x, 3), range(20))) + FROM numbers(1000000) + + + SELECT sum(ignore(m[reinterpretAsUUID(toFixedString('0000000000000019', 16))])) FROM map_uuid_keys SETTINGS max_threads = 1 + + DROP TABLE map_uuid_keys + diff --git a/tests/performance/text_index_like.xml b/tests/performance/text_index_like.xml index 5f38d8101570..935b025a5165 100644 --- a/tests/performance/text_index_like.xml +++ b/tests/performance/text_index_like.xml @@ -1,4 +1,9 @@ + + 0 + 4 + + CREATE TABLE tab ( @@ -14,14 +19,23 @@ INSERT INTO tab SELECT number, - if(number % 100 = 0, + trimRight(if(number < 100000, repeat('clickhouse is a fast column oriented database system ', 4), - repeat('the quick brown fox jumps over the lazy dog near the river ', 4)) + repeat('the quick brown fox jumps over the lazy dog near the river ', 4))) FROM numbers(10000000) SELECT count() FROM tab WHERE message LIKE '%clickhouse%' SELECT count() FROM tab WHERE message ILIKE '%CLICKHOUSE%' + SELECT count() FROM tab WHERE message LIKE 'clickhouse%' + SELECT count() FROM tab WHERE message ILIKE 'CLICKHOUSE%' + + SELECT count() FROM tab WHERE message LIKE '%system' + SELECT count() FROM tab WHERE message ILIKE '%SYSTEM' + + SELECT count() FROM tab WHERE startsWith(message, 'clickhouse') + SELECT count() FROM tab WHERE endsWith(message, 'system') + DROP TABLE IF EXISTS tab diff --git a/tests/queries/0_stateless/01780_column_sparse.reference b/tests/queries/0_stateless/01780_column_sparse.reference index 3393361a19fb..7452be01648e 100644 --- a/tests/queries/0_stateless/01780_column_sparse.reference +++ b/tests/queries/0_stateless/01780_column_sparse.reference @@ -132,7 +132,7 @@ SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; 4 190 5 330 6 270 -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; [1] [1,3] [1,3,5] diff --git a/tests/queries/0_stateless/01780_column_sparse.sql b/tests/queries/0_stateless/01780_column_sparse.sql index 8e3c4372d05a..c5b98b1fd422 100644 --- a/tests/queries/0_stateless/01780_column_sparse.sql +++ b/tests/queries/0_stateless/01780_column_sparse.sql @@ -27,7 +27,7 @@ SELECT * FROM t_sparse WHERE arr2 != [] ORDER BY id; SELECT sum(u) FROM t_sparse; SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; CREATE TABLE t_sparse_1 (id UInt64, v Int64) ENGINE = MergeTree ORDER BY tuple() diff --git a/tests/queries/0_stateless/01890_stem.reference b/tests/queries/0_stateless/01890_stem.reference index 7b5f7af34489..6b0de99cce39 100644 --- a/tests/queries/0_stateless/01890_stem.reference +++ b/tests/queries/0_stateless/01890_stem.reference @@ -103,6 +103,17 @@ run bless bless disguis +- Lengthening stemmers. +-- Turkish stems a 3-byte word to a 5-byte word, so the output is longer than the input. +uagı +-- Over a multi-block scan the output must stay correct even when it overflows the input-sized estimate. +0 +-- Turkish also lengthens via a multi-byte substitution (3 bytes to 4 bytes). +aaç +-- Estonian lengthens by appending ASCII letters (4 bytes to 5 bytes), a different mechanism. +keesi +-- Estonian over a multi-block scan must also stay correct past the input-sized estimate. +0 - Negative tests. -- Whitespace in a String input raises BAD_ARGUMENTS. -- Whitespace in an Array element raises BAD_ARGUMENTS. diff --git a/tests/queries/0_stateless/01890_stem.sql b/tests/queries/0_stateless/01890_stem.sql index cc5dd09fac70..c678bc88fb96 100644 --- a/tests/queries/0_stateless/01890_stem.sql +++ b/tests/queries/0_stateless/01890_stem.sql @@ -144,6 +144,27 @@ INSERT INTO stem_test_lc VALUES ('blessing'), ('disguise'), ('blessing'); SELECT stem(word, 'en') FROM stem_test_lc ORDER BY word; DROP TABLE stem_test_lc; +SELECT '- Lengthening stemmers.'; + +SELECT '-- Turkish stems a 3-byte word to a 5-byte word, so the output is longer than the input.'; +SELECT stem('uag', 'tr'); + +SELECT '-- Over a multi-block scan the output must stay correct even when it overflows the input-sized estimate.'; +SELECT countIf(s != 'uagı') +FROM (SELECT stem(materialize('uag'), 'tr') AS s FROM numbers(6800)) +SETTINGS max_block_size = 1700; + +SELECT '-- Turkish also lengthens via a multi-byte substitution (3 bytes to 4 bytes).'; +SELECT stem('aac', 'tr'); + +SELECT '-- Estonian lengthens by appending ASCII letters (4 bytes to 5 bytes), a different mechanism.'; +SELECT stem('keeb', 'et'); + +SELECT '-- Estonian over a multi-block scan must also stay correct past the input-sized estimate.'; +SELECT countIf(s != 'keesi') +FROM (SELECT stem(materialize('keeb'), 'et') AS s FROM numbers(6800)) +SETTINGS max_block_size = 1700; + SELECT '- Negative tests.'; SELECT '-- Whitespace in a String input raises BAD_ARGUMENTS.'; diff --git a/tests/queries/0_stateless/02346_text_index_function_like_affix.reference b/tests/queries/0_stateless/02346_text_index_function_like_affix.reference new file mode 100644 index 000000000000..89bb276d2e31 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_function_like_affix.reference @@ -0,0 +1,123 @@ +Test results are same with/without the optimization +-- without optimization +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[] +[] +[1] +[1,2,3,5] +[1] +[1,3] +[2,5] +[1] +[6] +[1,3] +[2,5] +[1,3] +[2,5] +-- with optimization +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[] +[] +[1] +[1,2,3,5] +[1] +[1,3] +[2,5] +[1] +[6] +[1,3] +[2,5] +[1,3] +[2,5] +-- with optimization but without hints +[1,3] +[2,5] +[1,2,3,4,5,6] +[2,4,5,6,7] +[1,3,4,6,7] +[1,3] +[2,5] +Prefix and suffix patterns keep the original condition, infix patterns do not +prefix 1 1 +suffix 1 1 +prefix, no rewrite 1 1 +suffix, no rewrite 1 1 +infix 1 0 +ilike prefix 1 1 +Needles shorter than text_index_like_min_pattern_length are not evaluated by a dictionary scan +prefix 0 1 +[1,3] +[1] +Text index analysis +-- Prefix pattern should choose 1 part and 1024 granules out of 4 parts and 4096 granules +Description: text GRANULARITY 100000000 +Parts: 1/4 +Granules: 1024/4096 +-- Suffix pattern should choose 3 parts and 3072 granules out of 4 parts and 4096 granules +Description: text GRANULARITY 100000000 +Parts: 3/4 +Granules: 3072/4096 +-- Prefix pattern with a non-existent token should choose none +Description: text GRANULARITY 100000000 +Parts: 0/4 +Granules: 0/4096 +1024 +1024 +2048 +2048 +Test results are same with/without the optimization with array tokenizer +-- without optimization +[1,3] +[3] +[2,4] +[1,3] +[3] +[1,2,3,4] +[3] +-- with optimization +[1,3] +[3] +[2,4] +[1,3] +[3] +[1,2,3,4] +[3] +With the array tokenizer a token is the whole value, so an affix is exact +prefix 1 0 +suffix 1 0 +ilike prefix 1 0 +A nullable value is left to the original condition +[1,3] +[1,3] +[2] +[2] +[1,2] +[1,2] +[] +[] +[2] +[2] +[] +[] +An affix hint that prunes nothing is discarded, granule pruning is kept +1000 +0 +affix_hint_nonselective 0 1 1 +affix_hint_selective 1 0 1 +A nullable value reached through mapValues is left to the original condition +499 +499 +499 +499 +499 +499 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_index_function_like_affix.sql b/tests/queries/0_stateless/02346_text_index_function_like_affix.sql new file mode 100644 index 000000000000..bf57731ff680 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_function_like_affix.sql @@ -0,0 +1,338 @@ +-- Tags: no-parallel-replicas +-- Tests that affix LIKE/ILIKE patterns, i.e. prefix ('value%') and suffix ('%value'), use the text index as a hint. +-- By default the analyzer rewrites such patterns into startsWith/endsWith (optimize_rewrite_like_perfect_affix), +-- so both spellings are covered here. +SET explain_query_plan_default = 'legacy'; + +SET enable_analyzer = 1; +SET use_skip_indexes_on_data_read = 1; +SET query_plan_direct_read_from_text_index = 1; +SET query_plan_text_index_add_hint = 1; +-- Pinned because the queries below assert which of the two spellings the plan ends up with. +SET optimize_rewrite_like_perfect_affix = 1; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY (id); + +-- The dictionary of this table contains tokens that match the prefix/suffix patterns below in rows +-- where the whole value does not match them, so a hint that is not verified would return extra rows. +INSERT INTO tab(id, message) VALUES + (1, 'foobar baz'), + (2, 'baz foobar'), + (3, 'foobarqux end'), + (4, 'end foobarqux'), + (5, 'quuxfoobar'), + (6, 'quuxfoobar tail'), + (7, 'nothing here'); + +SELECT 'Test results are same with/without the optimization'; + +SELECT '-- without optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 0; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'nonexistent%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%nonexistent'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND message LIKE '%baz'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' OR message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND hasToken(message, 'baz'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar baz'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'quuxfoobar tail'); +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0; + +SELECT '-- with optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 1; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'nonexistent%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%nonexistent'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND message LIKE '%baz'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' OR message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' AND hasToken(message, 'baz'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'foobar'); +SELECT groupArray(id) FROM tab WHERE startsWith(message, 'foobar baz'); +SELECT groupArray(id) FROM tab WHERE endsWith(message, 'quuxfoobar tail'); +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0; + +SELECT '-- with optimization but without hints'; + +SET query_plan_text_index_add_hint = 0; + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE 'foobar%'; +SELECT groupArray(id) FROM tab WHERE message NOT LIKE '%foobar'; +SELECT groupArray(id) FROM tab WHERE message ILIKE 'FOOBAR%'; +SELECT groupArray(id) FROM tab WHERE message ILIKE '%FOOBAR'; + +SET query_plan_text_index_add_hint = 1; + +SELECT 'Prefix and suffix patterns keep the original condition, infix patterns do not'; + +-- The columns are: whether the query plan contains a text index virtual column, and whether it still +-- evaluates the original search function. The latter is absent only for an exact direct read. +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foobar%'); + +SELECT 'suffix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION endsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar'); + +SELECT 'prefix, no rewrite', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foobar%' SETTINGS optimize_rewrite_like_perfect_affix = 0); + +SELECT 'suffix, no rewrite', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar' SETTINGS optimize_rewrite_like_perfect_affix = 0); + +SELECT 'infix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION like(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE '%foobar%'); + +SELECT 'ilike prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION ilike(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message ILIKE 'FOOBAR%'); + +SELECT 'Needles shorter than text_index_like_min_pattern_length are not evaluated by a dictionary scan'; + +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE message LIKE 'foo%'); + +SELECT groupArray(id) FROM tab WHERE message LIKE 'foo%'; +SELECT groupArray(id) FROM tab WHERE message LIKE '%baz'; + +DROP TABLE tab; + +SELECT 'Text index analysis'; + +CREATE TABLE tab +( + id UInt32, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY (id) +SETTINGS index_granularity = 1; + +INSERT INTO tab SELECT number, 'Hello ClickHouse' FROM numbers(1024); +INSERT INTO tab SELECT number, 'Hello World, ClickHouse is fast!' FROM numbers(1024); +INSERT INTO tab SELECT number, 'Hallo xClickHouse' FROM numbers(1024); +INSERT INTO tab SELECT number, 'ClickHousez rocks' FROM numbers(1024); + +SELECT '-- Prefix pattern should choose 1 part and 1024 granules out of 4 parts and 4096 granules'; +-- Only 'ClickHousez rocks' has a token starting with 'ClickHousez'. +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE 'ClickHousez%' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +SELECT '-- Suffix pattern should choose 3 parts and 3072 granules out of 4 parts and 4096 granules'; +-- 'ClickHouse' and 'xClickHouse' are tokens ending with 'ClickHouse', only 'ClickHousez' is not. +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE '%ClickHouse' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +SELECT '-- Prefix pattern with a non-existent token should choose none'; +SELECT trimLeft(explain) AS explain FROM ( + EXPLAIN indexes=1 + SELECT count() FROM tab WHERE message LIKE 'random%' +) WHERE explain LIKE '%Description:%' OR explain LIKE '%Parts:%' OR explain LIKE '%Granules:%' +LIMIT 2, 3; + +-- Three parts have a token starting with 'ClickHouse' but only one of them has rows starting with it, +-- so the hint must be verified by the original condition. +SELECT count() FROM tab WHERE message LIKE 'ClickHouse%'; +SELECT count() FROM tab WHERE message LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; + +SELECT count() FROM tab WHERE message LIKE '%ClickHouse'; +SELECT count() FROM tab WHERE message LIKE '%ClickHouse' SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; + +SELECT 'Test results are same with/without the optimization with array tokenizer'; + +CREATE TABLE tab +( + id UInt32, + tag String, + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, 'CLICKHOUSE_SQL'); + +SELECT '-- without optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 0; + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE '%Cloud'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE startsWith(tag, 'ClickHouse'); +SELECT groupArray(id) FROM tab WHERE endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE tag ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag ILIKE '%cloud'; + +SELECT '-- with optimization'; + +SET use_text_index_like_evaluation_by_dictionary_scan = 1; + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE '%Cloud'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE startsWith(tag, 'ClickHouse'); +SELECT groupArray(id) FROM tab WHERE endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE tag ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag ILIKE '%cloud'; + +DROP TABLE tab; + +SELECT 'With the array tokenizer a token is the whole value, so an affix is exact'; + +DROP TABLE IF EXISTS tab; + +CREATE TABLE tab +( + id UInt32, + tag String, + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, 'CLICKHOUSE_SQL'); + +SELECT 'prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION startsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag LIKE 'ClickHouse%'); + +SELECT 'suffix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION endsWith(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag LIKE '%Cloud'); + +SELECT 'ilike prefix', countIf(explain LIKE '%\_\_text_index\_%') > 0, countIf(explain LIKE '%FUNCTION ilike(%') > 0 +FROM (EXPLAIN actions = 1 SELECT count() FROM tab WHERE tag ILIKE 'clickhouse%'); + +DROP TABLE tab; + +SELECT 'A nullable value is left to the original condition'; + +CREATE TABLE tab +( + id UInt32, + tag Nullable(String), + INDEX idx(tag) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY (id); + +INSERT INTO tab(id, tag) VALUES + (1, 'ClickHouseServer'), + (2, 'clickhouseClient'), + (3, 'ClickHouseCloud'), + (4, NULL); + +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE 'ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE NOT endsWith(tag, 'Cloud'); +SELECT groupArray(id) FROM tab WHERE NOT endsWith(tag, 'Cloud') SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE 'clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE 'clickhouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE '%ClickHouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT LIKE '%ClickHouse%' SETTINGS use_skip_indexes = 0; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE '%clickhouse%'; +SELECT groupArray(id) FROM tab WHERE tag NOT ILIKE '%clickhouse%' SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; + +SELECT 'An affix hint that prunes nothing is discarded, granule pruning is kept'; + +CREATE TABLE tab +( + id UInt64, + message String, + INDEX idx(message) TYPE text(tokenizer = splitByNonAlpha) GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab SELECT number, multiIf(number < 1000, 'clickhouse is fast', number < 30000, 'bank of the river', 'alpha beta gamma') FROM numbers(100000); + +SELECT count() FROM tab WHERE message LIKE 'clickhouse%' SETTINGS log_comment = 'affix_hint_selective'; +SELECT count() FROM tab WHERE message LIKE 'river%' SETTINGS log_comment = 'affix_hint_nonselective'; + +SYSTEM FLUSH LOGS query_log; + +SELECT log_comment, + max(ProfileEvents['TextIndexUseHint'] > 0) AS hint_used, + max(ProfileEvents['TextIndexDiscardHint'] > 0) AS hint_discarded, + max(read_rows < 100000) AS granules_pruned +FROM system.query_log +WHERE event_date >= yesterday() AND event_time >= now() - 600 AND current_database = currentDatabase() + AND type = 'QueryFinish' AND log_comment IN ('affix_hint_selective', 'affix_hint_nonselective') +GROUP BY log_comment +ORDER BY log_comment; + +DROP TABLE tab; + +SELECT 'A nullable value reached through mapValues is left to the original condition'; + +CREATE TABLE tab +( + id UInt64, + m Map(String, Nullable(String)), + INDEX idx mapValues(m) TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id; + +INSERT INTO tab SELECT number, map('k', if(number = 0, 'foobar value', 'zulu yankee')) FROM numbers(500); +INSERT INTO tab VALUES (1000, map('k', NULL)); + +SELECT count() FROM tab WHERE NOT startsWith(m['k'], 'foobar'); +SELECT count() FROM tab WHERE NOT startsWith(m['k'], 'foobar') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE NOT m['k'] LIKE 'foobar%'; +SELECT count() FROM tab WHERE NOT m['k'] LIKE 'foobar%' SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE NOT endsWith(m['k'], 'value'); +SELECT count() FROM tab WHERE NOT endsWith(m['k'], 'value') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab WHERE startsWith(m['k'], 'foobar'); +SELECT count() FROM tab WHERE startsWith(m['k'], 'foobar') SETTINGS use_skip_indexes = 0; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference new file mode 100644 index 000000000000..7f82e6a6845d --- /dev/null +++ b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.reference @@ -0,0 +1,35 @@ +Granules: 1/2 +1 +1 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +1 +1 +Granules: 1/2 +1 +7 +1 +7 +1 +Granules: 1/2 +2 +3 +1 +1 +2 +2 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql new file mode 100644 index 000000000000..6127b05004cd --- /dev/null +++ b/tests/queries/0_stateless/02346_text_token_index_transform_null_in.sql @@ -0,0 +1,136 @@ +-- Tests that the `text`, `tokenbf_v1`, `ngrambf_v1` and `sparse_grams` skip indexes prune an `IN` +-- when `transform_null_in = 1`, where the predicate arrives as `nullIn`/`globalNullIn`, and that a +-- set holding a NULL element is refused because `nullIn` also matches the column's NULL rows. +-- +-- `index_granularity = 4` over 8 rows, so every granule is mixed: granule 0 holds word0..word3. + +SET enable_full_text_index = 1; +SET transform_null_in = 1; + +DROP TABLE IF EXISTS tab; + +-- The same block runs for every index type. `tokenbf_v1`, `ngrambf_v1` and `sparse_grams` reject a +-- `Nullable` column at DDL, so the common block uses `String` and NULL enters through the set only. + +-- text + +CREATE TABLE tab (s String, INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- The text index does not model NOT IN at all, unlike the token bloom filter family below. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- tokenbf_v1 + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE tokenbf_v1(256, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- ngrambf_v1 + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE ngrambf_v1(3, 256, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- A FixedString element has its NUL padding stripped before tokenization, so it keeps pruning. +SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12)) SETTINGS force_data_skipping_indices = 'idx'; +SELECT (SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12))) = (SELECT count() FROM tab WHERE s IN (SELECT toFixedString('word1', 12)) SETTINGS use_skip_indexes = 0); + +-- sparse_grams + +DROP TABLE tab; +CREATE TABLE tab (s String, INDEX idx s TYPE sparse_grams(3, 100, 512, 2, 0)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT 'word' || toString(number) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- NOT IN never prunes on these indexes, but the index must still be used. +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS force_data_skipping_indices = 'idx'; +-- Without transform_null_in the same predicates keep the globalIn/globalNotIn spellings, which must prune too. +SELECT count() FROM tab WHERE s GLOBAL IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s GLOBAL NOT IN ('word1') SETTINGS transform_null_in = 0, force_data_skipping_indices = 'idx'; +-- A Nullable element type is what transform_null_in = 1 adds, so a null-free set of it must prune. +SELECT count() FROM tab WHERE s IN (SELECT CAST('word1', 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE s IN (SELECT CAST(NULL, 'Nullable(String)')) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } + +-- A Nullable column: only the text index accepts one, and only there can a NULL element of the set +-- meet a NULL row of the column. The set is a disjunction, so one NULL element refuses the index +-- whatever else the set holds. + +DROP TABLE tab; +CREATE TABLE tab (s Nullable(String), INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4; +INSERT INTO tab SELECT if(number = 7, NULL, 'word' || toString(number)) FROM numbers(8); + +SELECT extract(explain, 'Granules: \\d+/\\d+') FROM (EXPLAIN indexes = 1 SELECT count() FROM tab WHERE s IN ('word1')) WHERE explain LIKE '%Granules: %/%'; +SELECT count() FROM tab WHERE s IN ('word1', NULL) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE s IN ('word1', NULL); +SELECT count() FROM tab WHERE s IN ('word1', NULL, 'word2') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE s IN ('word1', NULL, 'word2'); + +-- A tuple set is one Tuple column unpacked by position, so the indexed component is addressed by it. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, s String, INDEX idx s TYPE text(tokenizer = splitByNonAlpha)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 4; +INSERT INTO tab SELECT number, 'word' || toString(number) FROM numbers(8); + +SELECT count() FROM tab WHERE (id, s) IN ((1, 'word1')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE (id, s) IN (SELECT tuple(number, 'word1') FROM numbers(8)) SETTINGS force_data_skipping_indices = 'idx'; + +-- An absent map key reads the value type's default, which mapValues never stores. The FixedString default +-- is all NUL and the array tokenizer keeps the padding, so it must be recognised as the default. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, m Map(String, FixedString(4)), INDEX idx mapValues(m) TYPE text(tokenizer = array)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO tab VALUES (0, {'k':'val0'}), (1, {'other':'xxxx'}), (2, {'k':'val2'}), (3, {}); + +SELECT count() FROM tab WHERE m['k'] IN (toFixedString('', 4)) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] IN (toFixedString('', 4)); +SELECT count() FROM tab WHERE m['k'] = toFixedString('', 4) SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] = toFixedString('', 4); +SELECT count() FROM tab WHERE m['k'] IN ('val0') SETTINGS force_data_skipping_indices = 'idx'; + +-- A String map value defaults to '' only, so an all-NUL value is a real value and must prune. + +DROP TABLE tab; +CREATE TABLE tab (id UInt64, m Map(String, String), INDEX idx mapValues(m) TYPE text(tokenizer = array)) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity = 1; +INSERT INTO tab SELECT 0, map('k', unhex('00')); +INSERT INTO tab SELECT number, map('k', 'val' || toString(number)) FROM numbers(1, 2); +INSERT INTO tab SELECT 3, map('other', 'x'); + +SELECT count() FROM tab WHERE m['k'] = unhex('00') SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE m['k'] IN (unhex('00')) SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM tab WHERE m['k'] = '' SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] IN ('') SETTINGS force_data_skipping_indices = 'idx'; -- { serverError INDEX_NOT_USED } +SELECT count() FROM tab WHERE m['k'] = ''; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql index e0e49b060904..25efb3237844 100644 --- a/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql +++ b/tests/queries/0_stateless/02354_vector_search_rescoring_distance_in_select_list.sql @@ -62,6 +62,8 @@ INSERT INTO tab_bf16 VALUES (0, [1.0, 0.0]), (9, [0.0, 2.4]); -- The nearest neighbours to [0.0, 0.2] are 5,6,7,8 +-- Round distances: ARM sanitizer builds disable SimSIMD and scalar USearch +-- differs in the 3rd decimal (0.203125 vs 0.1875). Neighbor order is unchanged. SELECT 'Column: Array(Float32)'; diff --git a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql index 6c42607cc565..79d53d6a2e77 100644 --- a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql +++ b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql @@ -16,6 +16,6 @@ SELECT '--'; SELECT * FROM t1 AS t2, (SELECT 1) AS t1; -SELECT * FROM (SELECT 1) AS t1, t1.nested AS t2; -- { serverError UNKNOWN_IDENTIFIER } +SELECT * FROM (SELECT 1) AS t1, t1.nested AS t2; -- { serverError UNKNOWN_DATABASE } -SELECT * FROM t1.nested AS t2, (SELECT 1) AS t1; -- { serverError UNKNOWN_IDENTIFIER } +SELECT * FROM t1.nested AS t2, (SELECT 1) AS t1; -- { serverError UNKNOWN_DATABASE } diff --git a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference index 4f8c3ae7dc13..80083db7617a 100644 --- a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference +++ b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference @@ -20,7 +20,7 @@ 8 1500 9 1500 -- -CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', \'Disk(\\\'backups\\\', \\\'default_inner\\\')\') +CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', Disk(\'backups\', \'default_inner\')) test_table_1 15000 test_table_2 15000 -- diff --git a/tests/queries/0_stateless/03300_ai_functions.reference b/tests/queries/0_stateless/03300_ai_functions.reference index 2e1f69e2d613..7fd7da027037 100644 --- a/tests/queries/0_stateless/03300_ai_functions.reference +++ b/tests/queries/0_stateless/03300_ai_functions.reference @@ -44,9 +44,9 @@ String -- Setting defaults ai_function_embedding_default_credentials ai_function_embedding_max_batch_size 100 -ai_function_max_api_calls_per_query 1000 -ai_function_max_input_tokens_per_query 1000000 -ai_function_max_output_tokens_per_query 500000 +ai_function_max_api_calls_per_query 0 +ai_function_max_input_tokens_per_query 0 +ai_function_max_output_tokens_per_query 0 ai_function_max_retries 1 ai_function_request_timeout_sec 60 ai_function_retry_initial_delay_ms 1000 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference index a0f46fa1a31f..148993579eca 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_1.reference @@ -4,10 +4,10 @@ eq_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 3))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, 3)\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 5))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_flip -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(3, __table1.i), equals(__table1.i, 5))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_cross_same 3 30 3 c y 2024-06-15 12:00:00 @@ -15,7 +15,7 @@ eq_eq_cross_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, toUInt8(3)))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, 3)\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_cross_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, toUInt8(5)))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 eq_eq_float_same 3 30 3 c y 2024-06-15 12:00:00 @@ -194,5 +194,5 @@ multi_expr SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(greater(__table1.i, 1), less(__table1.i, 5), greater(__table1.f, 2.), less(__table1.f, 6.))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(greater(__table1.i, 1), less(__table1.i, 5), greater(__table1.f, 2.), less(__table1.f, 6.))\nSETTINGS optimize_redundant_comparisons = 1 multi_expr_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, 5), greater(__table1.f, 1.))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference index 08f40bf45839..4e7ebdf27a6d 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_optimization_2.reference @@ -15,7 +15,7 @@ lc_eq_eq SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.lc, \'y\'), equals(__table1.lc, \'y\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE _CAST(equals(__table1.lc, \'y\'), \'UInt8\')\nSETTINGS optimize_redundant_comparisons = 1 lc_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.lc, \'y\'), equals(__table1.lc, \'z\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 str_eq_eq 3 30 3 c y 2024-06-15 12:00:00 @@ -23,10 +23,10 @@ str_eq_eq SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.s, \'c\'), equals(__table1.s, \'c\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.s, \'c\')\nSETTINGS optimize_redundant_comparisons = 1 str_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.s, \'c\'), equals(__table1.s, \'a\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 transitive_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, 3), equals(__table1.i, __table1.u), equals(__table1.u, 5), equals(__table1.i, 5), equals(__table1.u, 3))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 transitive_no_redundant SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(less(__table1.i, 3), greater(__table1.u, 3), less(__table1.u, 10))\nSETTINGS optimize_redundant_comparisons = 0 diff --git a/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference b/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference index bceb5ddeb7ed..76b4c36132c4 100644 --- a/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference +++ b/tests/queries/0_stateless/04032_and_comparison_filter_type_conversion.reference @@ -4,7 +4,7 @@ int_str_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, \'3\'), equals(__table1.i, 3))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.i, \'3\')\nSETTINGS optimize_redundant_comparisons = 1 int_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.i, \'3\'), equals(__table1.i, \'5\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 int_str_range 3 30 3 c y 2024-06-15 12:00:00 @@ -17,7 +17,7 @@ float_int_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, 3), equals(__table1.f, 3.))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.f, 3)\nSETTINGS optimize_redundant_comparisons = 1 float_int_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, 3), equals(__table1.f, 4))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 float_int_range 1 10 1.5 a x 2024-01-01 00:00:00 @@ -42,7 +42,7 @@ float_str_and_int SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, \'3.0\'), less(__table1.f, 5))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.f, \'3.0\')\nSETTINGS optimize_redundant_comparisons = 1 float_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.f, \'3.0\'), equals(__table1.f, \'5.0\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 float_str_range 1 10 1.5 a x 2024-01-01 00:00:00 @@ -57,7 +57,7 @@ dt_str_eq_same SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, \'2024-06-15 12:00:00\'), equals(__table1.dt, \'2024-06-15 12:00:00\'))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.dt, \'2024-06-15 12:00:00\')\nSETTINGS optimize_redundant_comparisons = 1 dt_str_eq_diff -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, \'2024-06-15 12:00:00\'), equals(__table1.dt, \'2025-01-01 00:00:00\'))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 dt_str_range 3 30 3 c y 2024-06-15 12:00:00 @@ -78,7 +78,7 @@ dt_int_eq_prune SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, 1718452800), greater(__table1.dt, 1704067200))\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE equals(__table1.dt, 1718452800)\nSETTINGS optimize_redundant_comparisons = 1 dt_int_eq_conflict -SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE and(equals(__table1.dt, 1718452800), equals(__table1.dt, 1735689600))\nSETTINGS optimize_redundant_comparisons = 0 +SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 0 SELECT\n __table1.i AS i,\n __table1.u AS u,\n __table1.f AS f,\n __table1.s AS s,\n __table1.lc AS lc,\n __table1.dt AS dt\nFROM default.`04032_t` AS __table1\nWHERE 0\nSETTINGS optimize_redundant_comparisons = 1 dt_int_range 3 30 3 c y 2024-06-15 12:00:00 diff --git a/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference b/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference index 51f83ff27583..6090a2c07c4d 100644 --- a/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference +++ b/tests/queries/0_stateless/04050_text_index_starts_ends_with.reference @@ -6,7 +6,13 @@ Expression ((Project names + Projection)) Condition: true Parts: 3/3 Granules: 3/3 - Ranges: 3 + Skip + Name: idx_name_fts + Description: text GRANULARITY 100000000 + Condition: (mode: All; tokens: []) + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -15,7 +21,13 @@ Expression ((Project names + Projection)) Condition: true Parts: 3/3 Granules: 3/3 - Ranges: 3 + Skip + Name: idx_name_fts + Description: text GRANULARITY 100000000 + Condition: (mode: All; tokens: []) + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -112,9 +124,9 @@ Expression ((Project names + Projection)) Name: idx_name_fts Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) - Parts: 3/3 - Granules: 3/3 - Ranges: 3 + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) @@ -127,9 +139,9 @@ Expression ((Project names + Projection)) Name: idx_name_fts Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) - Parts: 3/3 - Granules: 3/3 - Ranges: 3 + Parts: 1/3 + Granules: 1/3 + Ranges: 1 2 Full-text search is now generally available Expression ((Project names + Projection)) ReadFromMergeTree (default.test_fts) diff --git a/tests/queries/0_stateless/04056_ast_json_limits.sql b/tests/queries/0_stateless/04056_ast_json_limits.sql index e448e8ff8bb7..6aac3b420051 100644 --- a/tests/queries/0_stateless/04056_ast_json_limits.sql +++ b/tests/queries/0_stateless/04056_ast_json_limits.sql @@ -91,3 +91,17 @@ SELECT formatQueryFromJSON('{"type":"RefreshStrategy","schedule_kind":"EVERY"}') -- `clickhouse_json` client/server entry points (so a huge shallow document is rejected before Poco -- materializes it). Here `parseQueryToJSON('SELECT 1')` is well over 10 bytes. SELECT formatQueryFromJSON(parseQueryToJSON('SELECT 1')) SETTINGS max_query_size = 10; -- { serverError SYNTAX_ERROR } + +-- Reading an AST back from JSON descends over the document, so raising `max_ast_depth` must not turn +-- a deep payload into a stack overflow: each shape below must fail with a controlled error, never +-- crash. The depths sit far past every bound so that holds in any build; which of the descents +-- reports it first follows the build's stack budget, so no arm asserts a particular one. +-- A chain of nested AST nodes. +SELECT formatQueryFromJSON(concat(repeat('{"type":"ExpressionList","children":[', 20000), '{"type":"Literal","value":{"field_type":"UInt64","value":1}}', repeat(']}', 20000))) SETTINGS max_ast_depth = 100000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- A nested `field_type` value inside a single literal, which adds no AST nodes of its own. +SELECT formatQueryFromJSON(concat('{"type":"Literal","value":', repeat('{"field_type":"Array","value":[', 25000), '{"field_type":"UInt64","value":1}', repeat(']}', 25000), '}')) SETTINGS max_ast_depth = 200000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } + +-- The JSON text is parsed before any AST node exists, so a deep member that the reader never descends +-- into still has to be bounded while the text itself is being read. +SELECT formatQueryFromJSON(concat('{"type":"Literal","value":{"field_type":"UInt64","value":1},"junk":', repeat('[', 200000), repeat(']', 200000), '}')) SETTINGS max_ast_depth = 100000, max_ast_elements = 0, max_query_size = 100000000; -- { serverError TOO_DEEP_RECURSION } diff --git a/tests/queries/0_stateless/04061_text_index_json_all_values.reference b/tests/queries/0_stateless/04061_text_index_json_all_values.reference index 494d66d4afc2..1bb69101acef 100644 --- a/tests/queries/0_stateless/04061_text_index_json_all_values.reference +++ b/tests/queries/0_stateless/04061_text_index_json_all_values.reference @@ -22,8 +22,8 @@ SELECT id FROM tab WHERE startsWith(data.key1, 'lazy') ORDER BY id 1 Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) -Parts: 4/4 -Granules: 4/4 +Parts: 1/4 +Granules: 1/4 SELECT id FROM tab WHERE endsWith(data.key1, 'fox') ORDER BY id 0 Description: text GRANULARITY 100000000 @@ -61,8 +61,8 @@ SELECT id FROM tab WHERE startsWith(data.key1::String, 'lazy') ORDER BY id 1 Description: text GRANULARITY 100000000 Condition: (mode: All; tokens: []) -Parts: 4/4 -Granules: 4/4 +Parts: 1/4 +Granules: 1/4 SELECT id FROM tab WHERE hasToken(data.key1::String, 'quick') ORDER BY id 0 2 diff --git a/tests/queries/0_stateless/04070_url_base_setting.reference b/tests/queries/0_stateless/04070_url_base_setting.reference index 8c23601338af..958e5fdfc6c7 100644 --- a/tests/queries/0_stateless/04070_url_base_setting.reference +++ b/tests/queries/0_stateless/04070_url_base_setting.reference @@ -28,3 +28,5 @@ url = 'http://base.invalid/dir/persist.csv' 0 0 must contain a scheme +must contain a scheme +0 diff --git a/tests/queries/0_stateless/04070_url_base_setting.sh b/tests/queries/0_stateless/04070_url_base_setting.sh index 46289ca8b2c1..9127cb87d066 100755 --- a/tests/queries/0_stateless/04070_url_base_setting.sh +++ b/tests/queries/0_stateless/04070_url_base_setting.sh @@ -172,3 +172,10 @@ DROP TABLE IF EXISTS ${CLICKHOUSE_TEST_UNIQUE_NAME}_url_pos_creds; # Invalid url_base (no scheme) should produce an error $CLICKHOUSE_CLIENT --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'example.invalid/def/', $FAST" 2>&1 | grep -oF 'must contain a scheme' | head -1 + +# The rejected value is not echoed back: it can carry a credential, and the message reaches the client, +# the exception column of the query log and the server log, none of which the display-secrets setting +# gates. (last line counts the leaks; clickhouse-local is used because the client also prints back the +# query it was given, which is the caller's own input rather than something the message disclosed) +$CLICKHOUSE_LOCAL --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:SEKRIT_PW@example.invalid/def/', $FAST" 2>&1 | grep -oF 'must contain a scheme' | head -1 +$CLICKHOUSE_LOCAL --query "SELECT * FROM url('data.csv', CSV, 'c String') SETTINGS url_base = 'user:SEKRIT_PW@example.invalid/def/', $FAST" 2>&1 | grep -c SEKRIT_PW ||: diff --git a/tests/queries/0_stateless/04072_jit_decimal_expressions.sql b/tests/queries/0_stateless/04072_jit_decimal_expressions.sql index e735f68165d7..eda70438d6ae 100644 --- a/tests/queries/0_stateless/04072_jit_decimal_expressions.sql +++ b/tests/queries/0_stateless/04072_jit_decimal_expressions.sql @@ -34,6 +34,7 @@ SELECT toFloat32(d32), toFloat64(d64) FROM test_jit_dec_expr ORDER BY d32; SELECT 'Test integer to Decimal conversions'; SELECT toDecimal32(i64, 2), toDecimal64(i64, 4) FROM test_jit_dec_expr ORDER BY i64; +-- The `Float ->` direction is evaluated by the interpreter: see #117442. SELECT 'Test float to Decimal conversions'; SELECT toDecimal64(f64, 4) FROM test_jit_dec_expr ORDER BY f64; diff --git a/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql b/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql index 0900b1e0533e..d54a37585d5a 100644 --- a/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql +++ b/tests/queries/0_stateless/04143_if_decimal_int_literal_jit.sql @@ -45,7 +45,7 @@ WITH materialize(3::Decimal(38, 30)) AS r, materialize(2) AS k SELECT sum(multiI WITH materialize(1::Decimal(76, 60)) AS r, materialize(1) AS k SELECT sum(if(k != 1, r, 1)); WITH materialize(1::Decimal(76, 73)) AS r, materialize(1) AS k SELECT sum(if(k != 1, r, 1)); --- `Float` -> `Decimal` branch lift (the analyzer promotes the result to `Decimal`). +-- A float branch is never lifted: the result is `Variant(Decimal(18, 4), Float64)`, which is not compiled. WITH materialize(toDecimal64(1.5, 4)) AS r, materialize(1) AS k SELECT if(k != 1, r, 2.5::Float64); -- Sanity check: the non-JIT path produces the same answers. diff --git a/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql b/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql index ca1d5926f830..ac79ea0cb10b 100644 --- a/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql +++ b/tests/queries/0_stateless/04205_jit_decimal_cast_branches_parity.sql @@ -40,11 +40,9 @@ -- the analyzer promotes the result type to `Variant` because -- `use_variant_as_common_type = 1` by default, and `canBeNativeType` -- excludes `Variant`. `FunctionIfBase::isCompilableImpl` returns `false` --- and the helper is never called. The helper still has correct code paths --- for these (the `pow10_fp_const` helper avoids 64-bit narrowing via --- `APFloat::convertFromAPInt` so high-bit-width `Decimal128`/`Decimal256` --- factors stay accurate) so it is forward-compatible if the analyzer ever --- stops promoting to `Variant`. +-- and the helper is never called. `Float` -> `Decimal` is asserted against +-- rather than supported: an out-of-range float to integer conversion has no +-- defined result once compiled. -- -- * `Decimal` -> `Decimal` with different scales (scale increase OR decrease): -- `FunctionIf::executeImpl` rejects this with `NOT_IMPLEMENTED: diff --git a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference index 67a5306eb758..102257aae8b5 100644 --- a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference +++ b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.reference @@ -1,18 +1,5 @@ --- JIT --- ---- Float -> 128-bit / 256-bit integers --- -2 -2 -2 -2 -2 -2 -2 -2 ---- 128-bit / 256-bit integers -> Float --- -2 -2 -2 -2 +--- 128-bit integers -> Float --- 2 2 2 @@ -36,3 +23,4 @@ 2 2 2 +1 diff --git a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql index 54923031e8c2..71acd02410de 100644 --- a/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql +++ b/tests/queries/0_stateless/04240_jit_float_to_big_int_libcalls.sql @@ -6,25 +6,15 @@ SELECT '--- JIT ---'; SET compile_expressions = 1, min_count_to_compile_expression = 0; -SELECT '--- Float -> 128-bit / 256-bit integers ---'; -SELECT toInt128 (materialize(1.5) + materialize(0.5)); -SELECT toUInt128(materialize(1.5) + materialize(0.5)); -SELECT toInt256 (materialize(1.5) + materialize(0.5)); -SELECT toUInt256(materialize(1.5) + materialize(0.5)); -SELECT toInt128 (materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toUInt128(materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toInt256 (materialize(1.5)::Float32 + materialize(0.5)::Float32); -SELECT toUInt256(materialize(1.5)::Float32 + materialize(0.5)::Float32); - -SELECT '--- 128-bit / 256-bit integers -> Float ---'; +-- Only the 128-bit integer to float direction is compiled, and it reaches `__floattisf` and +-- `__floattidf`. A float source with an integer destination is declined, and a 256-bit value is not +-- a native JIT type, so both have an interpreted arm only: the first is covered by +-- `05055_jit_float_to_integer_cast`, the second by the block below. +SELECT '--- 128-bit integers -> Float ---'; SELECT toFloat32(materialize(2::Int128) + materialize(0::Int128)); SELECT toFloat64(materialize(2::Int128) + materialize(0::Int128)); SELECT toFloat32(materialize(2::UInt128) + materialize(0::UInt128)); SELECT toFloat64(materialize(2::UInt128) + materialize(0::UInt128)); -SELECT toFloat32(materialize(2::Int256) + materialize(0::Int256)); -SELECT toFloat64(materialize(2::Int256) + materialize(0::Int256)); -SELECT toFloat32(materialize(2::UInt256) + materialize(0::UInt256)); -SELECT toFloat64(materialize(2::UInt256) + materialize(0::UInt256)); SELECT '--- no JIT ---'; SET compile_expressions = 0; @@ -48,3 +38,24 @@ SELECT toFloat32(materialize(2::Int256) + materialize(0::Int256)); SELECT toFloat64(materialize(2::Int256) + materialize(0::Int256)); SELECT toFloat32(materialize(2::UInt256) + materialize(0::UInt256)); SELECT toFloat64(materialize(2::UInt256) + materialize(0::UInt256)); + +-- Every row above compares a compiled value against an interpreted one, so all of them would still +-- pass if the JIT arm stopped compiling. This pins that the 128-bit conversion is compiled wherever +-- the control is, which also holds in a build without the embedded compiler. The control is plain +-- arithmetic, so it stays at 1 even if every conversion stops being compilable. +SELECT toFloat64(materialize(2::Int128) + materialize(0::Int128)) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '04240_int128' FORMAT Null; +SELECT materialize(2.0) + materialize(0.0) + materialize(1.0) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '04240_control' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +WITH shapes AS +( + SELECT log_comment, argMax(ProfileEvents['CompiledFunctionExecute'] > 0, event_time_microseconds) AS compiled + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE '04240_%' + GROUP BY log_comment +) +SELECT (SELECT compiled FROM shapes WHERE log_comment = '04240_int128') + = (SELECT compiled FROM shapes WHERE log_comment = '04240_control'); diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference index 24970f73243f..fa1f28c18666 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.reference @@ -1,9 +1,17 @@ 1 0 +1 +0 +0 +0 0 0 1 0 1 5 1 5 +1 5 a +false +true +true diff --git a/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql b/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql index 587399f309b5..c26c4b44f01e 100644 --- a/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql +++ b/tests/queries/0_stateless/04366_filter_push_down_limit_by.sql @@ -34,6 +34,89 @@ FROM ( ) WHERE key = '5' ); +-- Same query with query_plan_filter_push_down_below_limit_by = 0: the conjunct stays above the +-- LIMIT BY, so the primary key gets no condition -> the plan shape used before 26.7. +SELECT countIf(match(explain, 'Condition: \(key in ')) > 0 AS pushed +FROM ( + EXPLAIN indexes = 1 + SELECT * FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0 +); + +-- `filter_above_limitby` = 1 iff a Filter step exists AND its plan line precedes LimitBy's. +-- Absence of a primary key condition alone cannot tell "filter above LimitBy" from "filter +-- pushed below it but index analysis produced nothing", so the OFF path needs this placement +-- oracle too. Same shape as 04512_filter_push_down_limit_by_nondeterministic. +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT * FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0, enable_parallel_replicas = 0 + ) + ) +); + +-- The push-down pass also runs with `query_plan_filter_push_down = 0` once a JOIN runtime +-- filter has been added, so the setting has to be honoured on that path as well: at its default +-- the key conjunct still reaches below LimitBy (no Filter step above it -> 0), at 0 the conjunct +-- stays above LimitBy (-> 1). The two arms differ only in the new setting. +-- `query_plan_optimize_join_order_randomize = 0` (its default) is pinned in both arms because a +-- non-zero seed feeds the join order optimizer random cardinalities, which can reverse the two +-- join inputs; the reversed plan prints the right-hand branch first and `fl < ll` then compares +-- lines from different branches. +DROP TABLE IF EXISTS t_04366_rhs; +CREATE TABLE t_04366_rhs (key String) ENGINE = MergeTree ORDER BY key AS SELECT '5'; + +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT l.key FROM ( + SELECT key, ts FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) AS l INNER JOIN t_04366_rhs AS r ON l.key = r.key + WHERE l.key = '5' + SETTINGS query_plan_filter_push_down = 0, query_plan_filter_push_down_below_limit_by = 1, + enable_join_runtime_filters = 1, enable_parallel_replicas = 0, + query_plan_optimize_join_order_randomize = 0 + ) + ) +); + +SELECT (has_filter AND fl < ll) AS filter_above_limitby +FROM ( + SELECT countIf(explain LIKE '%Filter (%') > 0 AS has_filter, + minIf(ln, explain LIKE '%Filter (%') AS fl, + minIf(ln, explain LIKE '%LimitBy%') AS ll + FROM ( + SELECT explain, rowNumberInAllBlocks() AS ln + FROM ( + EXPLAIN + SELECT l.key FROM ( + SELECT key, ts FROM t_04366 ORDER BY key, ts LIMIT 1 BY key + ) AS l INNER JOIN t_04366_rhs AS r ON l.key = r.key + WHERE l.key = '5' + SETTINGS query_plan_filter_push_down = 0, query_plan_filter_push_down_below_limit_by = 0, + enable_join_runtime_filters = 1, enable_parallel_replicas = 0, + query_plan_optimize_join_order_randomize = 0 + ) + ) +); + +DROP TABLE t_04366_rhs; + -- LIMIT n OFFSET m BY (m > 0): NOT pushed. A group of size <= m is fully dropped, so a -- pushed throwing key predicate could be evaluated on rows the query never reaches. SELECT countIf(match(explain, 'Condition: \(key in ')) > 0 AS pushed @@ -94,6 +177,11 @@ SELECT count(), sum(val) FROM ( SELECT key, ts, val FROM t_04366 WHERE key = '5' ORDER BY key, ts LIMIT 1 BY key ); +-- Same result with the push-down declined, so the OFF path is not covered by EXPLAIN alone. +SELECT count(), sum(val) FROM ( + SELECT key, ts, val FROM t_04366 ORDER BY key, ts LIMIT 1 BY key +) WHERE key = '5' SETTINGS query_plan_filter_push_down_below_limit_by = 0; + DROP TABLE t_04366; -- Exception-semantics regression: a singleton group '0' dropped by OFFSET 1 must NOT be @@ -129,3 +217,12 @@ SELECT key FROM ( SELECT key, x FROM t_04366_nonkey ORDER BY key, ord LIMIT 1 BY key ) WHERE intDiv(1, x) > 0; DROP TABLE t_04366_nonkey; + +-- `compatibility` with a version before the one the setting was added in turns it off, so the +-- filter stays above the `LIMIT BY`; from that version on, and unset, it is on. +SET compatibility = '26.7'; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); +SET compatibility = '26.8'; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); +SET compatibility = ''; +SELECT getSetting('query_plan_filter_push_down_below_limit_by'); diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference new file mode 100644 index 000000000000..876d94d1857a --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference @@ -0,0 +1,4 @@ +TTLDropMerge Horizontal 0 +0 +TTLDropMerge Horizontal 0 +0 diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh new file mode 100755 index 000000000000..d34ea2f74dc6 --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +TABLE=t_ttl_drop_not_vertical + +function wait_for_ttl_drop_merge() +{ + for _ in $(seq 1 600); do + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS part_log" + local merge_count + merge_count=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge'") + + if [ "${merge_count}" -gt "0" ]; then + return + fi + sleep 0.1 + done + + echo "Timed out waiting for a TTLDropMerge of ${TABLE}" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" + +TABLE=t_ttl_drop_not_vertical_mixed_ttl + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY, d + INTERVAL 2 DAY RECOMPRESS CODEC(ZSTD) + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" diff --git a/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference b/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference index 1545bdb2fa43..7caab58d91b6 100644 --- a/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference +++ b/tests/queries/0_stateless/04510_distributed_final_many_partitions.reference @@ -1,3 +1,3 @@ split groups all partitions into the target tasks 1 -distributed plan 15000 112567500 -plain plan 15000 112567500 +distributed plan 6500 21154250 +plain plan 6500 21154250 diff --git a/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh b/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh index 5c4cfb379b5e..1e4009602a45 100755 --- a/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh +++ b/tests/queries/0_stateless/04510_distributed_final_many_partitions.sh @@ -18,28 +18,28 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # not expose the split, so the check reads the planner's trace message. $CLICKHOUSE_CLIENT --multiquery -q " -SET max_partitions_per_insert_block = 150; --- Statistics play no role here, and writing a statistics blob for each of the 300 tiny parts --- makes the setup exceed the test time limit on sanitizer builds with object storage. +-- Statistics play no role here, and writing a statistics blob for each tiny part slows the setup +-- down on sanitizer builds with object storage. SET materialize_statistics_on_insert = 0; DROP TABLE IF EXISTS t_final_many_partitions; CREATE TABLE t_final_many_partitions (k UInt64, p UInt16, v UInt64, ver UInt64) ENGINE = ReplacingMergeTree(ver) PARTITION BY p ORDER BY k SETTINGS index_granularity = 64, auto_statistics_types = ''; SYSTEM STOP MERGES t_final_many_partitions; --- 150 partitions, far more than the 4 tasks x 16 layers per task the planner aims for. Every key +-- 65 partitions, one more than the 4 tasks x 16 layers per task the planner aims for. Every key -- appears in every partition and must survive per-partition dedup, and its newer version must win. -INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number, 1 FROM numbers(15000); -INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number + 5, 2 FROM numbers(15000); +INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number, 1 FROM numbers(6500); +INSERT INTO t_final_many_partitions SELECT number % 100, intDiv(number, 100), number + 5, 2 FROM numbers(6500); " -SETTINGS="enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, max_rows_to_group_by = 0, +# The table is small enough to be broadcast; force the bucketed read regardless of size. +SETTINGS="enable_parallel_replicas = 0, automatic_parallel_replicas_mode = 0, max_rows_to_group_by = 0, distributed_plan_max_rows_to_broadcast = 0, distributed_plan_default_reader_bucket_count = 4, do_not_merge_across_partitions_select_final = 1" echo -n "split groups all partitions into the target tasks " if $CLICKHOUSE_CLIENT --send_logs_level=trace -q " SELECT count(), sum(v) FROM t_final_many_partitions FINAL FORMAT Null SETTINGS make_distributed_plan = 1, $SETTINGS" 2>&1 \ - | grep -q "Distributed FINAL read bucketed: 150 layers in 38 lanes per task make 4 tasks" + | grep -q "Distributed FINAL read bucketed: 65 layers in 17 lanes per task make 4 tasks" then echo 1; else echo 0; fi # Both plans must return identical results. diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index eb36b31ef3f9..5aa390bf9bb0 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -138,7 +138,7 @@ UNION id: 0, union_mode: UNION_ALL CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8 JOIN TREE IDENTIFIER id: 15, identifier: system.one --- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers\n-- (role_arn, role_session_name) stay visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; +-- session_token, the Google ADC secrets (google_adc_client_secret, google_adc_refresh_token) and\n-- the extra_credentials assume-role material (external_id) passed to the explicit-url or\n-- named-collection S3 form must be masked like secret_access_key. Every secret value below is tagged\n-- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE\n-- and logged query text.\n\n-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifier\n-- (role_arn) stays visible while external_id is hidden.\nDROP TABLE IF EXISTS t_04510; CREATE TABLE t_04510 (`x` UInt8) ENGINE = S3(\'http://localhost:11111/test/04510\', \'ak\', \'[HIDDEN]\', session_token = \'[HIDDEN]\', google_adc_client_secret = \'[HIDDEN]\', google_adc_refresh_token = \'[HIDDEN]\', extra_credentials(role_arn = \'visible_role_arn\', external_id = \'[HIDDEN]\'), format = \'TSV\') SHOW CREATE TABLE t_04510 SETTINGS format_display_secrets_in_show_and_select = 0; DROP TABLE t_04510; @@ -197,6 +197,46 @@ BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_pos\', \'[HIDDEN]\', \'[HIDDEN]\' BACKUP TABLE nonexistent_04510 TO S3(nc_bkp_missing, \'visible_bkp_dir\', \'[HIDDEN]\') BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, secret_access_key = \'[HIDDEN]\', \'visible_bkp_dir2\') BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_mixed\', equals(access_key_id, \'ak\'), \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Memory(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO `Null`(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Foo(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Null(); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureQueue(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'cont\', \'visible_04510_dir/b.zip\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/acct\', \'visible_04510_cont\', \'visible_04510_dir/b.zip\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, connection_string = \'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=[HIDDEN];\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, equals(storage_account_url, \'http://localhost:11111/visible_04510_url\'), account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = \'visible_04510_c1\', container = \'visible_04510_c2\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/visible_04510_acct5\', \'visible_04510_cont5\', \'visible_04510_blob5\', \'visible_04510_acctname5\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte1 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte2 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte3 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte4 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte5 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte6 (`x` UInt8) ENGINE = AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'visible_04510_teco\', \'visible_04510_teblob\') +CREATE TABLE t_04510_azte7 (`x` UInt8) ENGINE = AzureBlobStorage(\'http://localhost:11111/visible_04510_teacct\', \'visible_04510_tec5\', \'visible_04510_teb5\', \'visible_04510_teacctname\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte8 (`x` UInt8) ENGINE = AzureQueue(\'http://localhost:11111/visible_04510_teq/cont/*\', \'[HIDDEN]\') SETTINGS mode = \'unordered\' CREATE DATABASE db_04510_ec ENGINE = Backup(\'\', S3(\'url_dbec\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) CREATE DATABASE db_04510_postok ENGINE = Backup(\'\', S3(\'url_dbpostok\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_ncpos ENGINE = Backup(\'\', S3(nc_dbnc_missing, \'visible_dbnc_dir\', \'[HIDDEN]\')) @@ -207,6 +247,21 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup(\'\', S3(\'url_dbmixed\', access_ CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = \'[HIDDEN]\')) CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_quoted ENGINE = Backup(\'\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tail ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tmap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_lone ENGINE = Backup(\'[HIDDEN]\') +CREATE DATABASE db_04510_swap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ident ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ftail ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fover ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_dover ENGINE = Backup(\'src_04510\', Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_mover ENGINE = Backup(\'src_04510\', Memory(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fvalid ENGINE = Backup(\'src_04510\', File(\'nonexistent_04510\')); +CREATE DATABASE db_04510_dvalid ENGINE = Backup(\'src_04510\', Disk(\'backups\', \'nonexistent_04510\')); +CREATE DATABASE db_04510_eval ENGINE = Backup(\'\', S3(\'http://localhost:11111/test/04510eval\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) +CREATE DATABASE db_04510_nonlit ENGINE = Backup(\'\', S3(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_azure ENGINE = Backup(\'\', AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') DROP DATABASE IF EXISTS default_1 CREATE DATABASE default_1 ENGINE = S3(\'url_dbenv\', \'ak\', \'[HIDDEN]\', use_environment_credentials = 1) @@ -218,4 +273,4 @@ EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(nc_04510_missing, url = \'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', structure = \'x UInt8\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') UNION ALL SELECT 1 -1 0 +1 0 0 diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index 9cded2e5e118..4f7eee6fb510 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -7,8 +7,8 @@ -- so the final assertion can prove none of them leaks. They used to leak in plaintext in SHOW CREATE -- and logged query text. --- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifiers --- (role_arn, role_session_name) stay visible while external_id is hidden. +-- Engine form: SHOW CREATE hides every secret; the non-secret extra_credentials identifier +-- (role_arn) stays visible while external_id is hidden. DROP TABLE IF EXISTS t_04510; CREATE TABLE t_04510 (x UInt8) ENGINE = S3('http://localhost:11111/test/04510', 'ak', 'SEKRIT_SAK', @@ -234,6 +234,138 @@ BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, BACKUP TABLE nonexistent_04510 TO S3('url_bkp_mixed', access_key_id = 'ak', 'SEKRIT_BKPMIX'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- The credential-free engines (Disk, File, Memory, Null) are credential-free only in the shape they +-- read: File and Memory take one argument, Disk two, Null none. A surplus argument is rejected after +-- the statement is logged, so it reaches the log holding whatever was written in it, an argument that is +-- no string is read by none of these engines and can carry a string of its own, and an engine name that +-- is not registered at all has no known shape to trust; all three must be hidden. The last statement is +-- the control: the shape Null does read carries nothing to hide, so it stays visible verbatim and gets +-- as far as resolving the table. +BACKUP TABLE nonexistent_04510 TO File('nonexistent_04510', + 'SEKRIT_TOFILEOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Disk('backups', 'nonexistent_04510', + 'SEKRIT_TODISKOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Memory('nonexistent_04510', + 'SEKRIT_TOMEMOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO File(['SEKRIT_TOFILEARR']); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO Null('SEKRIT_TONULLOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Foo('SEKRIT_TOUNKNOWN'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO Null(); -- { serverError UNKNOWN_TABLE } + +-- The AzureBlobStorage backup destination reads a different signature than the table engine of the same +-- name: a named collection with an optional filename, three arguments (connection string or account url, +-- container, path), or five (adding account_name and account_key). An argument outside those shapes is +-- rejected only after the statement is logged, and AzureQueue has no backup engine at all. The last +-- two statements are the controls: a connection string hides its AccountKey, and the three-argument +-- shape has nothing to hide, so it stays visible verbatim. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZTO4'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, 'dir', + 'SEKRIT_AZTONCPOS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureQueue('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZQTO'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTOCSKEY==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTOCS5'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + 'cont', 'visible_04510_dir/b.zip'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'visible_04510_cont', 'visible_04510_dir/b.zip'); -- { serverError BAD_ARGUMENTS } + +-- A named collection can be overridden per statement, and the destination evaluates those overrides as +-- constant expressions. An override this rule cannot read may hold either credential, and hiding a +-- connection string replaces the whole argument, which cannot be combined with hiding account_key, so +-- both shapes hide the locator whole. The last three statements are the controls: a connection string +-- alone still hides only its AccountKey, account_key alone is hidden by itself, and an account url +-- override does not take the replacement path, so it stays visible next to a hidden account_key. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = concat('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=', + 'SEKRIT_AZNCEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + account_key = 'SEKRIT_AZNCBOTH'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = concat('https://a.blob.core.windows.net/c?sig=', + 'SEKRIT_AZNCURLEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=SEKRIT_AZNCCS==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + account_key = 'SEKRIT_AZNCKEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://localhost:11111/visible_04510_url', + account_key = 'SEKRIT_AZNCURLKEY'); -- { serverError BAD_ARGUMENTS } + +-- An override key can be an expression too, and one this rule cannot read hides which credential the +-- value is: both the computed key and a malformed override hide the locator whole, as the S3 form above +-- already does for a key it cannot read. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + concat('account_', 'key') = 'SEKRIT_AZNCKEYEXPR'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + equals('account_key', 'SEKRIT_AZNCEQ3', 'surplus')); -- { serverError BAD_ARGUMENTS } + +-- A readable key does not make the override readable: the destination evaluates its value as a constant +-- expression, so a value that is no plain literal or identifier can nest a credential of its own and is +-- formatted verbatim before that evaluation rejects it, as the S3 form above already is for such a value. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + container = headers('Authorization' = 'SEKRIT_AZNCHDR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + blob_path = concat('SEKRIT_AZNCPATHEXPR', '/b.zip')); -- { serverError BAD_ARGUMENTS } + +-- connection_string and storage_account_url are mutually exclusive, and the destination reads at most +-- one of them, so an override it never reads holds whatever was written there. The last statement is +-- the control: a key that carries no credential stays visible however often it is repeated. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + storage_account_url = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCPAIR==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCDUP==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = 'visible_04510_c1', container = 'visible_04510_c2'); -- { serverError BAD_ARGUMENTS } + +-- An account url is shown only when it is a plain storage account URL, which is what the destination +-- requires beside explicit credentials: userinfo, a query string (a SAS is a credential) and a fragment +-- each carry a credential of their own, in any shape and under any scheme spelling. A connection value +-- that is no string is read by neither this rule nor the destination, so it is hidden as well. The last +-- statement is the control: a plain url keeps the account url, container, path and account name visible +-- while account_key is hidden, and reaches the account_key decoding that rejects it. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://user:SEKRIT_AZNCUSERINFO@localhost:11111/acct'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'HTTPS://localhost:11111/acct?sig=SEKRIT_AZNCSAS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://user:SEKRIT_AZ3USERINFO@localhost:11111/acct', + 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct#f', 'cont', 'blob', + 'acct', 'SEKRIT_AZTO5KEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(['SEKRIT_AZ3ARR'], 'cont', 'blob'); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/visible_04510_acct5', + 'visible_04510_cont5', 'visible_04510_blob5', 'visible_04510_acctname5', + 'SEKRIT_AZ5PLAINKEY'); -- { serverError STD_EXCEPTION } + +-- The AzureBlobStorage table engine reads its arguments the same way: an override whose value or key +-- this rule cannot read may hold a credential the engine does read (the parser evaluates both as +-- constant expressions), an account url is shown only when it is a plain storage account url, and a +-- connection value that is no string is read by neither. A connection string is hidden by replacing +-- its whole argument, which cannot be combined with hiding a following account_key. The last two +-- statements are the controls: the connection string alone still hides only its AccountKey, and a +-- plain account url keeps url, container, path and account name visible beside a hidden account_key. +CREATE TABLE t_04510_azte1 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + connection_string = headers('Authorization' = 'SEKRIT_AZTEHDR')); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte2 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + upper('account_key') = 'SEKRIT_AZTEKEY'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte3 (x UInt8) ENGINE = AzureBlobStorage(['SEKRIT_AZTEARR'], 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_04510_azte4 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/acct?sig=SEKRIT_AZTESAS', + 'cont', 'blob', 'acct', 'SEKRIT_AZTE5KEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte5 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECS==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTEMIXKEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte6 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECTLCS==;', + 'visible_04510_teco', 'visible_04510_teblob'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte7 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/visible_04510_teacct', + 'visible_04510_tec5', 'visible_04510_teb5', 'visible_04510_teacctname', + 'SEKRIT_AZTECTLKEY'); -- { serverError STD_EXCEPTION } + +-- The two-argument signature takes container and path from the url and a shared access signature +-- beside it, so its second argument is a credential wherever the same rule serves the engine. +CREATE TABLE t_04510_azte8 (x UInt8) ENGINE = AzureQueue('http://localhost:11111/visible_04510_teq/cont/*', + 'SEKRIT_AZTESAS2') SETTINGS mode = 'unordered'; -- { serverError UNKNOWN_FORMAT } + -- Backup database engine reconstructs the nested S3 destination; extra_credentials must be masked. CREATE DATABASE db_04510_ec ENGINE = Backup('', S3('url_dbec', 'ak', 'SEKRIT_SAK', extra_credentials(external_id = 'SEKRIT_EID'))); -- { serverError BAD_ARGUMENTS } @@ -272,14 +404,82 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup('', S3('url_dbmixed', CREATE DATABASE db_04510_ncurl ENGINE = Backup('', S3(nc_dburl_missing, url = concat('https://user:SEKRIT_PW@', 'localhost/x?X-Amz-Signature=SEKRIT_SIG'))); -- { serverError BAD_ARGUMENTS } --- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim. +-- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim, and the +-- rejection message must not echo it either: a nested map's values are hidden only by its parent's +-- formatter, so such a node formatted on its own carries them in plaintext. CREATE DATABASE db_04510_hdr ENGINE = Backup('', S3('url_dbhdr', 'ak', 'SEKRIT_SAK', - headers('X-Auth' = 'SEKRIT_HDR'))); -- { serverError BAD_ARGUMENTS } + headers('X-Auth' = 'SEKRIT_DBHDR'))); -- { serverError BAD_ARGUMENTS } -- The reconstructor must also fail closed on a constant-expression extra_credentials key. CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT_SAK', extra_credentials(concat('extern', 'al_id') = 'SEKRIT_EXPR'))); -- { serverError BAD_ARGUMENTS } +-- A locator held in a string literal is text this finder cannot parse, and that text can carry an +-- access key or a presigned URL. The `Backup` engine accepts that form only while replaying its own +-- metadata, but a statement carrying it is formatted - by `PARALLEL WITH`, by the distributed DDL +-- queue, by `query_log` - before the engine rejects it, so it must be masked whole. +CREATE DATABASE db_04510_quoted ENGINE = Backup('', + 'S3(\'https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG\', \'ak\', \'SEKRIT_QUOTED\')'); -- { serverError BAD_ARGUMENTS } + +-- `Backup(database_name, locator)` is the only valid shape. The statement is logged before the arity +-- is validated, so a credential parked in a surplus argument must be hidden as well. +CREATE DATABASE db_04510_tail ENGINE = Backup('', S3('url_dbtail', 'ak', 'SEKRIT_SAK'), + 'SEKRIT_DBTAIL'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_tmap ENGINE = Backup('', S3('url_dbtmap', 'ak', 'SEKRIT_SAK'), + extra_credentials(external_id = 'SEKRIT_DBTMAP')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_lone ENGINE = Backup(S3('url_dblone', 'ak', + 'SEKRIT_DBLONE')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- A locator is also a credential carrier in the database-name position, where the argument is neither +-- masked by the S3 rule below nor safe to echo: the message that rejects it formats it standalone. +CREATE DATABASE db_04510_swap ENGINE = Backup(S3('url_dbswap', 'ak', 'SEKRIT_DBSWAP'), + ''); -- { serverError BAD_ARGUMENTS } + +-- An identifier is not a literal either, and the parsers evaluate one as a literal, so the +-- database-name position must hold a literal before the rest of the shape is trusted. +CREATE DATABASE db_04510_ident ENGINE = Backup(SEKRIT_DBIDENT, + S3('url_dbident', 'ak', 'SEKRIT_SAK')); -- { serverError BAD_ARGUMENTS } + +-- A credential-free locator (Disk, File, Memory, Null) names its destination with literals and holds +-- no credential, but it keeps a named override or nested map that it never reads, and that can carry +-- one. Both the logged text and the not-found message identify the locator, so both must hide it. +CREATE DATABASE db_04510_ftail ENGINE = Backup('src_04510', + File('nonexistent_04510', extra_credentials(external_id = 'SEKRIT_DBFTAIL'))); -- { serverError BACKUP_NOT_FOUND } + +-- A surplus literal argument reaches the same locator through the shape check instead of the tail, and +-- it is rejected only after the statement is logged, so it has to be hidden as well. The two controls +-- that follow hold the argument counts these engines do read (File one, Disk two), where the locator +-- names a destination only: nothing is masked, so both the logged text and the not-found message keep +-- it visible verbatim, and the transcript records the statement as sent rather than a re-formatted AST. +CREATE DATABASE db_04510_fover ENGINE = Backup('src_04510', + File('nonexistent_04510', 'SEKRIT_DBFILEOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_dover ENGINE = Backup('src_04510', + Disk('backups', 'nonexistent_04510', 'SEKRIT_DBDISKOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_mover ENGINE = Backup('src_04510', + Memory('nonexistent_04510', 'SEKRIT_DBMEMOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_fvalid ENGINE = Backup('src_04510', File('nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } +CREATE DATABASE db_04510_dvalid ENGINE = Backup('src_04510', Disk('backups', 'nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } + +-- A credential value given as an expression is hidden in the logged text, but evaluating it can fail +-- with a message that quotes its input, so the rejection must not carry that message either. The url +-- has to parse for the locator to reach credential evaluation at all. +CREATE DATABASE db_04510_eval ENGINE = Backup('', S3('http://localhost:11111/test/04510eval', 'ak', 'SEKRIT_SAK', + extra_credentials(external_id = toUInt64('SEKRIT_DBEVAL')))); -- { serverError BAD_ARGUMENTS } + +-- A non-tail argument that is neither a literal nor `key = value` cannot be reconstructed either. It +-- is rejected by a different message than the quoted locator above, and that message used to echo the +-- offending argument verbatim, so it needs its own tag. +CREATE DATABASE db_04510_nonlit ENGINE = Backup('', S3(concat('SEKRIT_NONLIT', 'x'), + 'url_dbnonlit')); -- { serverError BAD_ARGUMENTS } + +-- S3 is not the only backup engine whose locator carries credentials: AzureBlobStorage takes an +-- account_key and accepts connection strings and named-collection overrides that carry one too. +-- Only S3 is reconstructed above, so an Azure locator keeps its engine name and argument count +-- (neither is a secret) and every argument is hidden. The url carries a query string, which the +-- engine rejects before it reaches the network. +CREATE DATABASE db_04510_azure ENGINE = Backup('', AzureBlobStorage('http://localhost:11111/acct?sig=x', + 'cont', 'blob', 'account', 'SEKRIT_AZUREKEY')); -- { serverError BAD_ARGUMENTS } + -- The S3 database engine accepts no positional beyond secret_access_key; an extra positional must -- be masked in the logged query text. CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', @@ -338,7 +538,15 @@ ORDER BY event_time_microseconds; -- logs each DDL from the replay worker, which re-masks a rewritten AST independently, so assert -- the masking property over every row this test produced, replay rows included. count() > 0 keeps -- an empty row set from passing vacuously. -SELECT count() > 0, countIf(query LIKE '%SEKRIT%') +-- The third column covers the recorded exception messages of the seven unreconstructible-locator +-- statements above, whose rejections used to echo the locator verbatim - one tag per message, since +-- they are thrown at different sites. It is scoped to those tags because widening it to every +-- deliberately-failing statement here would report unrelated pre-existing echoes. +SELECT count() > 0, countIf(query LIKE '%SEKRIT%'), + countIf(exception LIKE '%SEKRIT_QUOTED%' OR exception LIKE '%SEKRIT_NONLIT%' + OR exception LIKE '%SEKRIT_DBHDR%' OR exception LIKE '%SEKRIT_DBSWAP%' + OR exception LIKE '%SEKRIT_DBEVAL%' OR exception LIKE '%SEKRIT_DBFTAIL%' + OR exception LIKE '%SEKRIT_DBIDENT%') FROM system.query_log WHERE current_database = currentDatabase() AND type != 'QueryStart' diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference new file mode 100644 index 000000000000..6cc8f3258bf9 --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference @@ -0,0 +1,3 @@ +1 +TOO_LARGE_ARRAY_SIZE +TOO_LARGE_ARRAY_SIZE diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh new file mode 100755 index 000000000000..627efc72b50a --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tests that a RowBinaryWithNames[AndTypes] header with a suspiciously large column +# count is rejected with TOO_LARGE_ARRAY_SIZE instead of amplifying into a huge +# allocation. See https://github.com/ClickHouse/clickhouse-private/issues/69219 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_rowbinary_header" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_rowbinary_header (x UInt64) ENGINE = Memory" + +# LEB128 encoder. +varint() { + python3 -c ' +import sys +n = int(sys.argv[1]) +out = bytearray() +while True: + b = n & 0x7F + n >>= 7 + if n: + b |= 0x80 + out.append(b) + if not n: + break +sys.stdout.buffer.write(bytes(out)) +' "$1" +} + +# Negative control: a well-formed one-column header (name "x", type "UInt64") plus one +# 8-byte UInt64 row inserts fine. +{ varint 1; printf '\x01x\x06UInt64'; printf '\x01\x00\x00\x00\x00\x00\x00\x00'; } \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_rowbinary_header" + +# Attack: a header claiming far more columns than the ceiling (1'000'000). +# The tiny body must be rejected up-front, not amplified. +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +# Same for the WithNames variant (the names header is a separate read path). +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNames" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_rowbinary_header" diff --git a/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference new file mode 100644 index 000000000000..394d98eaf66f --- /dev/null +++ b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference @@ -0,0 +1,14 @@ +0 1000 0 +1 1000 1 +[[1000],[1000]] +[2000,2000] +['a'] [2000] ['b','c'] [2000,2000] +[2000] [2000,2000] +[2000] +['a','b','c'] [0,1,1] [2000,0,0] ['x','y'] [0,0] +[1000,1000] [1000,1000] +[999,1000] [999,1000] +[1000,1000] [1000,1000] +[1000,1000] [1000,1000] +[1000,1000] [1000,1000] +[1000,1000] [1000,1000] diff --git a/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql new file mode 100644 index 000000000000..8fc70d385953 --- /dev/null +++ b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql @@ -0,0 +1,102 @@ +-- Tags: no-parallel +-- no-parallel -- enables a process-global failpoint that fires in every -State finalization + +-- A combinator that transfers several sub-states in one insertResultInto call aliases them into the +-- result column one by one. When the transfer throws part-way through, the aggregator destroys the +-- whole place while the result column still owns the already-transferred sub-states, so they are +-- freed twice. The failpoint throws from the -State transfer once the destination column already +-- holds an aliased state, which is exactly that partial transfer. +-- Each sub-state below holds more than 4096 bytes of groupArray data, so it is allocated outside the +-- arena and the second destroy reaches a real deallocation. + +-- The second failpoint throws at the start of a -Tuple element whose predecessor transferred in full, +-- so the child being undone is a COMPLETED one rather than the partially transferred innermost -State. + +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; + +-- Plain -State: the transfer is atomic, so this must throw cleanly on every version. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT number % 2 AS k, groupArrayState(number) FROM numbers(2000) GROUP BY k ORDER BY k SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT k, length(finalizeAggregation(s)), finalizeAggregation(s)[1] FROM (SELECT number % 2 AS k, groupArrayState(number) AS s FROM numbers(2000) GROUP BY k) ORDER BY k SETTINGS max_threads = 1; + +-- -Resample over -ForEach, one element per bucket: the throw is in the second bucket, so -Resample +-- has to undo a completed -ForEach child. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEachResample(0, 2, 1)([number], number % 2) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> arrayMap(y -> length(finalizeAggregation(y)), x), groupArrayStateForEachResample(0, 2, 1)([number], number % 2)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -ForEach alone: the throw is in its own loop, with the first element already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEach([number, number + 1]) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateForEach([number, number + 1])) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -Tuple over -Map with String keys: the first element completes, the second throws on its second +-- key, so -Tuple has to undo a completed -Map child in a different subcolumn. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateMapTuple((map('a', number), map('b', number, 'c', number))) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT mapKeys(t.1), arrayMap(x -> length(finalizeAggregation(x)), mapValues(t.1)), mapKeys(t.2), arrayMap(x -> length(finalizeAggregation(x)), mapValues(t.2)) FROM (SELECT groupArrayStateMapTuple((map('a', number), map('b', number, 'c', number))) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- -Tuple over -ForEach, first element shorter than the second. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEachTuple(([number], [number, number])) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateForEachTuple(([number], [number, number])) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- Exactly one bucket: the -State transfer cannot throw (the column is still empty), so the throw +-- comes from -Resample appending its own offset with the single sub-state already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResample(0, 1, 1)(number, 0) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateResample(0, 1, 1)(number, 0)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- A -Map whose value rows are not uniform: the key with non-NULL values aliases a sub-state, the +-- all-NULL keys make the null adapter insert a default the column itself owns. Undoing such a map in +-- any order other than the reverse of the sorted-key append order applies the wrong row's semantics. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateTupleMapTuple((CAST(map('b', NULL, 'a', (number, number + 1), 'c', NULL), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'), CAST(map('x', (number, number + 1), 'y', (number + 2, number + 3)), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'))) FROM numbers(2000) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT mapKeys(t.1), arrayMap(x -> isNull(x), mapValues(t.1)), arrayMap(x -> length(finalizeAggregation(assumeNotNull(x).1)), mapValues(t.1)), mapKeys(t.2), arrayMap(x -> isNull(x), mapValues(t.2)) FROM (SELECT groupArrayStateTupleMapTuple((CAST(map('b', NULL, 'a', (number, number + 1), 'c', NULL), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'), CAST(map('x', (number, number + 1), 'y', (number + 2, number + 3)), 'Map(String, Nullable(Tuple(UInt64, UInt64)))'))) AS t FROM numbers(2000)) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1; + +-- -Tuple over -Resample. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same, through a transparent -If wrapper. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleIfTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2), (number > 0, number > 0)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleIfTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2), (number > 0, number > 0)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- -OrNull around a Tuple of -State results: Tuple can be inside Nullable, so the transfer goes into +-- the nested column of a ColumnNullable. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTupleOrNullTuple(0, 2, 1)(((number, number + 1), (number + 2, number + 3)), ((number % 2, number % 2), (number % 2, number % 2))) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.1).1), arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.2).2) FROM (SELECT groupArrayStateResampleTupleOrNullTuple(0, 2, 1)(((number, number + 1), (number + 2, number + 3)), ((number % 2, number % 2), (number % 2, number % 2))) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same window through the implicit null adapter instead of -OrNull, plus a -Distinct wrapper. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResampleTupleDistinctTuple(0, 2, 1)(CAST(((number, number + 1), (number + 2, number + 3)), 'Tuple(Nullable(Tuple(UInt64, UInt64)), Nullable(Tuple(UInt64, UInt64)))'), ((number % 2, number % 2), (number % 2, number % 2))) FROM numbers(2000) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.1).1), arrayMap(x -> length(finalizeAggregation(x)), assumeNotNull(t.2).2) FROM (SELECT groupArrayStateResampleTupleDistinctTuple(0, 2, 1)(CAST(((number, number + 1), (number + 2, number + 3)), 'Tuple(Nullable(Tuple(UInt64, UInt64)), Nullable(Tuple(UInt64, UInt64)))'), ((number % 2, number % 2), (number % 2, number % 2))) AS t FROM numbers(2000)) SETTINGS max_threads = 1, enable_nullable_tuple_type = 1; + +-- A completed -Resample child undone by its parent: with the second failpoint the first tuple element +-- transfers all of its buckets, and the throw lands before the second element. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; + +-- The same completed child behind an -OrNull that forwards: Array cannot be inside Nullable, so -OrFill +-- passes the array column straight through and has to forward the undo as well. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT groupArrayStateResampleOrNullTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw_after_child; +SELECT arrayMap(x -> length(finalizeAggregation(x)), t.1), arrayMap(x -> length(finalizeAggregation(x)), t.2) FROM (SELECT groupArrayStateResampleOrNullTuple(0, 2, 1)((number, number + 1), (number % 2, number % 2)) AS t FROM numbers(2000)) SETTINGS max_threads = 1; diff --git a/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference new file mode 100644 index 000000000000..b170576ac2c0 --- /dev/null +++ b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.reference @@ -0,0 +1,19 @@ +1 +1 +1 +[10] +[10] +[10] +[10] +1 +1 +1 +1 +1 +1 +1 +1 +[10] +[10] +1 +1 diff --git a/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql new file mode 100644 index 000000000000..4bb4cb072734 --- /dev/null +++ b/tests/queries/0_stateless/04626_and_compare_chain_nothing_collapsed_and.sql @@ -0,0 +1,131 @@ +-- The `optimize_and_compare_chain` / `optimize_redundant_comparisons` passes prune and fold an +-- AND-chain of comparisons. Both skip a Nullable-typed AND, because a comparison over a Nullable +-- operand cannot participate in range pruning. That guard checks only the AND's own result type, +-- which is unsound: when one AND operand is `Nothing`-typed, the function resolver collapses the +-- whole AND's result type to bare `Nothing` (`Nothing::isNullable()` is false), so the guard passes +-- even though another operand is a comparison over a directly-Nullable expression. That operand then +-- reached the pruning path and hit `chassert(!raw_type->isNullable())`. + +SET enable_analyzer = 1; + +-- 1) No logical error. These queries are otherwise invalid (a `Nothing`-typed value cannot be +-- materialized), so they must fail with a normal handled exception, never a logical error. +-- optimize_and_compare_chain path (chassert reached via the seed loop of tryOptimizeAndCompareChain): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(1::Int32)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 1; -- { serverError ILLEGAL_COLUMN } +-- optimize_redundant_comparisons path (tryOptimizeAndCompareNotEqualsChain), independent of the chain setting: +SELECT tuple((materialize(toNullable(1::Int32)) = 1) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(1::Int32)) != 1) AND (assumeNotNull(materialize(toNullable(NULL))) != 2)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } + +-- 2) The optimizer still runs on such a `Nothing`-collapsed AND: the directly-Nullable operand is +-- kept as-is (the new opaque-filter fallback) while a redundant non-null comparison in the SAME +-- AND is still folded. In `x > 3 AND x > 5`, `x > 3` is redundant and pruned, so exactly one +-- `greater` survives when enabled against both when disabled. Pin both counts: a relative +-- comparison also holds when the predicates disappear entirely. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain LIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 0) WHERE explain LIKE '%function_name: greater,%'; +-- The exact predicate that used to hit the assertion is `equals(...) -> Nullable(Nothing)`; assert it +-- survives (exactly one such node) rather than being silently dropped. The chain's other `equals` +-- returns `Nothing`, so match the `Nullable(Nothing)` result type specifically. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = 2)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain LIKE '%function_name: equals, function_type: ordinary, result_type: Nullable(Nothing)%'; + +-- 3) Correctness guard: a genuine (non-collapsed) AND-compare chain over a Nullable column must still +-- be pruned/folded correctly, i.e. the optimization result matches the unoptimized one, and NULLs +-- are excluded by the comparisons. `optimize_and_compare_chain` / `optimize_redundant_comparisons` +-- must not change results. +DROP TABLE IF EXISTS t_and_chain_nullable; +CREATE TABLE t_and_chain_nullable (x Nullable(Int32)) ENGINE = Memory; +INSERT INTO t_and_chain_nullable VALUES (1), (5), (NULL), (10); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x > 3) AND (x > 5) ORDER BY x SETTINGS optimize_and_compare_chain = 1); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x > 3) AND (x > 5) ORDER BY x SETTINGS optimize_and_compare_chain = 0); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 1); +SELECT groupArray(x) FROM (SELECT x FROM t_and_chain_nullable WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 0); +DROP TABLE t_and_chain_nullable; +-- The checks above compare results, which stay equal even if `optimize_and_compare_chain` stops +-- deriving anything. Pin the derivation itself with exact node counts: `a < b AND b < 5` gains the +-- transitive `a < 5`, so the enabled tree holds exactly 3 `less` nodes against 2 when disabled. +SELECT count() = 3 FROM (EXPLAIN QUERY TREE SELECT a, b FROM values('a Int32, b Int32', (1, 2), (4, 9)) WHERE (a < b) AND (b < 5) SETTINGS optimize_and_compare_chain = 1) WHERE explain ILIKE '%function_name: less,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT a, b FROM values('a Int32, b Int32', (1, 2), (4, 9)) WHERE (a < b) AND (b < 5) SETTINGS optimize_and_compare_chain = 0) WHERE explain ILIKE '%function_name: less,%'; + +-- 4) In a `Nothing`-collapsed AND, an operand may be a comparison whose constant side is itself +-- NULL-valued (e.g. `expr = NULL`), found by the AST fuzzer mutating case 2's `= 2` to `= NULL`. +-- A NULL-valued constant carries no comparable value, so `tryOptimizeAndCompareNotEqualsChain` +-- must not treat it as the constant side (it used to hit `chassert(!literal->getValue().isNull())`). +-- The whole AND is still `Nothing`-typed, so these queries fail with a normal handled exception. +-- RHS-NULL constant: +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (assumeNotNull(materialize(toNullable(NULL))) != NULL)) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- LHS-NULL constant (the other assertion branch): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (NULL = assumeNotNull(materialize(toNullable(NULL))))) SETTINGS optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- Independent of the pruning setting (the classification loop runs unconditionally): +SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } + +-- 5) Liveness of the keep-as-is mechanism for the NULL-valued-constant operand: the optimizer must +-- still run on this `Nothing`-collapsed AND (so a redundant sibling is pruned) while keeping the +-- NULL-valued `equals` operands rather than skipping classification entirely. In the tree with +-- pruning on, `x > 3` is redundant given `x > 5`, so exactly one `greater` node survives; with +-- pruning off both survive; both NULL-valued `equals` operands are kept in either case. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((materialize(toNullable(NULL)) = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))) = NULL)) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: equals,%'; + +-- 6) A comparison whose RESULT is nullable must be kept as-is too, even when the raw operand type does +-- not report `isNullable`: `LowCardinality(Nullable(T))` (nested nullability) and the NULL-capable +-- carriers `Dynamic` / `Variant` all yield a nullable comparison result. They used to slip past the +-- guard, fold the contradictory `x = 1 AND x = 2`, and change the handled exception depending on the +-- setting. The error must now be the same regardless of the pruning / chain settings. +SET allow_suspicious_low_cardinality_types = 1; +SET allow_experimental_dynamic_type = 1; +SET allow_experimental_variant_type = 1; +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Dynamic', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Dynamic', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Variant(Int32, String)', NULL) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; -- { serverError ILLEGAL_COLUMN } +SELECT tuple((x = 1) AND (x = 2) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x Variant(Int32, String)', NULL) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError ILLEGAL_COLUMN } +-- Liveness for the LC(Nullable) operand: a redundant NON-nullable sibling (`> 3` given `> 5`) is still +-- pruned to one `greater` node while the LC-nullable `equals` operand is kept, proving the optimizer +-- runs and keeps only that operand opaque rather than declining the whole collapsed AND. +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 1 FROM (EXPLAIN QUERY TREE SELECT tuple((x = 1) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) AND (assumeNotNull(materialize(toNullable(NULL))))) FROM values('x LowCardinality(Nullable(Int32))', NULL) SETTINGS optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: equals,%'; +-- A non-collapsed `LowCardinality(Nullable(T))` chain must still return correct results, unchanged by +-- the optimization (NULL excluded, `1`/`5` filtered out; only `10` remains). +SELECT groupArray(x) FROM (SELECT x FROM values('x LowCardinality(Nullable(Int32))', 1, 5, NULL, 10) WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 1); +SELECT groupArray(x) FROM (SELECT x FROM values('x LowCardinality(Nullable(Int32))', 1, 5, NULL, 10) WHERE (x != 1) AND (x != 5) ORDER BY x SETTINGS optimize_redundant_comparisons = 0); + +-- 7) The carrier the AST fuzzer keeps rediscovering on master (`Logical error: +-- '!raw_type->isNullable()'`, STID `2508-50fe`, e.g. +-- https://s3.amazonaws.com/clickhouse-test-reports/json.html?REF=master&sha=d469feea5f342065ffe8b2384d4ddf354dae3978&name_0=MasterCI&name_1=Stress%20test%20%28arm_asan_ubsan%29 ). +-- It reaches the same `addComparisonFilter` through a different route than the cases above: the +-- `Nothing`-typed operand comes from `ARRAY JOIN []` (whose element type is `Nothing`) instead of +-- `assumeNotNull(materialize(toNullable(NULL)))`, and the collapsed AND sits in a `JOIN ON` +-- section. `JOIN ON` is load-bearing - the same AND in a `WHERE` is rejected earlier, so only the +-- join expression lets a `Nothing`-typed AND reach the optimizer. +DROP TABLE IF EXISTS t_and_chain_array_join; +CREATE TABLE t_and_chain_array_join (c0 Int32) ENGINE = MergeTree() ORDER BY tuple(); +INSERT INTO t_and_chain_array_join VALUES (1), (2); + +-- `t2.c0 = a0` is `Nothing`-typed and collapses the AND's own result type, while +-- `toNullable(t2.c0) > 0` stays `Nullable(UInt8)` and reaches the pruning path. `ARRAY JOIN []` +-- produces no rows, so the query is valid and returns nothing; assert the empty result on every +-- combination of the two settings, since either entry point alone reaches the assertion. +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 1; +SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) SETTINGS optimize_and_compare_chain = 0, optimize_redundant_comparisons = 0; +-- A result-only check stays green if the queries stop running the optimizer at all, so pin the +-- pruning as well: given `> 5`, the sibling `> 3` is redundant and folded away, leaving the +-- Nullable-result `greater` plus one surviving constant comparison. Both settings are pinned on +-- every query below because `clickhouse-test` randomizes `optimize_and_compare_chain`. +SELECT count() = 2 FROM (EXPLAIN QUERY TREE SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1) WHERE explain ILIKE '%function_name: greater,%'; +SELECT count() = 3 FROM (EXPLAIN QUERY TREE SELECT 1 FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join AS t2 ON (t2.c0 = a0) AND (toNullable(t2.c0) > 0) AND (materialize(toInt32(5)) > 3) AND (materialize(toInt32(5)) > 5) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0) WHERE explain ILIKE '%function_name: greater,%'; +-- The exact fuzzer query: here the Nullable-result comparison is over a correlated scalar +-- subquery rather than a column. `optimize_and_compare_chain` does not gate this one - +-- `tryOptimizeAndCompareChain` skips a chain holding a correlated subquery, while +-- `tryOptimizeAndCompareNotEqualsChain` has no such guard - so it arrives only via +-- `optimize_redundant_comparisons`. A correlated subquery is not supported in a join expression, +-- so the query must report that handled exception instead of aborting. +SELECT 1 AS x FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join ON (t_and_chain_array_join.c0 = a0) AND (t_and_chain_array_join.c0 != a0) AND (0 > (SELECT t_and_chain_array_join.c0)) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 1; -- { serverError NOT_IMPLEMENTED } +SELECT 1 AS x FROM t_and_chain_array_join AS tx ARRAY JOIN [] AS a0 LEFT JOIN t_and_chain_array_join ON (t_and_chain_array_join.c0 = a0) AND (t_and_chain_array_join.c0 != a0) AND (0 > (SELECT t_and_chain_array_join.c0)) SETTINGS optimize_and_compare_chain = 1, optimize_redundant_comparisons = 0; -- { serverError NOT_IMPLEMENTED } +DROP TABLE t_and_chain_array_join; diff --git a/tests/queries/0_stateless/04626_s3_base_setting.reference b/tests/queries/0_stateless/04626_s3_base_setting.reference index 6b40f3b84623..97a52a1f9a5c 100644 --- a/tests/queries/0_stateless/04626_s3_base_setting.reference +++ b/tests/queries/0_stateless/04626_s3_base_setting.reference @@ -20,3 +20,5 @@ 5 --- s3_base without a scheme is an error must contain a scheme +must contain a scheme +0 diff --git a/tests/queries/0_stateless/04626_s3_base_setting.sh b/tests/queries/0_stateless/04626_s3_base_setting.sh index e8793ce77cab..88ef59858add 100755 --- a/tests/queries/0_stateless/04626_s3_base_setting.sh +++ b/tests/queries/0_stateless/04626_s3_base_setting.sh @@ -61,3 +61,10 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE test_s3_base_nc" && ${CLICKHOUSE_CLIENT} -q echo '--- s3_base without a scheme is an error' ${CLICKHOUSE_CLIENT} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'localhost:11111/test/'" 2>&1 | grep -oF 'must contain a scheme' | head -1 + +# The rejected value is not echoed back: it can carry a credential, and the message reaches the client, +# the exception column of the query log and the server log, none of which the display-secrets setting +# gates. (last line counts the leaks; clickhouse-local is used because the client also prints back the +# query it was given, which is the caller's own input rather than something the message disclosed) +${CLICKHOUSE_LOCAL} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'user:SEKRIT_PW@localhost:11111/test/'" 2>&1 | grep -oF 'must contain a scheme' | head -1 +${CLICKHOUSE_LOCAL} -q "SELECT * FROM s3('${FILE}', 'test', 'testtest', 'TSV', 'n UInt32, s String') SETTINGS s3_base = 'user:SEKRIT_PW@localhost:11111/test/'" 2>&1 | grep -c SEKRIT_PW ||: diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference index a6800ceefe7f..078656c7dc9d 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference @@ -1,5 +1,5 @@ -- Current defaults -false 1000 1 +false 0 1 -- compatibility = 26.6 restores the legacy defaults true 0 0 0 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql index cf37018c792d..1fe196405fc4 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql @@ -3,10 +3,10 @@ -- no-replicated-database: named collections are server-global, not database-scoped -- ============================================================================= --- Three AI function defaults were flipped: `ai_function_allow_insecure_endpoint` from 1 to 0 --- and `ai_function_max_api_calls_per_query` from 0 (unlimited) to 1000 in 26.8, and --- `ai_function_max_retries` from 0 to 1 in 26.9. `compatibility = 26.6` predates all three --- and restores them, which pins the previous_value/new_value pairs in `SettingsChangesHistory`. +-- Two AI function defaults were flipped in 26.8: `ai_function_allow_insecure_endpoint` from 1 +-- to 0 and `ai_function_max_retries` from 0 to 1. `ai_function_max_api_calls_per_query` stays 0 +-- (unlimited). `compatibility = 26.6` predates the flips and restores them, which pins the +-- previous_value/new_value pairs in `SettingsChangesHistory`. -- -- The endpoint check runs in `resolveAIParams`, before the zero-row early return -- in `executeImpl`, so an empty source table exercises it without any real HTTP diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference index cd9aa6da303c..daa2aafc0de8 100644 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.reference +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.reference @@ -12,6 +12,14 @@ SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', 'u', '[HIDDEN]', ssl_ca_pem = ' CREATE TABLE t (`x` Int32) ENGINE = MySQL('127.0.0.1:3306', 'db', 't', 'u', '[HIDDEN]', ssl_cert_pem = '[HIDDEN]') --- MySQL database engine, positional arguments CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '[HIDDEN]', ssl_ca_pem = '[HIDDEN]') +--- mysql table function, credentials before the positional arguments +SELECT * FROM mysql(ssl_key_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +--- MySQL database engine, credentials before the positional arguments +CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +--- mysql table function, named password without a collection +SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', password = '[HIDDEN]') +--- MySQL database engine, named password without a collection +CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '[HIDDEN]') --- key given as a constant expression SELECT * FROM mysql(creds, concat('ssl_ca', '_pem') = '[HIDDEN]', `table` = 't') --- key given as a constant expression, positional arguments diff --git a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh index 0fcb5994fd45..0bd7978918bb 100755 --- a/tests/queries/0_stateless/04648_mysql_tls_credentials.sh +++ b/tests/queries/0_stateless/04648_mysql_tls_credentials.sh @@ -49,6 +49,22 @@ format "MySQL table engine, positional arguments" \ format "MySQL database engine, positional arguments" \ "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', 'u', '${SECRET}', ssl_ca_pem = '${SECRET}')" +# The same credentials written before the positional arguments: the call is invalid, but it is +# formatted for logging before it is rejected, so the scan must not start past the first argument, +# and the positional password must be located among the positional arguments rather than at a fixed +# argument index, which a named argument written first moves. +format "mysql table function, credentials before the positional arguments" \ + "SELECT * FROM mysql(ssl_key_pem = '${SECRET}', '127.0.0.1:3306', 'db', 't', 'u', '${SECRET}')" +format "MySQL database engine, credentials before the positional arguments" \ + "CREATE DATABASE d ENGINE = MySQL(ssl_ca_pem = '${SECRET}', '127.0.0.1:3306', 'db', 'u', '${SECRET}')" + +# A named `password` override with no named collection: the parsers reject the mix, but the statement is +# formatted for logging first, and the key is readable, so only the named scan can hide this value. +format "mysql table function, named password without a collection" \ + "SELECT * FROM mysql('127.0.0.1:3306', 'db', 't', password = '${SECRET}')" +format "MySQL database engine, named password without a collection" \ + "CREATE DATABASE d ENGINE = MySQL('127.0.0.1:3306', 'db', password = '${SECRET}')" + # The key of a named argument is not required to be a plain identifier or literal: the named # collection parser evaluates it as a constant expression, so `concat('ssl_ca', '_pem')` names a TLS # credential too. The formatter cannot evaluate it, so it hides the value of every argument whose key diff --git a/tests/queries/0_stateless/04651_system_sync_transaction_log_access.reference b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql new file mode 100644 index 000000000000..44814f2c966e --- /dev/null +++ b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql @@ -0,0 +1,20 @@ +-- Tags: no-parallel, zookeeper +-- zookeeper: SYSTEM SYNC TRANSACTION LOG reads the transaction log from Keeper after access is granted. +-- no-parallel: creates a temporary user; avoid concurrent GRANT/DROP races on the same name pattern. + +DROP USER IF EXISTS user_test_04651; +CREATE USER user_test_04651; + +-- Before the fix, local SYSTEM SYNC TRANSACTION LOG never called checkAccess, so a user +-- without SYSTEM SYNC TRANSACTION LOG could still run the command (only the feature flag +-- allow_experimental_transactions was checked). ON CLUSTER already required the privilege. +-- Without the privilege, expect ACCESS_DENIED (checked before the transactions feature flag). +EXECUTE AS user_test_04651 SYSTEM SYNC TRANSACTION LOG; -- { serverError ACCESS_DENIED } + +-- Grant the privilege that ON CLUSTER already required. +GRANT SYSTEM SYNC TRANSACTION LOG ON *.* TO user_test_04651; + +-- With the privilege, the command must get past access control and sync the transaction log. +EXECUTE AS user_test_04651 SYSTEM SYNC TRANSACTION LOG; + +DROP USER user_test_04651; diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference index 515699bb5704..c3c10ae1556f 100644 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.reference @@ -34,13 +34,13 @@ partition_record_count_null_count: [('{\'eu\'}',1,0),('{\'eu\'}',1,1),('{\'us\'} 0 --- per-entry pairing against the referenced Parquet file --- id=1 rows=1/1 score_nulls=0/0 paired=yes lower=[(1,1),(2,10)] upper=[(1,1),(2,10)] -id=2 rows=1/1 score_nulls=1/1 paired=yes lower=[] upper=[] -id=3 rows=1/1 score_nulls=1/1 paired=yes lower=[] upper=[] +id=2 rows=1/1 score_nulls=1/1 paired=yes lower=[(1,2)] upper=[(1,2)] +id=3 rows=1/1 score_nulls=1/1 paired=yes lower=[(1,3)] upper=[(1,3)] --- per-entry bounds describe only their own file --- id=1 lower=[(1,1),(2,100)] upper=[(1,1),(2,100)] bounds_are_own_row=yes id=2 lower=[(1,2),(2,200)] upper=[(1,2),(2,200)] bounds_are_own_row=yes id=3 lower=[(1,3),(2,300)] upper=[(1,3),(2,300)] bounds_are_own_row=yes --- per-entry column_sizes describe only their own file --- -id=1 width=4 id_size=4 s_size=12 -id=2 width=8 id_size=4 s_size=16 -id=3 width=12 id_size=4 s_size=20 +id=1 width=10000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes +id=2 width=100000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes +id=3 width=1000000 id_size_positive=yes s_size_follows_own_width=yes sizes_fit_own_file=yes diff --git a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh index dc05b1e1ec7e..58fcd64bfd73 100755 --- a/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh +++ b/tests/queries/0_stateless/04652_iceberg_insert_per_file_statistics.sh @@ -153,11 +153,10 @@ ${CLICKHOUSE_CLIENT} --query " echo '--- per-entry pairing against the referenced Parquet file ---' for manifest in $(find "${PAIRED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not -name 'snap-*.avro' -type f | sort); do # `lower_bounds`/`upper_bounds` hold raw little-endian bytes (`dumpValue` in `IcebergWrites.cpp`), - # so they are decoded with `reinterpretAsInt32`; both key columns are `Int32` here. An entry whose - # `score` is all-NULL legitimately has NO bounds at all: `canWriteStatistics` is all-or-nothing - # across the entry's columns and `ColumnNullable::getExtremes` yields NULL extremes for an - # all-NULL column, which `canDumpIcebergStats` rejects. That is pre-existing behaviour, so it is - # asserted here rather than fixed. + # so they are decoded with `reinterpretAsInt32`; both key columns are `Int32` here. Bounds are + # per-field: an entry keeps a bound for every column whose extreme is serializable and omits the + # rest. `ColumnNullable::getExtremes` yields NULL extremes for an all-NULL column, which + # `canDumpIcebergStats` rejects, so an all-NULL `score` contributes no bound while `id` still does. ${CLICKHOUSE_CLIENT} --query " WITH entries AS ( SELECT @@ -190,10 +189,9 @@ for manifest in $(find "${PAIRED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not " done -# The bounds above are present on one entry only, because the two all-NULL files legitimately carry no -# bounds at all. This companion has no nullable column, so every entry carries bounds and the decoded -# value of each is checked against the single row its own file holds. Without it the bounds half of -# this change would be pinned on a single entry. +# Only one entry above carries a `score` bound, because the other two files hold an all-NULL `score`. +# This companion has no nullable column, so every entry carries a bound for both of its columns and the +# decoded value of each is checked against the single row its own file holds. ${CLICKHOUSE_CLIENT} --query " ${ONE_ROW_PER_FILE} CREATE TABLE bounded (id Int32, v Int32) @@ -235,16 +233,16 @@ done # The two scenarios above pin `record_count`, `null_value_counts` and the bounds to the file each # entry names, but not `column_sizes`, whose only other assertion is a cross-table `max()` inequality # that constrains no individual entry. Both of their fixtures also give every file the same per-file -# sizes, so a permutation cannot move a size value there. This scenario gives each file a `String` of -# a different width, which makes the sizes distinct and therefore permutation-sensitive. The values -# are in-memory `IColumn::byteSize` sums, not Parquet file sizes, so they are deterministic: -# `ColumnString::byteSize` is `chars.size() + offsets.size() * sizeof(offsets[0])`, and `insertData` -# appends exactly `length` bytes with no terminator. +# sizes, so a permutation cannot move a size value there. This scenario gives each file an +# incompressible `String` of a different order of magnitude, which makes the sizes distinct and +# therefore permutation-sensitive. `column_sizes` holds the post-compression size of the column chunk +# inside the Parquet file, so the exact byte count is a property of the encoder; what is asserted is +# that each entry's size follows the width of the row its own file holds and fits inside that file. ${CLICKHOUSE_CLIENT} --query " ${ONE_ROW_PER_FILE} CREATE TABLE sized (id Int32, s String) ENGINE = IcebergLocal('${SIZED_PATH}', 'Parquet') ORDER BY (id); - INSERT INTO sized SELECT number + 1, repeat('x', (number + 1) * 4) FROM numbers(3); + INSERT INTO sized SELECT number + 1, randomPrintableASCII(toUInt32(pow(10, number + 4))) FROM numbers(3); " echo '--- per-entry column_sizes describe only their own file ---' @@ -252,8 +250,9 @@ for manifest in $(find "${SIZED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not ${CLICKHOUSE_CLIENT} --query " WITH entries AS ( SELECT - replaceRegexpOne(tupleElement(data_file, 'file_path'), '^.*/', '') AS base, - CAST(tupleElement(data_file, 'column_sizes'), 'Map(Int32, Int64)') AS entry_sizes + replaceRegexpOne(tupleElement(data_file, 'file_path'), '^.*/', '') AS base, + CAST(tupleElement(data_file, 'column_sizes'), 'Map(Int32, Int64)') AS entry_sizes, + tupleElement(data_file, 'file_size_in_bytes') AS entry_file_size FROM file('${manifest}', Avro) ), files AS ( @@ -267,8 +266,11 @@ for manifest in $(find "${SIZED_PATH}/metadata" -maxdepth 1 -name '*.avro' -not SELECT 'id=' || toString(f.own_id) || ' width=' || toString(f.own_width) - || ' id_size=' || toString(e.entry_sizes[1]) - || ' s_size=' || toString(e.entry_sizes[2]) AS entry + || ' id_size_positive=' || if(e.entry_sizes[1] > 0, 'yes', 'no') + || ' s_size_follows_own_width=' || if( + e.entry_sizes[2] > f.own_width / 2 AND e.entry_sizes[2] < f.own_width * 2, 'yes', 'no') + || ' sizes_fit_own_file=' || if( + e.entry_sizes[1] + e.entry_sizes[2] <= e.entry_file_size, 'yes', 'no') AS entry FROM entries AS e INNER JOIN files AS f ON e.base = f.base ORDER BY f.own_id FORMAT TSV; diff --git a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference index 670c62cb8acb..0259a7a6a904 100644 --- a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference +++ b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.reference @@ -30,10 +30,10 @@ -- an explicit PREWHERE through both views keeps working -- 10 10 --- re-attached tables are lazy proxies -- -lazy_inner TableProxy +-- only the re-attached MergeTree is a lazy proxy -- +lazy_inner Merge lazy_leaf TableProxy -lazy_outer TableProxy +lazy_outer Merge -- the proxy must still reject the mismatched column, not abort -- -- a matching column still supports PREWHERE through the proxy -- 10 diff --git a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql index 412183279c35..b0e1d21115ef 100644 --- a/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql +++ b/tests/queries/0_stateless/04652_nested_merge_prewhere_type_mismatch_2.sql @@ -133,7 +133,7 @@ ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; -- as the style check requires (a `{...:String}` parameter is not recognized by it). USE {CLICKHOUSE_DATABASE_1:Identifier}; -SELECT '-- re-attached tables are lazy proxies --'; +SELECT '-- only the re-attached MergeTree is a lazy proxy --'; SELECT name, engine FROM system.tables WHERE database = currentDatabase() ORDER BY name; SELECT '-- the proxy must still reject the mismatched column, not abort --'; diff --git a/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql b/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql index efb4a79a398a..d514602e88b4 100644 --- a/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql +++ b/tests/queries/0_stateless/04653_merging_aggregated_deserialized_thread_count.sql @@ -22,6 +22,9 @@ SET max_parallel_replicas = 3; SET parallel_replicas_plan_based = 1; SET parallel_replicas_for_non_replicated_merge_tree = 1; SET cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; +-- The test verifies a plan packet on a same-version local cluster. Hedged connections retain +-- the SQL fallback for a future, unverified rolling-upgrade peer, so avoid that unrelated route. +SET use_hedged_requests = 0; -- The parallel replicas optimizer rewrite builds a `MergingAggregatedStep` from the params of a -- deserialized `AggregatingStep`, which carry the "resolve locally later" sentinel 0 as the thread @@ -34,6 +37,11 @@ GROUP BY k ORDER BY k SETTINGS serialize_query_plan = 1, distributed_aggregation_memory_efficient = 0, enable_memory_bound_merging_of_aggregation_results = 0, + -- This test asserts the serialized-plan path itself. Keep plan-level limits at their + -- defaults: a hedged connection must deliberately use SQL instead when a future replica + -- could be an older peer that cannot receive serialized execution limits. + max_threads = 0, + use_concurrency_control = 0, -- Required by the second firing oracle below. It defaults true, but it flipped false to -- true in the 24.3 block of `SettingsChangesHistory.cpp`, so a `compatibility` draw -- below 24.3 turns it off and that oracle would silently read 0. diff --git a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference index 0fb27808c3ca..73b717c7d255 100644 --- a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference +++ b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.reference @@ -1,6 +1,6 @@ TOO_DEEP_AST TOO_DEEP_AST -Structured Field value exceeds maximum AST depth limit +Structured Field value rejected Field dump payload exceeds maximum AST depth limit 1 TOO_DEEP_AST diff --git a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh index 455c50c4a2d5..67c08b16c6ba 100755 --- a/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh +++ b/tests/queries/0_stateless/04654_ast_json_depth_guard_max_ast_depth_zero.sh @@ -27,14 +27,16 @@ ${CLICKHOUSE_CLIENT} --max_ast_depth 0 --param_json "$BRACKET_BOMB" \ --query "SELECT formatQueryFromJSON({json:String})" 2>&1 | grep -om1 'TOO_DEEP_AST' # 3. A deeply nested structured `Field` value adds no AST nodes and stays under the bracket -# budget, so only the `Field` depth bound rejects it. +# budget, so it is rejected by its own depth bound or by the recursion's stack check. Which of +# the two reports it depends on the build's stack budget, so assert only that it is rejected. ${CLICKHOUSE_CLIENT} --max_ast_depth 0 --query " SELECT formatQueryFromJSON(concat( '{\"type\":\"Literal\",\"value\":', repeat('{\"field_type\":\"Array\",\"value\":[', 2000), '{\"field_type\":\"UInt64\",\"value\":1}', repeat(']}', 2000), '}'))" 2>&1 | - grep -om1 'Structured Field value exceeds maximum AST depth limit' + grep -qE 'Structured Field value exceeds maximum AST depth limit|TOO_DEEP_RECURSION' && + echo 'Structured Field value rejected' # 4. A deeply nested `Field` dump hides its nesting inside a JSON string, so the bracket # pre-scan does not see it either; `Field::restoreFromDump` must not recurse unbounded. diff --git a/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference b/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference index 209f37b46ed4..f726717f90db 100644 --- a/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference +++ b/tests/queries/0_stateless/04665_valueless_setting_ast_json_and_secret_parts.reference @@ -8,7 +8,7 @@ TYPE_MISMATCH 1 SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:pass@localhost\' TYPE_MISMATCH -SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:[HIDDEN]@localhost\' 1 +SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://[HIDDEN]@localhost\' 1 TYPE_MISMATCH TYPE_MISMATCH SELECT 1 SETTINGS format_avro_schema_registry_url = \'http://user:pass@localhost\' diff --git a/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference new file mode 100644 index 000000000000..1a595af2079a --- /dev/null +++ b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.reference @@ -0,0 +1,39 @@ +--- WHERE on the evolved column --- +4 4 +5 five +--- PREWHERE on the evolved column --- +4 4 +5 five +--- PREWHERE IS NULL / IS NOT NULL --- +none +6 +--- PREWHERE on an untouched column --- +4 4 +--- declared type and full scan --- +7 2 Nullable(Int64) +0 0 +1 1 +2 2 +3 3 +4 4 +5 five +\N none +--- MODIFY COLUMN to Nullable plus RENAME COLUMN --- +4 4 Nullable(Int64) +5 five Nullable(Int64) +--- widening plus nullability (int required to long optional) --- +4 Nullable(Int64) +--- optional to required is still rejected --- +Iceberg spec doesn't allow change type from nullable to non-nullable +--- String column made Nullable --- +4 Nullable(String) +--- externally authored optional to required stays a passthrough --- +id Int64 +s String +none +one +three +none +one +three +three diff --git a/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh new file mode 100755 index 000000000000..064a0c290e54 --- /dev/null +++ b/tests/queries/0_stateless/04737_iceberg_prewhere_modify_column_nullable.sh @@ -0,0 +1,153 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel-replicas +# `no-parallel-replicas`: see comment in `04071_iceberg_orc_prewhere_crash.sh`. +# `StorageObjectStorageCluster` (used when `parallel_replicas_for_cluster_engines = 1`, +# default) does not delegate `supportsPrewhere` to its underlying configuration. +# +# Regression test for issue #85029: filtering a column that `ALTER TABLE ... MODIFY COLUMN` +# made `Nullable` fails on the Iceberg data files written before the `ALTER`. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# `optimize_move_to_prewhere=1` + `query_plan_optimize_prewhere=1` are pinned on every +# discriminating statement: the failure only appears when the predicate is pushed into the +# reader, the runner randomizes both, and with either one off the pre-fix result is already +# correct, so the test would stop exercising the fix. +PREWHERE_SETTINGS="--optimize_move_to_prewhere=1 --query_plan_optimize_prewhere=1" + +TABLE="t_null_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_REN="t_ren_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_WID="t_wid_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_REJ="t_rej_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_STR="t_str_${CLICKHOUSE_DATABASE}_${RANDOM}" + +drop_table() { + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $1" + rm -rf "${USER_FILES_PATH}/$1/" +} + +# Rows 0..4 are written while `id` is still required, rows 5 and NULL after the `ALTER`, so the +# table mixes pre- and post-evolution data files. Only the pre-evolution ones carry the defect. +create_mixed_nullability_table() { + local table="$1" + local table_path="${USER_FILES_PATH}/${table}/" + rm -rf "${table_path}" + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${table}" + ${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${table} (id Int64, s String) ENGINE = IcebergLocal('${table_path}', 'Parquet')" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${table} SELECT number, toString(number) FROM numbers(5)" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${table} MODIFY COLUMN id Nullable(Int64)" + ${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${table} SELECT * FROM values('id Nullable(Int64), s String', (5, 'five'), (NULL, 'none'))" +} + +create_mixed_nullability_table "${TABLE}" + +echo "--- WHERE on the evolved column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} WHERE id > 3 ORDER BY id" + +# Explicit PREWHERE probes the reader-side path directly, without depending on the +# WHERE->PREWHERE mover. +echo "--- PREWHERE on the evolved column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} PREWHERE id > 3 ORDER BY id" + +echo "--- PREWHERE IS NULL / IS NOT NULL ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT s FROM ${TABLE} PREWHERE id IS NULL ORDER BY s" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT count() FROM ${TABLE} PREWHERE id IS NOT NULL" + +echo "--- PREWHERE on an untouched column ---" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, s FROM ${TABLE} PREWHERE s = '4' ORDER BY id" + +echo "--- declared type and full scan ---" +${CLICKHOUSE_CLIENT} --query "SELECT count(), countIf(id > 3), toTypeName(any(id)) FROM ${TABLE}" +${CLICKHOUSE_CLIENT} --query "SELECT id, s FROM ${TABLE} ORDER BY id, s" + +# Composition with a rename: the transform must apply the new name and the new nullability in one +# node. Before the fix the rename branch renamed the column and dropped the type change. +echo "--- MODIFY COLUMN to Nullable plus RENAME COLUMN ---" +create_mixed_nullability_table "${TABLE_REN}" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_REN} RENAME COLUMN id TO idx" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT idx, s, toTypeName(idx) FROM ${TABLE_REN} PREWHERE idx > 3 ORDER BY idx" + +# Widening and nullability at once already took the type-conversion branch and was already +# correct; assert it stays correct. +echo "--- widening plus nullability (int required to long optional) ---" +rm -rf "${USER_FILES_PATH}/${TABLE_WID}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_WID} (id Int32, s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_WID}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_WID} SELECT toInt32(number), toString(number) FROM numbers(5)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_WID} MODIFY COLUMN id Nullable(Int64)" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT id, toTypeName(id) FROM ${TABLE_WID} PREWHERE id > 3 ORDER BY id" + +# The reverse direction is not legal evolution and must keep being rejected. +echo "--- optional to required is still rejected ---" +rm -rf "${USER_FILES_PATH}/${TABLE_REJ}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_REJ} (id Nullable(Int64), s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_REJ}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_REJ} SELECT number, toString(number) FROM numbers(3)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_REJ} MODIFY COLUMN id Int64" 2>&1 \ + | grep -oF "Iceberg spec doesn't allow change type from nullable to non-nullable" | head -1 + +# A non-numeric type reaches the same branch through a different comparison function. +echo "--- String column made Nullable ---" +rm -rf "${USER_FILES_PATH}/${TABLE_STR}/" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_STR} (v String, s String) ENGINE = IcebergLocal('${USER_FILES_PATH}/${TABLE_STR}/', 'Parquet')" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_STR} SELECT toString(number), toString(number) FROM numbers(5)" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "ALTER TABLE ${TABLE_STR} MODIFY COLUMN v Nullable(String)" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT v, toTypeName(v) FROM ${TABLE_STR} PREWHERE v = '4' ORDER BY v" + +# The reverse direction (optional -> required) cannot be produced by `ALTER`, which rejects it +# above, so the passthrough branch is reached only through metadata written by another engine. +# Appending a schema leaves schema 0 byte-identical, so the schema-id immutability check still +# passes, and the read selects the pair with `iceberg_metadata_file_path`. +echo "--- externally authored optional to required stays a passthrough ---" +TABLE_REV="t_rev_${CLICKHOUSE_DATABASE}_${RANDOM}" +REV_PATH="${USER_FILES_PATH}/${TABLE_REV}/" +rm -rf "${REV_PATH}" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_REV} (id Nullable(Int64), s String) ENGINE = IcebergLocal('${REV_PATH}', 'Parquet')" +# The NULL row is what makes the assertion non-vacuous: a cast to the required type only +# misbehaves when a NULL actually has to pass through it. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query "INSERT INTO ${TABLE_REV} SELECT * FROM values('id Nullable(Int64), s String', (1, 'one'), (NULL, 'none'), (3, 'three'))" + +LATEST_REV=$(ls "${REV_PATH}metadata/" | grep -E '^v[0-9]+\.metadata\.json$' | sort -t v -k2 -n | tail -1) +REV_META=$(python3 - "${REV_PATH}metadata" "${LATEST_REV}" <<'PYEOF' +import copy, json, os, re, sys + +metadata_dir, latest_file = sys.argv[1], sys.argv[2] + +with open(os.path.join(metadata_dir, latest_file)) as fh: + metadata = json.load(fh) + +current = next(s for s in metadata["schemas"] if s["schema-id"] == metadata["current-schema-id"]) +tightened = copy.deepcopy(current) +tightened["schema-id"] = max(s["schema-id"] for s in metadata["schemas"]) + 1 +for field in tightened["fields"]: + if field["name"] == "id": + field["required"] = True + +metadata["schemas"].append(tightened) +metadata["current-schema-id"] = tightened["schema-id"] +metadata["last-updated-ms"] = metadata.get("last-updated-ms", 0) + 60000 + +version = int(re.match(r"v(\d+)\.metadata\.json", latest_file).group(1)) + 1 +tmp_file = os.path.join(metadata_dir, ".tmp_next") +with open(tmp_file, "w") as fh: + json.dump(metadata, fh) +os.rename(tmp_file, os.path.join(metadata_dir, f"v{version}.metadata.json")) +print(f"metadata/v{version}.metadata.json") +PYEOF +) + +REV_TF="icebergLocal('${REV_PATH}', 'Parquet', SETTINGS iceberg_metadata_file_path = '${REV_META}')" +# The tightened schema is what the reader resolves against. +${CLICKHOUSE_CLIENT} --query "DESCRIBE ${REV_TF}" | cut -f1,2 +# Each of these three shapes fails if the transform casts the old optional column to the new +# required type instead of passing it through. +${CLICKHOUSE_CLIENT} --query "SELECT s FROM ${REV_TF} ORDER BY s" +${CLICKHOUSE_CLIENT} --query "SELECT s FROM ${REV_TF} WHERE id IS NOT NULL ORDER BY s" +${CLICKHOUSE_CLIENT} ${PREWHERE_SETTINGS} --query "SELECT s FROM ${REV_TF} PREWHERE id > 1 ORDER BY s" + +drop_table "${TABLE}" +drop_table "${TABLE_REN}" +drop_table "${TABLE_WID}" +drop_table "${TABLE_REJ}" +drop_table "${TABLE_STR}" +drop_table "${TABLE_REV}" diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference new file mode 100644 index 000000000000..415eb3c1f59a --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference @@ -0,0 +1,8 @@ +--- A: key Int32 + arr Array(Int32) +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- B: key Int32 + opt Nullable(Int32) all NULL +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- C: key Int32 + val String (control, all supported) +lower_ids=[1,2] upper_ids=[1,2] key_lower=[0] key_upper=[4] +--- D: key Int32 + f Nullable(Float64) with values +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh new file mode 100755 index 000000000000..8acdb1edb91b --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +PREFIX="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +ROOT="${USER_FILES_PATH}/${PREFIX}" + +cleanup() +{ + for suffix in a b c d; do + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${PREFIX}_${suffix}" + done + rm -rf "${ROOT}" +} +trap cleanup EXIT + +# Prints the field ids present in lower_bounds / upper_bounds, plus the decoded Int32 bounds of +# field id 1 (the `key` column, always Int32 and always serializable here). +report() +{ + local table_dir="$1" + for manifest in $(find "${table_dir}/metadata" -maxdepth 1 -name '*.avro' \ + -not -name 'snap-*.avro' -type f | sort); do + ${CLICKHOUSE_CLIENT} --query " + SELECT + 'lower_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'lower_bounds')))), + 'upper_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'upper_bounds')))), + 'key_lower=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'lower_bounds')))), + 'key_upper=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'upper_bounds')))) + FROM file('${manifest}', Avro) + ORDER BY 1, 2, 3, 4 + " + done +} + +# Case A: an unsupported column type (Array(Int32)) must not suppress the bounds of `key`. +echo '--- A: key Int32 + arr Array(Int32)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_a (key Int32, arr Array(Int32)) + ENGINE = IcebergLocal('${ROOT}/a/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_a SELECT number::Int32, [number, number] FROM numbers(5)" +report "${ROOT}/a" + +# Case B: an entirely-NULL Nullable column yields a Null bound, which must not suppress `key` either. +echo '--- B: key Int32 + opt Nullable(Int32) all NULL' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_b (key Int32, opt Nullable(Int32)) + ENGINE = IcebergLocal('${ROOT}/b/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_b SELECT number::Int32, NULL FROM numbers(5)" +report "${ROOT}/b" + +# Case C: control. All columns serializable, so both field ids keep their bounds exactly as before. +echo '--- C: key Int32 + val String (control, all supported)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_c (key Int32, val String) + ENGINE = IcebergLocal('${ROOT}/c/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_c SELECT number::Int32, 'val-' || toString(number) FROM numbers(5)" +report "${ROOT}/c" + +# Case D: wrapper composition with a non-NULL value. Float64 is unsupported for bounds even though +# the byte dumper handles it, so the filter must gate on the bounds predicate and skip only `f`. +echo '--- D: key Int32 + f Nullable(Float64) with values' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_d (key Int32, f Nullable(Float64)) + ENGINE = IcebergLocal('${ROOT}/d/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_d SELECT number::Int32, number + 0.5 FROM numbers(5)" +report "${ROOT}/d" diff --git a/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference new file mode 100644 index 000000000000..48531a4780d3 --- /dev/null +++ b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.reference @@ -0,0 +1,95 @@ +-- { echo } + +-- --------------------------------------------------------------------------- +-- Correctness: per-partition evaluation must agree with the merged evaluation. Each arm +-- prints the forced count and then the merged count; they must be equal. The two settings +-- apply to the outer query, so they reach the aggregation under test. +-- --------------------------------------------------------------------------- + +-- Date + INTERVAL MONTH collapses the 29th, 30th and 31st of a month into one key. 2001 and +-- 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_month; +CREATE TABLE t_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS force_distinct_partitions_independently = 1; +2 +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS allow_distinct_partitions_independently = 0; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 0; +2 +DROP TABLE t_month; +-- the minus direction collapses the same way, going back into a shorter month +DROP TABLE IF EXISTS t_month_minus; +CREATE TABLE t_month_minus (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month_minus SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-03-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +DROP TABLE t_month_minus; +-- DateTime + INTERVAL DAY collapses across a DST spring-forward, so the interval kind alone +-- does not decide safety +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (t DateTime('Europe/Moscow'), x UInt32) ENGINE = MergeTree ORDER BY t PARTITION BY t; +INSERT INTO t_dst SELECT toDateTime('2010-03-27 00:00:00', 'Europe/Moscow') + (intDiv(number, 100) * 1800) AS t, number FROM numbers_mt(600); +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +4 +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +4 +DROP TABLE t_dst; +-- a Float64 addend collapses by mantissa rounding; a float column reaches the optimizer +-- through a non-float partition key +DROP TABLE IF EXISTS t_float; +CREATE TABLE t_float (f Float64, x UInt32) ENGINE = MergeTree ORDER BY f PARTITION BY toUInt64(f - 1e16); +INSERT INTO t_float SELECT 1e16 + intDiv(number, 100) AS f, number FROM numbers_mt(600); +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +2 +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +2 +-- the collapse the arm above depends on: fewer distinct sums than distinct addends +SELECT uniqExact(f), uniqExact(f + 1.0) FROM t_float; +3 2 +DROP TABLE t_float; +-- a constant of a narrower date type narrows the result: every multiple of 65536 maps to +-- 1970-01-01. A NULL constant maps every value to NULL. +DROP TABLE IF EXISTS t_narrow; +CREATE TABLE t_narrow (x UInt32, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_narrow SELECT intDiv(number, 100) * 65536 AS x, number FROM numbers_mt(600); +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +1 +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +1 +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS force_distinct_partitions_independently = 1; +1 +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS allow_distinct_partitions_independently = 0; +1 +SELECT uniqExact(x), uniqExact(x + toDate(0)) FROM t_narrow; +6 1 +DROP TABLE t_narrow; +-- a Decimal constant rescales the varying operand: every multiple of 2^32 maps to one +-- Decimal(9, 1) +DROP TABLE IF EXISTS t_decimal; +CREATE TABLE t_decimal (x UInt64, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_decimal SELECT intDiv(number, 100) * 4294967296 AS x, number FROM numbers_mt(400); +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +1 +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +1 +SELECT uniqExact(x), uniqExact(x + toDecimal32(0, 1)) FROM t_decimal; +4 1 +DROP TABLE t_decimal; +-- two varying operands are not injective either: many pairs share one sum +DROP TABLE IF EXISTS t_two_cols; +CREATE TABLE t_two_cols (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_two_cols SELECT intDiv(number, 100) AS x, number % 100 AS y FROM numbers_mt(1000); +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +109 +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +109 +DROP TABLE t_two_cols; diff --git a/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql new file mode 100644 index 000000000000..cf5290884f8d --- /dev/null +++ b/tests/queries/0_stateless/04833_injective_plus_minus_type_gate.sql @@ -0,0 +1,97 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +-- The values and plans below are the analyzer's, so pin it. +SET enable_analyzer = 1; + +-- { echo } + +-- --------------------------------------------------------------------------- +-- Correctness: per-partition evaluation must agree with the merged evaluation. Each arm +-- prints the forced count and then the merged count; they must be equal. The two settings +-- apply to the outer query, so they reach the aggregation under test. +-- --------------------------------------------------------------------------- + +-- Date + INTERVAL MONTH collapses the 29th, 30th and 31st of a month into one key. 2001 and +-- 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_month; +CREATE TABLE t_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM t_month GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS force_distinct_partitions_independently = 1; +SELECT count() FROM (SELECT DISTINCT d + INTERVAL 1 MONTH AS k FROM t_month) SETTINGS allow_distinct_partitions_independently = 0; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 1; +SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k FROM t_month LIMIT 1 BY k) SETTINGS allow_limit_by_partitions_independently = 0; +DROP TABLE t_month; + +-- the minus direction collapses the same way, going back into a shorter month +DROP TABLE IF EXISTS t_month_minus; +CREATE TABLE t_month_minus (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_month_minus SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-03-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT d - INTERVAL 1 MONTH AS k, count() FROM t_month_minus GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_month_minus; + +-- DateTime + INTERVAL DAY collapses across a DST spring-forward, so the interval kind alone +-- does not decide safety +DROP TABLE IF EXISTS t_dst; +CREATE TABLE t_dst (t DateTime('Europe/Moscow'), x UInt32) ENGINE = MergeTree ORDER BY t PARTITION BY t; +INSERT INTO t_dst SELECT toDateTime('2010-03-27 00:00:00', 'Europe/Moscow') + (intDiv(number, 100) * 1800) AS t, number FROM numbers_mt(600); +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT t + INTERVAL 1 DAY AS k, count() FROM t_dst GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_dst; + +-- a Float64 addend collapses by mantissa rounding; a float column reaches the optimizer +-- through a non-float partition key +DROP TABLE IF EXISTS t_float; +CREATE TABLE t_float (f Float64, x UInt32) ENGINE = MergeTree ORDER BY f PARTITION BY toUInt64(f - 1e16); +INSERT INTO t_float SELECT 1e16 + intDiv(number, 100) AS f, number FROM numbers_mt(600); +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT f + 1.0 AS k, count() FROM t_float GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +-- the collapse the arm above depends on: fewer distinct sums than distinct addends +SELECT uniqExact(f), uniqExact(f + 1.0) FROM t_float; +DROP TABLE t_float; + +-- a constant of a narrower date type narrows the result: every multiple of 65536 maps to +-- 1970-01-01. A NULL constant maps every value to NULL. +DROP TABLE IF EXISTS t_narrow; +CREATE TABLE t_narrow (x UInt32, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_narrow SELECT intDiv(number, 100) * 65536 AS x, number FROM numbers_mt(600); +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + toDate(0) AS k, count() FROM t_narrow GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS force_distinct_partitions_independently = 1; +SELECT count() FROM (SELECT DISTINCT x + CAST(NULL, 'Nullable(UInt32)') AS k FROM t_narrow) SETTINGS allow_distinct_partitions_independently = 0; +SELECT uniqExact(x), uniqExact(x + toDate(0)) FROM t_narrow; +DROP TABLE t_narrow; + +-- a Decimal constant rescales the varying operand: every multiple of 2^32 maps to one +-- Decimal(9, 1) +DROP TABLE IF EXISTS t_decimal; +CREATE TABLE t_decimal (x UInt64, v UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_decimal SELECT intDiv(number, 100) * 4294967296 AS x, number FROM numbers_mt(400); +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + toDecimal32(0, 1) AS k, count() FROM t_decimal GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +SELECT uniqExact(x), uniqExact(x + toDecimal32(0, 1)) FROM t_decimal; +DROP TABLE t_decimal; + +-- two varying operands are not injective either: many pairs share one sum +DROP TABLE IF EXISTS t_two_cols; +CREATE TABLE t_two_cols (x UInt32, y UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x; +INSERT INTO t_two_cols SELECT intDiv(number, 100) AS x, number % 100 AS y FROM numbers_mt(1000); +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS force_aggregate_partitions_independently = 1; +SELECT count() FROM (SELECT x + y AS k, count() FROM t_two_cols GROUP BY k) SETTINGS allow_aggregate_partitions_independently = 0; +DROP TABLE t_two_cols; diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference new file mode 100644 index 000000000000..74b069ccd930 --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference @@ -0,0 +1,13 @@ +-- a valid frame still executes (proves the arms below fail for the intended reason) +1 +-- a codec that stores data uncompressed must not lie about the uncompressed size +1 +-- and neither may the other verbatim codec +1 +-- nor may it understate the uncompressed size +1 +-- engines keep working on ordinary data +1000 100000 +1000 100000 +1 +1 diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh new file mode 100755 index 000000000000..a0248bf8eb06 --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -o pipefail + +# Wire layout of a compressed frame: +# [16B checksum][1B method][4B LE size_compressed][4B LE size_decompressed][payload] +# size_compressed counts the 9-byte header. The checksum is an unkeyed CityHash128 of everything +# after it, so a peer can compute it: these frames are accepted at default settings, and +# http_native_compression_disable_checksumming_on_decompress is deliberately NOT used. +# +# checksum-for-compressed-block prints CityHash128 of every single-bit mutation of its input, so +# feeding it the body with bit 0 flipped yields CityHash128(body) on the line labelled "0, 0". The +# wire order is low64 then high64, each little-endian, i.e. the reverse of the printed hex. +emit() { # $1 = frame body as hex, checksum excluded + local checksum + checksum=$(python3 -c " +import sys +b = bytearray.fromhex(sys.argv[1]); b[0] ^= 1 +sys.stdout.buffer.write(bytes(b)) +" "$1" | $CLICKHOUSE_BINARY checksum-for-compressed-block | awk -F'\t' '$2 == "0, 0" { print $1; exit }') + python3 -c " +import sys +sys.stdout.buffer.write(bytearray.fromhex(sys.argv[1])[::-1] + bytearray.fromhex(sys.argv[2])) +" "$checksum" "$1" +} + +# Method bytes are from CompressionInfo.h: NONE is 0x02 = 2 and Quantized is 0x9e = 158. +frame() { # $1 = method byte (decimal), $2 = size_decompressed, $3 = payload + emit "$(python3 -c " +import struct, sys +payload = sys.argv[3].encode() +sys.stdout.buffer.write(bytes([int(sys.argv[1])]) + struct.pack('&1 | grep -c '(8) does not match size_decompressed (999)' +# Quantized is the other codec reporting isNone(), so it takes the same shortcut. The read path +# builds it from the method byte alone, without the enable_quantized_codec setting its DDL requires. +echo '-- and neither may the other verbatim codec' +frame 158 999 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (999)' + +# The comparison is for inequality, so arms that only ever declare more than the body pin one side of +# it: narrowed to "body shorter than declared", both frames above would still be refused. This one +# declares less instead, and without the check it is not refused at all, it executes its real body. +echo '-- nor may it understate the uncompressed size' +frame 2 3 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (3)' + +# No-regression control: only a NONE-coded frame takes the shortcut, so the column, the marks and +# the primary key are all stored uncompressed here. StripeLog is absent because it compresses its +# whole stream with the default codec and ignores the column codec, so its reads never take it. +echo '-- engines keep working on ordinary data' +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS t_log_bound; + DROP TABLE IF EXISTS t_mt_bound; + + CREATE TABLE t_log_bound (s String CODEC(NONE)) ENGINE = Log; + CREATE TABLE t_mt_bound (k UInt64, s String CODEC(NONE), INDEX idx_s s TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8, compress_marks = 1, compress_primary_key = 1, + min_bytes_for_wide_part = 0, packed_skip_index_max_bytes = 0, + marks_compression_codec = 'NONE', primary_key_compression_codec = 'NONE'; + + INSERT INTO t_log_bound SELECT repeat('a', 100) FROM numbers(1000); + INSERT INTO t_mt_bound SELECT number, repeat('a', 100) FROM numbers(1000); + + SELECT count(), sum(length(s)) FROM t_log_bound; + SELECT count(), sum(length(s)) FROM t_mt_bound WHERE s LIKE '%a%'; + SELECT count() FROM t_mt_bound WHERE k = 42; + -- packed_skip_index_max_bytes = 0 keeps the index in its own file, which is the skip-index class + -- checkDataPart iterates over; a packed one carries no per-file checksum entry to visit. + CHECK TABLE t_mt_bound SETTINGS check_query_single_value_result = 1; + + DROP TABLE t_log_bound; + DROP TABLE t_mt_bound; +" diff --git a/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference new file mode 100644 index 000000000000..3350bf0bdf33 --- /dev/null +++ b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.reference @@ -0,0 +1,12 @@ +reported greedy: +0 Join_1_Value_0 0 Join_2_Value_0 0 Join_3_Value_0 +reported dpsub: +0 Join_1_Value_0 0 Join_2_Value_0 0 Join_3_Value_0 +filter deep relation greedy: +0 Join_2_Value_0 0 Join_3_Value_0 0 Join_1_Value_0 +filter deep relation dpsub: +0 Join_2_Value_0 0 Join_3_Value_0 0 Join_1_Value_0 +four way filter last greedy: +0 0 0 0 Join_4_Value_0 +four way filter last dpsub: +0 0 0 0 Join_4_Value_0 diff --git a/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql new file mode 100644 index 000000000000..b0657008aedb --- /dev/null +++ b/tests/queries/0_stateless/04869_dpsub_single_table_filter_placement.sql @@ -0,0 +1,86 @@ +-- Tests that the DPsub join-order algorithm keeps single-table filter predicates that live in a +-- join's ON clause (e.g. `... JOIN t ON t.a = u.a AND t.b = 'x'`). DPsub attaches such a predicate +-- at the join that introduces its relation. Two earlier placement bugs silently dropped the +-- predicate, letting extra rows through: +-- 1. attaching only at two-relation joins dropped it when the relation was introduced against an +-- already-multi-relation subplan (e.g. `t1 JOIN (t2 JOIN t3)`); +-- 2. gating on fromLeft()/fromRight() dropped it for any filter on a relation whose id is >= 2, +-- because those helpers test relation ids 0 and 1 specifically. +-- For each shape we print the result with the 'greedy' algorithm (the reference, which places the +-- predicate correctly) and with 'dpsub'; the two must be identical row-for-row. +-- +-- `query_plan_enable_optimizations = 0` is required to expose the bug and does NOT disable join +-- reordering (that is controlled by `query_plan_optimize_join_order_algorithm`). It disables the +-- general plan-optimization passes, notably filter push-down; with push-down enabled the +-- single-table filter is applied independently of the join and masks the dropped ON-clause +-- conjunct, so the wrong-result would not surface. + +DROP TABLE IF EXISTS t1; +DROP TABLE IF EXISTS t2; +DROP TABLE IF EXISTS t3; +DROP TABLE IF EXISTS t4; + +CREATE TABLE t1 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t2 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t3 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE t4 (id UInt64, value String) ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t1 VALUES (0, 'Join_1_Value_0'), (1, 'Join_1_Value_1'), (2, 'Join_1_Value_2'); +INSERT INTO t2 VALUES (0, 'Join_2_Value_0'), (1, 'Join_2_Value_1'), (3, 'Join_2_Value_3'); +INSERT INTO t3 VALUES (0, 'Join_3_Value_0'), (1, 'Join_3_Value_1'), (4, 'Join_3_Value_4'); +INSERT INTO t4 VALUES (0, 'Join_4_Value_0'), (1, 'Join_4_Value_1'), (5, 'Join_4_Value_5'); + +SET enable_analyzer = 1; +SET query_plan_optimize_join_order_limit = 10; + +-- The reported case: filter on the first relation, INNER then LEFT join. +SELECT 'reported greedy:'; +SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' +LEFT JOIN t3 ON t2.id = t3.id ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'reported dpsub:'; +SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' +LEFT JOIN t3 ON t2.id = t3.id ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +-- Filter on a relation that is introduced last, against an already-joined subplan (relation id >= 2). +SELECT 'filter deep relation greedy:'; +SELECT t2.id, t2.value, t3.id, t3.value, t1.id, t1.value +FROM t2 INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t1 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'filter deep relation dpsub:'; +SELECT t2.id, t2.value, t3.id, t3.value, t1.id, t1.value +FROM t2 INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t1 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +-- Four-way all-inner chain with the filter on the last relation (id = 3). +SELECT 'four way filter last greedy:'; +SELECT t1.id, t2.id, t3.id, t4.id, t4.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id +INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t4 ON t3.id = t4.id AND t4.value = 'Join_4_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'greedy', + query_plan_enable_optimizations = 0; + +SELECT 'four way filter last dpsub:'; +SELECT t1.id, t2.id, t3.id, t4.id, t4.value +FROM t1 INNER JOIN t2 ON t1.id = t2.id +INNER JOIN t3 ON t2.id = t3.id +INNER JOIN t4 ON t3.id = t4.id AND t4.value = 'Join_4_Value_0' ORDER BY ALL +SETTINGS query_plan_optimize_join_order_algorithm = 'dpsub', + query_plan_enable_optimizations = 0; + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference index 494e4eb3e207..1809b85a1aac 100644 --- a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference +++ b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.reference @@ -1,5 +1,5 @@ t_lazy TableProxy -t_lazy_memory TableProxy +t_lazy_log TableProxy CREATE TABLE default_1.t_lazy\n(\n `key` UInt64,\n `v` Float64 STATISTICS(tdigest),\n `s` String TTL d + toIntervalMonth(1),\n `d` DateTime\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 TableProxy CREATE TABLE default_1.t_lazy\n(\n `key` UInt64,\n `v` Float64 STATISTICS(tdigest),\n `s` String TTL d + toIntervalMonth(1),\n `d` DateTime,\n `v2` Float64 STATISTICS(uniq)\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql index de3fda3e1185..80ff346f1857 100644 --- a/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql +++ b/tests/queries/0_stateless/04870_column_declaration_statistics_lazy_load.sql @@ -14,7 +14,7 @@ DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; CREATE DATABASE {CLICKHOUSE_DATABASE_1:Identifier} ENGINE = Atomic SETTINGS lazy_load_tables = 1; CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy (key UInt64, v Float64, s String, d DateTime) ENGINE = MergeTree ORDER BY key; -CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy_memory (x UInt64) ENGINE = Memory; +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t_lazy_log (x UInt64) ENGINE = Log; DETACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; ATTACH DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; @@ -38,7 +38,7 @@ SHOW CREATE TABLE t_lazy; -- The proxy forwards the nested answer rather than a blanket `true`: a lazily loaded engine without -- statistics support is still rejected. -ALTER TABLE t_lazy_memory MODIFY COLUMN x UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } -ALTER TABLE t_lazy_memory ADD COLUMN y UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_lazy_log MODIFY COLUMN x UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_lazy_log ADD COLUMN y UInt64 STATISTICS(tdigest); -- { serverError NOT_IMPLEMENTED } DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.python b/tests/queries/0_stateless/04909_client_hints_history_navigation.python new file mode 100644 index 000000000000..54cd43293f00 --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.python @@ -0,0 +1,392 @@ +import multiprocessing +import os +import pty +import re +import select +import shlex +import sys +import time + +TIMEOUT_SECONDS = 30 + +# How long to keep reading after the last keystroke while waiting for the expected output. The +# committed-query case only sees the result once the query has executed, which can take longer +# than the inter-keystroke drain on slow (sanitizer) builds. +FINAL_WAIT_SECONDS = 20 + +# How long to wait for the client to print the next `:) ` prompt after a committed query. This is +# the time a query of this test may take, and it is generous because the query does not have to be +# fast for the test to mean anything: in an MSan build a trivial `SELECT` that fails to resolve an +# identifier has been seen taking 36 seconds. +PROMPT_WAIT_SECONDS = 120 + +# Idle window after every keystroke, long enough for `replxx` to repaint the line. +DRAIN_IDLE_SECONDS = 0.4 + +# Extra slack for the supervisor on top of everything the worker itself may legitimately wait for. +SUPERVISOR_SLACK_SECONDS = 15 + +# A pseudo-keystroke: wait for the next prompt instead of writing anything. Sending the keys of the +# next line while the previous query is still running would let the terminal driver echo them +# instead of `replxx` interpreting them - a bracketed paste would then appear verbatim as +# `^[[200~...^[[201~` in the output and never reach the line editor. +WAIT_FOR_PROMPT = object() + +# `clickhouse-test` dumps this file into the failure report, but only under the name it derives +# from the testcase (the `.sh` wrapper), so take the path from there. +DEBUG_LOG = os.environ["CLICKHOUSE_TEST_DEBUG_LOG"] + +# The gray SGR sequence replxx uses for the as-you-type hint (Color::GRAY -> "0;90"). +HINT_COLOR = "\x1b[0;90m" + + +def make_history_file(name, entries): + """Write a replxx-format history file with the given entries, oldest first.""" + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + + f".{name}.history", + ) + with open(path, "w") as f: + for i, entry in enumerate(entries): + f.write(f"### 2020-01-01 00:00:{i:02}.000\n{entry}\n") + return path + + +def read_until(master, debug_log_fd, predicate, timeout): + output = "" + deadline = time.time() + timeout + while time.time() < deadline: + r, _, _ = select.select([master], [], [], 0.3) + if not r: + if predicate(output): + return output + continue + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + if predicate(output): + return output + return output + + +def drain(master, debug_log_fd, idle): + output = "" + while True: + r, _, _ = select.select([master], [], [], idle) + if not r: + break + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + return output + + +def report_failure(name, shell_pid, output, debug_log_fd): + """A PTY case that fails prints just `FAIL`, which says nothing about why. Record what the + terminal actually showed and whether the client is still alive - the client dying (the read + loop hitting EOF) looks exactly like the expected text never being printed.""" + try: + reaped_pid, wait_status = os.waitpid(shell_pid, os.WNOHANG) + except ChildProcessError: + reaped_pid, wait_status = shell_pid, -1 + running = reaped_pid == 0 + banner = ( + f"=== {name}: FAIL ===\n" + f"client still running: {running}, wait status: {wait_status}\n" + f"terminal output: {output[-8000:]!r}\n" + ) + debug_log_fd.write(banner) + debug_log_fd.flush() + sys.stderr.write(banner) + sys.stderr.flush() + + +def run_case(program, argv, name, keystrokes, check, state=None): + """Fork a PTY, wait for the prompt, send keystrokes (with a small pause between + them so replxx repaints), then run `check` over the collected output.""" + shell_pid, master = pty.fork() + if shell_pid == 0: + os.environ["TERM"] = "xterm" + os.execv(program, argv) + return + + debug_log_fd = open(DEBUG_LOG, "a") + try: + read_until(master, debug_log_fd, lambda o: ":)" in o, TIMEOUT_SECONDS) + + output = "" + # Only what arrived after the last keystroke - a prompt seen earlier in `output` says + # nothing about the prompt this step is waiting for. + since_last_keystroke = "" + for keys in keystrokes: + if keys is WAIT_FOR_PROMPT: + extra = read_until( + master, + debug_log_fd, + lambda more, seen=since_last_keystroke: ":)" in seen + more, + PROMPT_WAIT_SECONDS, + ) + output += extra + since_last_keystroke += extra + # `read_until` returns on its deadline as well as on a match, and going on from a + # deadline would write the next keys into a terminal that is still running the + # previous query: the driver echoes them instead of `replxx` interpreting them, so + # a bracketed paste ends up in the output verbatim and the case fails on its + # content, saying nothing about the prompt that never came. Stop here instead. + if ":)" not in since_last_keystroke: + print(f"{name}: FAIL") + report_failure(name, shell_pid, output, debug_log_fd) + state.value = 1 + return + continue + os.write(master, keys) + since_last_keystroke = drain(master, debug_log_fd, DRAIN_IDLE_SECONDS) + output += since_last_keystroke + + # The expected output may arrive only after the committed query executes, which can + # outlast the inter-keystroke drain on slow builds. Keep reading until the check passes + # instead of giving up after a single drain window. + if not check(output): + output += read_until( + master, + debug_log_fd, + lambda extra: check(output + extra), + FINAL_WAIT_SECONDS, + ) + + ok = check(output) + print(f"{name}: {'OK' if ok else 'FAIL'}") + if not ok: + report_failure(name, shell_pid, output, debug_log_fd) + state.value = 0 if ok else 1 + finally: + os.close(master) + debug_log_fd.close() + + +def worker_budget(keystrokes): + """Everything `run_case` may legitimately spend: the wait for the first prompt, the idle window + after every keystroke, every explicit prompt wait, and the final wait for the expected output. + The supervisor must not expire before that, or ordinary slowness looks like a failure.""" + prompt_waits = sum(1 for keys in keystrokes if keys is WAIT_FOR_PROMPT) + return ( + TIMEOUT_SECONDS + + PROMPT_WAIT_SECONDS * prompt_waits + + DRAIN_IDLE_SECONDS * len(keystrokes) + + FINAL_WAIT_SECONDS + + SUPERVISOR_SLACK_SECONDS + ) + + +def run_with_timeout(program, argv, name, history, keystrokes, check): + """Run a case with a history file holding the `history` entries. The file is written anew for + every attempt: an attempt killed after its final Enter may already have appended the query to + it, and a retry starting from that tail would recall a different entry than the case intends + while still printing the expected result.""" + for attempt in range(5): + history_file = make_history_file(name, history) + state = multiprocessing.Value("i", -1) + process = multiprocessing.Process( + target=run_case, + args=(program, argv + [f"--history_file={history_file}"], name, keystrokes, check), + kwargs={"state": state}, + ) + process.start() + process.join(worker_budget(keystrokes)) + if process.is_alive(): + process.terminate() + if state.value in (0, 1): + return + # transient timeout on a loaded machine - retry + print(f"{name}: FAIL") + sys.stderr.write(f"=== {name}: FAIL === the case timed out on every attempt\n") + + +def main(): + program = os.environ["CLICKHOUSE_LOCAL"] + base = shlex.split(program) + args = base + ["--wait_for_suggestions_to_load", "--hints", "1"] + + # A line displayed programmatically (recalled from history, pasted) must not pop the + # as-you-type hints by itself: with hints visible, the next Up/Down press would navigate + # the hints instead of the history. An edit shows the hints again. + + # 1. Up recalls an entry; no hint appears at its end ("ive" would be the ghost suffix of + # "concatAssumeInjective"). + run_with_timeout( + base[0], + args, + "up_recall_no_hints", + ["SELECT concatAssumeInject"], + [b"\x1b[A"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # Meta-Up is replxx's direct history-previous binding. It must use the same suppression as + # plain Up, rather than leaving the default action to regenerate hints. + run_with_timeout( + base[0], + args, + "meta_up_recall_no_hints", + ["SELECT concatAssumeInject"], + [b"\x1b\x1b[A"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # Ctrl-G restores the currently recalled history entry after an edit. It is another whole + # line display, so Down must return to the scratch line rather than navigate the hint list. + run_with_timeout( + base[0], + args, + "restore_current_down_navigates_history", + ["SELECT concatAssumeInjec"], + [b"SELECT 111 AS rand", b"\x1b[A", b"t", b"\x07", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + + # 2. Up, Up, Down walks the history (the entries end in "rand", which has many hint + # matches - Down must not step into the hint list); Enter then runs the recalled entry, + # so its result (222) is printed. The result can use either Pretty or TabSeparated output + # depending on the client's default format. + run_with_timeout( + base[0], + args, + "down_navigates_history", + ["SELECT 111 AS rand", "SELECT 222 AS rand"], + [b"\x1b[A", b"\x1b[A", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*222\s*│|\x1b\[\?25h222\r?\n)", o) is not None, + ) + + # 3. Recalling a history entry with the same text as the scratch line is still a programmatic + # display. Down must return from that entry to the scratch line instead of selecting a hint. + run_with_timeout( + base[0], + args, + "same_text_recall_down_navigates_history", + ["SELECT 222 AS rand", "SELECT 111 AS rand"], + [b"SELECT 111 AS rand", b"\x1b[A", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + + # 3b. An incremental history search (Ctrl-T, the ClickHouse binding for the regular non-fuzzy + # reverse search) that accepts an entry is another whole-line display: it goes through a + # different `replxx` code path than the plain history recall, so it needs its own proof that + # Down keeps walking the history. Ctrl-T, then the search text, then Ctrl-E leaves the search + # keeping the found line ("SELECT 111 AS rand") and puts the cursor at its end, where the hints + # would render. The accepted entry is committed as the most recent recall, so the first Down + # re-displays it (`replxx` emulates the Windows down-arrow there) and the second one moves on + # to "SELECT 222 AS rand", which Enter then runs. + run_with_timeout( + base[0], + args, + "incremental_search_down_navigates_history", + ["SELECT 111 AS rand", "SELECT 222 AS rand"], + [b"\x14", b"111", b"\x05", b"\x1b[B", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*222\s*│|\x1b\[\?25h222\r?\n)", o) is not None, + ) + + # 4. Editing a recalled entry shows the hints again: after Up, typing the "t" completes the + # prefix "concatAssumeInject" and the ghost "ive" appears. + run_with_timeout( + base[0], + args, + "edit_after_recall_shows_hints", + ["SELECT concatAssumeInjec"], + [b"\x1b[A", b"t"], + lambda o: (HINT_COLOR + "ive") in o, + ) + + # 5. A (bracketed) paste is also a whole new line displayed at once - no hint at its end. + run_with_timeout( + base[0], + args, + "paste_no_hints", + [], + [b"\x1b[200~SELECT concatAssumeInject\x1b[201~"], + lambda o: "concatAssumeInject" in o and (HINT_COLOR + "ive") not in o, + ) + + # 5b. Pasting the exact text that carried a visible hint on an earlier prompt: replxx caches + # the hints by the buffer text and the cache outlives the prompt, so without invalidating it + # the paste would redisplay the stale ghost without ever asking our hint callback. The first + # line is typed (so the hints for it are generated and cached) and committed - it is not a + # valid query, which is irrelevant here, only the cache state matters - and then the very same + # text is pasted on the next prompt, where no hint may appear. Only the output after the last + # prompt is examined: the first, typed line legitimately shows the ghost. + run_with_timeout( + base[0], + args, + "same_text_paste_no_hints", + [], + [ + b"SELECT concatAssumeInject", + b"\r", + WAIT_FOR_PROMPT, + b"\x1b[200~SELECT concatAssumeInject\x1b[201~", + ], + lambda o: "concatAssumeInject" in o.rsplit(":)", 1)[-1] + and (HINT_COLOR + "ive") not in o.rsplit(":)", 1)[-1], + ) + + # 6. An unchanged editor round-trip redisplays the line programmatically. It must invalidate + # replxx's cached hints, so Down remains history navigation. + editor = os.path.join(os.environ["CLICKHOUSE_TMP"], "client_hints_unchanged_editor.sh") + with open(editor, "w") as f: + f.write("#!/bin/sh\nexit 0\n") + os.chmod(editor, 0o755) + previous_editor = os.environ.get("EDITOR") + os.environ["EDITOR"] = editor + try: + run_with_timeout( + base[0], + args, + "unchanged_editor_down_navigates_history", + [], + [b"SELECT 111 AS rand", b"\x1bE", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + finally: + if previous_editor is None: + del os.environ["EDITOR"] + else: + os.environ["EDITOR"] = previous_editor + + # 7. An editor that exits unsuccessfully brings the original line back - also a whole-line + # programmatic display, and one that happens while the hints of the line typed before the + # editor was opened are still live. Down must remain history navigation there too. + editor = os.path.join(os.environ["CLICKHOUSE_TMP"], "client_hints_failing_editor.sh") + with open(editor, "w") as f: + f.write("#!/bin/sh\nexit 1\n") + os.chmod(editor, 0o755) + previous_editor = os.environ.get("EDITOR") + os.environ["EDITOR"] = editor + try: + run_with_timeout( + base[0], + args, + "failing_editor_down_navigates_history", + [], + [b"SELECT 111 AS rand", b"\x1bE", b"\x1b[B", b"\r"], + lambda o: re.search(r"(?:│\s*111\s*│|\x1b\[\?25h111\r?\n)", o) is not None, + ) + finally: + if previous_editor is None: + del os.environ["EDITOR"] + else: + os.environ["EDITOR"] = previous_editor + + +if __name__ == "__main__": + # The check lambdas are not picklable, so the subprocesses must be forked (not spawned, + # which became the default on newer Python). + multiprocessing.set_start_method("fork") + main() diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.reference b/tests/queries/0_stateless/04909_client_hints_history_navigation.reference new file mode 100644 index 000000000000..bcd45174864a --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.reference @@ -0,0 +1,11 @@ +up_recall_no_hints: OK +meta_up_recall_no_hints: OK +restore_current_down_navigates_history: OK +down_navigates_history: OK +same_text_recall_down_navigates_history: OK +incremental_search_down_navigates_history: OK +edit_after_recall_shows_hints: OK +paste_no_hints: OK +same_text_paste_no_hints: OK +unchanged_editor_down_navigates_history: OK +failing_editor_down_navigates_history: OK diff --git a/tests/queries/0_stateless/04909_client_hints_history_navigation.sh b/tests/queries/0_stateless/04909_client_hints_history_navigation.sh new file mode 100755 index 000000000000..04878aed0389 --- /dev/null +++ b/tests/queries/0_stateless/04909_client_hints_history_navigation.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: long, no-debug + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-test` dumps `$CLICKHOUSE_TMP/.debuglog` into the failure report, and the +# testcase name is the name of this shell script - the Python part cannot derive it from its own +# file name, so pass the path down. +export CLICKHOUSE_TEST_DEBUG_LOG="$CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog" + +python3 "$CUR_DIR"/04909_client_hints_history_navigation.python diff --git a/tests/queries/0_stateless/04952_projection_argumentless_function_json.reference b/tests/queries/0_stateless/04952_projection_argumentless_function_json.reference new file mode 100644 index 000000000000..15a67caf6921 --- /dev/null +++ b/tests/queries/0_stateless/04952_projection_argumentless_function_json.reference @@ -0,0 +1,2 @@ +NUMBER_OF_ARGUMENTS_DOESNT_MATCH +1 diff --git a/tests/queries/0_stateless/04952_projection_argumentless_function_json.sh b/tests/queries/0_stateless/04952_projection_argumentless_function_json.sh new file mode 100755 index 000000000000..37b6ea150204 --- /dev/null +++ b/tests/queries/0_stateless/04952_projection_argumentless_function_json.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Malformed JSON AST: a function without an `arguments` list inside a projection expression slot +# (`query` for a SELECT projection, `index` for an INDEX projection) must not crash the server: +# `count` accepts no arguments, `plus` fails with NUMBER_OF_ARGUMENTS_DOESNT_MATCH. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# $1: projection definition, $2: name of the function whose `arguments` are removed +function check() +{ + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_04952 SYNC" + + # The valid statement round-trips through the JSON dialect. + JSON=$(${CLICKHOUSE_CLIENT} -q "SELECT parseQueryToJSON('CREATE TABLE t_04952 (x UInt8, PROJECTION $1) ENGINE = MergeTree ORDER BY x') FORMAT TSVRaw") + ${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json -q "$JSON" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_04952 SYNC" + + JSON_BAD=$(printf '%s' "$JSON" | python3 -c ' +import json, sys + +def strip_args(node, name): + if isinstance(node, dict): + if node.get("type") == "Function" and node.get("name") == name: + node.pop("arguments", None) + for v in node.values(): + strip_args(v, name) + elif isinstance(node, list): + for v in node: + strip_args(v, name) + +ast = json.load(sys.stdin) +strip_args(ast, sys.argv[1]) +print(json.dumps(ast)) +' "$2") + + OUT=$(${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json -q "$JSON_BAD" 2>&1 || true) + echo "$OUT" | grep -oE 'NUMBER_OF_ARGUMENTS_DOESNT_MATCH' | head -1 + + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_04952 SYNC" +} + +check "p (SELECT count() GROUP BY x)" count +check "p INDEX x + 1 TYPE basic" plus + +# The server is still alive to serve a plain query. +${CLICKHOUSE_CLIENT} -q 'SELECT 1' diff --git a/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference new file mode 100644 index 000000000000..a19952f23e73 --- /dev/null +++ b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.reference @@ -0,0 +1,42 @@ +saturate +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +2149-06-06 2149-06-06 2149-06-06 +2149-06-06 2149-06-06 +ignore +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +throw +throw, in range +2149-06-06 1970-01-01 2106-02-07 06:28:15 1970-01-01 00:00:00 +2299-12-31 1900-01-01 2299-12-31 23:59:59.999 +throw, OrNull and OrZero still fall back +\N 1970-01-01 \N 1970-01-01 00:00:00 +throw, input formats +2149-06-06 +2106-02-07 06:28:15 +throw, tentative parsers must not accept a clamped value +2150-12-31 \N 2150-12-31 +2149-06-06 +throw, a digit-only timestamp is text too +2106-02-07 06:28:15 2023-11-14 22:13:20 \N 1970-01-01 00:00:00 +throw, Date32 rejects what it cannot represent instead of substituting a default +\N 2299-12-31 1900-01-01 +2000-13-01 +throw, an unquoted numeric token is checked too +2023-11-14 22:13:20 +2106-02-07 06:28:15 +2023-12-23 20:37:33 +throw, the last second keeps its fractional ticks +9999-12-31 23:59:59.500 +9999-12-31 23:59:59.999 +9999-12-31 23:59:59.500 +throw, the range is checked after the timezone offset is applied +2106-02-07 06:28:15 1970-01-01 00:00:00 +2106-02-07 06:28:15 2106-02-07 06:28:15 +throw, typed JSON columns are checked like declared ones +2149-06-06 +2299-12-31 23:59:59.999 +throw, JSONExtract keeps returning a default or NULL +1970-01-01 \N 2149-06-06 +throw, an explicitly written year 0000 is not silently replaced +\N 1970-01-01 00:00:00 +3 diff --git a/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql new file mode 100644 index 000000000000..95e7f651a503 --- /dev/null +++ b/tests/queries/0_stateless/05019_date_time_overflow_behavior_from_string.sql @@ -0,0 +1,105 @@ +-- date_time_overflow_behavior was ignored when the value came from text instead of a typed column +SET session_timezone = 'UTC'; + +SELECT 'saturate'; +SET date_time_overflow_behavior = 'saturate'; +SELECT toDate('9999-12-31'), toDate('1969-12-31'), toDateTime('2106-02-07 06:28:16'), toDateTime('1969-12-31 23:59:59'); +SELECT CAST('9999-12-31' AS Date), CAST(materialize('9999-12-31') AS Date), CAST('9999-12-31'::FixedString(10) AS Date); +SELECT toDateOrNull('9999-12-31'), toDateOrZero('9999-12-31'); + +SELECT 'ignore'; +SET date_time_overflow_behavior = 'ignore'; +SELECT toDate('9999-12-31'), toDate('1969-12-31'), toDateTime('2106-02-07 06:28:16'), toDateTime('1969-12-31 23:59:59'); + +SELECT 'throw'; +SET date_time_overflow_behavior = 'throw'; +SELECT toDate('2149-06-07'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDate('1969-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('2106-02-07 06:28:16'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('1969-12-31 23:59:59'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('2149-06-07' AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('2149-06-07') AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('2149-06-07'::FixedString(10) AS Date); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('2106-02-07 06:28:16') AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } + +SELECT 'throw, in range'; +SELECT toDate('2149-06-06'), toDate('1970-01-01'), toDateTime('2106-02-07 06:28:15'), toDateTime('1970-01-01 00:00:00'); +-- Date32 and DateTime64 are not covered yet: a high-scale DateTime64 text parse still raises DECIMAL_OVERFLOW +-- in every mode, because the tick range is only checked in DecimalUtils +SELECT toDate32('2299-12-31'), toDate32('1900-01-01'), toDateTime64('2299-12-31 23:59:59.999', 3); + +SELECT 'throw, OrNull and OrZero still fall back'; +SELECT toDateOrNull('2149-06-07'), toDateOrZero('2149-06-07'), toDateTimeOrNull('2106-02-07 06:28:16'), toDateTimeOrZero('1969-12-31 23:59:59'); + +SELECT 'throw, input formats'; +SELECT * FROM format(CSV, 'v Date', '2150-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v Date', '1960-01-01'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(TSV, 'v Date', '2150-12-31'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v Date', '{"v":"2150-12-31"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v DateTime', '2106-02-07 06:28:16'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(TSV, 'v DateTime', '1960-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"2106-02-07 06:28:16"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(CSV, 'v Date', '2149-06-06'); +SELECT * FROM format(TSV, 'v DateTime', '2106-02-07 06:28:15'); + +SELECT 'throw, tentative parsers must not accept a clamped value'; +SELECT v, variantElement(v, 'Date') AS d, variantElement(v, 'String') AS s +FROM format(CSV, 'v Variant(Date, String)', '2150-12-31') SETTINGS allow_experimental_variant_type = 1; +SELECT v FROM format(CSV, 'v Variant(Date, String)', '2149-06-06') SETTINGS allow_experimental_variant_type = 1; + +SELECT 'throw, a digit-only timestamp is text too'; +SELECT toDateTime('4294967296'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST('4294967296' AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT CAST(materialize('4294967296') AS DateTime); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"4294967296"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(Values, 'v DateTime', '(\'4294967296\')'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('4294967295'), toDateTime('1700000000'), toDateTimeOrNull('4294967296'), toDateTimeOrZero('4294967296'); + +SELECT 'throw, Date32 rejects what it cannot represent instead of substituting a default'; +SELECT toDate32('2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT toDate32('99999999'); -- { serverError CANNOT_PARSE_DATE } +SELECT CAST(materialize('2000-13-01') AS Date32); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(CSV, 'v Date32', '2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(TSV, 'v Date32', '2000-13-01'); -- { serverError CANNOT_PARSE_DATE } +SELECT * FROM format(JSONEachRow, 'v Date32', '{"v":"2000-13-01"}'); -- { serverError CANNOT_PARSE_DATE } +SELECT toDate32OrNull('2000-13-01'), toDate32('2299-12-31'), toDate32('1900-01-01'); +SELECT v FROM format(CSV, 'v Variant(Date32, String)', '2000-13-01') SETTINGS allow_experimental_variant_type = 1; + +SELECT 'throw, an unquoted numeric token is checked too'; +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":4294967296}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":-1}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(Values, 'v DateTime', '(4294967296)'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":1700000000}'); +SELECT * FROM format(Values, 'v DateTime', '(4294967295)'); +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":1703363853.5}'); + +SELECT 'throw, the last second keeps its fractional ticks'; +SELECT * FROM format(JSONEachRow, 'v DateTime64(3)', '{"v":253402300799.5}'); +SELECT * FROM format(JSONEachRow, 'v DateTime64(3)', '{"v":253402300799.999}'); +SELECT * FROM format(Values, 'v DateTime64(3)', '(253402300799.5)'); + +SELECT 'throw, the range is checked after the timezone offset is applied'; +SELECT parseDateTimeBestEffort('2106-02-07 07:28:15+01:00', 'UTC'), parseDateTimeBestEffort('1969-12-31 23:00:00-01:00', 'UTC'); +SELECT parseDateTimeBestEffortOrNull('2106-02-07 07:28:15+01:00', 'UTC'), toDateTime('2106-02-07 07:28:15+01:00', 'UTC'); +SELECT parseDateTimeBestEffort('2106-02-07 08:28:15+01:00', 'UTC'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } + +SELECT 'throw, typed JSON columns are checked like declared ones'; +SELECT j.d FROM format(JSONEachRow, 'j JSON(d Date)', '{"j":{"d":"2150-12-31"}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":"2106-02-07 06:28:16"}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":"2106-02-07 06:28:16"}}') SETTINGS date_time_input_format = 'basic'; -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime)', '{"j":{"d":4294967296}}'); -- { serverError INCORRECT_DATA } +SELECT j.d FROM format(JSONEachRow, 'j JSON(d Date)', '{"j":{"d":"2149-06-06"}}'); +SELECT j.d FROM format(JSONEachRow, 'j JSON(d DateTime64(3))', '{"j":{"d":"2299-12-31 23:59:59.999"}}'); + +SELECT 'throw, JSONExtract keeps returning a default or NULL'; +SELECT JSONExtract('{"d":"2150-12-31"}', 'd', 'Date'), JSONExtract('{"d":"2150-12-31"}', 'd', 'Nullable(Date)'), JSONExtract('{"d":"2149-06-06"}', 'd', 'Date'); + +SELECT 'throw, an explicitly written year 0000 is not silently replaced'; +SELECT parseDateTimeBestEffort('0000-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffort('00000101'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffortUS('01/01/0000'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT toDateTime('0000-01-01 00:00:00'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT * FROM format(JSONEachRow, 'v DateTime', '{"v":"0000-01-01 00:00:00"}'); -- { serverError VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE } +SELECT parseDateTimeBestEffortOrNull('0000-01-01 00:00:00'), parseDateTimeBestEffortOrZero('0000-01-01 00:00:00'); +-- An absent year is a documented best-effort feature, not an overflow +SELECT toMonth(parseDateTimeBestEffort('Mar 3 01:33:48')); diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python new file mode 100644 index 000000000000..072594e6baef --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.python @@ -0,0 +1,338 @@ +"""A query prepopulated into the next input line (the `??` AI SQL generation flow) is displayed +programmatically, so - like a recalled or pasted line - it must not pop the as-you-type hints: +with hints visible, the next Up/Down press would navigate the hints instead of the history. + +The AI provider is a local HTTP server speaking just enough of the OpenAI chat completion protocol +to answer with a canned query, so the test needs no network and no API key. +""" + +import http.server +import json +import multiprocessing +import os +import pty +import select +import shlex +import sys +import threading +import time + +TIMEOUT_SECONDS = 60 + +# How long to keep reading after the last keystroke while waiting for the expected output. +FINAL_WAIT_SECONDS = 30 + +# How long an explicit `wait_for` of a keystroke may take. It is generous because it covers the +# wait for the next `:) ` prompt after a committed query, and a query does not have to be fast for +# this test to mean anything: in an MSan build a trivial `SELECT` that fails to resolve an +# identifier has been seen taking 36 seconds. +STEP_WAIT_SECONDS = 120 + +# Idle window after a keystroke that has no explicit `wait_for`, long enough for `replxx` to +# repaint the line. +DRAIN_IDLE_SECONDS = 0.4 + +# Extra slack for the supervisor on top of everything the worker itself may legitimately wait for. +SUPERVISOR_SLACK_SECONDS = 15 + +# `clickhouse-test` dumps this file into the failure report, but only under the name it derives +# from the testcase (the `.sh` wrapper), so take the path from there. +DEBUG_LOG = os.environ["CLICKHOUSE_TEST_DEBUG_LOG"] + +# The gray SGR sequence replxx uses for the as-you-type hint (Color::GRAY -> "0;90"). +HINT_COLOR = "\x1b[0;90m" + +# The query the fake provider "generates". Its last word is a prefix of `concatAssumeInjective`, +# so an unsuppressed hint would render the gray ghost suffix "ive" right after it. +GENERATED_QUERY = "SELECT concatAssumeInject" + +# The prepopulated line is syntax-highlighted, so only its last word survives as a plain substring +# of the terminal output. +GENERATED_QUERY_LAST_WORD = "concatAssumeInject" + + +class CompletionHandler(http.server.BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + self.rfile.read(length) + body = json.dumps( + { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": f"{GENERATED_QUERY}", + }, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + +def write_config(port): + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + ".config.xml", + ) + with open(path, "w") as f: + f.write( + "\n" + " \n" + " openai\n" + " test\n" + f" http://127.0.0.1:{port}\n" + " test-model\n" + " false\n" + " \n" + "\n" + ) + return path + + +def make_history_file(): + path = os.path.join( + os.environ["CLICKHOUSE_TMP"], + os.path.splitext(os.path.basename(os.path.abspath(__file__)))[0] + ".history", + ) + with open(path, "w") as f: + pass + return path + + +def read_until(master, debug_log_fd, predicate, timeout): + """Read until the predicate holds, the deadline passes, or the client is gone. Returns whether + the predicate matched along with what was read: a deadline is not a match, and treating it as + one lets a case go on from a state it never reached.""" + output = "" + deadline = time.time() + timeout + while time.time() < deadline: + r, _, _ = select.select([master], [], [], 0.3) + if not r: + if predicate(output): + return True, output + continue + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + if predicate(output): + return True, output + return predicate(output), output + + +def drain(master, debug_log_fd, idle): + output = "" + while True: + r, _, _ = select.select([master], [], [], idle) + if not r: + break + try: + chunk = os.read(master, 4096) + except OSError: + break + debug_log_fd.write(repr(chunk) + "\n") + debug_log_fd.flush() + output += chunk.decode(errors="replace") + return output + + +def report_failure(name, shell_pid, output, debug_log_fd): + """A PTY case that fails prints just `FAIL`, which says nothing about why. Record what the + terminal actually showed and whether the client is still alive - the client dying (the read + loop hitting EOF) looks exactly like the expected text never being printed.""" + try: + reaped_pid, wait_status = os.waitpid(shell_pid, os.WNOHANG) + except ChildProcessError: + reaped_pid, wait_status = shell_pid, -1 + running = reaped_pid == 0 + banner = ( + f"=== {name}: FAIL ===\n" + f"client still running: {running}, wait status: {wait_status}\n" + f"terminal output: {output[-8000:]!r}\n" + ) + debug_log_fd.write(banner) + debug_log_fd.flush() + sys.stderr.write(banner) + sys.stderr.flush() + + +def run_case(program, argv, name, keystrokes, check, state=None, check_from=0): + """Each keystroke is a pair of the bytes to send and an optional predicate to wait for before + sending the next one (in place of a fixed idle drain). Only the output produced from the + keystroke with index `check_from` on is passed to `check`, so the earlier keystrokes may + legitimately pop hints without failing a "no hints" check.""" + shell_pid, master = pty.fork() + if shell_pid == 0: + os.environ["TERM"] = "xterm" + os.execv(program, argv) + return + + debug_log_fd = open(DEBUG_LOG, "a") + try: + matched, transcript = read_until( + master, debug_log_fd, lambda o: ":)" in o, TIMEOUT_SECONDS + ) + if not matched: + print(f"{name}: FAIL") + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 1 + return + + # Everything the terminal showed, for the failure transcript - `output` holds only what + # the check is allowed to look at. + output = "" + for index, (keys, wait_for) in enumerate(keystrokes): + os.write(master, keys) + if wait_for is not None: + matched, segment = read_until( + master, debug_log_fd, wait_for, STEP_WAIT_SECONDS + ) + else: + matched, segment = True, drain(master, debug_log_fd, DRAIN_IDLE_SECONDS) + transcript += segment + if index >= check_from: + output += segment + # A `wait_for` that expired means the state the next keystroke assumes was never + # reached: the hint that arms the stale cache never appeared, or the prompt never came + # back and the next line would be typed into a still running query, where the terminal + # driver echoes the bytes instead of `replxx` interpreting them. Either way the case + # would prove nothing while still being able to pass, so end it here with its + # transcript. + if not matched: + print(f"{name}: FAIL") + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 1 + return + + # The prepopulated line only appears after the (faked) generation round trip, which can + # outlast the inter-keystroke drain on slow builds. + if not check(output): + _, extra = read_until( + master, + debug_log_fd, + lambda more: check(output + more), + FINAL_WAIT_SECONDS, + ) + output += extra + transcript += extra + + ok = check(output) + print(f"{name}: {'OK' if ok else 'FAIL'}") + if not ok: + report_failure(name, shell_pid, transcript, debug_log_fd) + state.value = 0 if ok else 1 + finally: + os.close(master) + debug_log_fd.close() + + +def worker_budget(keystrokes): + """Everything `run_case` may legitimately spend: the wait for the first prompt, every explicit + `wait_for`, the idle window after every other keystroke, and the final wait for the expected + output. The supervisor must not expire before that, or ordinary slowness looks like a + failure.""" + waits = sum(1 for _, wait_for in keystrokes if wait_for is not None) + return ( + TIMEOUT_SECONDS + + STEP_WAIT_SECONDS * waits + + DRAIN_IDLE_SECONDS * (len(keystrokes) - waits) + + FINAL_WAIT_SECONDS + + SUPERVISOR_SLACK_SECONDS + ) + + +def run_with_timeout(program, argv, name, keystrokes, check, check_from=0): + for attempt in range(5): + state = multiprocessing.Value("i", -1) + process = multiprocessing.Process( + target=run_case, + args=(program, argv, name, keystrokes, check), + kwargs={"state": state, "check_from": check_from}, + ) + process.start() + process.join(worker_budget(keystrokes)) + if process.is_alive(): + process.terminate() + if state.value in (0, 1): + return + # transient timeout on a loaded machine - retry + print(f"{name}: FAIL") + sys.stderr.write(f"=== {name}: FAIL === the case timed out on every attempt\n") + + +def main(): + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), CompletionHandler) + server.daemon_threads = True + threading.Thread(target=server.serve_forever, daemon=True).start() + + config = write_config(server.server_address[1]) + history = make_history_file() + + program = os.environ["CLICKHOUSE_LOCAL"] + base = shlex.split(program) + args = base + [ + "--wait_for_suggestions_to_load", + "--hints", + "1", + f"--config-file={config}", + f"--history_file={history}", + ] + + no_hints = lambda o: GENERATED_QUERY_LAST_WORD in o and (HINT_COLOR + "ive") not in o + + try: + run_with_timeout( + base[0], + args, + "prepopulated_query_no_hints", + [(b"?? show me a query\r", None)], + no_hints, + ) + # Same-text regression: replxx keeps the hint cache across prompts, so when the generated + # query is byte-for-byte equal to the previously displayed line, the hint callback may not + # even be called - the stale cached hint must not be reused either. Type the query manually + # first (and wait for the hint ghost, which poisons the cache), execute it, then have the + # AI "generate" the very same text. The `??` request is kept as short as possible so that + # replxx's rapid-refresh throttling (1 ms) can swallow its keystrokes without regenerating + # the hint seed - only then does the preloaded line hit the stale cache. Only the output + # after the `??` keystroke is checked, as the manually typed line legitimately shows the + # hint. + run_with_timeout( + base[0], + args, + "same_text_prepopulated_query_no_hints", + [ + (GENERATED_QUERY.encode(), lambda o: (HINT_COLOR + "ive") in o), + (b"\r", lambda o: ":)" in o), + (b"??q\r", None), + ], + no_hints, + check_from=2, + ) + finally: + server.shutdown() + + +if __name__ == "__main__": + # The check lambdas are not picklable, so the subprocesses must be forked (not spawned, + # which became the default on newer Python). + multiprocessing.set_start_method("fork") + main() diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference new file mode 100644 index 000000000000..fccb0eff5b81 --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.reference @@ -0,0 +1,2 @@ +prepopulated_query_no_hints: OK +same_text_prepopulated_query_no_hints: OK diff --git a/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh new file mode 100755 index 000000000000..d48ace2e154e --- /dev/null +++ b/tests/queries/0_stateless/05025_client_hints_prepopulated_query.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: long, no-debug, no-fasttest +# no-fasttest: needs the AI SQL generator (`ENABLE_CLIENT_AI`), which is not built in the fast test. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-test` dumps `$CLICKHOUSE_TMP/.debuglog` into the failure report, and the +# testcase name is the name of this shell script - the Python part cannot derive it from its own +# file name, so pass the path down. +export CLICKHOUSE_TEST_DEBUG_LOG="$CLICKHOUSE_TMP/$(basename "${BASH_SOURCE[0]}").debuglog" + +python3 "$CUR_DIR"/05025_client_hints_prepopulated_query.python diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference index dff69852207d..ba2bb9215f91 100644 --- a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.reference @@ -1,13 +1,13 @@ ---- default: aws_external_id hidden, role identifiers visible +--- default: aws_external_id and aws_role_session_name hidden, aws_role_arn visible aws_external_id = '[HIDDEN]' aws_access_key_id = '[HIDDEN]' aws_secret_access_key = '[HIDDEN]' aws_role_arn = 'arn:aws:iam::1:role/r' -aws_role_session_name = 'sess' +aws_role_session_name = '[HIDDEN]' OK: no secret in formatted query ---- show_secrets: aws_external_id visible +--- show_secrets: aws_external_id and aws_role_session_name visible aws_external_id = 'SECRET_THAT_MUST_NOT_LEAK' aws_access_key_id = 'SECRET_THAT_MUST_NOT_LEAK' aws_secret_access_key = 'SECRET_THAT_MUST_NOT_LEAK' aws_role_arn = 'arn:aws:iam::1:role/r' -aws_role_session_name = 'sess' +aws_role_session_name = 'SECRET_THAT_MUST_NOT_LEAK' diff --git a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh index f33348fb92d4..d846eca5fa41 100755 --- a/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh +++ b/tests/queries/0_stateless/05030_datalake_catalog_hide_aws_external_id.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash -# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, so it must be -# redacted as [HIDDEN] when a DataLakeCatalog CREATE query is formatted (system.databases.engine_full, -# SHOW CREATE DATABASE), while aws_role_arn and aws_role_session_name are non-secret identifiers that -# stay visible. Uses clickhouse-format so it needs no live catalog and is safe to run in parallel. +# Regression test: aws_external_id is the shared secret of the AWS AssumeRole triple, and +# aws_role_session_name can be one too (a trust policy can require a specific value through the +# sts:RoleSessionName condition), so both must be redacted as [HIDDEN] when a DataLakeCatalog CREATE +# query is formatted (system.databases.engine_full, SHOW CREATE DATABASE), while aws_role_arn is a +# non-secret identifier that stays visible. Uses clickhouse-format so it needs no live catalog and is +# safe to run in parallel. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -17,7 +19,7 @@ aws_access_key_id = '${SECRET}', aws_secret_access_key = '${SECRET}', aws_external_id = '${SECRET}', aws_role_arn = 'arn:aws:iam::1:role/r', -aws_role_session_name = 'sess'" +aws_role_session_name = '${SECRET}'" show_settings() { local formatted="$1" @@ -32,9 +34,9 @@ show_settings() { done } -# Arm A: at the default the secret is redacted; arms C (non-secret identifiers stay visible) and +# Arm A: at the default the secrets are redacted; arms C (the non-secret role ARN stays visible) and # D (sibling AWS keys still redacted) are asserted by the same output. -echo "--- default: aws_external_id hidden, role identifiers visible" +echo "--- default: aws_external_id and aws_role_session_name hidden, aws_role_arn visible" formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline) show_settings "$formatted" if echo "$formatted" | grep -q "$SECRET"; then @@ -44,6 +46,6 @@ else fi # Arm B: an authorized caller can still retrieve the value, so the fix redacts rather than destroys. -echo "--- show_secrets: aws_external_id visible" +echo "--- show_secrets: aws_external_id and aws_role_session_name visible" formatted=$(echo "$query" | $CLICKHOUSE_FORMAT --oneline --show_secrets) show_settings "$formatted" diff --git a/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.reference b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.reference new file mode 100644 index 000000000000..3f5bade2d42e --- /dev/null +++ b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.reference @@ -0,0 +1,4 @@ +499500 1000 +499500 1000 +1 +0 diff --git a/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.sql b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.sql new file mode 100644 index 000000000000..f943a7cb3e78 --- /dev/null +++ b/tests/queries/0_stateless/05038_query_plan_limits_hedged_sql_fallback.sql @@ -0,0 +1,72 @@ +-- Tags: no-darwin +-- `HedgedConnections` are compiled only under `OS_LINUX`, so on macOS `use_hedged_requests` is a no-op +-- and the initiator never has to fall back to SQL. + +-- Regression test: a distributed query with `serialize_query_plan = 1` over hedged connections +-- must fall back to sending SQL when the plan carries non-default execution limits and a later +-- hedge could select a replica whose query-plan serialization version is unverified. Plan-level +-- `max_threads` and `concurrency_control` are serialized only since plan serialization version 10; +-- shipping the plan to an unverified peer would silently drop them during a rolling upgrade. + +DROP TABLE IF EXISTS t_plan_limits_hedged_fallback; + +CREATE TABLE t_plan_limits_hedged_fallback (a UInt64) ENGINE = MergeTree ORDER BY a; +INSERT INTO t_plan_limits_hedged_fallback SELECT number FROM numbers(1000); + +-- `serialize_query_plan` requires the analyzer; pin it against `compatibility` randomization. +SET enable_analyzer = 1; +-- The ordinary distributed path is under test; parallel replicas gate plan shipping differently. +SET enable_parallel_replicas = 0; +SET serialize_query_plan = 1; +-- Non-default plan-level execution limits, pinned against CI randomization: these are what the +-- fallback protects during a rolling upgrade. +SET max_threads = 4; +SET use_concurrency_control = 1; +-- Force a real remote connection: with the default the initiator would execute the only shard +-- locally and never open a connection, hedged or not. +SET prefer_localhost_replica = 0; + +-- Hedged connections: the first replica is established up front, but a later hedge may select any +-- remaining replica of the pool, whose version is unknown at send time. With three replicas in the +-- pool only one is verified, so the initiator must choose the SQL fallback. +SELECT sum(a), count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_plan_limits_hedged_fallback) +SETTINGS use_hedged_requests = 1, log_comment = '05038_hedged_plan_limits_fallback'; + +-- Control: with hedged connections off every connected replica is verified (same-version local +-- cluster), so the plan is shipped and the fallback must stay silent. +SELECT sum(a), count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t_plan_limits_hedged_fallback) +SETTINGS use_hedged_requests = 0, log_comment = '05038_hedged_plan_limits_no_fallback'; + +SYSTEM FLUSH LOGS query_log, text_log; + +-- The firing oracle: the hedged query logged the SQL fallback on the initiator. Resolve the newest +-- matching initial query by `log_comment` (CI reuses one database across executions in some jobs, +-- so aggregate over history could be satisfied by an earlier run). +SELECT count() > 0 +FROM system.text_log +WHERE query_id = ( + SELECT argMax(query_id, event_time_microseconds) + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment = '05038_hedged_plan_limits_fallback' + AND type = 'QueryFinish' + AND is_initial_query + AND event_date >= yesterday() AND event_time >= now() - 600) + AND message LIKE 'Sending query as SQL because a replica does not support query-plan serialization version%' + AND event_date >= yesterday() AND event_time >= now() - 600; + +-- The control oracle: the non-hedged query did not fall back. +SELECT count() +FROM system.text_log +WHERE query_id = ( + SELECT argMax(query_id, event_time_microseconds) + FROM system.query_log + WHERE current_database = currentDatabase() + AND log_comment = '05038_hedged_plan_limits_no_fallback' + AND type = 'QueryFinish' + AND is_initial_query + AND event_date >= yesterday() AND event_time >= now() - 600) + AND message LIKE 'Sending query as SQL because a replica does not support query-plan serialization version%' + AND event_date >= yesterday() AND event_time >= now() - 600; + +DROP TABLE t_plan_limits_hedged_fallback; diff --git a/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference new file mode 100644 index 000000000000..72c3262a309d --- /dev/null +++ b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.reference @@ -0,0 +1 @@ +Hello 1 diff --git a/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh new file mode 100755 index 000000000000..8385d6f97658 --- /dev/null +++ b/tests/queries/0_stateless/05043_direct_dictionary_in_merge.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# A `direct` dictionary reads through the source's own pipeline, and `merge` wraps every child in +# `materialize`, which rejects a header that carries rows. +$CLICKHOUSE_CLIENT -q " + CREATE DICTIONARY dict (word String, counter UInt32) + PRIMARY KEY word + SOURCE(HTTP(url '${CLICKHOUSE_URL}&query=SELECT+%27Hello%27,1+FORMAT+CSV' format 'CSV')) + LAYOUT(DIRECT())" + +$CLICKHOUSE_CLIENT -q "SELECT * FROM merge(currentDatabase(), '^dict\$')" diff --git a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference index 9f607859a6e8..586c62c10408 100644 --- a/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference +++ b/tests/queries/0_stateless/05045_merge_prefilter_lazy_load_tables.reference @@ -1,4 +1,4 @@ -TableProxy +Distributed 3 t05045_leaf 3 diff --git a/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference new file mode 100644 index 000000000000..d93a3042cfa8 --- /dev/null +++ b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.reference @@ -0,0 +1,97 @@ +-- { echo } + +-- per-partition set building reads the same predicate. The set fill deduplicates across +-- partitions anyway, so the merged answer stays correct and only the plan shape shows the +-- decline; the bare-key arm is the control that the fixture reaches the optimization. +-- 2001 and 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_set_month; +CREATE TABLE t_set_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +SYSTEM STOP MERGES t_set_month; +INSERT INTO t_set_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d + INTERVAL 1 MONTH FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +Pre-distinct: 1 +Read each partition through separate port: 1 +DROP TABLE t_set_month; +-- an integer key keeps per-partition set building; the partition key is a function of the set's +-- own output column, which the interval arm above cannot use because its key is the collapsing one +DROP TABLE IF EXISTS t_set_int; +CREATE TABLE t_set_int (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY tuple() PARTITION BY a % 8; +SYSTEM STOP MERGES t_set_int; +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +Pre-distinct: 1 +Read each partition through separate port: 1 +SELECT (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 0) = (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1); +1 +DROP TABLE t_set_int; +-- --------------------------------------------------------------------------- +-- The distributed sharding-key consumer reaches the same predicate through its own rejection +-- loop and its own direct call, so it gets its own arms. Dropping the merge step is only +-- correct when the group key determines the shard: a key that collapses distinct shard-key +-- values leaves each shard's partial groups unmerged, so the same key is returned twice. +-- Each view filters itself by shardNum() so the two shards hold the disjoint rows the +-- declared key implies - a declared key alone does not redistribute rows on a read, and +-- without the filter every shard holds every row and even a sound merge drop doubles the +-- answer. The first arm of each pair counts merge steps (1 = kept, 0 = dropped) and the +-- second compares the answer against the unoptimized one; the integer pair is the control +-- that the optimization still fires where it is sound. +-- --------------------------------------------------------------------------- + +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY s ORDER BY s; +1 10 +2 20 +SELECT count() FROM (EXPLAIN SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +1 +SELECT (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +1 +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY s ORDER BY s; +1 15 +2 15 +SELECT count() FROM (EXPLAIN SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +0 +SELECT (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +1 +-- --------------------------------------------------------------------------- +-- The window consumer reaches the same predicate through the stream-disjointness +-- propagation, at two sites: the per-partition read request and the scatter skip above it. +-- INTERVAL MONTH collapses the 29th, 30th and 31st into one key, so one logical window +-- partition spans the table partitions those days live in and must not be evaluated per +-- table partition. The default arm carries no setting: the cost heuristic accepts this +-- fixture, so the answer has to be right without opting out. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_win_month; +CREATE TABLE t_win_month (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_win_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d FROM numbers(60); +-- the collapse the arms below depend on: more table partitions than window keys +SELECT uniqExact(_partition_id), uniqExact(d + INTERVAL 1 MONTH) FROM t_win_month; +6 2 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS force_window_partitions_independently = 1; +30 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +30 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c; +30 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +0 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +0 +DROP TABLE t_win_month; +-- the integer control: an injective addend keeps both sites firing, so the arms above +-- attribute to the operand type and not to the window shape or the fixture +DROP TABLE IF EXISTS t_win_int; +CREATE TABLE t_win_int (x UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x % 8; +INSERT INTO t_win_int SELECT number % 8 FROM numbers_mt(800); +SELECT uniqExact(_partition_id), uniqExact(x + 1) FROM t_win_int; +8 8 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS force_window_partitions_independently = 1; +100 +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +100 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +1 +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +1 +DROP TABLE t_win_int; diff --git a/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql new file mode 100644 index 000000000000..509e8a3e3d5e --- /dev/null +++ b/tests/queries/0_stateless/05053_injective_plus_minus_type_gate_sets_shards_windows.sql @@ -0,0 +1,97 @@ +-- Tags: distributed, no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +SET explain_query_plan_default = 'legacy'; +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +-- The values and plans below are the analyzer's, so pin it. +SET enable_analyzer = 1; + +-- { echo } + +-- per-partition set building reads the same predicate. The set fill deduplicates across +-- partitions anyway, so the merged answer stays correct and only the plan shape shows the +-- decline; the bare-key arm is the control that the fixture reaches the optimization. +-- 2001 and 2002 are not leap years, so all three days of each map to February 28. +DROP TABLE IF EXISTS t_set_month; +CREATE TABLE t_set_month (d Date, x UInt32) ENGINE = MergeTree ORDER BY d PARTITION BY d; +SYSTEM STOP MERGES t_set_month; +INSERT INTO t_set_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d, number FROM numbers(60); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d + INTERVAL 1 MONTH FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE toDate('2001-02-28') + number IN (SELECT d FROM t_set_month) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_set_month; + +-- an integer key keeps per-partition set building; the partition key is a function of the set's +-- own output column, which the interval arm above cannot use because its key is the collapsing one +DROP TABLE IF EXISTS t_set_int; +CREATE TABLE t_set_int (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY tuple() PARTITION BY a % 8; +SYSTEM STOP MERGES t_set_int; +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +INSERT INTO t_set_int SELECT number % 64, number FROM numbers_mt(400); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1) WHERE explain LIKE '%Pre-distinct%' OR explain LIKE '%Read each partition through separate port%'; +SELECT (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 0) = (SELECT count() FROM numbers(100) WHERE number IN (SELECT a + 1 FROM t_set_int) SETTINGS allow_creating_set_partitions_independently = 1); +DROP TABLE t_set_int; + +-- --------------------------------------------------------------------------- +-- The distributed sharding-key consumer reaches the same predicate through its own rejection +-- loop and its own direct call, so it gets its own arms. Dropping the merge step is only +-- correct when the group key determines the shard: a key that collapses distinct shard-key +-- values leaves each shard's partial groups unmerged, so the same key is returned twice. +-- Each view filters itself by shardNum() so the two shards hold the disjoint rows the +-- declared key implies - a declared key alone does not redistribute rows on a read, and +-- without the filter every shard holds every row and even a sound merge drop doubles the +-- answer. The first arm of each pair counts merge steps (1 = kept, 0 = dropped) and the +-- second compares the answer against the unoptimized one; the integer pair is the control +-- that the optimization still fires where it is sound. +-- --------------------------------------------------------------------------- + +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY s ORDER BY s; +SELECT count() FROM (EXPLAIN SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +SELECT (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT d + INTERVAL 1 MONTH AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE toYYYYMMDD(toDate('2001-01-29') + (number % 3)) % 2 = (shardNum() - 1)), toUInt64(toYYYYMMDD(d))) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); +SELECT shardNum() AS s, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY s ORDER BY s; +SELECT count() FROM (EXPLAIN SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1) WHERE explain ILIKE '%MergingAggregated%'; +SELECT (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 0)) = (SELECT count() FROM (SELECT x + 1 AS k, count() FROM remote('127.{1,2}', view(SELECT toDate('2001-01-29') + (number % 3) AS d, number AS x FROM numbers(30) WHERE number % 2 = (shardNum() - 1)), toUInt64(x)) GROUP BY k SETTINGS optimize_skip_unused_shards = 1, optimize_distributed_group_by_sharding_key = 1)); + +-- --------------------------------------------------------------------------- +-- The window consumer reaches the same predicate through the stream-disjointness +-- propagation, at two sites: the per-partition read request and the scatter skip above it. +-- INTERVAL MONTH collapses the 29th, 30th and 31st into one key, so one logical window +-- partition spans the table partitions those days live in and must not be evaluated per +-- table partition. The default arm carries no setting: the cost heuristic accepts this +-- fixture, so the answer has to be right without opting out. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_win_month; +CREATE TABLE t_win_month (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY d; +INSERT INTO t_win_month SELECT toDate(concat(toString(2001 + intDiv(number, 30)), '-01-', toString(29 + (intDiv(number, 10) % 3)))) AS d FROM numbers(60); +-- the collapse the arms below depend on: more table partitions than window keys +SELECT uniqExact(_partition_id), uniqExact(d + INTERVAL 1 MONTH) FROM t_win_month; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS force_window_partitions_independently = 1; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) AS c FROM t_win_month) ORDER BY c; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY d + INTERVAL 1 MONTH) FROM t_win_month SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +DROP TABLE t_win_month; + +-- the integer control: an injective addend keeps both sites firing, so the arms above +-- attribute to the operand type and not to the window shape or the fixture +DROP TABLE IF EXISTS t_win_int; +CREATE TABLE t_win_int (x UInt32) ENGINE = MergeTree ORDER BY x PARTITION BY x % 8; +INSERT INTO t_win_int SELECT number % 8 FROM numbers_mt(800); +SELECT uniqExact(_partition_id), uniqExact(x + 1) FROM t_win_int; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS force_window_partitions_independently = 1; +SELECT DISTINCT c FROM (SELECT count() OVER (PARTITION BY x + 1) AS c FROM t_win_int) ORDER BY c SETTINGS allow_window_partitions_independently = 0; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Read each partition through separate port: 1%'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT count() OVER (PARTITION BY x + 1) FROM t_win_int SETTINGS force_window_partitions_independently = 1) WHERE explain ILIKE '%Skip scatter by partition: 1%'; +DROP TABLE t_win_int; diff --git a/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference new file mode 100644 index 000000000000..d695e73641ae --- /dev/null +++ b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.reference @@ -0,0 +1,98 @@ +-- { echo } + +-- --------------------------------------------------------------------------- +-- Preservation: integer arithmetic with an integer constant stays injective, so the +-- optimization must keep firing. A correctness-only test would not notice this. Each arm +-- prints two plan lines; an arm printing nothing has lost the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_int; +CREATE TABLE t_int (a UInt32) ENGINE = MergeTree ORDER BY a PARTITION BY intDiv(a, 2) * 2 + 1; +INSERT INTO t_int SELECT number FROM numbers_mt(32); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) + 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- subtraction widens to a signed result, which is a different branch of the integer test +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) - 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- the constant may be either operand, so both positions have to be recognized +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 7 - intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 1 + intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +-- negate and tuple are not named by the query-plan consumer: they answer isInjective themselves, so +-- the optimization has to keep firing for them. +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT -intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT tuple(intDiv(a, 2)) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_int; +-- wide integers are exact, so the gate must not be narrowed to native widths +DROP TABLE IF EXISTS t_int128; +CREATE TABLE t_int128 (a Int128) ENGINE = MergeTree ORDER BY a PARTITION BY a % 8; +INSERT INTO t_int128 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT a + 7 AS a1 FROM t_int128 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_int128; +DROP TABLE IF EXISTS t_uint256; +CREATE TABLE t_uint256 (b UInt256) ENGINE = MergeTree ORDER BY b PARTITION BY b % 8; +INSERT INTO t_uint256 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT b + 7 AS b1 FROM t_uint256 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_uint256; +-- both type-wrapper orders must be stripped before the integer test. Contrast with the NULL +-- constant arm above: a Nullable column keeps the optimization, a NULL constant loses it. +DROP TABLE IF EXISTS t_nullable; +CREATE TABLE t_nullable (n Nullable(UInt32)) ENGINE = MergeTree ORDER BY n PARTITION BY n % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT n + 1 AS n1 FROM t_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_nullable; +DROP TABLE IF EXISTS t_lc; +CREATE TABLE t_lc (l LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY l PARTITION BY l % 8; +INSERT INTO t_lc SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT l + 1 AS l1 FROM t_lc SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_lc; +DROP TABLE IF EXISTS t_lc_nullable; +CREATE TABLE t_lc_nullable (ln LowCardinality(Nullable(UInt32))) ENGINE = MergeTree ORDER BY ln PARTITION BY ln % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT ln + 1 AS ln1 FROM t_lc_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_lc_nullable; +-- --------------------------------------------------------------------------- +-- Accepted narrowings. These three expressions are injective, but every type they would +-- admit also admits a non-injective case above that cannot be told apart: an Interval on +-- Date is not separable from one on DateTime by kind, a Date operand re-admits the +-- narrowing constant, and a Decimal operand re-admits the rescaling constant. Each arm +-- prints nothing; the bare-key arm after it is the control showing the fixture does reach +-- the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY toDayOfWeek(d); +INSERT INTO t_date SELECT toDate('2001-01-01') + number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + INTERVAL 1 DAY AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + 1 AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toDayOfWeek(d) AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_date; +DROP TABLE IF EXISTS t_dec; +CREATE TABLE t_dec (dec Decimal64(2)) ENGINE = MergeTree ORDER BY dec PARTITION BY toUInt32(dec) % 8; +INSERT INTO t_dec SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT dec + 1 AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toUInt32(dec) AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +Skip stream merging: 1 +Read each partition through separate port: 1 +DROP TABLE t_dec; diff --git a/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql new file mode 100644 index 000000000000..cb93bc55fbcc --- /dev/null +++ b/tests/queries/0_stateless/05054_injective_plus_minus_type_gate_plan_shape.sql @@ -0,0 +1,100 @@ +-- Tags: no-random-settings, no-random-merge-tree-settings +-- no-random-settings, no-random-merge-tree-settings: randomized settings and part counts +-- change both the plans and the values these arms count. + +SET explain_query_plan_default = 'legacy'; +-- max_threads is pinned because the cost heuristic accepts a fixture only when its partition +-- count is at least max_threads / 2; arms that must not depend on the heuristic force it instead. +SET max_threads = 8; +SET enable_parallel_replicas = 0; +SET max_rows_in_distinct = 0; +SET max_bytes_in_distinct = 0; +-- The stateless CI profile sets these to 10G, and a nonzero limit disables per-partition +-- evaluation outright. +SET max_rows_to_group_by = 0; +SET max_rows_to_sort = 0; +SET max_bytes_to_sort = 0; +SET optimize_use_implicit_projections = 0; +SET allow_suspicious_low_cardinality_types = 1; +-- The EXPLAIN QUERY TREE arms below are analyzer-only; old-analyzer jobs would error on them. +SET enable_analyzer = 1; + +-- { echo } + +-- --------------------------------------------------------------------------- +-- Preservation: integer arithmetic with an integer constant stays injective, so the +-- optimization must keep firing. A correctness-only test would not notice this. Each arm +-- prints two plan lines; an arm printing nothing has lost the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_int; +CREATE TABLE t_int (a UInt32) ENGINE = MergeTree ORDER BY a PARTITION BY intDiv(a, 2) * 2 + 1; +INSERT INTO t_int SELECT number FROM numbers_mt(32); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) + 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- subtraction widens to a signed result, which is a different branch of the integer test +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT intDiv(a, 2) - 1 AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- the constant may be either operand, so both positions have to be recognized +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 7 - intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT 1 + intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +-- negate and tuple are not named by the query-plan consumer: they answer isInjective themselves, so +-- the optimization has to keep firing for them. +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT -intDiv(a, 2) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT tuple(intDiv(a, 2)) AS a1 FROM t_int SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_int; + +-- wide integers are exact, so the gate must not be narrowed to native widths +DROP TABLE IF EXISTS t_int128; +CREATE TABLE t_int128 (a Int128) ENGINE = MergeTree ORDER BY a PARTITION BY a % 8; +INSERT INTO t_int128 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT a + 7 AS a1 FROM t_int128 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_int128; + +DROP TABLE IF EXISTS t_uint256; +CREATE TABLE t_uint256 (b UInt256) ENGINE = MergeTree ORDER BY b PARTITION BY b % 8; +INSERT INTO t_uint256 SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT b + 7 AS b1 FROM t_uint256 SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_uint256; + +-- both type-wrapper orders must be stripped before the integer test. Contrast with the NULL +-- constant arm above: a Nullable column keeps the optimization, a NULL constant loses it. +DROP TABLE IF EXISTS t_nullable; +CREATE TABLE t_nullable (n Nullable(UInt32)) ENGINE = MergeTree ORDER BY n PARTITION BY n % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT n + 1 AS n1 FROM t_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_nullable; + +DROP TABLE IF EXISTS t_lc; +CREATE TABLE t_lc (l LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY l PARTITION BY l % 8; +INSERT INTO t_lc SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT l + 1 AS l1 FROM t_lc SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_lc; + +DROP TABLE IF EXISTS t_lc_nullable; +CREATE TABLE t_lc_nullable (ln LowCardinality(Nullable(UInt32))) ENGINE = MergeTree ORDER BY ln PARTITION BY ln % 8 SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc_nullable SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT ln + 1 AS ln1 FROM t_lc_nullable SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_lc_nullable; + +-- --------------------------------------------------------------------------- +-- Accepted narrowings. These three expressions are injective, but every type they would +-- admit also admits a non-injective case above that cannot be told apart: an Interval on +-- Date is not separable from one on DateTime by kind, a Date operand re-admits the +-- narrowing constant, and a Decimal operand re-admits the rescaling constant. Each arm +-- prints nothing; the bare-key arm after it is the control showing the fixture does reach +-- the optimization. +-- --------------------------------------------------------------------------- + +DROP TABLE IF EXISTS t_date; +CREATE TABLE t_date (d Date) ENGINE = MergeTree ORDER BY d PARTITION BY toDayOfWeek(d); +INSERT INTO t_date SELECT toDate('2001-01-01') + number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + INTERVAL 1 DAY AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT d + 1 AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toDayOfWeek(d) AS d1 FROM t_date SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_date; + +DROP TABLE IF EXISTS t_dec; +CREATE TABLE t_dec (dec Decimal64(2)) ENGINE = MergeTree ORDER BY dec PARTITION BY toUInt32(dec) % 8; +INSERT INTO t_dec SELECT number FROM numbers_mt(200); +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT dec + 1 AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +SELECT replaceRegexpOne(explain, '^[ ]*(.*)', '\\1') FROM (EXPLAIN actions = 1 SELECT DISTINCT toUInt32(dec) AS dec1 FROM t_dec SETTINGS allow_distinct_partitions_independently = 1) WHERE explain LIKE '%Skip stream merging%' OR explain LIKE '%Read each partition through separate port%'; +DROP TABLE t_dec; diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference new file mode 100644 index 000000000000..19219d2f6180 --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference @@ -0,0 +1,8 @@ +Granules: 2/10 +Granules: 2/10 +Granules: 2/10 +3 15 +3 6 +6 105 +6 105 +9 120 diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql new file mode 100644 index 000000000000..aae7082ce17b --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql @@ -0,0 +1,33 @@ +-- The monotonicity of a conversion does not depend on the `Nullable` wrapper of its argument, so a +-- `toDate`/`toDateTime` predicate over a `Nullable` key must still be usable for index analysis. + +SET session_timezone = 'UTC'; + +DROP TABLE IF EXISTS t_nullable_key; +CREATE TABLE t_nullable_key (x Nullable(DateTime64(6)), y Int64) +ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 4, allow_nullable_key = 1; + +INSERT INTO t_nullable_key +SELECT if(number % 7 = 0, NULL, toDateTime64('2026-03-01 00:00:00', 6) + INTERVAL number * 6 HOUR), number +FROM numbers(40); + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate32(x) = toDate32('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDateTime(x) = toDateTime('2026-03-02 06:00:00') +) WHERE granules != ''; + +-- The rows the index analysis keeps must be the rows the predicate selects, `NULL`s included. +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) < toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IS NULL; +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IN (toDate('2026-03-02'), toDate('2026-03-08')); +SELECT count(), sum(y) FROM t_nullable_key WHERE isNull(x) OR toDate(x) = toDate('2026-03-02'); + +DROP TABLE t_nullable_key; diff --git a/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.reference b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.reference new file mode 100644 index 000000000000..9ad7af9de8a7 --- /dev/null +++ b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.reference @@ -0,0 +1,3 @@ +1 8 108 2 +0 8 108 1 +1 0 8 diff --git a/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.sql b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.sql new file mode 100644 index 000000000000..352f69510120 --- /dev/null +++ b/tests/queries/0_stateless/05055_clear_column_ephemeral_materialized.sql @@ -0,0 +1,50 @@ +-- CLEAR COLUMN recomputes the MATERIALIZED columns that read the cleared one. A MATERIALIZED column +-- derived from an EPHEMERAL column cannot be recomputed outside INSERT, so it must be left alone: +-- recomputing it fails to resolve the EPHEMERAL name and the whole mutation dies with +-- "There is no column or subcolumn e in table". + +SET mutations_sync = 2; + +DROP TABLE IF EXISTS t_clear_ephemeral; + +CREATE TABLE t_clear_ephemeral +( + x Int32, + y Int32, + e Int32 EPHEMERAL 0, + me Int32 MATERIALIZED x + e, + me2 Int32 MATERIALIZED me + 100, + mk Int32 MATERIALIZED x + 1 +) +ENGINE = MergeTree ORDER BY tuple() PARTITION BY tuple() +-- The runner randomizes both of these together with `min_bytes_for_wide_part`, and on a Wide part +-- with either block column on the mutation does not recompute MATERIALIZED columns at all, so `mk` +-- would keep its pre-clear value and the recompute this test is about would never run. +SETTINGS enable_block_number_column = 0, enable_block_offset_column = 0; + +INSERT INTO t_clear_ephemeral (x, y, e) VALUES (1, 0, 7); +SELECT x, me, me2, mk FROM t_clear_ephemeral; + +-- `mk` is recomputed from the cleared `x`. `me` reads the EPHEMERAL `e`, so it cannot be recomputed +-- and keeps its stored value. `me2` reads only `me`, so it keeps its value too. +ALTER TABLE t_clear_ephemeral CLEAR COLUMN x IN PARTITION tuple(); +SELECT x, me, me2, mk FROM t_clear_ephemeral; + +-- A cleared column that no MATERIALIZED column reads must not start a recompute at all. +DROP TABLE IF EXISTS t_clear_unrelated; + +CREATE TABLE t_clear_unrelated +( + x Int32, + y Int32, + e Int32 EPHEMERAL 0, + me Int32 MATERIALIZED x + e +) +ENGINE = MergeTree ORDER BY tuple() PARTITION BY tuple(); + +INSERT INTO t_clear_unrelated (x, y, e) VALUES (1, 5, 7); +ALTER TABLE t_clear_unrelated CLEAR COLUMN y IN PARTITION tuple(); +SELECT x, y, me FROM t_clear_unrelated; + +DROP TABLE t_clear_ephemeral; +DROP TABLE t_clear_unrelated; diff --git a/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference new file mode 100644 index 000000000000..1c4357e61ffd --- /dev/null +++ b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.reference @@ -0,0 +1,8 @@ +230 1 230 +250 1 250 +true true +true true +true true +true true +0 +1 1 1 diff --git a/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql new file mode 100644 index 000000000000..26e66cc4e70c --- /dev/null +++ b/tests/queries/0_stateless/05055_jit_float_to_integer_cast.sql @@ -0,0 +1,88 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/117442 +SET compile_expressions = 1; +SET min_count_to_compile_expression = 0; + +DROP TABLE IF EXISTS t_jit_float_cast; +CREATE TABLE t_jit_float_cast (c0 UInt8) ENGINE = Memory; +INSERT INTO t_jit_float_cast VALUES (230), (250); + +-- In-range conversions are exact and identical compiled or interpreted. +SELECT c0, + toFloat64(c0) / 10 <= CAST(toFloat64(c0) AS UInt8) AS in_range_by_cast, + CAST(toFloat64(c0) AS Decimal32(2)) AS in_range_decimal +FROM t_jit_float_cast +ORDER BY c0; + +-- A value the destination cannot hold raises. The non-finite one is built from the column so that +-- it is not constant folded before execution. +SELECT CAST(-toFloat64(c0) * 1e9 AS Decimal32(2)) FROM t_jit_float_cast; -- { serverError DECIMAL_OVERFLOW } +SELECT toDecimal32(-toFloat64(c0) * 1e9, 2) FROM t_jit_float_cast; -- { serverError DECIMAL_OVERFLOW } +SELECT toUInt8(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0))) FROM t_jit_float_cast; -- { serverError CANNOT_CONVERT_TYPE } +SELECT CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS UInt8) FROM t_jit_float_cast; -- { serverError CANNOT_CONVERT_TYPE } + +-- `Bool` is the one float to integer destination that stays compiled, because it is lowered as +-- `value != 0` rather than as a conversion, which is exact for every value. +SELECT CAST(-toFloat64(c0) * 1e30 AS Bool), + CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS Bool) +FROM t_jit_float_cast +ORDER BY c0; + +SET compile_expressions = 0; +SELECT CAST(-toFloat64(c0) * 1e30 AS Bool), + CAST(toFloat64(c0) / (toFloat64(c0) - toFloat64(c0)) AS Bool) +FROM t_jit_float_cast +ORDER BY c0; +SET compile_expressions = 1, min_count_to_compile_expression = 0; + +-- The value an out-of-range float converts to is not defined by the language, so compare the compiled +-- and the interpreted evaluation of the same expression instead of pinning a literal. +CREATE TABLE t_jit_float_cast_arms (c0 UInt8, lte UInt8) ENGINE = Memory; + +INSERT INTO t_jit_float_cast_arms +SELECT c0, toFloat64(c0) / 10 <= CAST(-toFloat64(c0) AS UInt8) FROM t_jit_float_cast; + +SET compile_expressions = 0; +INSERT INTO t_jit_float_cast_arms +SELECT c0, toFloat64(c0) / 10 <= CAST(-toFloat64(c0) AS UInt8) FROM t_jit_float_cast; +SET compile_expressions = 1, min_count_to_compile_expression = 0; + +SELECT count() FROM (SELECT c0 FROM t_jit_float_cast_arms GROUP BY c0 HAVING uniqExact(lte) > 1); + +DROP TABLE t_jit_float_cast_arms; + +-- Every row above is a value oracle, so all of them would still pass if the conversions silently +-- stopped or started being compiled. The shapes below pin which of them compiles. Each has one +-- compilable child, so once its `CAST` is declined nothing is left to compile: a declined shape +-- reaches zero even where the control compiles. `CompiledFunctionExecute` counts executions of an +-- already-compiled node, so a warm compiled cache does not change any of them. +SELECT CAST(toFloat64(number) AS Bool) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_bool' FORMAT Null; +SELECT CAST(toFloat64(number) AS UInt8) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_declined' FORMAT Null; +SELECT toFloat64(number) + 1 FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_control' FORMAT Null; +-- The reported shape: a comparison whose right side is a declined conversion. The conversion becomes +-- an input to the compiled expression, so the comparison around it must still compile. Neither side +-- can compile on its own, so the counter here belongs to the comparison and to nothing else. +SELECT toFloat64(number) <= CAST(toFloat64(number) AS UInt8) FROM numbers(2) + SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0, log_comment = '05055_parent' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +WITH shapes AS +( + SELECT log_comment, argMax(ProfileEvents['CompiledFunctionExecute'] > 0, event_time_microseconds) AS compiled + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment LIKE '05055_%' + GROUP BY log_comment +) +-- The control keeps the first column honest in a build without the embedded compiler, where every +-- shape is interpreted and an absolute assertion would go green on nothing being compiled. +SELECT + (SELECT compiled FROM shapes WHERE log_comment = '05055_bool') + = (SELECT compiled FROM shapes WHERE log_comment = '05055_control'), + (SELECT compiled FROM shapes WHERE log_comment = '05055_declined') = 0, + (SELECT compiled FROM shapes WHERE log_comment = '05055_parent') + = (SELECT compiled FROM shapes WHERE log_comment = '05055_control'); + +DROP TABLE t_jit_float_cast; diff --git a/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference b/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference new file mode 100644 index 000000000000..68961ec9d0d8 --- /dev/null +++ b/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference @@ -0,0 +1,9 @@ +t_json_flattened: accepted truncated blocks 0 +0 +12 18 6 51 +t_json_array_flattened: accepted truncated blocks 0 +0 +12 12 2 +t_dynamic_flattened: accepted truncated blocks 0 +0 +12 6 27 str8 diff --git a/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh b/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh new file mode 100755 index 000000000000..7577adec5191 --- /dev/null +++ b/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# no-old-analyzer: the queries building the payloads read JSON/Dynamic subcolumns, which it cannot resolve. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A flattened JSON/Dynamic Native block must be rejected when a stream ends before the declared row count. +# The data starts with rows that have no dynamic paths, so the Dynamic indexes stream starts with a run of +# NULL markers: truncated inside that run it demands no values at all and passes every per-type check. + +DATA_FILE=$CLICKHOUSE_TMP/flattened_native_$CLICKHOUSE_DATABASE.bin + +function check_truncated_blocks() +{ + local table=$1 + local query=$2 + + $CLICKHOUSE_CLIENT --output_format_native_use_flattened_dynamic_and_json_serialization=1 -q "$query FORMAT Native" > "$DATA_FILE" + + local size + size=$(stat -c%s "$DATA_FILE") + local accepted=0 + local n + # Sent over HTTP: a client process per truncation is what made this test time out under sanitizers. + for ((n = 1; n < size; n += 5)); do + if head -c "$n" "$DATA_FILE" \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+$table+FORMAT+Native" --data-binary @- \ + | grep -q "Exception" + then + continue + fi + accepted=$((accepted + 1)) + done + + echo "$table: accepted truncated blocks $accepted" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $table" + + $CLICKHOUSE_CLIENT -q "INSERT INTO $table FORMAT Native" < "$DATA_FILE" + rm "$DATA_FILE" +} + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t_json_flattened (json JSON(a UInt64, max_dynamic_paths=2)) ENGINE = Memory; + CREATE TABLE t_json_array_flattened (arr Array(JSON)) ENGINE = Memory; + CREATE TABLE t_dynamic_flattened (dyn Dynamic) ENGINE = Memory; +" + +check_truncated_blocks t_json_flattened " + SELECT (number < 6 ? '{\"a\":1}' : '{\"a\":2,\"b\":\"str' || toString(number) || '\",\"c\":' || toString(number) || '}')::JSON(a UInt64, max_dynamic_paths=2) AS json + FROM numbers(12)" +$CLICKHOUSE_CLIENT -q "SELECT count(), sum(json.a), countIf(json.b IS NULL), sum(json.c.:Int64) FROM t_json_flattened" + +check_truncated_blocks t_json_array_flattened " + SELECT arrayMap(x -> (number < 6 ? '{}' : '{\"a\":' || toString(x) || '}')::JSON, range(number % 3)) AS arr + FROM numbers(12)" +$CLICKHOUSE_CLIENT -q "SELECT count(), sum(length(arr)), sum(arraySum(arrayMap(x -> x.a::UInt64, arr))) FROM t_json_array_flattened" + +check_truncated_blocks t_dynamic_flattened " + SELECT json.a AS dyn + FROM (SELECT (number < 6 ? '{}' : (number % 2 ? '{\"a\":' || toString(number) || '}' : '{\"a\":\"str' || toString(number) || '\"}'))::JSON AS json FROM numbers(12))" +$CLICKHOUSE_CLIENT -q "SELECT count(), countIf(dyn IS NULL), sum(dyn.Int64), max(dyn.String) FROM t_dynamic_flattened" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE t_json_flattened; + DROP TABLE t_json_array_flattened; + DROP TABLE t_dynamic_flattened; +" diff --git a/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.reference b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.reference new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql new file mode 100644 index 000000000000..526e1f167e6d --- /dev/null +++ b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql @@ -0,0 +1,5 @@ +-- The FLATTENED (version 3) path list is sized from an untrusted count in the Native prefix, so its +-- container must be memory-tracked like the non-flat sibling (version 2) below. +-- Frame: 1 column, 1 row, name "j", type "JSON", 8-byte LE structure version, VarUInt path count 1000000. +SELECT * FROM format(Native, 'j JSON', unhex('0101016A044A534F4E0300000000000000C0843D')) SETTINGS max_memory_usage = 10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } +SELECT * FROM format(Native, 'j JSON', unhex('0101016A044A534F4E0200000000000000C0843D')) SETTINGS max_memory_usage = 10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference new file mode 100644 index 000000000000..d50220788b0b --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference @@ -0,0 +1,20 @@ +--- A1 control: the boundary the injection bypasses is enforced when the path is named directly +PATH_ACCESS_DENIED +--- A2 injection through the remote table name +no such table +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A3 injection through a column name +no column named +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A4 control: a table and column name that legitimately contain a double quote round-trip +NO_ERROR +11 +--- A5 control: a table name that legitimately contains a backslash round-trips +NO_ERROR +22 +--- A6 a NUL in the remote table name fails loudly, it is never silently truncated +unrecognized token diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh new file mode 100755 index 000000000000..9ec9abb0b9ba --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Fast tests don't build external libraries (SQLite) + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ANCHOR="${USER_FILES_PATH}/05055_anchor_${CLICKHOUSE_DATABASE}.db" +# Outside the user_files confinement. +OUTSIDE="${CLICKHOUSE_TMP}/05055_outside_${CLICKHOUSE_DATABASE}.db" + +cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_name_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_col_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_quote_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_backslash_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_nul_name" + rm -f "${ANCHOR}" "${OUTSIDE}" +} +trap cleanup EXIT +cleanup + +# Odd names: the old backslash escaping produced them, so the injected script's first statement succeeds and the rest runs. +sqlite3 "${ANCHOR}" 'CREATE TABLE "target\"(id INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE col_anchor("c\" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "ta""ble" ("c""1" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "a\b" (x INTEGER)' +chmod ugo+rw "${ANCHOR}" + +# A database outside user_files, holding a canary the user must never be able to reach. +sqlite3 "${OUTSIDE}" "CREATE TABLE secrets(s TEXT); INSERT INTO secrets VALUES ('CANARY')" +chmod ugo+rw "${OUTSIDE}" + +# One stable token per failure, so the reference embeds no path and no server version. +classify() +{ + local out + out=$(${CLICKHOUSE_CLIENT} --query="$1" 2>&1 \ + | grep -oF -e 'no such table' -e 'no column named' -e 'unrecognized token' \ + -e 'syntax error' -e 'PATH_ACCESS_DENIED' \ + | sed -n 1p) + echo "${out:-NO_ERROR}" +} + +# Read out of band: the ClickHouse read path cannot address these names, so an in-band readback would be a false negative. +injected_state() +{ + sqlite3 "${ANCHOR}" "SELECT 'objects created in the anchor: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" + sqlite3 "${ANCHOR}" "SELECT 'canary read from outside user_files: ' || coalesce((SELECT group_concat(s) FROM stolen_marker), 'none')" 2>/dev/null \ + || echo 'canary read from outside user_files: none' + sqlite3 "${OUTSIDE}" "SELECT 'objects written outside user_files: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" +} + +echo '--- A1 control: the boundary the injection bypasses is enforced when the path is named directly' +classify "CREATE TABLE t_ctl (s String) ENGINE = SQLite('${OUTSIDE}', 'secrets')" + +echo '--- A2 injection through the remote table name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_name_inj (id UInt32) ENGINE = SQLite('${ANCHOR}', \$\$target\" (id) VALUES (999); ATTACH DATABASE '${OUTSIDE}' AS v; CREATE TABLE stolen_marker AS SELECT s FROM v.secrets; CREATE TABLE v.written_marker(z); DETACH v; --\$\$)" +classify "INSERT INTO t_name_inj VALUES (0)" +injected_state + +echo '--- A3 injection through a column name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_col_inj (\`c') VALUES (1); CREATE TABLE col_marker(z); --\` UInt32) ENGINE = SQLite('${ANCHOR}', 'col_anchor')" +classify "INSERT INTO t_col_inj VALUES (0)" +injected_state + +echo '--- A4 control: a table and column name that legitimately contain a double quote round-trip' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_quote_id (\`c\"1\` UInt32) ENGINE = SQLite('${ANCHOR}', 'ta\"ble')" +classify "INSERT INTO t_quote_id VALUES (11)" +sqlite3 "${ANCHOR}" 'SELECT "c""1" FROM "ta""ble"' + +echo '--- A5 control: a table name that legitimately contains a backslash round-trips' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_backslash_id (x UInt32) ENGINE = SQLite('${ANCHOR}', 'a\\\\b')" +classify "INSERT INTO t_backslash_id VALUES (22)" +sqlite3 "${ANCHOR}" 'SELECT x FROM "a\b"' + +echo '--- A6 a NUL in the remote table name fails loudly, it is never silently truncated' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_nul_name (id UInt32) ENGINE = SQLite('${ANCHOR}', 'a\0b')" +classify "INSERT INTO t_nul_name VALUES (0)" diff --git a/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference new file mode 100644 index 000000000000..2c297afeb42e --- /dev/null +++ b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference @@ -0,0 +1,12 @@ +object_v3: Version 4 of Object structure serialization is written only into MergeTree data parts and is not allowed in Native format. (INCORRECT_DATA) +dynamic_v3: Version 4 of Dynamic structure serialization is written only into MergeTree data parts and is not allowed in Native format. (INCORRECT_DATA) +dynamic_duplicate_types: Duplicate type Int64 in the list of types of a flattened Dynamic column. (INCORRECT_DATA) +dynamic_nothing_type: Type Nothing is not allowed in the list of types of a flattened Dynamic column. (INCORRECT_DATA) +object_duplicate_flattened_path: Duplicate path x in the list of paths of a flattened Object column. (INCORRECT_DATA) +object_duplicate_dynamic_path: Duplicate path in the list of dynamic paths of an Object column. (INCORRECT_DATA) +object_typed_path_collision_flattened: Path x of an Object column is stored as a dynamic path, but it is a typed path in the type of the column. (INCORRECT_DATA) +object_typed_path_collision: Path x of an Object column is stored as a dynamic path, but it is a typed path in the type of the column. (INCORRECT_DATA) +object_duplicate_path_after_realloc: Duplicate path p0 in the list of paths of a flattened Object column. (INCORRECT_DATA) +0 +0 +0 diff --git a/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh new file mode 100755 index 000000000000..1078ca8f04e0 --- /dev/null +++ b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Malformed structure prefixes of JSON/Dynamic columns in a Native block. Each of them used to be +# accepted and then read out of bounds, or to escape as an internal error instead of a data error. + +DATA_DIR=$CLICKHOUSE_TMP/native_structure_$CLICKHOUSE_DATABASE +rm -rf "$DATA_DIR" +mkdir -p "$DATA_DIR" + +python3 - "$DATA_DIR" <<'EOF' +import struct +import sys + +data_dir = sys.argv[1] + +def s(b): + return bytes([len(b)]) + b + +def block(column_type, tail, rows=2): + """One Native block with a single column and a hand-built serialization prefix.""" + return bytes([1, rows]) + s(b'j') + s(column_type) + tail + +def varuint(n): + out = b'' + while True: + byte = n & 0x7f + n >>= 7 + out += bytes([byte | 0x80]) if n else bytes([byte]) + if not n: + return out + +def object_prefix(version, paths): + return struct.pack('&1 | head -10 + fi +} + +function read_part() +{ + $CLICKHOUSE_LOCAL --path "$DATA_DIR" -q "SELECT count(), sum(length(toString(json))) FROM t" 2>&1 \ + | grep -m1 -oE "ATTEMPT_TO_READ_AFTER_EOF|CANNOT_READ_ALL_DATA" +} + +create_part +truncate_streams "paths indexes stream" "$PART"/*object_shared_data*paths_indexes.bin +read_part + +create_part +truncate_streams "structure stream" "$PART"/*object_shared_data*structure.bin +read_part + +rm -rf "$DATA_DIR" diff --git a/tests/queries/0_stateless/05081_text_index_fixed_string_constant.reference b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.reference new file mode 100644 index 000000000000..97ca403d83b5 --- /dev/null +++ b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.reference @@ -0,0 +1,112 @@ +Ground truth +1 +Text Index +-- ngrams tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- ngrams tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- splitByNonAlpha tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- splitByNonAlpha tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- array tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- array tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- sparseGrams tokenizer on a String column +---- FixedString comparison +2 +2 +2 +2 +-- sparseGrams tokenizer on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +Bloom Filter Index +-- tokenbf_v1 on a String column +---- FixedString comparison +2 +2 +2 +2 +-- tokenbf_v1 on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +-- ngrambf_v1 on a String column +---- FixedString comparison +2 +2 +2 +2 +-- ngrambf_v1 on a FixedString column +---- FixedString comparison +2 +2 +2 +2 +Functions +-- hasAny and hasAll ignore the padding +2 +2 +2 +2 +2 +2 +2 +2 +-- has keeps the padding +0 +0 +2 +2 +-- mapContainsKey and mapContainsValue keep the padding +0 +0 +0 +0 +2 +2 +-- startsWith and endsWith keep the padding +0 +0 +0 +0 +2 +2 +2 +2 +-- an unpadded constant still prunes +2 +0 +2 +0 diff --git a/tests/queries/0_stateless/05081_text_index_fixed_string_constant.sql b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.sql new file mode 100644 index 000000000000..11ef86b681cb --- /dev/null +++ b/tests/queries/0_stateless/05081_text_index_fixed_string_constant.sql @@ -0,0 +1,403 @@ +-- Fixes https://github.com/ClickHouse/ClickHouse/issues/117021. + +-- `String = FixedString(N)` ignores the constant's trailing zero padding, but the index terms were +-- extracted from the padded bytes, so every granule looked unmatched and matching rows disappeared. + +SELECT 'Ground truth'; +SELECT 'hello' = toFixedString('hello', 10); + +SELECT 'Text Index'; + +SELECT '-- ngrams tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = ngrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- ngrams tokenizer on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = ngrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- splitByNonAlpha tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- splitByNonAlpha tokenizer on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = splitByNonAlpha) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- array tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- array tokenizer on a FixedString column'; + +-- The `array` tokenizer stores the whole padded value as one term, so the constant keeps its padding +-- and only a constant as wide as the column matches. +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 6); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 6) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 6)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 6)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- sparseGrams tokenizer on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = sparseGrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- sparseGrams tokenizer on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE text(tokenizer = sparseGrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT 'Bloom Filter Index'; + +SELECT '-- tokenbf_v1 on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE tokenbf_v1(512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- tokenbf_v1 on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE tokenbf_v1(512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT '-- ngrambf_v1 on a String column'; + +DROP TABLE IF EXISTS tab_string; +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_string; + +SELECT '-- ngrambf_v1 on a FixedString column'; + +DROP TABLE IF EXISTS tab_fixed_string; +CREATE TABLE tab_fixed_string +( + id UInt32, + s FixedString(6), + INDEX idx s TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_fixed_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); + +SELECT '---- FixedString comparison'; +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10); +SELECT count() FROM tab_fixed_string WHERE s = toFixedString('hello', 10) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)); +SELECT count() FROM tab_fixed_string WHERE s IN (SELECT toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_fixed_string; + +SELECT 'Functions'; + +DROP TABLE IF EXISTS tab_string; +DROP TABLE IF EXISTS tab_string_ngrambf; +DROP TABLE IF EXISTS tab_array; +DROP TABLE IF EXISTS tab_array_ngrambf; +DROP TABLE IF EXISTS tab_map; + +CREATE TABLE tab_string +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = ngrams(3)) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_string_ngrambf +( + id UInt32, + s String, + INDEX idx s TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_array +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_array_ngrambf +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE ngrambf_v1(3, 512, 2, 0) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +CREATE TABLE tab_map +( + id UInt32, + m Map(String, String), + INDEX idx_keys mapKeys(m) TYPE text(tokenizer = array), + INDEX idx_values mapValues(m) TYPE text(tokenizer = array) +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 1; + +INSERT INTO tab_string VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); +INSERT INTO tab_string_ngrambf VALUES (1, 'hello'), (2, 'world'), (3, 'hello'), (4, 'foobar'); +INSERT INTO tab_array VALUES (1, ['hello']), (2, ['world']), (3, ['hello']); +INSERT INTO tab_array_ngrambf VALUES (1, ['hello']), (2, ['world']), (3, ['hello']); +INSERT INTO tab_map VALUES (1, map('hello', 'world')), (2, map('foo', 'bar')), (3, map('hello', 'world')); + +SELECT '-- hasAny and hasAll ignore the padding'; +SELECT count() FROM tab_array WHERE hasAny(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array WHERE hasAny(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_array WHERE hasAll(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array WHERE hasAll(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_array_ngrambf WHERE hasAny(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array_ngrambf WHERE hasAny(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_array_ngrambf WHERE hasAll(arr, [toFixedString('hello', 10)]); +SELECT count() FROM tab_array_ngrambf WHERE hasAll(arr, [toFixedString('hello', 10)]) SETTINGS use_skip_indexes = 0; + +-- The functions below compare the raw padded bytes, so their terms must keep the padding. +-- `text(tokenizer = array)` answers them by exact direct read, where a stripped term would return +-- rows the predicate rejects. +SELECT '-- has keeps the padding'; +SELECT count() FROM tab_array WHERE has(arr, toFixedString('hello', 10)); +SELECT count() FROM tab_array WHERE has(arr, toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_array WHERE has(arr, 'hello'); +SELECT count() FROM tab_array WHERE has(arr, 'hello') SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; + +SELECT '-- mapContainsKey and mapContainsValue keep the padding'; +SELECT count() FROM tab_map WHERE mapContainsKey(m, toFixedString('hello', 10)); +SELECT count() FROM tab_map WHERE mapContainsKey(m, toFixedString('hello', 10)) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_map WHERE mapContainsValue(m, toFixedString('world', 10)); +SELECT count() FROM tab_map WHERE mapContainsValue(m, toFixedString('world', 10)) SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; +SELECT count() FROM tab_map WHERE mapContainsKey(m, 'hello'); +SELECT count() FROM tab_map WHERE mapContainsKey(m, 'hello') SETTINGS use_skip_indexes = 0, query_plan_direct_read_from_text_index = 0; + +SELECT '-- startsWith and endsWith keep the padding'; +SELECT count() FROM tab_string WHERE startsWith(s, toFixedString('hel', 10)); +SELECT count() FROM tab_string WHERE startsWith(s, toFixedString('hel', 10)) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE endsWith(s, toFixedString('llo', 10)); +SELECT count() FROM tab_string WHERE endsWith(s, toFixedString('llo', 10)) SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE startsWith(s, 'hel'); +SELECT count() FROM tab_string WHERE startsWith(s, 'hel') SETTINGS use_skip_indexes = 0; +SELECT count() FROM tab_string WHERE endsWith(s, 'llo'); +SELECT count() FROM tab_string WHERE endsWith(s, 'llo') SETTINGS use_skip_indexes = 0; + +SELECT '-- an unpadded constant still prunes'; +SELECT count() FROM tab_string WHERE s = 'hello'; +SELECT count() FROM tab_string WHERE s = 'nosuch'; +SELECT count() FROM tab_string_ngrambf WHERE s = 'hello'; +SELECT count() FROM tab_string_ngrambf WHERE s = 'nosuch'; + +DROP TABLE tab_string; +DROP TABLE tab_string_ngrambf; +DROP TABLE tab_array; +DROP TABLE tab_array_ngrambf; +DROP TABLE tab_map; diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference new file mode 100644 index 000000000000..e9efacb33c69 --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference @@ -0,0 +1,12 @@ +KILL TRANSACTION -> KILL TRANSACTION ON *.* +KILL PART_MOVE_TO_SHARD -> SELECT ON system.part_moves_between_shards +SYSTEM STOP THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM START THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM RESET COVERAGE -> SYSTEM ON *.* +KILL TRANSACTION without system.transactions -> SELECT ON system.transactions +KILL PART_MOVE_TO_SHARD without move privileges -> ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* +no CLUSTER grant -> CLUSTER ON *.* +KILL TRANSACTION local -> allowed +KILL TRANSACTION on cluster -> allowed +KILL PART_MOVE_TO_SHARD local -> allowed +KILL PART_MOVE_TO_SHARD on cluster -> allowed diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh new file mode 100755 index 000000000000..9f246c25da6d --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Holds CLUSTER and nothing else. +cluster_user="cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Holds every statement privilege but not CLUSTER. +no_cluster_user="no_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold the complete privilege set of one statement each. +kill_txn_user="kill_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +move_user="move_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold all but one privilege of one statement each. +partial_txn_user="partial_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +partial_move_user="partial_move_$CLICKHOUSE_TEST_UNIQUE_NAME" + +function cleanup() +{ + $CLICKHOUSE_CLIENT -mq " + DROP USER IF EXISTS $cluster_user; + DROP USER IF EXISTS $no_cluster_user; + DROP USER IF EXISTS $kill_txn_user; + DROP USER IF EXISTS $move_user; + DROP USER IF EXISTS $partial_txn_user; + DROP USER IF EXISTS $partial_move_user; + " +} +cleanup +trap cleanup EXIT + +# CLUSTER is a global privilege, so it cannot share a GRANT with a table-scoped one. +$CLICKHOUSE_CLIENT -mq " + CREATE USER $cluster_user, $no_cluster_user, $kill_txn_user, $move_user, + $partial_txn_user, $partial_move_user IDENTIFIED WITH no_password; + + GRANT CLUSTER ON *.* TO $cluster_user; + + GRANT KILL TRANSACTION, SYSTEM THREAD FUZZER, SYSTEM ON *.* TO $no_cluster_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $kill_txn_user; + GRANT SELECT ON system.transactions TO $kill_txn_user; + + GRANT CLUSTER, ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* TO $move_user; + GRANT SELECT ON system.part_moves_between_shards TO $move_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $partial_txn_user; + + GRANT CLUSTER ON *.* TO $partial_move_user; + GRANT SELECT ON system.part_moves_between_shards TO $partial_move_user; +" + +cluster="test_shard_localhost" +# Match no live transaction and no live part move, so every allowed arm below is a no-op. The +# transaction predicate deliberately does not compare `tid`: its tuple shape is not the same in +# every build, so a literal tuple fails type analysis before the access check this test is about. +txn_predicate="tid_hash = 0" +task_uuid="'00000000-0000-0000-0000-000000000000'" + +# Report the privilege the server asked for and let the reference hold the expected mapping. The +# privilege name is what discriminates: an ACCESS_DENIED-only assertion would also pass when the +# CLUSTER check, which runs before the required-access check, is what fired. +required_privilege() { + $CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1 | + sed -n "/necessary to have the grant/{s/.*grant \(.*\)\. (ACCESS_DENIED).*/\1/p;q;}" +} + +# Each statement must name over ON CLUSTER the same privilege its local spelling names. +while IFS= read -r statement; do + echo "${statement%% ON CLUSTER*} -> $(required_privilege "$cluster_user" "$statement")" +done < $(required_privilege "$partial_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" +echo "KILL PART_MOVE_TO_SHARD without move privileges -> $(required_privilege "$partial_move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid")" + +# In-range control: holding the statement privileges without CLUSTER is refused by the earlier check, +# so a mapping that refuses everything would not produce the five lines above. +echo "no CLUSTER grant -> $(required_privilege "$no_cluster_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" + +# The two statements whose privileges can be granted in full are allowed in both spellings, which +# proves the new elements are a gate rather than an unconditional refusal. The local half is asserted +# too: a user who is cluster-allowed while locally refused is the bypass this test exists to catch. +# The thread fuzzer and coverage statements get no allowed arm, because executing them would change +# server-global state that concurrent tests read and neither takes an argument that lets the host +# reject them harmlessly. +allowed() { + local out + out=$($CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1) + if [ -z "$out" ]; then + echo "$3 -> allowed" + else + echo "$3 -> FAIL: $out" + fi +} + +allowed "$kill_txn_user" "KILL TRANSACTION WHERE $txn_predicate" "KILL TRANSACTION local" +allowed "$kill_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate" "KILL TRANSACTION on cluster" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD local" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD on cluster" diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference new file mode 100644 index 000000000000..ca36e0892c1a --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference @@ -0,0 +1,9 @@ +CREATE DICTIONARY default.d_05141\n(\n `id` UInt64,\n `v` String\n)\nPRIMARY KEY id\nSOURCE(HTTP(URL \'http://localhost:11111/x.tsv\' FORMAT \'TabSeparated\' CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\')))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(FLAT()) +0 0 1 +d_05141_flat HEADERS \'[HIDDEN]\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_foo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_key HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nested HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nobr HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_typo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +2 14 0 diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql new file mode 100644 index 000000000000..0e1468fdee93 --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql @@ -0,0 +1,68 @@ +-- The values of custom HTTP headers of an `HTTP` dictionary source often carry credentials, +-- so they must be hidden in `SHOW CREATE DICTIONARY`, `system.tables` and `system.query_log`, +-- the same way as the password. They are hidden as a whole, header names included. + +SET format_display_secrets_in_show_and_select = 0; + +DROP DICTIONARY IF EXISTS d_05141; +CREATE DICTIONARY d_05141 (id UInt64, v String) +PRIMARY KEY id +SOURCE(HTTP( + url 'http://localhost:11111/x.tsv' + format 'TabSeparated' + credentials(user 'user' password 'SEKRIT_PW') + headers( + header(name 'API-KEY' value 'SEKRIT_TOKEN_1') + header(name 'X-Other' value 'SEKRIT_TOKEN_2') + ) +)) +LIFETIME(0) LAYOUT(FLAT()); + +SHOW CREATE DICTIONARY d_05141; + +SELECT create_table_query LIKE concat('%', 'SEKRIT', '%'), create_table_query LIKE '%API-KEY%', create_table_query LIKE '%HEADERS (\'[HIDDEN]\')%' +FROM system.tables WHERE database = currentDatabase() AND name = 'd_05141'; + +DROP DICTIONARY d_05141; + +-- The query is logged before the dictionary source validates its structure, so malformed header +-- definitions must not leak either. +CREATE DICTIONARY d_05141_typo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' vaule 'SEKRIT_TYPO')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_key (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(secret 'SEKRIT_KEY')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_nested (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name(foo 'SEKRIT_NESTED'))))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_func (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name concat('X-', 'SEKRIT_FUNC') value 'SEKRIT_FUNC_VALUE')))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError INCORRECT_DICTIONARY_DEFINITION } +CREATE DICTIONARY d_05141_array (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name ['SEKRIT_ARRAY'])))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError BAD_ARGUMENTS } +CREATE DICTIONARY d_05141_nobr (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header 'SEKRIT_NOBR'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_foo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(foo 'SEKRIT_FOO'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_flat (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers 'SEKRIT_HEADERS')) +LIFETIME(0) LAYOUT(FLAT()); + +SELECT name, extract(create_table_query, 'HEADERS.*$') +FROM system.tables WHERE database = currentDatabase() AND name LIKE 'd\\_05141\\_%' ORDER BY name; + +SYSTEM FLUSH LOGS query_log; +SELECT countIf(query LIKE '%d\\_05141 %'), countIf(query LIKE '%d\\_05141\\_%'), countIf(query LIKE concat('%', 'SEKRIT', '%')) +FROM system.query_log +WHERE current_database = currentDatabase() AND query_kind = 'Create' AND event_date >= yesterday(); + +DROP DICTIONARY d_05141_typo; +DROP DICTIONARY d_05141_key; +DROP DICTIONARY d_05141_nested; +DROP DICTIONARY d_05141_nobr; +DROP DICTIONARY d_05141_foo; +DROP DICTIONARY d_05141_flat; diff --git a/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference new file mode 100644 index 000000000000..f7c7e2e15325 --- /dev/null +++ b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.reference @@ -0,0 +1,3 @@ +HEADERS (\'[HIDDEN]\'))) +CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\'))) +1 0 diff --git a/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh new file mode 100755 index 000000000000..11eae7fc24b4 --- /dev/null +++ b/tests/queries/0_stateless/05142_http_dictionary_hide_header_values_json.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# The HTTP headers of an `HTTP` dictionary source must also be hidden when the query is submitted +# as a JSON AST (`dialect = 'clickhouse_json'`). + +CLICKHOUSE_CLIENT_JSON="${CLICKHOUSE_CLIENT} --enable_json_ast_dialect 1 --dialect clickhouse_json" + +function to_json() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT parseQueryToJSON(\$\$$1\$\$) FORMAT TSVRaw" +} + +# A definition submitted as a JSON AST is masked like the SQL one. +JSON=$(to_json "CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_05142 (id UInt64, v String) PRIMARY KEY id + SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' value 'SEKRIT_JSON')))) + LIFETIME(0) LAYOUT(FLAT())") +${CLICKHOUSE_CLIENT_JSON} --query "$JSON" +${CLICKHOUSE_CLIENT} --query "SELECT extract(create_table_query, 'HEADERS.*\\)\\)\\)') FROM system.tables WHERE database = currentDatabase() AND name = 'd_05142' SETTINGS format_display_secrets_in_show_and_select = 0" + +# The SQL parser lower-cases the keys, but a JSON AST can spell them in any case; the secret keys must +# be recognized anyway. +JSON=$(to_json "CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_05142_case (id UInt64, v String) PRIMARY KEY id + SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' credentials(user 'user' password 'SEKRIT_JSON_CASE_PW') + headers(header(name 'API-KEY' value 'SEKRIT_JSON_CASE')))) + LIFETIME(0) LAYOUT(FLAT())") +for key in headers header value password +do + JSON=${JSON//\"first\":\"$key\"/\"first\":\"${key^^}\"} +done +${CLICKHOUSE_CLIENT_JSON} --query "$JSON" +${CLICKHOUSE_CLIENT} --query "SELECT extract(create_table_query, 'CREDENTIALS.*\\)\\)\\)') FROM system.tables WHERE database = currentDatabase() AND name = 'd_05142_case' SETTINGS format_display_secrets_in_show_and_select = 0" + +# The JSON queries are logged without the headers and the password. +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" +${CLICKHOUSE_CLIENT} --query " + SELECT count() > 0, countIf(query LIKE concat('%', 'SEKRIT', '%')) + FROM system.query_log + WHERE current_database = currentDatabase() AND query_kind = 'Create' AND query LIKE '%d_05142%' AND event_date >= yesterday()" + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY d_05142" +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY d_05142_case" diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.reference b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference new file mode 100644 index 000000000000..b3863579a954 --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference @@ -0,0 +1,4 @@ +10 +10 +a comment +refused diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.sh b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh new file mode 100755 index 000000000000..4af56ac37b84 --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; + +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +# A comment change rewrites the metadata file of the database, so the locator has to survive the +# round trip: loading it back is what the server does at every start. +$CLICKHOUSE_CLIENT -q """ +ALTER DATABASE $ATTACHED_DATABASE_NAME MODIFY COMMENT 'a comment'; +DETACH DATABASE $ATTACHED_DATABASE_NAME; +ATTACH DATABASE $ATTACHED_DATABASE_NAME; + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +SELECT comment FROM system.databases WHERE name = '$ATTACHED_DATABASE_NAME'; +""" + +# A locator serialized as a string literal is accepted only while loading metadata an older server +# rewrote: in a statement a user writes it must be the function it is, because that is the form the +# secret masker redacts. +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05153_clear_column_stale_materialized_warning.reference b/tests/queries/0_stateless/05153_clear_column_stale_materialized_warning.reference new file mode 100644 index 000000000000..ddffde0b0b3a --- /dev/null +++ b/tests/queries/0_stateless/05153_clear_column_stale_materialized_warning.reference @@ -0,0 +1,2 @@ +0 1 9 +warned about m2 1 diff --git a/tests/queries/0_stateless/05153_clear_column_stale_materialized_warning.sh b/tests/queries/0_stateless/05153_clear_column_stale_materialized_warning.sh new file mode 100755 index 000000000000..ff9cfe533ed2 --- /dev/null +++ b/tests/queries/0_stateless/05153_clear_column_stale_materialized_warning.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A MATERIALIZED column that reads an EPHEMERAL column cannot be recomputed by a mutation, so +# `CLEAR COLUMN` of one of its regular inputs silently leaves the on-disk value stale. The server +# warns about it, and the warning has to be decided over the same transitive closure the recompute +# uses: `m2` below reaches the cleared `x` only through `m1`, which the mutation does recompute, so +# a check against the directly cleared columns alone stays silent. + +${CLICKHOUSE_CLIENT} -q " +DROP TABLE IF EXISTS t_stale_warning; + +CREATE TABLE t_stale_warning +( + x Int32, + e Int32 EPHEMERAL 0, + m1 Int32 MATERIALIZED x + 1, + m2 Int32 MATERIALIZED m1 + e +) +ENGINE = MergeTree ORDER BY tuple() PARTITION BY tuple() +-- The runner randomizes both of these together with \`min_bytes_for_wide_part\`, and on a Wide part +-- with either block column on the mutation does not recompute MATERIALIZED columns at all, so the +-- \`m1\` hop this test walks through would never run. +SETTINGS enable_block_number_column = 0, enable_block_offset_column = 0; + +INSERT INTO t_stale_warning (x, e) VALUES (1, 7); + +SET mutations_sync = 2; +ALTER TABLE t_stale_warning CLEAR COLUMN x IN PARTITION tuple(); + +-- \`m1\` is recomputed from the cleared \`x\`; \`m2\` keeps the value stored at INSERT time. +SELECT x, m1, m2 FROM t_stale_warning; +" + +# The interpreter runs inside the background mutate task, so its warning never reaches the client +# that issued the ALTER. The logger name carries the table, which isolates the rows well enough. +${CLICKHOUSE_CLIENT} -q " +SYSTEM FLUSH LOGS text_log; +SELECT 'warned about m2', count() >= 1 +FROM system.text_log +WHERE logger_name = 'MutationsInterpreter(${CLICKHOUSE_DATABASE}.t_stale_warning)' + AND level = 'Warning' + AND message LIKE 'MATERIALIZED column \'m2\' depends on both EPHEMERAL and regular%'; + +DROP TABLE t_stale_warning; +" diff --git a/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference new file mode 100644 index 000000000000..88d32f1ade59 --- /dev/null +++ b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.reference @@ -0,0 +1,9 @@ +2 +2 +2 +2 +2 +7002 +7 +2 +2 diff --git a/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql new file mode 100644 index 000000000000..c7408aab6b1f --- /dev/null +++ b/tests/queries/0_stateless/05189_update_cte_out_of_scope_database.sql @@ -0,0 +1,98 @@ +-- A common table expression is expanded before the database is filled into a mutation command, so a +-- table identifier hidden by an expression alias of the same name is left unqualified. With +-- `enable_global_with_statement` disabled the alias is not visible in the subquery, so the identifier +-- names a table there and has to be resolved in the database of the updated table, not of the session. +-- The source table exists in both databases with a different row, so an expression resolved in the +-- database of the session silently reads the wrong row instead of failing. +-- The reference is one `SELECT` deeper than the alias: only a lookup in an enclosing scope is +-- disabled, so a reference in the select that declares the alias reads it either way. +-- The old analyzer resolves a common table expression in a subquery of a mutation as a table, so the +-- analyzer is requested explicitly, as in `04693_merge_table_function_in_mutation`. + +CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DATABASE_1:Identifier}; + +CREATE TABLE src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO src VALUES (99); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.src (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.src VALUES (2); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.u (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.u VALUES (1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0), (8, 0), (99, 0); + +CREATE TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO {CLICKHOUSE_DATABASE_1:Identifier}.t VALUES (1, 0), (2, 0), (3, 0), (99, 0); + +-- Every row starts at 0 and every expected value is non-zero, so a statement that does not update the +-- row at all also fails. The row `id = 99` exists so that a predicate resolved in the database of the +-- session updates the wrong row rather than no row. + +-- An assignment reads the table of the updated database (2), not the table of the session (99). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 1 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 1; + +-- The `ALTER TABLE ... UPDATE` spelling of the same expression answers the same. +ALTER TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t + UPDATE v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 1 SETTINGS mutations_sync = 2, enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.t WHERE id = 1; + +-- The predicate is expanded by a separate call, so it is asserted separately: it marks the row the +-- updated database names (2), not the one the session database names (99). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = 11 + WHERE id IN (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + SETTINGS enable_analyzer = 1; +SELECT id FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE v = 11; + +ALTER TABLE {CLICKHOUSE_DATABASE_1:Identifier}.t + UPDATE v = 11 + WHERE id IN (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS enable_global_with_statement = 0)) + SETTINGS mutations_sync = 2, enable_analyzer = 1; +SELECT id FROM {CLICKHOUSE_DATABASE_1:Identifier}.t WHERE v = 11; + +-- The effective value of the setting is read, so a profile that carries it answers the same as the +-- setting written out. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src SETTINGS compatibility = '20.3')) + WHERE id = 3 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 3; + +-- A clause is read per arm of a union: the arm carrying it names a table there, resolved in the +-- updated database (2), while the arm without it keeps reading the alias (7 * 1000). The two arms +-- are weighted differently so that reading the clause for the wrong arm, for both arms or for +-- neither answers 2007, 2002 and 7007 rather than the expected value. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT sum(m) FROM ( + SELECT max(id) * 1000 AS m FROM src + UNION ALL + SELECT max(id) AS m FROM src SETTINGS enable_global_with_statement = 0)) + WHERE id = 8 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 8; + +-- With the setting at its default the alias IS visible in the subquery, so the same reference reads +-- the common table expression (7) and must not be qualified as a table. +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH src AS (SELECT 7 AS id) SELECT (SELECT max(id) FROM src)) + WHERE id = 4 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 4; + +-- A reference in the select that declares the alias keeps reading the alias, and the table its body +-- names is resolved in the updated database (2). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (WITH c AS (SELECT max(id) AS m FROM src) SELECT m FROM c) + WHERE id = 5 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 5; + +-- An identifier that no alias hides was always resolved in the updated database (2). +UPDATE {CLICKHOUSE_DATABASE_1:Identifier}.u + SET v = (SELECT max(id) FROM src) + WHERE id = 6 SETTINGS enable_analyzer = 1; +SELECT v FROM {CLICKHOUSE_DATABASE_1:Identifier}.u WHERE id = 6; + +DROP TABLE src; +DROP DATABASE {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference new file mode 100644 index 000000000000..a3a948fe5178 --- /dev/null +++ b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.reference @@ -0,0 +1,7 @@ +strict read +ICEBERG_SPECIFICATION_VIOLATION +tolerant compaction +3 +DateTime64(6, \'UTC\') 2024-01-01 00:00:00.000000 1 +DateTime64(6, \'UTC\') 2024-01-02 00:00:00.000000 2 +DateTime64(6, \'UTC\') 2024-01-03 00:00:00.000000 3 diff --git a/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh new file mode 100755 index 000000000000..9a72f718081a --- /dev/null +++ b/tests/queries/0_stateless/05212_iceberg_manifest_compaction_conflicting_manifest_schema.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) +# - no-parallel: uses DETACH/ATTACH which serializes per database + +# A manifest file header keeps a copy of the schema the data was written with, while metadata.json is +# the authoritative source. The conflict below is produced the way broken writers produce it: the +# manifest headers keep the `timestamp` type, while metadata.json binds the same schema-id to +# `timestamptz`. +# +# A read registers the metadata.json schemas before it walks the manifests, so a strict read meets a +# header conflicting with a known metadata.json copy and fails with `ICEBERG_SPECIFICATION_VIOLATION`. +# `OPTIMIZE TABLE ... MANIFEST` walks the current manifests first and registers the metadata.json +# schemas only afterwards, so on a freshly attached table the schemas that came from the manifests are registered before +# metadata.json: the metadata.json copy has to replace them and the compaction has to succeed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The data is written as `timestamp` (parsed in the session time zone) and read back as `timestamptz` +# (rendered in UTC), so the session time zone has to be UTC for the values to round-trip unchanged. +CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --session_timezone UTC" + +TABLE="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +TABLE_PATH="${USER_FILES_PATH}/${TABLE}/" + +trap 'rm -rf "${TABLE_PATH}" 2>/dev/null' EXIT + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${TABLE} (ts DateTime64(6), v Int32) + ENGINE = IcebergLocal('${TABLE_PATH}', 'Parquet') +" + +# One INSERT per manifest, so the table has enough manifests to compact. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --use_iceberg_metadata_files_cache=0 -m --query " + INSERT INTO ${TABLE} VALUES ('2024-01-01 00:00:00', 1); + INSERT INTO ${TABLE} VALUES ('2024-01-02 00:00:00', 2); + INSERT INTO ${TABLE} VALUES ('2024-01-03 00:00:00', 3); +" + +# Rebind the schema-id in metadata.json to a schema that differs from the schemas that came from the manifests. +LATEST_METADATA=$(ls "${TABLE_PATH}"metadata/v*.metadata.json | sed 's#.*/v##;s#\.metadata.json##' | sort -n | tail -1) +python3 - "${TABLE_PATH}metadata/v${LATEST_METADATA}.metadata.json" <<'PY' +import json, sys +path = sys.argv[1] +meta = json.load(open(path)) +for schema in meta["schemas"]: + for field in schema["fields"]: + if field["type"] == "timestamp": + field["type"] = "timestamptz" +json.dump(meta, open(path, "w")) +PY + +# Drop the in-memory metadata and the shared schema processor, so the read starts from an empty +# processor and registers the metadata.json schemas before it meets the schemas that came from the manifests. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query "DETACH TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --send_logs_level=fatal --query "ATTACH TABLE ${TABLE}" + +echo "strict read" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --iceberg_tolerate_conflicting_manifest_schemas=0 \ + --query "SELECT count() FROM ${TABLE}" 2>&1 \ + | grep -oF 'ICEBERG_SPECIFICATION_VIOLATION' | head -n1 + +# Start from an empty processor again, so the compaction registers the schemas that came from the manifests first +# and the metadata.json schemas replace them. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --query "DETACH TABLE ${TABLE}" +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --send_logs_level=fatal --query "ATTACH TABLE ${TABLE}" + +echo "tolerant compaction" +${CLICKHOUSE_CLIENT} --allow_experimental_iceberg_compaction=1 --use_iceberg_metadata_files_cache=0 \ + --iceberg_tolerate_conflicting_manifest_schemas=1 --send_logs_level=error \ + --query "OPTIMIZE TABLE ${TABLE} MANIFEST SETTINGS iceberg_manifest_min_count_to_compact=2" + +# The compaction rewrote three data manifests into one, and the data is intact. +${CLICKHOUSE_CLIENT} --use_iceberg_metadata_files_cache=0 --iceberg_tolerate_conflicting_manifest_schemas=1 --query " + SELECT count() FROM ${TABLE}; + SELECT toTypeName(ts), ts, v FROM ${TABLE} ORDER BY v; +" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TABLE} SYNC" diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference new file mode 100644 index 000000000000..a9796c468d97 --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference @@ -0,0 +1,3 @@ +the maximum size of multipart/form-data +30 +50 diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh new file mode 100755 index 000000000000..4d497382264e --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `http_max_multipart_form_data_size` cannot be set through URL parameters, so it comes from a user. +user="u_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} -q "CREATE USER $user IDENTIFIED WITH plaintext_password BY 'pw' SETTINGS http_max_multipart_form_data_size = 100" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, SELECT ON *.* TO $user" + +url="${CLICKHOUSE_URL}&user=${user}&password=pw" +part="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_part.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 30)' > "$part" + +# Two parts of 60 bytes: each one fits the 100 byte limit on its own, the form as a whole does not. +${CLICKHOUSE_CURL} -sS -F "a=@$part" -F "b=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&b_structure=s+String&a_format=TSV&b_format=TSV" \ + | grep -o -m1 'the maximum size of multipart/form-data' + +# One part of the same size stays under the limit. +${CLICKHOUSE_CURL} -sS -F "a=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +# A part that ends exactly at the limit is not over it. +exact="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_exact.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 50)' > "$exact" +${CLICKHOUSE_CURL} -sS -F "a=@$exact" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +${CLICKHOUSE_CLIENT} -q "DROP USER $user" +rm -f "$part" "$exact" diff --git a/tests/queries/0_stateless/05215_uniq_theta_short_state.reference b/tests/queries/0_stateless/05215_uniq_theta_short_state.reference new file mode 100644 index 000000000000..f5f31343b08b --- /dev/null +++ b/tests/queries/0_stateless/05215_uniq_theta_short_state.reference @@ -0,0 +1,16 @@ +char(8, 0, 1, 3, 0, 0, 0, 0, 0) -> at least 24 bytes expected, actual 8 +char(8, 0, 2, 3, 0, 0, 0, 0, 0) -> seed hash mismatch +char(8, 0, 3, 3, 0, 0, 0, 0, 0) -> at least 16 bytes expected, actual 8 +char(8, 0, 4, 3, 0, 0, 0, 0, 0) -> seed hash mismatch +char(8, 0, 5, 3, 0, 0, 0, 0, 0) -> unsupported serial version 5 +char(8, 2, 2, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +char(8, 3, 2, 3, 0, 0, 0, 204, 147) -> at least 24 bytes expected, actual 8 +char(8, 2, 3, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +char(8, 3, 3, 3, 0, 0, 0, 204, 147) -> at least 24 bytes expected, actual 8 +char(8, 2, 4, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +1000 +1000 +100 +1 0 +17 1 +33 2 diff --git a/tests/queries/0_stateless/05215_uniq_theta_short_state.sh b/tests/queries/0_stateless/05215_uniq_theta_short_state.sh new file mode 100755 index 000000000000..81eced6b8d12 --- /dev/null +++ b/tests/queries/0_stateless/05215_uniq_theta_short_state.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest -- compiled w/o datasketches + +# `compact_theta_sketch_parser::parse` verifies that the buffer holds 8 bytes and then reads header +# fields that lie beyond them - `num_entries` at offset 8, `theta` at offset 16 - before it checks the +# size again. A state shorter than that, which any `CAST` from a string can produce, was therefore read +# past its end (an ASan heap-buffer-overflow), and the out-of-bounds value decided the size the parser +# reported as missing: the same query answered `at least 86160 bytes expected` on one run and something +# else on the next. Every size below is now the one the parser really needs before it can validate +# anything, and the same on every run. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The outcome of one state, reduced to the part that used to come from outside the buffer. The client +# prints the message of the exception rather than only its code. +outcome() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT finalizeAggregation(CAST($1, 'AggregateFunction(uniqTheta, UInt64)'))" 2>&1 \ + | grep -o -E "at least [0-9]+ bytes expected, actual [0-9]+|seed hash mismatch|unsupported serial version [0-9]+|^[0-9]+$" \ + | head -n 1 +} + +# The leading byte of each `char` is the length of the state that follows it. All of these states are +# eight bytes long - the most the parser checks for before it reads further out. +# +# The four states the fuzzer produced come first, and their seed hash is zero. Serial versions 2 and 4 +# compare that against the one they compute and stop at `check_seed_hash`; serial version 1 computes +# the seed hash itself instead of comparing it, and serial version 3 reads `num_entries` before it +# compares, so those two are refused for their size. Serial version 5 does not exist and is refused +# outright. +for STATE in \ + "char(8, 0, 1, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 2, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 3, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 4, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 5, 3, 0, 0, 0, 0, 0)" +do + echo "${STATE} -> $(outcome "${STATE}")" +done + +# `204, 147` at offset 6 is the seed hash of the default seed, so these states get past +# `check_seed_hash` and reach every one of the reads that happen before the size is validated: +# `num_entries` alone for two preamble longs, `num_entries` and `theta` for three. In particular a +# serial version 2 state with two preamble longs is refused rather than taken for an empty sketch on +# the strength of bytes it does not have. +for STATE in \ + "char(8, 2, 2, 3, 0, 0, 0, 204, 147)" \ + "char(8, 3, 2, 3, 0, 0, 0, 204, 147)" \ + "char(8, 2, 3, 3, 0, 0, 0, 204, 147)" \ + "char(8, 3, 3, 3, 0, 0, 0, 204, 147)" \ + "char(8, 2, 4, 3, 0, 0, 0, 204, 147)" +do + echo "${STATE} -> $(outcome "${STATE}")" +done + +# A state that a `uniqTheta` aggregation wrote still reads back. +${CLICKHOUSE_CLIENT} --query " + SELECT uniqTheta(number) FROM numbers(1000); + SELECT uniqThetaMerge(s) FROM (SELECT uniqThetaState(number) AS s FROM numbers(1000)); + SELECT finalizeAggregation(CAST(unhex(hex(uniqThetaState(number))), 'AggregateFunction(uniqTheta, UInt64)')) FROM numbers(100); +" + +# A short state cannot be refused for its length alone: a single-entry sketch is sixteen bytes long, +# below the twenty-four the parser reads for other headers before it validates anything. The length of +# each state is printed next to the value it reads back as - it counts the one-byte length prefix, so +# the sketch itself is one byte smaller. +${CLICKHOUSE_CLIENT} --query " + SELECT + intDiv(length(hex(s)), 2) AS serialized_size, + finalizeAggregation(CAST(unhex(hex(s)), 'AggregateFunction(uniqTheta, UInt64)')) + FROM + ( + SELECT rows, uniqThetaStateIf(number, number < rows) AS s + FROM numbers(2) AS n, (SELECT arrayJoin([0, 1, 2]) AS rows) AS r + GROUP BY rows + ORDER BY rows + )" diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference new file mode 100644 index 000000000000..1f63a15bb084 --- /dev/null +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.reference @@ -0,0 +1,7 @@ +local +1 +no settings clause +1 +constraints +1 +unrelated change allowed diff --git a/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh new file mode 100755 index 000000000000..a0cae4db32a7 --- /dev/null +++ b/tests/queries/0_stateless/05217_custom_disk_local_path_alter.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: custom disks are not configured in fasttest +# Tag no-replicated-database: creates a user and a settings profile + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every location on the local filesystem that a disk defined in SQL names has to be inside +# `custom_local_disks_base_directory`. `CREATE TABLE` checks that; `ALTER TABLE ... MODIFY SETTING disk` +# has to check it the same way. +# In this release the check runs after the disk is created, and does not cover object storage disks over +# the local filesystem, so only the rejection of the statement is checked here. + +OUTSIDE="${USER_FILES_PATH}/05217_outside_${CLICKHOUSE_DATABASE}" +INSIDE="${CLICKHOUSE_DISKS_FILES}/05217_${CLICKHOUSE_DATABASE}" + +# Prints 1 if the statement was rejected by the fence. +function check_rejected() +{ + local query=$1 + $CLICKHOUSE_CLIENT -q "$query" 2>&1 | grep -m1 -c -F "must be inside" +} + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_05217" +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t_05217 (x UInt64) ENGINE = MergeTree ORDER BY x + SETTINGS disk = disk(name = '05217_ok_${CLICKHOUSE_DATABASE}', type = local, path = '${INSIDE}_ok/')" + +# `ALTER TABLE ... MODIFY SETTING disk = disk(...)` used to resolve the definition as if it came from +# stored metadata, which registered the disk unchecked, before the check of the statement ran. +echo "local" +check_rejected " + ALTER TABLE t_05217 MODIFY SETTING disk = disk(name = '05217_local_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_local/')" + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_05217" + +# A table created without a SETTINGS clause goes through the same check. +echo "no settings clause" +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_05217_plain (x UInt64) ENGINE = MergeTree ORDER BY x" +check_rejected " + ALTER TABLE t_05217_plain MODIFY SETTING disk = disk(name = '05217_plain_${CLICKHOUSE_DATABASE}', type = local, path = '${OUTSIDE}_plain/')" +$CLICKHOUSE_CLIENT -q "DROP TABLE t_05217_plain" + +# The settings constraints compare the `disk` the table has after the `ALTER`, not one left unresolved: +# a constant `merge_tree_disk` rejects a change of the disk and nothing else. +echo "constraints" +user="u_05217_${CLICKHOUSE_DATABASE}" +profile="p_05217_${CLICKHOUSE_DATABASE}" +$CLICKHOUSE_CLIENT -q " + DROP USER IF EXISTS ${user}; + DROP SETTINGS PROFILE IF EXISTS ${profile}; + CREATE USER ${user} IDENTIFIED WITH no_password; + GRANT ALTER ON ${CLICKHOUSE_DATABASE}.* TO ${user}; + CREATE SETTINGS PROFILE ${profile} SETTINGS merge_tree_disk CONST TO ${user}; + CREATE TABLE t_05217_default (x UInt64) ENGINE = MergeTree ORDER BY x; + CREATE TABLE t_05217_disk (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS disk = 'local_disk';" + +$CLICKHOUSE_CLIENT --user "${user}" -q "ALTER TABLE t_05217_default MODIFY SETTING disk = 'local_disk'" 2>&1 \ + | grep -m1 -c -F "Setting disk should not be changed" +$CLICKHOUSE_CLIENT --user "${user}" -q "ALTER TABLE t_05217_disk MODIFY SETTING min_bytes_for_wide_part = 0" && echo "unrelated change allowed" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE t_05217_default; + DROP TABLE t_05217_disk; + DROP SETTINGS PROFILE ${profile}; + DROP USER ${user};" + +# In this release a rejected disk still creates its directory, so remove what the statements above left. +rm -rf "${OUTSIDE}"_* diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference new file mode 100644 index 000000000000..97d96022f020 --- /dev/null +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.reference @@ -0,0 +1,80 @@ +-- the analyzer wraps the subcolumn into a cast to String +1 +-- mapValues: the index prunes granules through the cast +Granules: 2/2 +Name: idx +Granules: 1/2 +Granules: 2/2 +Name: idx +Granules: 1/2 +-- mapValues: results match the scan +idx 1 +idx 3 +scan 1 +scan 3 +idx 1 +idx 2 +scan 1 +scan 2 +idx 3 +scan 3 +-- mapValues is only a hint: a value that belongs to another key does not match +idx 0 +-- mapValues on LowCardinality(Nullable) values: the cast to Nullable(String) is looked through +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 3 +scan 3 +-- String column: conversions that only add Nullable or LowCardinality are looked through +CAST Nullable Granules: 2/2 +CAST Nullable Name: idx +CAST Nullable Granules: 1/2 +CAST LowCardinality Granules: 2/2 +CAST LowCardinality Name: idx +CAST LowCardinality Granules: 1/2 +toNullable Granules: 2/2 +toNullable Name: idx +toNullable Granules: 1/2 +toLowCardinality Granules: 2/2 +toLowCardinality Name: idx +toLowCardinality Granules: 1/2 +nested Granules: 2/2 +nested Name: idx +nested Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 +not idx 3 +not idx 4 +not scan 3 +not scan 4 +in idx 1 +in idx 3 +in scan 1 +in scan 3 +-- join_use_nulls pushes the filter down to the outer side as toNullable(column) +1 +3 +-- Nullable column: adding LowCardinality is looked through +Granules: 2/2 +Name: idx +Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 +-- Nullable column: dropping Nullable is not looked through, the cast still throws on the NULL row +0 +-- Array column: a cast that only wraps the elements is looked through +Array(Nullable) Granules: 2/2 +Array(Nullable) Name: idx +Array(Nullable) Granules: 1/2 +Array(LowCardinality) Granules: 2/2 +Array(LowCardinality) Name: idx +Array(LowCardinality) Granules: 1/2 +idx 1 +idx 2 +scan 1 +scan 2 diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql new file mode 100644 index 000000000000..9c2f4b4409a6 --- /dev/null +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql @@ -0,0 +1,156 @@ +-- Tags: no-parallel-replicas +-- Tag no-parallel-replicas -- direct read is not compatible with parallel replicas + +-- `arrayElement` on a `Map(K, LowCardinality(V))` returns `V`, while the subcolumn `m.key_` is `LowCardinality(V)`. +-- Hence `optimize_functions_to_subcolumns` rewrites `m['key'] = 'value'` into `_CAST(m.key_, 'V') = 'value'`, +-- and the text index has to look through the cast: for an index on `mapValues(m)`. +-- The same applies to every conversion that cannot change a value or throw: adding or dropping `LowCardinality`, +-- adding `Nullable` (also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. + +SET explain_query_plan_default = 'legacy'; +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET use_skip_indexes = 1; +SET query_plan_direct_read_from_text_index = 1; + +DROP TABLE IF EXISTS tab_values; + +CREATE TABLE tab_values +( + id UInt32, + m Map(String, LowCardinality(String)), + INDEX idx mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_values VALUES (1, {'level':'error','msg':'disk is full'}), (2, {'level':'warn','msg':'disk is slow'}), (3, {'level':'error','msg':'network is down'}), (4, {}); + +SELECT '-- the analyzer wraps the subcolumn into a cast to String'; +SELECT count() FROM (EXPLAIN QUERY TREE run_passes = 1 SELECT id FROM tab_values WHERE m['level'] = 'warn') WHERE explain LIKE '%function_name: _CAST%'; + +SELECT '-- mapValues: the index prunes granules through the cast'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values WHERE m['level'] = 'warn') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values WHERE hasToken(m['msg'], 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; + +SELECT '-- mapValues: results match the scan'; +SELECT 'idx', id FROM tab_values WHERE m['level'] = 'error' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE m['level'] = 'error' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'idx', id FROM tab_values WHERE hasToken(m['msg'], 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE hasToken(m['msg'], 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'idx', id FROM tab_values WHERE m['msg'] LIKE '%down%' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values WHERE m['msg'] LIKE '%down%' ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT '-- mapValues is only a hint: a value that belongs to another key does not match'; +SELECT 'idx', count() FROM tab_values WHERE m['level'] = 'disk' SETTINGS force_data_skipping_indices = 'idx'; + +DROP TABLE tab_values; + +-- A `mapValues` index on `LowCardinality(Nullable(String))` values: the analyzer casts to `Nullable(String)`. +DROP TABLE IF EXISTS tab_values_n; + +CREATE TABLE tab_values_n +( + id UInt32, + m Map(String, LowCardinality(Nullable(String))), + INDEX idx mapValues(m) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_values_n VALUES (1, {'level':'error'}), (2, {'level':'error'}), (3, {'level':'warn'}), (4, {'level':NULL}); + +SELECT '-- mapValues on LowCardinality(Nullable) values: the cast to Nullable(String) is looked through'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_values_n WHERE m['level'] = 'warn') WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_values_n WHERE m['level'] = 'warn' ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_values_n WHERE m['level'] = 'warn' ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_values_n; + +-- Plain columns with explicit conversions. +DROP TABLE IF EXISTS tab_s; + +CREATE TABLE tab_s +( + id UInt32, + s String, + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_s VALUES (1, 'disk is full'), (2, 'disk is slow'), (3, 'network is down'), (4, ''); + +SELECT '-- String column: conversions that only add Nullable or LowCardinality are looked through'; +SELECT 'CAST Nullable', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(CAST(s, 'Nullable(String)'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'CAST LowCardinality', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(CAST(s, 'LowCardinality(String)'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'toNullable', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toNullable(s), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'toLowCardinality', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toLowCardinality(s), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'nested', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_s WHERE hasToken(toNullable(toLowCardinality(s)), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_s WHERE hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_s WHERE hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'not idx', id FROM tab_s WHERE NOT hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'not scan', id FROM tab_s WHERE NOT hasToken(toNullable(s), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; +SELECT 'in idx', id FROM tab_s WHERE toNullable(s) IN ('disk is full', 'network is down') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'in scan', id FROM tab_s WHERE toNullable(s) IN ('disk is full', 'network is down') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- join_use_nulls pushes the filter down to the outer side as toNullable(column)'; +DROP TABLE IF EXISTS tab_ids; +CREATE TABLE tab_ids (id UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO tab_ids SELECT number FROM numbers(1, 4); +-- `query_plan_convert_outer_join_to_inner_join` is pinned because the filter is pushed below the join only +-- when the LEFT JOIN becomes an INNER one; otherwise it stays above the join and no index is consulted. +SELECT count() FROM (EXPLAIN indexes = 1 SELECT l.id FROM tab_ids AS l LEFT JOIN tab_s AS r ON l.id = r.id WHERE hasToken(r.s, 'network') SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1) WHERE explain LIKE '%Name: idx%'; +SELECT l.id FROM tab_ids AS l LEFT JOIN tab_s AS r ON l.id = r.id WHERE hasToken(r.s, 'network') ORDER BY l.id SETTINGS join_use_nulls = 1; +DROP TABLE tab_ids; + +DROP TABLE tab_s; + +DROP TABLE IF EXISTS tab_ns; + +CREATE TABLE tab_ns +( + id UInt32, + s Nullable(String), + INDEX idx s TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_ns VALUES (1, 'disk is full'), (2, 'disk is slow'), (3, 'network is down'), (4, NULL); + +SELECT '-- Nullable column: adding LowCardinality is looked through'; +SELECT trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_ns WHERE hasToken(CAST(s, 'LowCardinality(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; + +SELECT '-- Nullable column: dropping Nullable is not looked through, the cast still throws on the NULL row'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network')) WHERE explain LIKE '%Name: idx%'; +SELECT id FROM tab_ns WHERE hasToken(CAST(s, 'String'), 'network'); -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } + +DROP TABLE tab_ns; + +DROP TABLE IF EXISTS tab_arr; + +CREATE TABLE tab_arr +( + id UInt32, + arr Array(String), + INDEX idx arr TYPE text(tokenizer = 'array') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY id +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; + +INSERT INTO tab_arr VALUES (1, ['disk', 'full']), (2, ['disk', 'slow']), (3, ['network', 'down']), (4, []); + +SELECT '-- Array column: a cast that only wraps the elements is looked through'; +SELECT 'Array(Nullable)', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'Array(LowCardinality)', trimLeft(explain) FROM (EXPLAIN indexes = 1 SELECT id FROM tab_arr WHERE has(CAST(arr, 'Array(LowCardinality(String))'), 'network')) WHERE explain LIKE '%Name:%' OR explain LIKE '%Granules:%'; +SELECT 'idx', id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'disk') ORDER BY id SETTINGS force_data_skipping_indices = 'idx'; +SELECT 'scan', id FROM tab_arr WHERE has(CAST(arr, 'Array(Nullable(String))'), 'disk') ORDER BY id SETTINGS use_skip_indexes = 0; + +DROP TABLE tab_arr; diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.reference b/tests/queries/0_stateless/05218_allow_executable_tables.reference new file mode 100644 index 000000000000..604b15353012 --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.reference @@ -0,0 +1 @@ +x UInt32 diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.sql b/tests/queries/0_stateless/05218_allow_executable_tables.sql new file mode 100644 index 000000000000..f56bfcf22caa --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.sql @@ -0,0 +1,35 @@ +-- `allow_executable_tables` gates reading through the `executable` table function and from +-- `Executable` and `ExecutablePool` tables. + +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +DESCRIBE executable('nonexist.sh', 'TSV', 'x UInt32'); + +-- A definition is still accepted and can be managed; only reading it is refused. +CREATE TABLE t_exec_gate (x UInt32) ENGINE = Executable('nonexist.sh', 'TSV'); +CREATE VIEW v_exec_gate AS SELECT * FROM t_exec_gate; +CREATE MATERIALIZED VIEW mv_exec_gate ENGINE = MergeTree ORDER BY x AS SELECT * FROM t_exec_gate; +CREATE TABLE t_exec_pool_gate (x UInt32) ENGINE = ExecutablePool('nonexist.sh', 'TSV'); +DETACH TABLE t_exec_gate; +ATTACH TABLE t_exec_gate; + +SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM v_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM t_exec_pool_gate; -- { serverError SUPPORT_IS_DISABLED } + +-- The gate is evaluated on the node that runs the read, so a distributed wrapper does not lift it. +SELECT * FROM remote('127.0.0.1', executable('nonexist.sh', 'TSV', 'x UInt32')); -- { serverError SUPPORT_IS_DISABLED } + +-- A statement that reads as part of its own execution is refused too. +CREATE TABLE t_exec_gate_copy ENGINE = MergeTree ORDER BY x AS SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +CREATE MATERIALIZED VIEW mv_exec_gate_populate ENGINE = MergeTree ORDER BY x POPULATE AS SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } + +SET allow_executable_tables = 1; +SELECT * FROM t_exec_gate; -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +DROP VIEW mv_exec_gate; +DROP VIEW v_exec_gate; +DROP TABLE t_exec_gate; diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference new file mode 100644 index 000000000000..8b916e9e9712 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference @@ -0,0 +1,5 @@ +refused +refused +refused +0 1 +10 diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh new file mode 100755 index 000000000000..454cce40ebff --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached +OTHER_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_other + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; +""" + +# `PARALLEL WITH` runs its statements as internal queries, and the full `ATTACH DATABASE ... ENGINE = ...` +# form runs in `ATTACH` mode - the same pair the server's own replay of stored metadata shows. The quoted +# locator is accepted only on that replay, never in a statement a user writes, however it is wrapped. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +# Each block below must start from the same state: whether the sibling `CREATE DATABASE` of a +# `PARALLEL WITH` whose other statement threw is committed or rolled back depends on how the +# statements were scheduled, and with `max_threads = 1` it is committed. Drop it in between, so the +# refusal under test is what the block observes, not a leftover "database already exists". +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +# A quoted locator can carry credentials, and `PARALLEL WITH` formats its statements before the engine +# refuses them, so the formatted text must hide it: neither the logged query text (the same text an +# `ON CLUSTER` statement puts into the distributed DDL payload) nor the refusal message may carry it. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'S3(\\'http://localhost:11111/05218\\', \\'ak\\', \\'SEKRIT_05218\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q """ +SELECT countIf(query LIKE '%SEKRIT_05218%' OR exception LIKE '%SEKRIT_05218%'), countIf(query LIKE '%Backup(%[HIDDEN]%') > 0 +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE; +""" + +# The function form goes through the same wrapper. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE IF NOT EXISTS $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference new file mode 100644 index 000000000000..4cd3790c6b88 --- /dev/null +++ b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.reference @@ -0,0 +1,8 @@ +--- url --- +60 30 +--- s3 --- +60 30 +--- forwarded queries of url --- +3 3 60 +--- forwarded queries of s3 --- +3 3 60 diff --git a/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh new file mode 100755 index 000000000000..c956ed1d537a --- /dev/null +++ b/tests/queries/0_stateless/05218_insert_select_auto_cluster_function_parallel_replicas.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper +# Tag no-fasttest: needs MinIO and a Replicated table + +# https://github.com/ClickHouse/ClickHouse/issues/120485 +# +# `INSERT INTO SELECT * FROM url(...)` (or `s3(...)`) with parallel replicas enabled: +# `parallel_replicas_for_cluster_engines` converts the plain table function into a cluster storage on the +# initiator, so the INSERT takes the distributed `parallel_distributed_insert_select` path and forwards the +# query to every replica of the cluster. The forwarded query text still named the plain function, and a replica +# running it as a secondary query created a plain storage that expanded the globs and read every file on its +# own instead of taking its share of the read tasks from the initiator, so N replicas inserted the data N times. +# The forwarded query must name the `*Cluster` variant, the same way the SELECT path does. +# +# The three "replicas" of `test_cluster_one_shard_three_replicas_localhost` are all this server, so the +# secondary queries are visible in the local query log. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -eu + +S3_DIR="http://localhost:11111/test/${CLICKHOUSE_DATABASE}/05218" + +# Three files of different sizes: 10 + 20 + 30 = 60 rows, so a duplicated file changes the count. +for i in 1 2 3 +do + $CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/part_${i}.tsv', 'TSV', 'x UInt32') SELECT number FROM numbers(${i} * 10)" +done + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS dst_05218 SYNC; + CREATE TABLE dst_05218 (x UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/dst_05218', 'r1') ORDER BY x; +" + +SETTINGS="enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', + parallel_replicas_for_cluster_engines = 1, parallel_distributed_insert_select = 2, log_queries = 1" + +# The query ids and the query log lookup below must isolate this run: a re-run on the same server +# must not pick up the secondary queries of a previous run. +QUERY_ID_SUFFIX="${CLICKHOUSE_DATABASE}_$(date +%s%N)_${RANDOM}" +QUERY_ID_URL="05218_url_${QUERY_ID_SUFFIX}" +QUERY_ID_S3="05218_s3_${QUERY_ID_SUFFIX}" + +echo "--- url ---" +$CLICKHOUSE_CLIENT --query_id "${QUERY_ID_URL}" -q " + INSERT INTO dst_05218 SELECT * FROM url('${S3_DIR}/part_{1..3}.tsv', 'TSV', 'x UInt32') SETTINGS ${SETTINGS}" +$CLICKHOUSE_CLIENT -q "SELECT count(), uniqExact(x) FROM dst_05218" + +echo "--- s3 ---" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE dst_05218" +$CLICKHOUSE_CLIENT --query_id "${QUERY_ID_S3}" -q " + INSERT INTO dst_05218 SELECT * FROM s3('${S3_DIR}/part_{1..3}.tsv', 'TSV', 'x UInt32') SETTINGS ${SETTINGS}" +$CLICKHOUSE_CLIENT -q "SELECT count(), uniqExact(x) FROM dst_05218" + +# The INSERT must really have been distributed: every replica ran the forwarded INSERT, the forwarded +# query names the `*Cluster` function, and the replicas together read every file exactly once. +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +for pair in "${QUERY_ID_URL} urlCluster" "${QUERY_ID_S3} s3Cluster" +do + query_id="${pair%% *}" + cluster_function="${pair##* }" + echo "--- forwarded queries of ${cluster_function%Cluster} ---" + $CLICKHOUSE_CLIENT -q " + WITH initial AS + ( + SELECT query_id + FROM system.query_log + WHERE current_database = currentDatabase() + AND query_id = '${query_id}' + AND is_initial_query = 1 + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - INTERVAL 10 MINUTE + ) + SELECT + count() AS replicas, + countIf(query ILIKE '%${cluster_function}(%') AS cluster_function_queries, + sum(read_rows) AS rows_read_by_replicas + FROM system.query_log + WHERE initial_query_id IN (SELECT query_id FROM initial) + AND is_initial_query = 0 + AND query_kind = 'Insert' + AND type = 'QueryFinish' + AND event_date >= yesterday() + AND event_time >= now() - INTERVAL 10 MINUTE" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE dst_05218 SYNC" diff --git a/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference new file mode 100644 index 000000000000..06f2d273f643 --- /dev/null +++ b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.reference @@ -0,0 +1,102 @@ +present key, subcolumns=0 1 b1 +present key, subcolumns=0 2 +present key, subcolumns=0 3 +present key, subcolumns=0 4 FIRST +present key, subcolumns=1 1 b1 +present key, subcolumns=1 2 +present key, subcolumns=1 3 +present key, subcolumns=1 4 FIRST +absent key, subcolumns=0 1 +absent key, subcolumns=0 2 +absent key, subcolumns=0 3 +absent key, subcolumns=0 4 +absent key, subcolumns=1 1 +absent key, subcolumns=1 2 +absent key, subcolumns=1 3 +absent key, subcolumns=1 4 +empty key, subcolumns=0 1 +empty key, subcolumns=0 2 +empty key, subcolumns=0 3 +empty key, subcolumns=0 4 empty +empty key, subcolumns=1 1 +empty key, subcolumns=1 2 +empty key, subcolumns=1 3 +empty key, subcolumns=1 4 empty +direct subcolumn 1 b1 +direct subcolumn 2 +direct subcolumn 3 +direct subcolumn 4 FIRST +compact, subcolumns=0 1 b1 +compact, subcolumns=0 2 +compact, subcolumns=0 3 +compact, subcolumns=0 4 FIRST +compact, subcolumns=1 1 b1 +compact, subcolumns=1 2 +compact, subcolumns=1 3 +compact, subcolumns=1 4 FIRST +fixed string key, subcolumns=0 1 b1 +fixed string key, subcolumns=0 2 +fixed string key, subcolumns=1 1 b1 +fixed string key, subcolumns=1 2 +wide map, subcolumns=0 17633783096591347703 10736917531793588130 +wide map, subcolumns=1 17633783096591347703 10736917531793588130 +high dictionary position, subcolumns=0 100000 100 +high dictionary position, subcolumns=1 100000 100 +random maps, subcolumns=0 9043450906938593318 +random maps, subcolumns=1 9043450906938593318 +UUID key, subcolumns=0 1 hit +UUID key, subcolumns=0 2 +UUID key, subcolumns=1 1 hit +UUID key, subcolumns=1 2 +IPv4 key, subcolumns=0 1 hit +IPv4 key, subcolumns=0 2 +IPv4 key, subcolumns=1 1 hit +IPv4 key, subcolumns=1 2 +IPv6 key, subcolumns=0 1 hit +IPv6 key, subcolumns=0 2 +IPv6 key, subcolumns=1 1 hit +IPv6 key, subcolumns=1 2 +Int128 key, subcolumns=0 1 hit +Int128 key, subcolumns=0 2 +Int128 key, subcolumns=1 1 hit +Int128 key, subcolumns=1 2 +UInt256 key, subcolumns=0 1 hit +UInt256 key, subcolumns=0 2 +UInt256 key, subcolumns=1 1 hit +UInt256 key, subcolumns=1 2 +Decimal64 key subcolumn 1 hit +Decimal64 key subcolumn 2 +DateTime64 key subcolumn 1 hit +DateTime64 key subcolumn 2 +UUID value, subcolumns=0 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +UUID value, subcolumns=0 2 00000000-0000-0000-0000-000000000000 +UUID value, subcolumns=1 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +UUID value, subcolumns=1 2 00000000-0000-0000-0000-000000000000 +IPv6 value, subcolumns=0 1 ::1 +IPv6 value, subcolumns=0 2 :: +IPv6 value, subcolumns=1 1 ::1 +IPv6 value, subcolumns=1 2 :: +Int128 value, subcolumns=0 1 -123456789012345678901234567890 +Int128 value, subcolumns=0 2 0 +Int128 value, subcolumns=1 1 -123456789012345678901234567890 +Int128 value, subcolumns=1 2 0 +Decimal64 value, subcolumns=0 1 1.25 +Decimal64 value, subcolumns=0 2 0 +Decimal64 value, subcolumns=1 1 1.25 +Decimal64 value, subcolumns=1 2 0 +DateTime64 value, subcolumns=0 1 2020-01-01 00:00:00.500 +DateTime64 value, subcolumns=0 2 1970-01-01 00:00:00.000 +DateTime64 value, subcolumns=1 1 2020-01-01 00:00:00.500 +DateTime64 value, subcolumns=1 2 1970-01-01 00:00:00.000 +Nullable(UUID) value, subcolumns=0 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +Nullable(UUID) value, subcolumns=0 2 \N +Nullable(UUID) value, subcolumns=0 3 \N +Nullable(UUID) value, subcolumns=1 1 61f0c404-5cb3-11e7-907b-a6006ad3dba0 +Nullable(UUID) value, subcolumns=1 2 \N +Nullable(UUID) value, subcolumns=1 3 \N +Nullable(Decimal64) value, subcolumns=0 1 1.25 +Nullable(Decimal64) value, subcolumns=0 2 \N +Nullable(Decimal64) value, subcolumns=0 3 \N +Nullable(Decimal64) value, subcolumns=1 1 1.25 +Nullable(Decimal64) value, subcolumns=1 2 \N +Nullable(Decimal64) value, subcolumns=1 3 \N diff --git a/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql new file mode 100644 index 000000000000..925533b788e1 --- /dev/null +++ b/tests/queries/0_stateless/05218_map_lowcardinality_key_lookup.sql @@ -0,0 +1,203 @@ +-- m[key] on a Map with LowCardinality keys is resolved through dictionary positions instead of +-- comparing the key values. The result must be the same as with the generic comparison, and the +-- same for both the arrayElement path (optimize_functions_to_subcolumns = 0) and the subcolumn +-- path (optimize_functions_to_subcolumns = 1). + +DROP TABLE IF EXISTS t_map_lc; + +CREATE TABLE t_map_lc (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +-- Row 2 has no 'b' at all, row 3 is an empty map, row 4 has 'b' twice and '' as a key. +INSERT INTO t_map_lc VALUES (1, map('a', 'a1', 'b', 'b1')), (2, map('a', 'a2')), (3, map()), (4, map('', 'empty', 'b', 'FIRST', 'b', 'SECOND')); + +SELECT 'present key, subcolumns=0', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'present key, subcolumns=1', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- A key that is in no row is absent from the dictionary as well, and is resolved without +-- looking at the rows. +SELECT 'absent key, subcolumns=0', id, m['zz'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'absent key, subcolumns=1', id, m['zz'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- The empty string is kept in the reserved default position of the dictionary, not in its index. +SELECT 'empty key, subcolumns=0', id, m[''] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'empty key, subcolumns=1', id, m[''] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +-- Reading the subcolumn directly goes through the same key lookup. +SELECT 'direct subcolumn', id, m.key_b FROM t_map_lc ORDER BY id; + +DROP TABLE t_map_lc; + +-- Compact parts exercise the other reader path. +CREATE TABLE t_map_lc (id UInt64, m Map(LowCardinality(String), String)) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_map_lc VALUES (1, map('a', 'a1', 'b', 'b1')), (2, map('a', 'a2')), (3, map()), (4, map('', 'empty', 'b', 'FIRST', 'b', 'SECOND')); + +SELECT 'compact, subcolumns=0', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'compact, subcolumns=1', id, m['b'] FROM t_map_lc ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc; + +-- LowCardinality(FixedString) keys use the same dictionary lookup. +CREATE TABLE t_map_lc_fixed (id UInt64, m Map(LowCardinality(FixedString(3)), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_lc_fixed VALUES (1, map('aaa', 'a1', 'bbb', 'b1')), (2, map('aaa', 'a2')); + +SELECT 'fixed string key, subcolumns=0', id, m['bbb'], m['zzz'] FROM t_map_lc_fixed ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'fixed string key, subcolumns=1', id, m['bbb'], m['zzz'] FROM t_map_lc_fixed ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_fixed; + +-- A wide map, where the looked up key is the last one in every row: the shape the dictionary +-- lookup is meant to make cheap. +DROP TABLE IF EXISTS t_map_lc_wide; + +CREATE TABLE t_map_lc_wide (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_wide +SELECT number, mapFromArrays(arrayMap(x -> concat('k', leftPad(toString(x), 2, '0')), range(20)), arrayMap(x -> concat('v', toString(number), '_', toString(x)), range(20))) +FROM numbers(1000); + +SELECT 'wide map, subcolumns=0', sum(cityHash64(m['k19'])), sum(cityHash64(m['k00'])) FROM t_map_lc_wide SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'wide map, subcolumns=1', sum(cityHash64(m['k19'])), sum(cityHash64(m['k00'])) FROM t_map_lc_wide SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_wide; + +-- A key at a high dictionary position looked up over rows that do not contain it. The dictionary of +-- a part is shared by all of its blocks, so a dictionary position can be out of the range of the +-- index type of an individual block, and must not be truncated into a false match. +DROP TABLE IF EXISTS t_map_lc_many_keys; + +CREATE TABLE t_map_lc_many_keys (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_many_keys +SELECT number, map(concat('k', leftPad(toString(number % 1000), 4, '0')), concat('v', toString(number))) +FROM numbers(100000); + +SELECT 'high dictionary position, subcolumns=0', count(), countIf(m['k0999'] != '') FROM t_map_lc_many_keys SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'high dictionary position, subcolumns=1', count(), countIf(m['k0999'] != '') FROM t_map_lc_many_keys SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_many_keys; + +-- Pseudo-random wide maps with duplicate keys: both paths must agree, which is the property that +-- the first-match fix of issue #111203 established. +DROP TABLE IF EXISTS t_map_lc_random; + +CREATE TABLE t_map_lc_random (id UInt64, m Map(LowCardinality(String), String)) +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO t_map_lc_random +SELECT + number, + mapFromArrays( + arrayMap(x -> concat('k', toString(cityHash64(number, x) % 12)), range(1 + (number % 15))), + arrayMap(x -> concat('v', toString(cityHash64(number, x, 'value'))), range(1 + (number % 15)))) +FROM numbers(20000); + +SELECT 'random maps, subcolumns=0', sum(cityHash64(m['k0'], m['k5'], m['k11'], m['k99'])) FROM t_map_lc_random SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'random maps, subcolumns=1', sum(cityHash64(m['k0'], m['k5'], m['k11'], m['k99'])) FROM t_map_lc_random SETTINGS optimize_functions_to_subcolumns = 1; + +DROP TABLE t_map_lc_random; + +-- Key types that used to be compared through virtual compareAt. + +DROP TABLE IF EXISTS t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(UUID, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('61f0c404-5cb3-11e7-907b-a6006ad3dba0', 'hit')), (2, map('00000000-0000-0000-0000-000000000001', 'other')); +SELECT 'UUID key, subcolumns=0', id, m[toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UUID key, subcolumns=1', id, m[toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(IPv4, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('1.2.3.4', 'hit')), (2, map('5.6.7.8', 'other')); +SELECT 'IPv4 key, subcolumns=0', id, m[toIPv4('1.2.3.4')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv4 key, subcolumns=1', id, m[toIPv4('1.2.3.4')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(IPv6, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types VALUES (1, map('::1', 'hit')), (2, map('::2', 'other')); +SELECT 'IPv6 key, subcolumns=0', id, m[toIPv6('::1')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv6 key, subcolumns=1', id, m[toIPv6('::1')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +-- The wide integers are built from strings, because a literal that wide goes through Float64 and +-- loses precision. +CREATE TABLE t_map_key_types (id UInt64, m Map(Int128, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toInt128('-123456789012345678901234567890'), 'hit'); +INSERT INTO t_map_key_types SELECT 2, map(toInt128(1), 'other'); +SELECT 'Int128 key, subcolumns=0', id, m[toInt128('-123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Int128 key, subcolumns=1', id, m[toInt128('-123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +CREATE TABLE t_map_key_types (id UInt64, m Map(UInt256, String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toUInt256('12345678901234567890123456789012345678901234567890'), 'hit'); +INSERT INTO t_map_key_types SELECT 2, map(toUInt256(1), 'other'); +SELECT 'UInt256 key, subcolumns=0', id, m[toUInt256('12345678901234567890123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UInt256 key, subcolumns=1', id, m[toUInt256('12345678901234567890123456789012345678901234567890')] FROM t_map_key_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_key_types; + +-- Decimal and DateTime64 keys are rejected by m[key], but reachable through the subcolumn name. + +CREATE TABLE t_map_key_types (id UInt64, m Map(Decimal64(2), String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toDecimal64(1.5, 2), 'hit', toDecimal64(2.5, 2), 'other'); +INSERT INTO t_map_key_types SELECT 2, map(toDecimal64(2.5, 2), 'other'); +SELECT 'Decimal64 key subcolumn', id, `m.key_1.50` FROM t_map_key_types ORDER BY id; +DROP TABLE t_map_key_types; + +-- The timezone is pinned, because the key is rendered into the subcolumn name. +CREATE TABLE t_map_key_types (id UInt64, m Map(DateTime64(3, 'UTC'), String)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_key_types SELECT 1, map(toDateTime64('2020-01-01 00:00:00.500', 3, 'UTC'), 'hit', toDateTime64('2021-01-01 00:00:00.500', 3, 'UTC'), 'other'); +INSERT INTO t_map_key_types SELECT 2, map(toDateTime64('2021-01-01 00:00:00.500', 3, 'UTC'), 'other'); +SELECT 'DateTime64 key subcolumn', id, `m.key_2020-01-01 00:00:00.500` FROM t_map_key_types ORDER BY id; +DROP TABLE t_map_key_types; + +-- Value types that used to be copied through virtual insertFrom. The rows not holding the key +-- cover the default-insert branch, the Nullable variants the null map. + +DROP TABLE IF EXISTS t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, UUID)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '61f0c404-5cb3-11e7-907b-a6006ad3dba0')), (2, map('b', '00000000-0000-0000-0000-000000000001')); +SELECT 'UUID value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'UUID value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, IPv6)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '::1')), (2, map('b', '::2')); +SELECT 'IPv6 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'IPv6 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Int128)) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types SELECT 1, map('a', toInt128('-123456789012345678901234567890')); +INSERT INTO t_map_value_types SELECT 2, map('b', toInt128(1)); +SELECT 'Int128 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Int128 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Decimal64(3))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', 1.25)), (2, map('b', 2.5)); +SELECT 'Decimal64 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Decimal64 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +-- The timezone is pinned, because the missing key renders the epoch as the default value. +CREATE TABLE t_map_value_types (id UInt64, m Map(String, DateTime64(3, 'UTC'))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '2020-01-01 00:00:00.500')), (2, map('b', '2021-01-01 00:00:00.500')); +SELECT 'DateTime64 value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'DateTime64 value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Nullable(UUID))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', '61f0c404-5cb3-11e7-907b-a6006ad3dba0')), (2, map('a', NULL)), (3, map('b', NULL)); +SELECT 'Nullable(UUID) value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Nullable(UUID) value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; + +CREATE TABLE t_map_value_types (id UInt64, m Map(String, Nullable(Decimal64(3)))) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_map_value_types VALUES (1, map('a', 1.25)), (2, map('a', NULL)), (3, map('b', NULL)); +SELECT 'Nullable(Decimal64) value, subcolumns=0', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT 'Nullable(Decimal64) value, subcolumns=1', id, m['a'] FROM t_map_value_types ORDER BY id SETTINGS optimize_functions_to_subcolumns = 1; +DROP TABLE t_map_value_types; diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference new file mode 100644 index 000000000000..018a9a02fa76 --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference @@ -0,0 +1,60 @@ +-- user granted only pub +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,2,3,4,5,6,7] +SELECT pub FROM t_prewhere_alias PREWHERE pub = 2 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5) +1 +-- user granted pub and the alias columns, but not their source +SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2 +200 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +-- user granted pub and the source column, but not the alias columns +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +[4,5,6,7] +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +0 +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501) +1 +-- row policy over a column the user is not granted +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,3,4,5,6,7] +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1 +[3,4,5,6,7] diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh new file mode 100755 index 000000000000..89efe0535064 --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer, no-parallel-replicas +# no-parallel-replicas: reading an ALIAS column with a grant on only that ALIAS column is denied under +# parallel replicas. That is an unrelated pre-existing bug (reproducible on master without this test's +# subject), so exclude the setting rather than encode the wrong behaviour in the reference. + +# Column-level SELECT grants must be enforced for columns that the planner resolves away before the +# access check runs, so that PREWHERE cannot be used as an oracle over a column the user cannot read: +# +# 1. An ALIAS column referenced in PREWHERE is replaced by its expression, so it used to never reach +# the list of selected columns and was not access checked at all, while the same alias in SELECT or +# WHERE was correctly denied. +# 2. A column used only as an `indexHint` argument is never read, only used for index analysis, but +# which granules survive the analysis is observable in the result. +# +# In both cases the required privilege is the one on the name written in the query, exactly as in +# WHERE: referencing an ALIAS column requires a grant on the ALIAS column itself, and a grant on the +# physical columns its expression reads is neither sufficient nor required. Those source columns are +# still read from disk to compute the ALIAS - the administrator authored the ALIAS expression and so +# chose what it exposes - but their values never reach the user. + +# Most of the queries below are expected to be denied, and each denial would otherwise add an +# log line and a stack trace to the output, because the harness streams server-side logs to the client. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_PUB="user_pub_${CLICKHOUSE_DATABASE}" +USER_ALIAS="user_alias_${CLICKHOUSE_DATABASE}" +USER_SOURCE="user_source_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP USER IF EXISTS ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + + CREATE TABLE t_prewhere_alias + ( + secret Int32, + pub Int32, + secret_alias Int32 ALIAS secret, + secret_alias_expression Int32 ALIAS secret_alias + 1 + ) + ENGINE = MergeTree ORDER BY secret SETTINGS index_granularity = 1, min_bytes_for_wide_part = 0; + + INSERT INTO t_prewhere_alias (secret, pub) SELECT number * 100, number FROM numbers(8); + + CREATE USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + GRANT SELECT(pub) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_PUB}; + GRANT SELECT(pub, secret_alias, secret_alias_expression) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_ALIAS}; + GRANT SELECT(pub, secret) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_SOURCE}; +" + +# Run all the queries of one user in a single session, echoing each query before its own result so +# that every line of the reference is attributable. --ignore-error keeps the session going past the +# ACCESS_DENIED answers that most of these queries are expected to produce; without it the first +# denial would end the session and silently drop every query after it. +# +# An exception is printed by the client as exactly three lines - the "Received exception" banner, the +# "Code: ..." message and the echoed query - of which only the error name is stable: the banner holds +# the server version and the message holds the user name, which embeds ${CLICKHOUSE_DATABASE}. Reduce +# the block to that name, keeping it distinct per error so that a change of error shows up as a diff +# rather than being masked. +run_all() +{ + local user=$1 + shift + + local sql="" + local query + for query in "$@" + do + # The echoed query is derived from the query itself, so the two cannot drift apart. + sql+="SELECT '${query//\'/\'\'}'; +${query}; +" + done + + ${CLICKHOUSE_CLIENT} --user "${user}" --multiquery --ignore-error --query "${sql}" 2>&1 \ + | sed -E '/^Received exception/d; /^\(query:/d; s/^Code: [0-9]+\..*\(([A-Z_]+)\)$/\1/' +} + +echo "-- user granted only pub" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT pub FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5)" + +echo "-- user granted pub and the alias columns, but not their source" +run_all "${USER_ALIAS}" \ + "SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" + +echo "-- user granted pub and the source column, but not the alias columns" +run_all "${USER_SOURCE}" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501)" + +# A row policy is defined by an administrator, so it may reference columns the user cannot read. +echo "-- row policy over a column the user is not granted" +${CLICKHOUSE_CLIENT} --query "CREATE ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias USING secret_alias_expression != 201 TO ${USER_PUB}" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias; + DROP TABLE t_prewhere_alias; + DROP USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; +" diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference new file mode 100644 index 000000000000..17c209230693 --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference @@ -0,0 +1 @@ +7 8 diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh new file mode 100755 index 000000000000..e4d11a0e11bf --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# A `SET` statement that changes `profile` installs a new constraint set halfway through itself, and the +# assignments and resets after that point must pass it, as they would in a separate statement. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CONST_PROFILE="profile_const_$CLICKHOUSE_DATABASE" +READONLY_PROFILE="profile_readonly_$CLICKHOUSE_DATABASE" +MAX_PROFILE="profile_max_$CLICKHOUSE_DATABASE" +RELAXED_PROFILE="profile_relaxed_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -m -q " +DROP SETTINGS PROFILE IF EXISTS $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE; +CREATE SETTINGS PROFILE $CONST_PROFILE SETTINGS max_execution_time = 10 CONST, SQL_tenant_id = 1 CONST; +CREATE SETTINGS PROFILE $READONLY_PROFILE SETTINGS readonly = 1; +CREATE SETTINGS PROFILE $MAX_PROFILE SETTINGS max_memory_usage MAX 1000000; +CREATE SETTINGS PROFILE $RELAXED_PROFILE SETTINGS max_memory_usage MAX 1099511627776; +" + +# One session on purpose: a rejected statement must leave it untouched, which the statements after it prove. +$CLICKHOUSE_CLIENT -m -q " +SET profile = '$CONST_PROFILE', max_execution_time = 999; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$CONST_PROFILE', max_execution_time = DEFAULT; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$READONLY_PROFILE', max_memory_usage = 1099511627776; -- { serverError READONLY } +SET profile = '$MAX_PROFILE', max_memory_usage = 1099511627776; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- a later, looser profile does not lift the constraint in force at the assignment +SET profile = '$MAX_PROFILE', max_memory_usage = 5000000, profile = '$RELAXED_PROFILE'; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- none of the rejected statements applied its profile, which would have installed this setting +SELECT getSetting('SQL_tenant_id'); -- { serverError UNKNOWN_SETTING } +-- assigning the value the profile installs is a no-op and stays allowed +SET profile = '$CONST_PROFILE', max_execution_time = 10; +-- what is not constrained still applies, before and after the profile change +SET SQL_before = 7, profile = '$CONST_PROFILE', SQL_after = 8; +SELECT getSetting('SQL_before'), getSetting('SQL_after'); +" + +$CLICKHOUSE_CLIENT -q "DROP SETTINGS PROFILE $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE" diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference new file mode 100644 index 000000000000..405f09ad316d --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference @@ -0,0 +1,101 @@ +-- the inlined view body hides the key +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Identifier encrypted_secret + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Function hex (alias encrypted_secret) (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Identifier secret + Literal \'[HIDDEN]\' + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + TableIdentifier default.private_plaintext +-- the session setting alone does not disclose it + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' +-- the graph dump hides it too +Literal \'[HIDDEN]\' +-- a secret typed into the explained query itself is hidden as well +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 4) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Function leftPad (children 1) + ExpressionList (children 3) + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a positional secret written as a comparison collapses to one node, not to 'Function equals' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 1) + Literal \'[HIDDEN]\' +-- a mongodb url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function mongodb (children 1) + ExpressionList (children 2) + Literal \'[HIDDEN]\' + Literal \'c\' +-- the same url as a named override keeps its key, the value collapses +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 2) + Identifier creds + Function equals (children 1) + ExpressionList (children 2) + Identifier url + Literal \'[HIDDEN]\' +-- a nested map keeps its keys and hides its values + Literal \'http://x/f\' + Literal \'Authorization\' + Literal \'[HIDDEN]\' +-- the view itself stays usable for the restricted user +1 diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh new file mode 100755 index 000000000000..dcc3ca50f8c0 --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXPLAIN AST optimize = 1` inlines the body of a view the user may only SELECT from, and the dump +# prints every literal verbatim. The secret arguments must be hidden as `SHOW CREATE` hides them. +# The stateless test server keeps `display_secrets_in_show_and_select` off, so the gate always hides here. + +user="user_05219_${CLICKHOUSE_DATABASE}_$RANDOM" +db=${CLICKHOUSE_DATABASE} +key='Sixteen byte key' + +${CLICKHOUSE_CLIENT} <