From 9c33fc20fb9cd4aa98a9b5ed4cfd77ea39f5ad8b Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Mon, 15 Dec 2025 09:42:45 -0500 Subject: [PATCH 01/10] move stateless crossout logic into clickhouse-test --- ci/jobs/scripts/functional_tests_results.py | 126 ++------------------ tests/broken_tests.yaml | 18 +++ tests/clickhouse-test | 117 ++++++++++++++++++ 3 files changed, 146 insertions(+), 115 deletions(-) diff --git a/ci/jobs/scripts/functional_tests_results.py b/ci/jobs/scripts/functional_tests_results.py index eb62b5089683..58f497b3d638 100755 --- a/ci/jobs/scripts/functional_tests_results.py +++ b/ci/jobs/scripts/functional_tests_results.py @@ -14,6 +14,7 @@ TIMEOUT_SIGN = "[ Timeout! " UNKNOWN_SIGN = "[ UNKNOWN " SKIPPED_SIGN = "[ SKIPPED " +BROKEN_SIGN = "[ BROKEN " HUNG_SIGN = "Found hung queries in processlist" SERVER_DIED_SIGN = "Server died, terminating all processes" SERVER_DIED_SIGN2 = "Server does not respond to health check" @@ -33,101 +34,6 @@ # out.writerow(status) -def get_broken_tests_rules() -> dict: - broken_tests_file_path = "tests/broken_tests.yaml" - if ( - not os.path.isfile(broken_tests_file_path) - or os.path.getsize(broken_tests_file_path) == 0 - ): - raise ValueError( - "There is something wrong with getting broken tests rules: " - f"file '{broken_tests_file_path}' is empty or does not exist." - ) - - with open(broken_tests_file_path, "r", encoding="utf-8") as broken_tests_file: - broken_tests = yaml.safe_load(broken_tests_file) - - compiled_rules = {"exact": {}, "pattern": {}} - - for test in broken_tests: - regex = test.get("regex") is True - rule = { - "reason": test["reason"], - } - - if test.get("message"): - rule["message"] = re.compile(test["message"]) if regex else test["message"] - - if test.get("not_message"): - rule["not_message"] = ( - re.compile(test["not_message"]) if regex else test["not_message"] - ) - if test.get("check_types"): - rule["check_types"] = test["check_types"] - - if regex: - rule["regex"] = True - compiled_rules["pattern"][re.compile(test["name"])] = rule - else: - compiled_rules["exact"][test["name"]] = rule - - print( - f"INFO: Compiled {len(compiled_rules['exact'])} exact rules and {len(compiled_rules['pattern'])} pattern rules" - ) - - return compiled_rules - - -def test_is_known_fail(test_name, test_logs, known_broken_tests, test_options_string): - matching_rules = [] - - print(f"Checking known broken tests for failed test: {test_name}") - print("Potential matching rules:") - exact_rule = known_broken_tests["exact"].get(test_name) - if exact_rule: - print(f"{test_name} - {exact_rule}") - matching_rules.append(exact_rule) - - for name_re, data in known_broken_tests["pattern"].items(): - if name_re.fullmatch(test_name): - print(f"{name_re} - {data}") - matching_rules.append(data) - - if not matching_rules: - return False - - def matches_substring(substring, log, is_regex): - if log is None: - return False - if is_regex: - return bool(substring.search(log)) - return substring in log - - for rule_data in matching_rules: - if rule_data.get("check_types") and not any( - ct in test_options_string for ct in rule_data["check_types"] - ): - print( - f"Check types didn't match: '{rule_data['check_types']}' not in '{test_options_string}'" - ) - continue # check_types didn't match → skip rule - - is_regex = rule_data.get("regex", False) - not_message = rule_data.get("not_message") - if not_message and matches_substring(not_message, test_logs, is_regex): - print(f"Skip rule: Not message matched: '{rule_data['not_message']}'") - continue # not_message matched → skip rule - message = rule_data.get("message") - if message and not matches_substring(message, test_logs, is_regex): - print(f"Skip rule: Message didn't match: '{rule_data['message']}'") - continue - - print(f"Test {test_name} matched rule: {rule_data}") - return rule_data["reason"] - - return False - - class FTResultsProcessor: @dataclasses.dataclass class Summary: @@ -163,8 +69,6 @@ def _process_test_output(self): test_results = [] test_end = True - known_broken_tests = get_broken_tests_rules() - with open(self.tests_output_file, "r", encoding="utf-8") as test_file: for line in test_file: original_line = line @@ -183,7 +87,13 @@ def _process_test_output(self): retries = True if any( sign in line - for sign in (OK_SIGN, FAIL_SIGN, UNKNOWN_SIGN, SKIPPED_SIGN) + for sign in ( + OK_SIGN, + FAIL_SIGN, + UNKNOWN_SIGN, + SKIPPED_SIGN, + BROKEN_SIGN, + ) ): test_name = line.split(" ")[2].split(":")[0] @@ -216,6 +126,9 @@ def _process_test_output(self): elif SKIPPED_SIGN in line: skipped += 1 test_results.append((test_name, "SKIPPED", test_time, [])) + elif BROKEN_SIGN in line: + broken += 1 + test_results.append((test_name, "BROKEN", test_time, [])) else: success += int(OK_SIGN in line) test_results.append((test_name, "OK", test_time, [])) @@ -233,8 +146,6 @@ def _process_test_output(self): if DATABASE_SIGN in line: test_end = True - test_options_string = ", ".join(self.test_options) - test_results_ = [] for test in test_results: try: @@ -248,21 +159,6 @@ def _process_test_output(self): ) ) - if test[1] == "FAIL": - broken_message = test_is_known_fail( - test[0], - test_results_[-1].info, - known_broken_tests, - test_options_string, - ) - - if broken_message: - broken += 1 - failed -= 1 - test_results_[-1].set_status(Result.StatusExtended.BROKEN) - test_results_[-1].set_label(Result.Label.BROKEN) - test_results_[-1].info += "\nMarked as broken: " + broken_message - except Exception as e: print(f"ERROR: Failed to parse test results: {test}") traceback.print_exc() diff --git a/tests/broken_tests.yaml b/tests/broken_tests.yaml index cda3d3091d8c..00da04529efe 100644 --- a/tests/broken_tests.yaml +++ b/tests/broken_tests.yaml @@ -57,6 +57,24 @@ message: result differs with reference check_types: - msan +- name: 02313_filesystem_cache_seeks + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02286_drop_filesystem_cache + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02242_system_filesystem_cache_log_table + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02241_filesystem_cache_on_write_operations + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02240_system_filesystem_cache_table + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02226_filesystem_cache_profile_events + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' - name: 00024_random_counters reason: INVESTIGATE - random timeout message: Timeout! Killing process group diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 077abaa0cdb6..9ccd3dd9e97b 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -11,6 +11,10 @@ import copy import enum import glob +# For processing the broken tests rules +from functools import lru_cache +import yaml + # Not requests, to avoid requiring extra dependency. import http.client import io @@ -83,6 +87,100 @@ VERSION_PATTERN = r"^((\d+\.)?(\d+\.)?(\d+\.)?\d+)$" TEST_MAX_RUN_TIME_IN_SECONDS = 180 +@lru_cache(maxsize=1) +def get_broken_tests_rules() -> dict: + broken_tests_file_path = "tests/broken_tests.yaml" + if ( + not os.path.isfile(broken_tests_file_path) + or os.path.getsize(broken_tests_file_path) == 0 + ): + raise ValueError( + "There is something wrong with getting broken tests rules: " + f"file '{broken_tests_file_path}' is empty or does not exist." + ) + + with open(broken_tests_file_path, "r", encoding="utf-8") as broken_tests_file: + broken_tests = yaml.safe_load(broken_tests_file) + + compiled_rules = {"exact": {}, "pattern": {}} + + for test in broken_tests: + regex = test.get("regex") is True + rule = { + "reason": test["reason"], + } + + if test.get("message"): + rule["message"] = re.compile(test["message"]) if regex else test["message"] + + if test.get("not_message"): + rule["not_message"] = ( + re.compile(test["not_message"]) if regex else test["not_message"] + ) + if test.get("check_types"): + rule["check_types"] = test["check_types"] + + if regex: + rule["regex"] = True + compiled_rules["pattern"][re.compile(test["name"])] = rule + else: + compiled_rules["exact"][test["name"]] = rule + + return compiled_rules + + +def test_is_known_fail(test_name, test_logs, build_flags): + matching_rules = [] + + known_broken_tests = get_broken_tests_rules() + + # print(f"Checking known broken tests for failed test: {test_name}") + # print("Potential matching rules:") + exact_rule = known_broken_tests["exact"].get(test_name) + if exact_rule: + # print(f"{test_name} - {exact_rule}") + matching_rules.append(exact_rule) + + for name_re, data in known_broken_tests["pattern"].items(): + if name_re.fullmatch(test_name): + # print(f"{name_re} - {data}") + matching_rules.append(data) + + if not matching_rules: + return False + + def matches_substring(substring, log, is_regex): + if log is None: + return False + if is_regex: + return bool(substring.search(log)) + return substring in log + + for rule_data in matching_rules: + if rule_data.get("check_types") and not any( + ct in build_flags for ct in rule_data["check_types"] + ): + # print( + # f"Check types didn't match: '{rule_data['check_types']}' not in '{build_flags}'" + # ) + continue # check_types didn't match → skip rule + + is_regex = rule_data.get("regex", False) + not_message = rule_data.get("not_message") + if not_message and matches_substring(not_message, test_logs, is_regex): + # print(f"Skip rule: Not message matched: '{rule_data['not_message']}'") + continue # not_message matched → skip rule + message = rule_data.get("message") + if message and not matches_substring(message, test_logs, is_regex): + # print(f"Skip rule: Message didn't match: '{rule_data['message']}'") + continue + + # print(f"Test {test_name} matched rule: {rule_data}") + return rule_data["reason"] + + return False + + class SharedEngineReplacer: SPECIALIZED_ENGINES = "Collapsing|VersionedCollapsing|Summing|Replacing|Aggregating" ENGINES_NON_REPLICATED_REGEXP = ( @@ -918,6 +1016,7 @@ class TestStatus(enum.Enum): OK = "OK" SKIPPED = "SKIPPED" NOT_FAILED = "NOT_FAILED" + BROKEN = "BROKEN" class FailureReason(enum.Enum): @@ -2222,6 +2321,14 @@ class TestCase: if result.status == TestStatus.FAIL: result.description = self.add_info_about_settings(result.description) + # Check if the test is known to fail + known_fail_reason = test_is_known_fail( + self.name, result.description, args.build_flags + ) + if known_fail_reason: + result.status = TestStatus.BROKEN + result.description += f"Marked as broken: {known_fail_reason}" + if self.name in suite.blacklist_check: if result.status == TestStatus.OK: result.status = TestStatus.NOT_FAILED @@ -2726,6 +2833,11 @@ def run_tests_array( + colored(" NOT_FAILED ", args, "red", attrs=["bold"]) + CL_SQUARE_BRACKET ) + MSG_BROKEN = ( + OP_SQUARE_BRACKET + + colored(" BROKEN ", args, "cyan", attrs=["bold"]) + + CL_SQUARE_BRACKET + ) MESSAGES = { TestStatus.FAIL: MSG_FAIL, @@ -2733,12 +2845,14 @@ def run_tests_array( TestStatus.OK: MSG_OK, TestStatus.SKIPPED: MSG_SKIPPED, TestStatus.NOT_FAILED: MSG_NOT_FAILED, + TestStatus.BROKEN: MSG_BROKEN, } passed_total = 0 skipped_total = 0 failures_total = 0 failures_chain = 0 + broken_total = 0 start_time = datetime.now() client_options = get_additional_client_options(args) @@ -2833,6 +2947,8 @@ def run_tests_array( raise ServerDied("Server died") elif test_result.status == TestStatus.SKIPPED: skipped_total += 1 + elif test_result.status == TestStatus.BROKEN: + broken_total += 1 except KeyboardInterrupt as e: print(colored("Break tests execution", args, "red")) @@ -2848,6 +2964,7 @@ def run_tests_array( colored( f"\nHaving {failures_total} errors! {passed_total} tests passed." f" {skipped_total} tests skipped." + f" {broken_total} broken tests." f" {(datetime.now() - start_time).total_seconds():.2f} s elapsed" f" ({multiprocessing.current_process().name}).", args, From 6454dfd34dd3b2a8c1aa5a8d5f7c99c71a0f5215 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Mon, 15 Dec 2025 14:58:51 -0500 Subject: [PATCH 02/10] make clickhouse-test crossouts optional --- ci/jobs/functional_tests.py | 2 ++ tests/clickhouse-test | 27 ++++++++++++++++----------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index 02d148e2bd5f..408cd846b22a 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -354,6 +354,8 @@ def start(): ) res = results[-1].is_ok() + runner_options += f" --known-fails-file-path tests/broken_tests.yaml" + test_result = None if res and JobStages.TEST in stages: stop_watch_ = Utils.Stopwatch() diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 9ccd3dd9e97b..4a21a01be570 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -88,8 +88,7 @@ TEST_MAX_RUN_TIME_IN_SECONDS = 180 @lru_cache(maxsize=1) -def get_broken_tests_rules() -> dict: - broken_tests_file_path = "tests/broken_tests.yaml" +def get_broken_tests_rules(broken_tests_file_path: str) -> dict: if ( not os.path.isfile(broken_tests_file_path) or os.path.getsize(broken_tests_file_path) == 0 @@ -129,10 +128,10 @@ def get_broken_tests_rules() -> dict: return compiled_rules -def test_is_known_fail(test_name, test_logs, build_flags): +def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path): matching_rules = [] - known_broken_tests = get_broken_tests_rules() + known_broken_tests = get_broken_tests_rules(broken_tests_file_path) # print(f"Checking known broken tests for failed test: {test_name}") # print("Potential matching rules:") @@ -2321,13 +2320,17 @@ class TestCase: if result.status == TestStatus.FAIL: result.description = self.add_info_about_settings(result.description) - # Check if the test is known to fail - known_fail_reason = test_is_known_fail( - self.name, result.description, args.build_flags - ) - if known_fail_reason: - result.status = TestStatus.BROKEN - result.description += f"Marked as broken: {known_fail_reason}" + if args.known_fails_file_path: + # Check if the test is known to fail + known_fail_reason = test_is_known_fail( + self.name, + result.description, + args.build_flags, + args.known_fails_file_path, + ) + if known_fail_reason: + result.status = TestStatus.BROKEN + result.description += f"Marked as broken: {known_fail_reason}" if self.name in suite.blacklist_check: if result.status == TestStatus.OK: @@ -3818,6 +3821,8 @@ def parse_args(): parser.add_argument("-q", "--queries", help="Path to queries dir") parser.add_argument("--tmp", help="Path to tmp dir") + parser.add_argument("--known-fails-file-path", help="Path to known fails file") + parser.add_argument( "-b", "--binary", From 018897e3339b20d19402ca69d1e3e18b5e7b0800 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Tue, 16 Dec 2025 17:23:09 -0500 Subject: [PATCH 03/10] try to fix broken tests output capture --- ci/jobs/scripts/functional_tests_results.py | 2 +- tests/clickhouse-test | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/jobs/scripts/functional_tests_results.py b/ci/jobs/scripts/functional_tests_results.py index 58f497b3d638..1b038e0eb042 100755 --- a/ci/jobs/scripts/functional_tests_results.py +++ b/ci/jobs/scripts/functional_tests_results.py @@ -135,7 +135,7 @@ def _process_test_output(self): test_end = False elif ( len(test_results) > 0 - and test_results[-1][1] in ("FAIL", "SKIPPED") + and test_results[-1][1] in ("FAIL", "SKIPPED", "BROKEN") and not test_end ): test_results[-1][3].append(original_line) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 4a21a01be570..032da6e45b9c 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2330,7 +2330,15 @@ class TestCase: ) if known_fail_reason: result.status = TestStatus.BROKEN - result.description += f"Marked as broken: {known_fail_reason}" + if "Database: " in result.description.splitlines()[-1]: + result.description = result.description.replace( + "Database: ", + f"Marked as broken: {known_fail_reason}\nDatabase: ", + ) + else: + result.description += ( + f"\nMarked as broken: {known_fail_reason}" + ) if self.name in suite.blacklist_check: if result.status == TestStatus.OK: From 23a1102c7153d8339a5d6033d8c03be5a575e07a Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Wed, 17 Dec 2025 10:40:43 -0500 Subject: [PATCH 04/10] update 00024_random_counters crossout --- tests/broken_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/broken_tests.yaml b/tests/broken_tests.yaml index 00da04529efe..8778f72a7be0 100644 --- a/tests/broken_tests.yaml +++ b/tests/broken_tests.yaml @@ -77,7 +77,7 @@ message: 'DB::Exception: Unknown storage policy `azure_cache`' - name: 00024_random_counters reason: INVESTIGATE - random timeout - message: Timeout! Killing process group + message: Timeout! Processes left in process group - name: test_storage_s3_queue/test_5.py::test_migration[1-s3queue_] reason: KNOWN - Sometimes fails due to test order message: 'Failed: Timeout >900.0s' From 016243c350153771746197ee098aa92481729469 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 18 Dec 2025 10:47:54 -0500 Subject: [PATCH 05/10] place the crossout message at the log start --- tests/clickhouse-test | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 032da6e45b9c..f335380ae313 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2330,15 +2330,11 @@ class TestCase: ) if known_fail_reason: result.status = TestStatus.BROKEN - if "Database: " in result.description.splitlines()[-1]: - result.description = result.description.replace( - "Database: ", - f"Marked as broken: {known_fail_reason}\nDatabase: ", - ) - else: - result.description += ( - f"\nMarked as broken: {known_fail_reason}" - ) + + # Place the message on the second line + result.description.replace( + "\n", f"\nMarked as broken: {known_fail_reason}\n", count=1 + ) if self.name in suite.blacklist_check: if result.status == TestStatus.OK: From 6900c18768713988438914913422e8349481e48b Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 18 Dec 2025 11:29:26 -0500 Subject: [PATCH 06/10] add debug logging to stateless crossouts --- ci/jobs/functional_tests.py | 4 + ci/jobs/scripts/functional_tests_results.py | 5 -- tests/clickhouse-test | 91 ++++++++++++--------- 3 files changed, 55 insertions(+), 45 deletions(-) diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index 408cd846b22a..d34cf0b8d579 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -449,6 +449,10 @@ def collect_logs(): ) force_ok_exit = True + broken_tests_handler_log = os.path.join(temp_dir, "broken_tests_handler.log") + if os.path.exists(broken_tests_handler_log): + debug_files.append(broken_tests_handler_log) + Result.create_from( results=results, stopwatch=stop_watch, diff --git a/ci/jobs/scripts/functional_tests_results.py b/ci/jobs/scripts/functional_tests_results.py index 1b038e0eb042..cc3378d7b72f 100755 --- a/ci/jobs/scripts/functional_tests_results.py +++ b/ci/jobs/scripts/functional_tests_results.py @@ -1,11 +1,6 @@ import dataclasses -import json -import os import traceback from typing import List -import re - -import yaml from praktika.result import Result diff --git a/tests/clickhouse-test b/tests/clickhouse-test index f335380ae313..e34de858b0c5 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -131,23 +131,6 @@ def get_broken_tests_rules(broken_tests_file_path: str) -> dict: def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path): matching_rules = [] - known_broken_tests = get_broken_tests_rules(broken_tests_file_path) - - # print(f"Checking known broken tests for failed test: {test_name}") - # print("Potential matching rules:") - exact_rule = known_broken_tests["exact"].get(test_name) - if exact_rule: - # print(f"{test_name} - {exact_rule}") - matching_rules.append(exact_rule) - - for name_re, data in known_broken_tests["pattern"].items(): - if name_re.fullmatch(test_name): - # print(f"{name_re} - {data}") - matching_rules.append(data) - - if not matching_rules: - return False - def matches_substring(substring, log, is_regex): if log is None: return False @@ -155,29 +138,57 @@ def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path return bool(substring.search(log)) return substring in log - for rule_data in matching_rules: - if rule_data.get("check_types") and not any( - ct in build_flags for ct in rule_data["check_types"] - ): - # print( - # f"Check types didn't match: '{rule_data['check_types']}' not in '{build_flags}'" - # ) - continue # check_types didn't match → skip rule - - is_regex = rule_data.get("regex", False) - not_message = rule_data.get("not_message") - if not_message and matches_substring(not_message, test_logs, is_regex): - # print(f"Skip rule: Not message matched: '{rule_data['not_message']}'") - continue # not_message matched → skip rule - message = rule_data.get("message") - if message and not matches_substring(message, test_logs, is_regex): - # print(f"Skip rule: Message didn't match: '{rule_data['message']}'") - continue + broken_tests_log = "ci/tmp/broken_tests_handler.log" + + with open(broken_tests_log, "a") as log_file: + try: + known_broken_tests = get_broken_tests_rules(broken_tests_file_path) + except Exception as e: + log_file.write(f"Error getting broken tests rules: {e}\n") + return False - # print(f"Test {test_name} matched rule: {rule_data}") - return rule_data["reason"] + log_file.write(f"Checking known broken tests for failed test: {test_name}\n") + log_file.write("Potential matching rules:\n") + exact_rule = known_broken_tests["exact"].get(test_name) + if exact_rule: + log_file.write(f"{test_name} - {exact_rule}\n") + matching_rules.append(exact_rule) - return False + for name_re, data in known_broken_tests["pattern"].items(): + if name_re.fullmatch(test_name): + log_file.write(f"{name_re} - {data}\n") + matching_rules.append(data) + + if not matching_rules: + return False + + for rule_data in matching_rules: + if rule_data.get("check_types") and not any( + ct in build_flags for ct in rule_data["check_types"] + ): + log_file.write( + f"Check types didn't match: '{rule_data['check_types']}' not in '{build_flags}'\n" + ) + continue # check_types didn't match → skip rule + + is_regex = rule_data.get("regex", False) + not_message = rule_data.get("not_message") + if not_message and matches_substring(not_message, test_logs, is_regex): + log_file.write( + f"Skip rule: Not message matched: '{rule_data['not_message']}'\n" + ) + continue # not_message matched → skip rule + message = rule_data.get("message") + if message and not matches_substring(message, test_logs, is_regex): + log_file.write( + f"Skip rule: Message didn't match: '{rule_data['message']}'\n" + ) + continue + + log_file.write(f"Test {test_name} matched rule: {rule_data}\n") + return rule_data["reason"] + + return False class SharedEngineReplacer: @@ -2332,8 +2343,8 @@ class TestCase: result.status = TestStatus.BROKEN # Place the message on the second line - result.description.replace( - "\n", f"\nMarked as broken: {known_fail_reason}\n", count=1 + result.description = result.description.replace( + "\n", f"\nMarked as broken: {known_fail_reason}\n", 1 ) if self.name in suite.blacklist_check: From 573a73bf3ae0e1a632cabeceb9be6304ef12bfa9 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:22:11 -0500 Subject: [PATCH 07/10] improve new log messages --- tests/clickhouse-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index e34de858b0c5..0577acb17783 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -167,7 +167,7 @@ def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path ct in build_flags for ct in rule_data["check_types"] ): log_file.write( - f"Check types didn't match: '{rule_data['check_types']}' not in '{build_flags}'\n" + f"Skip rule: Check types didn't match: '{rule_data['check_types']}' not in '{build_flags}'\n" ) continue # check_types didn't match → skip rule @@ -185,7 +185,7 @@ def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path ) continue - log_file.write(f"Test {test_name} matched rule: {rule_data}\n") + log_file.write(f"Matched rule: {rule_data}\n") return rule_data["reason"] return False From 03854b9debfb937d0c062ebdc7b567d027cae4e2 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:30:11 -0500 Subject: [PATCH 08/10] fix crossout: consider both reason and description --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 0577acb17783..35358716dfbe 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2335,7 +2335,7 @@ class TestCase: # Check if the test is known to fail known_fail_reason = test_is_known_fail( self.name, - result.description, + result.reason.value + result.description, args.build_flags, args.known_fails_file_path, ) From 29d97c9b206e6ba227cbbad60337a48cb9e6e839 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 19 Dec 2025 06:25:03 -0500 Subject: [PATCH 09/10] add logging, why is prior not working? --- tests/clickhouse-test | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 35358716dfbe..6c52598e6b3b 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -162,6 +162,8 @@ def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path if not matching_rules: return False + log_file.write(f"First line of test logs: {test_logs.splitlines()[0]}\n") + for rule_data in matching_rules: if rule_data.get("check_types") and not any( ct in build_flags for ct in rule_data["check_types"] From 87758587f2602eb85cd42c1e41c7812ea7e17aa8 Mon Sep 17 00:00:00 2001 From: strtgbb <146047128+strtgbb@users.noreply.github.com> Date: Fri, 19 Dec 2025 21:39:58 -0500 Subject: [PATCH 10/10] A space was missing :) --- tests/clickhouse-test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 6c52598e6b3b..45f9d5719b00 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -2337,7 +2337,7 @@ class TestCase: # Check if the test is known to fail known_fail_reason = test_is_known_fail( self.name, - result.reason.value + result.description, + f"Reason: {result.reason.value} {result.description}", args.build_flags, args.known_fails_file_path, )