diff --git a/ci/jobs/functional_tests.py b/ci/jobs/functional_tests.py index 02d148e2bd5f..d34cf0b8d579 100644 --- a/ci/jobs/functional_tests.py +++ b/ci/jobs/functional_tests.py @@ -354,6 +354,8 @@ def start(): ) res = results[-1].is_ok() + runner_options += f" --known-fails-file-path tests/broken_tests.yaml" + test_result = None if res and JobStages.TEST in stages: stop_watch_ = Utils.Stopwatch() @@ -447,6 +449,10 @@ def collect_logs(): ) force_ok_exit = True + broken_tests_handler_log = os.path.join(temp_dir, "broken_tests_handler.log") + if os.path.exists(broken_tests_handler_log): + debug_files.append(broken_tests_handler_log) + Result.create_from( results=results, stopwatch=stop_watch, diff --git a/ci/jobs/scripts/functional_tests_results.py b/ci/jobs/scripts/functional_tests_results.py index eb62b5089683..cc3378d7b72f 100755 --- a/ci/jobs/scripts/functional_tests_results.py +++ b/ci/jobs/scripts/functional_tests_results.py @@ -1,11 +1,6 @@ import dataclasses -import json -import os import traceback from typing import List -import re - -import yaml from praktika.result import Result @@ -14,6 +9,7 @@ TIMEOUT_SIGN = "[ Timeout! " UNKNOWN_SIGN = "[ UNKNOWN " SKIPPED_SIGN = "[ SKIPPED " +BROKEN_SIGN = "[ BROKEN " HUNG_SIGN = "Found hung queries in processlist" SERVER_DIED_SIGN = "Server died, terminating all processes" SERVER_DIED_SIGN2 = "Server does not respond to health check" @@ -33,101 +29,6 @@ # out.writerow(status) -def get_broken_tests_rules() -> dict: - broken_tests_file_path = "tests/broken_tests.yaml" - if ( - not os.path.isfile(broken_tests_file_path) - or os.path.getsize(broken_tests_file_path) == 0 - ): - raise ValueError( - "There is something wrong with getting broken tests rules: " - f"file '{broken_tests_file_path}' is empty or does not exist." - ) - - with open(broken_tests_file_path, "r", encoding="utf-8") as broken_tests_file: - broken_tests = yaml.safe_load(broken_tests_file) - - compiled_rules = {"exact": {}, "pattern": {}} - - for test in broken_tests: - regex = test.get("regex") is True - rule = { - "reason": test["reason"], - } - - if test.get("message"): - rule["message"] = re.compile(test["message"]) if regex else test["message"] - - if test.get("not_message"): - rule["not_message"] = ( - re.compile(test["not_message"]) if regex else test["not_message"] - ) - if test.get("check_types"): - rule["check_types"] = test["check_types"] - - if regex: - rule["regex"] = True - compiled_rules["pattern"][re.compile(test["name"])] = rule - else: - compiled_rules["exact"][test["name"]] = rule - - print( - f"INFO: Compiled {len(compiled_rules['exact'])} exact rules and {len(compiled_rules['pattern'])} pattern rules" - ) - - return compiled_rules - - -def test_is_known_fail(test_name, test_logs, known_broken_tests, test_options_string): - matching_rules = [] - - print(f"Checking known broken tests for failed test: {test_name}") - print("Potential matching rules:") - exact_rule = known_broken_tests["exact"].get(test_name) - if exact_rule: - print(f"{test_name} - {exact_rule}") - matching_rules.append(exact_rule) - - for name_re, data in known_broken_tests["pattern"].items(): - if name_re.fullmatch(test_name): - print(f"{name_re} - {data}") - matching_rules.append(data) - - if not matching_rules: - return False - - def matches_substring(substring, log, is_regex): - if log is None: - return False - if is_regex: - return bool(substring.search(log)) - return substring in log - - for rule_data in matching_rules: - if rule_data.get("check_types") and not any( - ct in test_options_string for ct in rule_data["check_types"] - ): - print( - f"Check types didn't match: '{rule_data['check_types']}' not in '{test_options_string}'" - ) - continue # check_types didn't match → skip rule - - is_regex = rule_data.get("regex", False) - not_message = rule_data.get("not_message") - if not_message and matches_substring(not_message, test_logs, is_regex): - print(f"Skip rule: Not message matched: '{rule_data['not_message']}'") - continue # not_message matched → skip rule - message = rule_data.get("message") - if message and not matches_substring(message, test_logs, is_regex): - print(f"Skip rule: Message didn't match: '{rule_data['message']}'") - continue - - print(f"Test {test_name} matched rule: {rule_data}") - return rule_data["reason"] - - return False - - class FTResultsProcessor: @dataclasses.dataclass class Summary: @@ -163,8 +64,6 @@ def _process_test_output(self): test_results = [] test_end = True - known_broken_tests = get_broken_tests_rules() - with open(self.tests_output_file, "r", encoding="utf-8") as test_file: for line in test_file: original_line = line @@ -183,7 +82,13 @@ def _process_test_output(self): retries = True if any( sign in line - for sign in (OK_SIGN, FAIL_SIGN, UNKNOWN_SIGN, SKIPPED_SIGN) + for sign in ( + OK_SIGN, + FAIL_SIGN, + UNKNOWN_SIGN, + SKIPPED_SIGN, + BROKEN_SIGN, + ) ): test_name = line.split(" ")[2].split(":")[0] @@ -216,13 +121,16 @@ def _process_test_output(self): elif SKIPPED_SIGN in line: skipped += 1 test_results.append((test_name, "SKIPPED", test_time, [])) + elif BROKEN_SIGN in line: + broken += 1 + test_results.append((test_name, "BROKEN", test_time, [])) else: success += int(OK_SIGN in line) test_results.append((test_name, "OK", test_time, [])) test_end = False elif ( len(test_results) > 0 - and test_results[-1][1] in ("FAIL", "SKIPPED") + and test_results[-1][1] in ("FAIL", "SKIPPED", "BROKEN") and not test_end ): test_results[-1][3].append(original_line) @@ -233,8 +141,6 @@ def _process_test_output(self): if DATABASE_SIGN in line: test_end = True - test_options_string = ", ".join(self.test_options) - test_results_ = [] for test in test_results: try: @@ -248,21 +154,6 @@ def _process_test_output(self): ) ) - if test[1] == "FAIL": - broken_message = test_is_known_fail( - test[0], - test_results_[-1].info, - known_broken_tests, - test_options_string, - ) - - if broken_message: - broken += 1 - failed -= 1 - test_results_[-1].set_status(Result.StatusExtended.BROKEN) - test_results_[-1].set_label(Result.Label.BROKEN) - test_results_[-1].info += "\nMarked as broken: " + broken_message - except Exception as e: print(f"ERROR: Failed to parse test results: {test}") traceback.print_exc() diff --git a/tests/broken_tests.yaml b/tests/broken_tests.yaml index cda3d3091d8c..8778f72a7be0 100644 --- a/tests/broken_tests.yaml +++ b/tests/broken_tests.yaml @@ -57,9 +57,27 @@ message: result differs with reference check_types: - msan +- name: 02313_filesystem_cache_seeks + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02286_drop_filesystem_cache + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02242_system_filesystem_cache_log_table + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02241_filesystem_cache_on_write_operations + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02240_system_filesystem_cache_table + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' +- name: 02226_filesystem_cache_profile_events + reason: fails when azure storage is not set up + message: 'DB::Exception: Unknown storage policy `azure_cache`' - name: 00024_random_counters reason: INVESTIGATE - random timeout - message: Timeout! Killing process group + message: Timeout! Processes left in process group - name: test_storage_s3_queue/test_5.py::test_migration[1-s3queue_] reason: KNOWN - Sometimes fails due to test order message: 'Failed: Timeout >900.0s' diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 077abaa0cdb6..45f9d5719b00 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -11,6 +11,10 @@ import copy import enum import glob +# For processing the broken tests rules +from functools import lru_cache +import yaml + # Not requests, to avoid requiring extra dependency. import http.client import io @@ -83,6 +87,112 @@ VERSION_PATTERN = r"^((\d+\.)?(\d+\.)?(\d+\.)?\d+)$" TEST_MAX_RUN_TIME_IN_SECONDS = 180 +@lru_cache(maxsize=1) +def get_broken_tests_rules(broken_tests_file_path: str) -> dict: + if ( + not os.path.isfile(broken_tests_file_path) + or os.path.getsize(broken_tests_file_path) == 0 + ): + raise ValueError( + "There is something wrong with getting broken tests rules: " + f"file '{broken_tests_file_path}' is empty or does not exist." + ) + + with open(broken_tests_file_path, "r", encoding="utf-8") as broken_tests_file: + broken_tests = yaml.safe_load(broken_tests_file) + + compiled_rules = {"exact": {}, "pattern": {}} + + for test in broken_tests: + regex = test.get("regex") is True + rule = { + "reason": test["reason"], + } + + if test.get("message"): + rule["message"] = re.compile(test["message"]) if regex else test["message"] + + if test.get("not_message"): + rule["not_message"] = ( + re.compile(test["not_message"]) if regex else test["not_message"] + ) + if test.get("check_types"): + rule["check_types"] = test["check_types"] + + if regex: + rule["regex"] = True + compiled_rules["pattern"][re.compile(test["name"])] = rule + else: + compiled_rules["exact"][test["name"]] = rule + + return compiled_rules + + +def test_is_known_fail(test_name, test_logs, build_flags, broken_tests_file_path): + matching_rules = [] + + def matches_substring(substring, log, is_regex): + if log is None: + return False + if is_regex: + return bool(substring.search(log)) + return substring in log + + broken_tests_log = "ci/tmp/broken_tests_handler.log" + + with open(broken_tests_log, "a") as log_file: + try: + known_broken_tests = get_broken_tests_rules(broken_tests_file_path) + except Exception as e: + log_file.write(f"Error getting broken tests rules: {e}\n") + return False + + log_file.write(f"Checking known broken tests for failed test: {test_name}\n") + log_file.write("Potential matching rules:\n") + exact_rule = known_broken_tests["exact"].get(test_name) + if exact_rule: + log_file.write(f"{test_name} - {exact_rule}\n") + matching_rules.append(exact_rule) + + for name_re, data in known_broken_tests["pattern"].items(): + if name_re.fullmatch(test_name): + log_file.write(f"{name_re} - {data}\n") + matching_rules.append(data) + + if not matching_rules: + return False + + log_file.write(f"First line of test logs: {test_logs.splitlines()[0]}\n") + + for rule_data in matching_rules: + if rule_data.get("check_types") and not any( + ct in build_flags for ct in rule_data["check_types"] + ): + log_file.write( + f"Skip rule: Check types didn't match: '{rule_data['check_types']}' not in '{build_flags}'\n" + ) + continue # check_types didn't match → skip rule + + is_regex = rule_data.get("regex", False) + not_message = rule_data.get("not_message") + if not_message and matches_substring(not_message, test_logs, is_regex): + log_file.write( + f"Skip rule: Not message matched: '{rule_data['not_message']}'\n" + ) + continue # not_message matched → skip rule + message = rule_data.get("message") + if message and not matches_substring(message, test_logs, is_regex): + log_file.write( + f"Skip rule: Message didn't match: '{rule_data['message']}'\n" + ) + continue + + log_file.write(f"Matched rule: {rule_data}\n") + return rule_data["reason"] + + return False + + class SharedEngineReplacer: SPECIALIZED_ENGINES = "Collapsing|VersionedCollapsing|Summing|Replacing|Aggregating" ENGINES_NON_REPLICATED_REGEXP = ( @@ -918,6 +1028,7 @@ class TestStatus(enum.Enum): OK = "OK" SKIPPED = "SKIPPED" NOT_FAILED = "NOT_FAILED" + BROKEN = "BROKEN" class FailureReason(enum.Enum): @@ -2222,6 +2333,22 @@ class TestCase: if result.status == TestStatus.FAIL: result.description = self.add_info_about_settings(result.description) + if args.known_fails_file_path: + # Check if the test is known to fail + known_fail_reason = test_is_known_fail( + self.name, + f"Reason: {result.reason.value} {result.description}", + args.build_flags, + args.known_fails_file_path, + ) + if known_fail_reason: + result.status = TestStatus.BROKEN + + # Place the message on the second line + result.description = result.description.replace( + "\n", f"\nMarked as broken: {known_fail_reason}\n", 1 + ) + if self.name in suite.blacklist_check: if result.status == TestStatus.OK: result.status = TestStatus.NOT_FAILED @@ -2726,6 +2853,11 @@ def run_tests_array( + colored(" NOT_FAILED ", args, "red", attrs=["bold"]) + CL_SQUARE_BRACKET ) + MSG_BROKEN = ( + OP_SQUARE_BRACKET + + colored(" BROKEN ", args, "cyan", attrs=["bold"]) + + CL_SQUARE_BRACKET + ) MESSAGES = { TestStatus.FAIL: MSG_FAIL, @@ -2733,12 +2865,14 @@ def run_tests_array( TestStatus.OK: MSG_OK, TestStatus.SKIPPED: MSG_SKIPPED, TestStatus.NOT_FAILED: MSG_NOT_FAILED, + TestStatus.BROKEN: MSG_BROKEN, } passed_total = 0 skipped_total = 0 failures_total = 0 failures_chain = 0 + broken_total = 0 start_time = datetime.now() client_options = get_additional_client_options(args) @@ -2833,6 +2967,8 @@ def run_tests_array( raise ServerDied("Server died") elif test_result.status == TestStatus.SKIPPED: skipped_total += 1 + elif test_result.status == TestStatus.BROKEN: + broken_total += 1 except KeyboardInterrupt as e: print(colored("Break tests execution", args, "red")) @@ -2848,6 +2984,7 @@ def run_tests_array( colored( f"\nHaving {failures_total} errors! {passed_total} tests passed." f" {skipped_total} tests skipped." + f" {broken_total} broken tests." f" {(datetime.now() - start_time).total_seconds():.2f} s elapsed" f" ({multiprocessing.current_process().name}).", args, @@ -3701,6 +3838,8 @@ def parse_args(): parser.add_argument("-q", "--queries", help="Path to queries dir") parser.add_argument("--tmp", help="Path to tmp dir") + parser.add_argument("--known-fails-file-path", help="Path to known fails file") + parser.add_argument( "-b", "--binary",