Skip to content

Commit 6693d34

Browse files
authored
ci: benchmark tpch1000 with local disk cache (#19081)
1 parent df89407 commit 6693d34

File tree

4 files changed

+91
-19
lines changed

4 files changed

+91
-19
lines changed

.github/actions/benchmark_cloud/action.yml

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ inputs:
2222
size:
2323
description: "Small/Medium/Large"
2424
required: true
25+
cache_size:
26+
description: "Warehouse cache size"
27+
required: false
28+
default: "0"
29+
tries:
30+
description: "Number of attempts per query (1-3)"
31+
required: false
32+
default: ""
2533
version:
2634
description: "Databend version"
2735
required: true
@@ -53,22 +61,30 @@ runs:
5361
run: |
5462
if [[ "${{ inputs.dataset }}" == "load" ]]; then
5563
echo "database=load_test_${{ inputs.run_id }}" >> $GITHUB_OUTPUT
56-
echo "tries=1" >> $GITHUB_OUTPUT
5764
else
5865
database="${{ inputs.database }}"
5966
if [[ -z "$database" ]]; then
6067
database="clickbench"
6168
fi
6269
echo "database=$database" >> $GITHUB_OUTPUT
63-
echo "tries=3" >> $GITHUB_OUTPUT
6470
fi
71+
tries="${{ inputs.tries }}"
72+
if [[ -z "$tries" ]]; then
73+
if [[ "${{ inputs.dataset }}" == "load" ]]; then
74+
tries=1
75+
else
76+
tries=3
77+
fi
78+
fi
79+
echo "tries=$tries" >> $GITHUB_OUTPUT
6580
6681
- name: Run Benchmark
6782
working-directory: benchmark/clickbench
6883
env:
6984
BENCHMARK_ID: ${{ inputs.run_id }}
7085
BENCHMARK_DATASET: ${{ inputs.dataset }}
7186
BENCHMARK_SIZE: ${{ inputs.size }}
87+
BENCHMARK_CACHE_SIZE: ${{ inputs.cache_size }}
7288
BENCHMARK_VERSION: ${{ inputs.version }}
7389
BENCHMARK_DATABASE: ${{ steps.prepare.outputs.database }}
7490
BENCHMARK_TRIES: ${{ steps.prepare.outputs.tries }}
@@ -86,10 +102,10 @@ runs:
86102
- name: Upload artifact
87103
uses: actions/upload-artifact@v4
88104
with:
89-
name: benchmark-${{ inputs.dataset }}-${{ inputs.size }}
105+
name: benchmark-${{ inputs.dataset }}-${{ inputs.size }}-cache-${{ inputs.cache_size }}
90106
path: |
91-
benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}.json
92-
benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-*.ndjson
107+
benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-cache-${{ inputs.cache_size }}.json
108+
benchmark/clickbench/result-${{ inputs.dataset }}-cloud-${{ inputs.size }}-cache-${{ inputs.cache_size }}-*.ndjson
93109
94110
- name: Remove warehouse
95111
if: always()

.github/workflows/reuse.benchmark.yml

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ jobs:
115115
source: ${{ inputs.source }}
116116
source_id: ${{ inputs.source_id }}
117117
size: Small
118+
cache_size: "0"
119+
tries: 1
118120
version: ${{ inputs.version }}
119121
cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
120122
cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}
@@ -137,11 +139,12 @@ jobs:
137139
strategy:
138140
matrix:
139141
include:
140-
- { dataset: hits, size: Small, database: hits, timeout: 10 }
141-
- { dataset: hits, size: Large, database: hits, timeout: 10 }
142-
- { dataset: tpch100, size: Small, database: tpch_100, timeout: 20 }
143-
- { dataset: tpch100, size: Large, database: tpch_100, timeout: 20 }
144-
- { dataset: tpch1000, size: Large, database: tpch_1000, timeout: 60 }
142+
- { dataset: hits, size: Small, database: hits, timeout: 10, cache_size: 0, tries: 3 }
143+
- { dataset: hits, size: Large, database: hits, timeout: 10, cache_size: 0, tries: 3 }
144+
- { dataset: tpch100, size: Small, database: tpch_100, timeout: 20, cache_size: 0, tries: 3 }
145+
- { dataset: tpch100, size: Large, database: tpch_100, timeout: 20, cache_size: 0, tries: 3 }
146+
- { dataset: tpch1000, size: Large, database: tpch_1000, timeout: 60, cache_size: 0, tries: 1 }
147+
- { dataset: tpch1000, size: Large, database: tpch_1000, timeout: 60, cache_size: 300, tries: 1 }
145148
fail-fast: true
146149
max-parallel: 1
147150
steps:
@@ -165,6 +168,8 @@ jobs:
165168
source: ${{ inputs.source }}
166169
source_id: ${{ inputs.source_id }}
167170
size: ${{ matrix.size }}
171+
cache_size: ${{ matrix.cache_size }}
172+
tries: ${{ matrix.tries }}
168173
version: ${{ inputs.version }}
169174
cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
170175
cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}

benchmark/clickbench/benchmark_cloud.py

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ class ResultRecord:
5252
size: str
5353
tries: int
5454
storage: str
55+
cache_size: str
5556
load_time: Optional[float] = None
5657
data_size: Optional[int] = None
5758
system: Optional[str] = None
@@ -120,8 +121,8 @@ def load_config() -> BenchmarkConfig:
120121
logger.error("BENCHMARK_TRIES must be an integer, got %s", tries_raw)
121122
sys.exit(1)
122123

123-
if tries < 1:
124-
logger.error("BENCHMARK_TRIES must be positive, got %s", tries)
124+
if not 1 <= tries <= 3:
125+
logger.error("BENCHMARK_TRIES must be between 1 and 3, got %s", tries)
125126
sys.exit(1)
126127

127128
user = os.environ.get("CLOUD_USER", "")
@@ -234,11 +235,19 @@ def run_timed_query(runner: BendSQLRunner, sql: str) -> Optional[float]:
234235
return parse_time_output(output)
235236

236237

238+
def pad_attempts(values: List[float], target_size: int) -> None:
239+
if not values:
240+
return
241+
while len(values) < target_size:
242+
values.append(values[-1])
243+
244+
237245
def write_result_files(script_dir: Path, record: ResultRecord) -> None:
238246
result_path = script_dir / "result.json"
239-
final_result_path = script_dir / f"result-{record.dataset}-cloud-{record.size}.json"
247+
cache_suffix = f"-cache-{record.cache_size}" if record.cache_size else ""
248+
final_result_path = script_dir / f"result-{record.dataset}-cloud-{record.size}{cache_suffix}.json"
240249
ndjson_name = (
241-
f"result-{record.dataset}-cloud-{record.size}"
250+
f"result-{record.dataset}-cloud-{record.size}{cache_suffix}"
242251
f"-{record.run_id}.ndjson"
243252
)
244253
ndjson_path = script_dir / ndjson_name
@@ -272,7 +281,7 @@ def main() -> None:
272281
sys.exit(1)
273282

274283
run_date = datetime.now(timezone.utc).strftime("%Y-%m-%d")
275-
tags = ["s3"]
284+
tags = ["s3", f"cache-{config.cache_size}"]
276285
cluster_size = SIZE_MAPPING[config.size]["cluster_size"]
277286
machine = SIZE_MAPPING[config.size]["machine"]
278287
system: Optional[str] = None
@@ -306,6 +315,7 @@ def main() -> None:
306315
size=config.size,
307316
tries=config.tries,
308317
storage="s3",
318+
cache_size=config.cache_size,
309319
system=system,
310320
comment=comment,
311321
)
@@ -369,6 +379,9 @@ def main() -> None:
369379
logger.info("Q%s[%s] succeeded in %.3f seconds", query_num, attempt, q_time)
370380
record.result[query_num].append(round(q_time, 3))
371381
record.values[f"Q{query_num}"].append(round(q_time, 3))
382+
if config.tries < 3:
383+
pad_attempts(record.result[query_num], 3)
384+
pad_attempts(record.values[f"Q{query_num}"], 3)
372385

373386
cleanup_runner = BendSQLRunner()
374387
cleanup_runner.set_dsn(build_dsn(config, warehouse="default", login_disable=True))

benchmark/clickbench/index.jinja

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,12 @@
334334
<a id="select-all-cluster-sizes" class="selector selector-active">All</a>
335335
</td>
336336
</tr>
337+
<tr>
338+
<th>Cache size: </th>
339+
<td id="selectors_cache_size">
340+
<a id="select-all-cache-sizes" class="selector selector-active">All</a>
341+
</td>
342+
</tr>
337343
<tr>
338344
<th>Metric: </th>
339345
<td id="selectors_run">
@@ -386,10 +392,21 @@
386392
"type": {},
387393
"machine": {},
388394
"cluster_size": {},
395+
"cache_size": {},
389396
"metric": "hot",
390397
"queries": [],
391398
};
392399
400+
function normalizeSelectors() {
401+
selectors.system = selectors.system || {};
402+
selectors.type = selectors.type || {};
403+
selectors.machine = selectors.machine || {};
404+
selectors.cluster_size = selectors.cluster_size || {};
405+
selectors.cache_size = selectors.cache_size || {};
406+
selectors.metric = selectors.metric || 'hot';
407+
selectors.queries = selectors.queries || [];
408+
}
409+
393410
let theme = 'light';
394411
395412
function setTheme(new_theme) {
@@ -413,8 +430,16 @@
413430
let types = document.getElementById('selectors_type');
414431
let machines = document.getElementById('selectors_machine');
415432
let cluster_sizes = document.getElementById('selectors_cluster_size');
433+
let cache_sizes = document.getElementById('selectors_cache_size');
416434
417435
let unique_systems = [... new Set(data.map(elem => elem.system))];
436+
const default_cache_size = 'n/a';
437+
const getCacheLabel = entry => {
438+
if (entry.cache_size === undefined || entry.cache_size === null || entry.cache_size === '') {
439+
return default_cache_size;
440+
}
441+
return entry.cache_size.toString();
442+
};
418443
419444
function toggle(e, elem, selectors_map) {
420445
selectors_map[elem] = !selectors_map[elem];
@@ -483,11 +508,20 @@
483508
if (!(elem in selectors.cluster_size)) { selectors.cluster_size[elem] = true; }
484509
selector.addEventListener('click', e => toggle(e, elem, selectors.cluster_size));
485510
});
511+
[... new Set(data.map(getCacheLabel))].map(elem => {
512+
let selector = document.createElement('a');
513+
selector.className = 'selector selector-active';
514+
selector.appendChild(document.createTextNode(elem));
515+
cache_sizes.appendChild(selector);
516+
if (!(elem in selectors.cache_size)) { selectors.cache_size[elem] = true; }
517+
selector.addEventListener('click', e => toggle(e, elem, selectors.cache_size));
518+
});
486519
487520
document.getElementById('select-all-systems').addEventListener('click', e => toggleAll(e, selectors.system));
488521
document.getElementById('select-all-types').addEventListener('click', e => toggleAll(e, selectors.type));
489522
document.getElementById('select-all-machines').addEventListener('click', e => toggleAll(e, selectors.machine));
490523
document.getElementById('select-all-cluster-sizes').addEventListener('click', e => toggleAll(e, selectors.cluster_size));
524+
document.getElementById('select-all-cache-sizes').addEventListener('click', e => toggleAll(e, selectors.cache_size));
491525
492526
[...document.getElementById('selectors_run').querySelectorAll('a')].map(elem => elem.addEventListener('click', e => {
493527
[...e.target.parentElement.querySelectorAll('a')].map(elem => { elem.className = elem == e.target ? 'selector selector-active' : 'selector' });
@@ -505,6 +539,7 @@
505539
[...types.childNodes].map(elem => { elem.className = selectors.type[elem.innerText] ? 'selector selector-active' : 'selector' });
506540
[...machines.childNodes].map(elem => { elem.className = selectors.machine[elem.innerText] ? 'selector selector-active' : 'selector' });
507541
[...cluster_sizes.childNodes].map(elem => { elem.className = selectors.cluster_size[elem.innerText] ? 'selector selector-active' : 'selector' });
542+
[...cache_sizes.childNodes].map(elem => { elem.className = selectors.cache_size[elem.innerText] ? 'selector selector-active' : 'selector' });
508543
509544
[...document.getElementById('selectors_run').querySelectorAll('a')].map(elem => {
510545
elem.className = elem.id == 'selector-metric-' + selectors.metric ? 'selector selector-active' : 'selector'
@@ -713,10 +748,11 @@
713748
clearElement(details_body);
714749
715750
let filtered_data = data.filter(elem =>
716-
selectors.system[elem.system] &&
717-
selectors.machine[elem.machine] &&
718-
selectors.cluster_size[elem.cluster_size] &&
719-
elem.tags.filter(type => selectors.type[type]).length > 0);
751+
selectors.system[elem.system] &&
752+
selectors.machine[elem.machine] &&
753+
selectors.cluster_size[elem.cluster_size] &&
754+
selectors.cache_size[getCacheLabel(elem)] &&
755+
elem.tags.filter(type => selectors.type[type]).length > 0);
720756
721757
let nothing_selected_elem = document.getElementById('nothing-selected');
722758
if (filtered_data.length == 0) {
@@ -868,13 +904,15 @@
868904
window.onpopstate = function (event) {
869905
if (!event.state) { return; }
870906
selectors = event.state;
907+
normalizeSelectors();
871908
render();
872909
updateSelectors();
873910
};
874911
875912
if (window.location.hash) {
876913
try {
877914
selectors = JSON.parse(atob(window.location.hash.substring(1)));
915+
normalizeSelectors();
878916
} catch { }
879917
}
880918

0 commit comments

Comments
 (0)