Skip to content

Commit b4d980c

Browse files
TyriarTyriar
authored
Merge pull request microsoft#273081 from microsoft/tyriar/272780
Add tree sitter command parser tests
2 parents 70c31a4 + f649b7c commit b4d980c

File tree

1 file changed

+196
-0
lines changed

1 file changed

+196
-0
lines changed

‎src/vs/workbench/contrib/terminalContrib/chatAgentTools/test/electron-browser/treeSitterCommandParser.test.ts‎

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
/*---------------------------------------------------------------------------------------------
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for license information.
4+
*--------------------------------------------------------------------------------------------*/
5+
6+
import { deepStrictEqual } from 'assert';
7+
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../../base/test/common/utils.js';
8+
import type { TestInstantiationService } from '../../../../../../platform/instantiation/test/common/instantiationServiceMock.js';
9+
import { workbenchInstantiationService } from '../../../../../test/browser/workbenchTestServices.js';
10+
import { ITreeSitterLibraryService } from '../../../../../../editor/common/services/treeSitter/treeSitterLibraryService.js';
11+
import { TreeSitterLibraryService } from '../../../../../services/treeSitter/browser/treeSitterLibraryService.js';
12+
import { FileService } from '../../../../../../platform/files/common/fileService.js';
13+
import { NullLogService } from '../../../../../../platform/log/common/log.js';
14+
import { Schemas } from '../../../../../../base/common/network.js';
15+
import { TestIPCFileSystemProvider } from '../../../../../test/electron-browser/workbenchTestServices.js';
16+
import { TreeSitterCommandParser, TreeSitterCommandParserLanguage } from '../../browser/treeSitterCommandParser.js';
17+
18+
suite('TreeSitterCommandParser', () => {
19+
const store = ensureNoDisposablesAreLeakedInTestSuite();
20+
21+
let instantiationService: TestInstantiationService;
22+
let parser: TreeSitterCommandParser;
23+
24+
setup(() => {
25+
const logService = new NullLogService();
26+
const fileService = store.add(new FileService(logService));
27+
const fileSystemProvider = new TestIPCFileSystemProvider();
28+
store.add(fileService.registerProvider(Schemas.file, fileSystemProvider));
29+
30+
instantiationService = workbenchInstantiationService({
31+
fileService: () => fileService,
32+
}, store);
33+
34+
const treeSitterLibraryService = store.add(instantiationService.createInstance(TreeSitterLibraryService));
35+
treeSitterLibraryService.isTest = true;
36+
instantiationService.stub(ITreeSitterLibraryService, treeSitterLibraryService);
37+
38+
parser = instantiationService.createInstance(TreeSitterCommandParser);
39+
});
40+
41+
suite('Bash command parsing', () => {
42+
async function t(commandLine: string, expectedCommands: string[]) {
43+
const result = await parser.extractSubCommands(TreeSitterCommandParserLanguage.Bash, commandLine);
44+
deepStrictEqual(result, expectedCommands);
45+
}
46+
47+
test('simple commands', () => t('ls -la', ['ls -la']));
48+
test('commands with &&', () => t('echo hello && ls -la', ['echo hello', 'ls -la']));
49+
test('commands with ||', () => t('test -f file.txt || touch file.txt', ['test -f file.txt', 'touch file.txt']));
50+
test('commands with semicolons', () => t('cd /tmp; ls; pwd', ['cd /tmp', 'ls', 'pwd']));
51+
test('pipe chains', () => t('cat file.txt | grep pattern | sort | uniq', ['cat file.txt', 'grep pattern', 'sort', 'uniq']));
52+
test('commands with subshells', () => t('echo $(date +%Y) && ls', ['echo $(date +%Y)', 'date +%Y', 'ls']));
53+
test('complex quoting', () => t('echo "hello && world" && echo \'test\'', ['echo "hello && world"', 'echo \'test\'']));
54+
test('escaped characters', () => t('echo hello\\ world && ls', ['echo hello\\ world', 'ls']));
55+
test('background commands', () => t('sleep 10 & echo done', ['sleep 10', 'echo done']));
56+
test('variable assignments', () => t('VAR=value command1 && echo $VAR', ['VAR=value command1', 'echo $VAR']));
57+
test('redirections', () => t('echo hello > file.txt && cat < file.txt', ['echo hello', 'cat']));
58+
test('arithmetic expansion', () => t('echo $((1 + 2)) && ls', ['echo $((1 + 2))', 'ls']));
59+
test('nested command substitution', () => t('echo $(cat $(echo file.txt)) && ls', ['echo $(cat $(echo file.txt))', 'cat $(echo file.txt)', 'echo file.txt', 'ls']));
60+
test('mixed operators', () => t('cmd1 && cmd2 || cmd3; cmd4 | cmd5 & cmd6', ['cmd1', 'cmd2', 'cmd3', 'cmd4', 'cmd5', 'cmd6']));
61+
test('parameter expansion', () => t('echo ${VAR:-default} && echo ${#VAR}', ['echo ${VAR:-default}', 'echo ${#VAR}']));
62+
test('process substitution', () => t('diff <(sort file1) <(sort file2) && echo done', ['diff <(sort file1) <(sort file2)', 'sort file1', 'sort file2', 'echo done']));
63+
test('brace expansion', () => t('echo {a,b,c}.txt && ls', ['echo {a,b,c}.txt', 'ls']));
64+
test('tilde expansion', () => t('cd ~/Documents && ls ~/.bashrc', ['cd ~/Documents', 'ls ~/.bashrc']));
65+
66+
suite('control flow and structures', () => {
67+
test('if-then-else', () => t('if [ -f file.txt ]; then cat file.txt; else echo "not found"; fi', ['cat file.txt', 'echo "not found"']));
68+
test('simple iteration', () => t('for file in *.txt; do cat "$file"; done', ['cat "$file"']));
69+
test('function declaration and call', () => t('function test_func() { echo "inside function"; } && test_func', ['echo "inside function"', 'test_func']));
70+
test('heredoc with commands', () => t('cat << EOF\nhello\nworld\nEOF\necho done', ['cat', 'echo done']));
71+
test('while loop', () => t('while read line; do echo "$line"; done < file.txt', ['read line', 'echo "$line"']));
72+
test('case statement', () => t('case $var in pattern1) echo "match1" ;; pattern2) echo "match2" ;; esac', ['echo "match1"', 'echo "match2"']));
73+
test('until loop', () => t('until [ -f ready.txt ]; do sleep 1; done && echo ready', ['sleep 1', 'echo ready']));
74+
test('nested conditionals', () => t('if [ -f file ]; then if [ -r file ]; then cat file; fi; fi', ['cat file']));
75+
test('test command alternatives', () => t('[[ -f file ]] && cat file || echo missing', ['cat file', 'echo missing']));
76+
});
77+
78+
suite('edge cases', () => {
79+
test('malformed syntax', () => t('echo "unclosed quote && ls', ['echo']));
80+
test('unmatched parentheses', () => t('echo $(missing closing && ls', ['echo $(missing closing && ls', 'missing closing', 'ls']));
81+
test('very long command lines', () => t('echo ' + 'a'.repeat(10000) + ' && ls', ['echo ' + 'a'.repeat(10000), 'ls']));
82+
test('special characters', () => t('echo "πλαςε 测试 🚀" && ls', ['echo "πλαςε 测试 🚀"', 'ls']));
83+
test('multiline with continuations', () => t('echo hello \\\n&& echo world \\\n&& ls', ['echo hello', 'echo world', 'ls']));
84+
test('commands with comments', () => t('echo hello # this is a comment\nls # another comment', ['echo hello', 'ls']));
85+
test('empty command in chain', () => t('echo hello && && echo world', ['echo hello', 'echo world']));
86+
test('trailing operators', () => t('echo hello &&', ['echo hello', '']));
87+
test('only operators', () => t('&& || ;', []));
88+
test('nested quotes', () => t('echo "outer \"inner\" outer" && ls', ['echo "outer \"inner\" outer"', 'ls']));
89+
test('incomplete escape sequences', () => t('echo hello\\ && ls', ['echo hello\\ ', 'ls']));
90+
test('mixed quote types', () => t('echo "hello \`world\`" && echo \'test\'', ['echo "hello \`world\`"', 'world', 'echo \'test\'']));
91+
test('deeply nested structures', () => t('echo $(echo $(echo $(echo nested))) && ls', ['echo $(echo $(echo $(echo nested)))', 'echo $(echo $(echo nested))', 'echo $(echo nested)', 'echo nested', 'ls']));
92+
test('unicode command names', () => t('测试命令 && echo done', ['测试命令', 'echo done']));
93+
});
94+
95+
// TODO: These should be common but the pwsh grammar doesn't handle && yet https://github.com/microsoft/vscode/issues/272704
96+
suite('real-world scenarios', () => {
97+
test('complex Docker commands', () => t('docker run -it --rm -v $(pwd):/app ubuntu:latest bash -c "cd /app && npm install && npm test"', ['docker run -it --rm -v $(pwd):/app ubuntu:latest bash -c "cd /app && npm install && npm test"', 'pwd']));
98+
test('Git workflow commands', () => t('git add . && git commit -m "Update feature" && git push origin main', [
99+
'git add .',
100+
'git commit -m "Update feature"',
101+
'git push origin main'
102+
]));
103+
test('npm/yarn workflow commands', () => t('npm ci && npm run build && npm test && npm run lint', [
104+
'npm ci',
105+
'npm run build',
106+
'npm test',
107+
'npm run lint'
108+
]));
109+
test('build system commands', () => t('make clean && make -j$(nproc) && make install PREFIX=/usr/local', [
110+
'make clean',
111+
'make -j$(nproc)',
112+
'nproc',
113+
'make install PREFIX=/usr/local'
114+
]));
115+
test('deployment script', () => t('rsync -avz --delete src/ user@server:/path/ && ssh user@server "systemctl restart service" && echo "Deployed successfully"', [
116+
'rsync -avz --delete src/ user@server:/path/',
117+
'ssh user@server "systemctl restart service"',
118+
'echo "Deployed successfully"'
119+
]));
120+
test('database backup script', () => t('mysqldump -u user -p database > backup_$(date +%Y%m%d).sql && gzip backup_$(date +%Y%m%d).sql && echo "Backup complete"', [
121+
'mysqldump -u user -p database',
122+
'date +%Y%m%d',
123+
'gzip backup_$(date +%Y%m%d).sql',
124+
'date +%Y%m%d',
125+
'echo "Backup complete"'
126+
]));
127+
test('log analysis pipeline', () => t('tail -f /var/log/app.log | grep ERROR | while read line; do echo "$(date): $line" >> error.log; done', [
128+
'tail -f /var/log/app.log',
129+
'grep ERROR',
130+
'read line',
131+
'echo "$(date): $line"',
132+
'date'
133+
]));
134+
test('conditional installation', () => t('which docker || (curl -fsSL https://get.docker.com | sh && systemctl enable docker) && docker --version', [
135+
'which docker',
136+
'curl -fsSL https://get.docker.com',
137+
'sh',
138+
'systemctl enable docker',
139+
'docker --version'
140+
]));
141+
});
142+
});
143+
144+
suite('PowerShell command parsing', () => {
145+
async function t(commandLine: string, expectedCommands: string[]) {
146+
const result = await parser.extractSubCommands(TreeSitterCommandParserLanguage.PowerShell, commandLine);
147+
deepStrictEqual(result, expectedCommands);
148+
}
149+
150+
test('simple commands', () => t('Get-ChildItem -Path C:\\', ['Get-ChildItem -Path C:\\']));
151+
test('commands with semicolons', () => t('Get-Date; Get-Location; Write-Host "done"', ['Get-Date', 'Get-Location', 'Write-Host "done"']));
152+
test('pipeline commands', () => t('Get-Process | Where-Object {$_.CPU -gt 100} | Sort-Object CPU', ['Get-Process ', 'Where-Object {$_.CPU -gt 100} ', 'Sort-Object CPU']));
153+
test('command substitution', () => t('Write-Host $(Get-Date) ; Get-Location', ['Write-Host $(Get-Date)', 'Get-Date', 'Get-Location']));
154+
test('complex parameters', () => t('Get-ChildItem -Path "C:\\Program Files" -Recurse -Include "*.exe"', ['Get-ChildItem -Path "C:\\Program Files" -Recurse -Include "*.exe"']));
155+
test('splatting', () => t('$params = @{Path="C:\\"; Recurse=$true}; Get-ChildItem @params', ['Get-ChildItem @params']));
156+
test('here-strings', () => t('Write-Host @"\nhello\nworld\n"@ ; Get-Date', ['Write-Host @"\nhello\nworld\n"@', 'Get-Date']));
157+
test('method calls', () => t('"hello".ToUpper() ; Get-Date', ['Get-Date']));
158+
test('complex quoting', () => t('Write-Host "She said `"Hello`"" ; Write-Host \'Single quotes\'', ['Write-Host "She said `"Hello`""', 'Write-Host \'Single quotes\'']));
159+
test('array operations', () => t('$arr = @(1,2,3); $arr | ForEach-Object { $_ * 2 }', ['ForEach-Object { $_ * 2 }']));
160+
test('hashtable operations', () => t('$hash = @{key="value"}; Write-Host $hash.key', ['Write-Host $hash.key']));
161+
test('type casting', () => t('[int]"123" + [int]"456" ; Write-Host "done"', ['Write-Host "done"']));
162+
test('regex operations', () => t('"hello world" -match "w.*d" ; Get-Date', ['Get-Date']));
163+
test('comparison operators', () => t('5 -gt 3 -and "hello" -like "h*" ; Write-Host "true"', ['Write-Host "true"']));
164+
test('null-conditional operators', () => t('$obj?.Property?.SubProperty ; Get-Date', ['Get-Date']));
165+
test('string interpolation', () => t('$name="World"; "Hello $name" ; Get-Date', ['Get-Date']));
166+
test('expandable strings', () => t('$var="test"; "Value: $($var.ToUpper())" ; Get-Date', ['Get-Date']));
167+
168+
suite('Control flow and structures', () => {
169+
test('logical and', () => t('Test-Path "file.txt" -and Get-Content "file.txt"', ['Test-Path "file.txt" -and Get-Content "file.txt"']));
170+
test('foreach with script block', () => t('ForEach-Object { Write-Host $_.Name } ; Get-Date', ['ForEach-Object { Write-Host $_.Name }', 'Write-Host $_.Name', 'Get-Date']));
171+
test('if-else', () => t('if (Test-Path "file.txt") { Get-Content "file.txt" } else { Write-Host "not found" }', ['Test-Path "file.txt"', 'Get-Content "file.txt"', 'Write-Host "not found"']));
172+
test('error handling', () => t('try { Get-Content "file.txt" } catch { Write-Error "failed" }', ['Get-Content "file.txt"', 'Write-Error "failed"']));
173+
test('switch statement', () => t('switch ($var) { 1 { "one" } 2 { "two" } default { "other" } } ; Get-Date', ['Get-Date']));
174+
test('do-while loop', () => t('do { Write-Host $i; $i++ } while ($i -lt 5) ; Get-Date', ['Write-Host $i', 'Get-Date']));
175+
test('for loop', () => t('for ($i=0; $i -lt 5; $i++) { Write-Host $i } ; Get-Date', ['Write-Host $i', 'Get-Date']));
176+
test('foreach loop with range', () => t('foreach ($i in 1..5) { Write-Host $i } ; Get-Date', ['1..5', 'Write-Host $i', 'Get-Date']));
177+
test('break and continue', () => t('while ($true) { if ($condition) { break } ; Write-Host "running" } ; Get-Date', ['Write-Host "running"', 'Get-Date']));
178+
test('nested try-catch-finally', () => t('try { try { Get-Content "file" } catch { throw } } catch { Write-Error "outer" } finally { Write-Host "cleanup" }', ['Get-Content "file"', 'Write-Error "outer"', 'Write-Host "cleanup"']));
179+
test('parallel processing', () => t('1..10 | ForEach-Object -Parallel { Start-Sleep 1; Write-Host $_ } ; Get-Date', ['1..10 ', 'ForEach-Object -Parallel { Start-Sleep 1; Write-Host $_ }', 'Start-Sleep 1', 'Write-Host $_', 'Get-Date']));
180+
});
181+
});
182+
183+
suite('all shell command parsing', () => {
184+
async function t(commandLine: string, expectedCommands: string[]) {
185+
for (const shell of [TreeSitterCommandParserLanguage.Bash, TreeSitterCommandParserLanguage.PowerShell]) {
186+
const result = await parser.extractSubCommands(shell, commandLine);
187+
deepStrictEqual(result, expectedCommands);
188+
}
189+
}
190+
191+
suite('edge cases', () => {
192+
test('empty strings', () => t('', []));
193+
test('whitespace-only strings', () => t(' \n\t ', []));
194+
});
195+
});
196+
});

0 commit comments

Comments
 (0)