From 4d34b8c71573b8612c8a772084b72bb7bc59693e Mon Sep 17 00:00:00 2001 From: cruldra Date: Thu, 24 Sep 2026 21:16:21 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(vscode):=20=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E9=98=85=E8=AF=BB=E7=9A=84=E8=A7=A3=E6=9E=90=E4=B8=8E=E6=A8=A1?= =?UTF-8?q?=E6=9D=BF=E7=9B=B4=E8=AF=91=E6=A0=B8=E5=BF=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- vscode/esbuild.mjs | 22 ++ vscode/package.json | 1 + vscode/pnpm-lock.yaml | 8 + vscode/src/codeReader/blockView.test.ts | 111 ++++++ vscode/src/codeReader/blockView.ts | 119 +++++++ vscode/src/codeReader/contract.ts | 54 +++ vscode/src/codeReader/editText.test.ts | 46 +++ vscode/src/codeReader/editText.ts | 41 +++ vscode/src/codeReader/fixtures/envLock.ts.txt | 61 ++++ .../src/codeReader/fixtures/issueLock.ts.txt | 25 ++ vscode/src/codeReader/fixtures/sample.py | 40 +++ .../src/codeReader/languages/python.test.ts | 88 +++++ vscode/src/codeReader/languages/python.ts | 217 ++++++++++++ vscode/src/codeReader/languages/shared.ts | 122 +++++++ .../codeReader/languages/typescript.test.ts | 227 ++++++++++++ vscode/src/codeReader/languages/typescript.ts | 328 ++++++++++++++++++ vscode/src/codeReader/registry.test.ts | 88 +++++ vscode/src/codeReader/registry.ts | 86 +++++ vscode/src/codeReader/runtime.ts | 50 +++ vscode/src/codeReader/testSupport.ts | 45 +++ vscode/src/codeReader/translators/pythonZh.ts | 77 ++++ .../codeReader/translators/typescriptZh.ts | 90 +++++ .../codeReader/translators/zhTemplate.test.ts | 179 ++++++++++ .../src/codeReader/translators/zhTemplate.ts | 169 +++++++++ 24 files changed, 2294 insertions(+) create mode 100644 vscode/src/codeReader/blockView.test.ts create mode 100644 vscode/src/codeReader/blockView.ts create mode 100644 vscode/src/codeReader/contract.ts create mode 100644 vscode/src/codeReader/editText.test.ts create mode 100644 vscode/src/codeReader/editText.ts create mode 100644 vscode/src/codeReader/fixtures/envLock.ts.txt create mode 100644 vscode/src/codeReader/fixtures/issueLock.ts.txt create mode 100644 vscode/src/codeReader/fixtures/sample.py create mode 100644 vscode/src/codeReader/languages/python.test.ts create mode 100644 vscode/src/codeReader/languages/python.ts create mode 100644 vscode/src/codeReader/languages/shared.ts create mode 100644 vscode/src/codeReader/languages/typescript.test.ts create mode 100644 vscode/src/codeReader/languages/typescript.ts create mode 100644 vscode/src/codeReader/registry.test.ts create mode 100644 vscode/src/codeReader/registry.ts create mode 100644 vscode/src/codeReader/runtime.ts create mode 100644 vscode/src/codeReader/testSupport.ts create mode 100644 vscode/src/codeReader/translators/pythonZh.ts create mode 100644 vscode/src/codeReader/translators/typescriptZh.ts create mode 100644 vscode/src/codeReader/translators/zhTemplate.test.ts create mode 100644 vscode/src/codeReader/translators/zhTemplate.ts diff --git a/vscode/esbuild.mjs b/vscode/esbuild.mjs index ed18453..ec12dcb 100644 --- a/vscode/esbuild.mjs +++ b/vscode/esbuild.mjs @@ -1,4 +1,7 @@ /* eslint-disable no-console -- 构建脚本，向终端打印进度是它的正当输出 */ +import { copyFileSync, mkdirSync } from 'node:fs' +import { createRequire } from 'node:module' +import path from 'node:path' import process from 'node:process' import * as esbuild from 'esbuild' @@ -19,7 +22,26 @@ const options = { logLevel: 'info', } +// 代码阅读的解析运行时（web-tree-sitter，由 esbuild 打进 index.cjs）和它要加载的 wasm： +// 运行时按 __dirname 找 dist/tree-sitter/，语法包只带用到的几种语言 +const TREE_SITTER_WASM = [ + 'tree-sitter.wasm', + 'tree-sitter-typescript.wasm', + 'tree-sitter-tsx.wasm', + 'tree-sitter-javascript.wasm', + 'tree-sitter-python.wasm', +] + +function copyTreeSitterWasm() { + const from = path.dirname(createRequire(import.meta.url).resolve('@vscode/tree-sitter-wasm')) + const to = 'dist/tree-sitter' + mkdirSync(to, { recursive: true }) + for (const file of TREE_SITTER_WASM) + copyFileSync(path.join(from, file), path.join(to, file)) +} + async function main() { + copyTreeSitterWasm() if (watch) { const ctx = await esbuild.context(options) await ctx.watch() diff --git a/vscode/package.json b/vscode/package.json index 038b5d4..9c397a4 100644 --- a/vscode/package.json +++ b/vscode/package.json @@ -108,6 +108,7 @@ "ext:publish:vsce": "npm_config_proxy= npm_config_https_proxy= vsce publish --skip-duplicate" }, "dependencies": { + "@vscode/tree-sitter-wasm": "^0.3.1", "diff": "^9.0.0", "handlebars": "^4.7.9" }, diff --git a/vscode/pnpm-lock.yaml b/vscode/pnpm-lock.yaml index c9245a2..41e9d76 100644 --- a/vscode/pnpm-lock.yaml +++ b/vscode/pnpm-lock.yaml @@ -13,6 +13,9 @@ importers: .: dependencies: + '@vscode/tree-sitter-wasm': + specifier: ^0.3.1 + version: 0.3.1 diff: specifier: ^9.0.0 version: 9.0.0 @@ -1381,6 +1384,9 @@ packages: '@vitest/utils@4.0.16': resolution: {integrity: sha512-h8z9yYhV3e1LEfaQ3zdypIrnAg/9hguReGZoS7Gl0aBG5xgA410zBqECqmaF/+RkTggRsfnzc1XaAHA6bmUufA==} + '@vscode/tree-sitter-wasm@0.3.1': + resolution: {integrity: sha512-RJFoomET6FajjG511fmQxeBQfU6M24a0aFZPqpid+ttIxanWf1VGytBG0UmsGjt07qmIPJS8U31D+aecuCucsQ==} + '@vscode/vsce-sign-alpine-arm64@2.0.2': resolution: {integrity: sha512-E80YvqhtZCLUv3YAf9+tIbbqoinWLCO/B3j03yQPbjT3ZIHCliKZlsy1peNc4XNZ5uIb87Jn0HWx/ZbPXviuAQ==} cpu: [arm64] @@ -4834,6 +4840,8 @@ snapshots: '@vitest/pretty-format': 4.0.16 tinyrainbow: 3.0.3 + '@vscode/tree-sitter-wasm@0.3.1': {} + '@vscode/vsce-sign-alpine-arm64@2.0.2': optional: true diff --git a/vscode/src/codeReader/blockView.test.ts b/vscode/src/codeReader/blockView.test.ts new file mode 100644 index 0000000..12f5527 --- /dev/null +++ b/vscode/src/codeReader/blockView.test.ts @@ -0,0 +1,111 @@ +import type { BlockView } from './blockView' +import type { LanguageAdapter, Translator } from './contract' +import { describe, expect, it } from 'vitest' +import { readBlockViews, toBlockViews } from './blockView' +import { createPythonAdapter } from './languages/python' +import { createTypeScriptAdapter } from './languages/typescript' +import { readFixture, runtime, WASM_DIR, withBlocks } from './testSupport' +import { pythonZhTranslator } from './translators/pythonZh' +import { typescriptZhTranslator } from './translators/typescriptZh' + +const ts = createTypeScriptAdapter(WASM_DIR) +const py = createPythonAdapter(WASM_DIR) + +function views(adapter: LanguageAdapter, languageId: string, translator: Translator | undefined, source: string): Promise { + return withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, source, adapter, translator)) +} + +function flatten(list: BlockView[]): BlockView[] { + return list.flatMap(view => [view, ...flatten(view.children)]) +} + +function byId(list: BlockView[], id: string): BlockView { + const found = flatten(list).find(view => view.id === id) + if (!found) + throw new Error(`没有块 ${id}`) + return found +} + +const SAMPLES: [LanguageAdapter, string, Translator, string][] = [ + [ts, 'typescript', typescriptZhTranslator, 'envLock.ts.txt'], + [ts, 'typescript', typescriptZhTranslator, 'issueLock.ts.txt'], + [py, 'python', pythonZhTranslator, 'sample.py'], +] + +describe('blockView', () => { + it.each(SAMPLES)('%#: codeTokens 拼起来等于 code，code 是 header 整行去共同缩进', async (adapter, languageId, translator, fixture) => { + const source = readFixture(fixture) + const lines = source.split('\n') + for (const view of flatten(await views(adapter, languageId, translator, source))) { + expect(view.codeTokens.map(token => token.text).join(''), view.id).toBe(view.code) + const rows = lines.slice(view.startLine, view.endLine + 1) + const indent = Math.min(...rows.filter(row => row.trim()).map(row => row.length - row.trimStart().length)) + expect(view.code, view.id).toBe(rows.map(row => row.slice(indent)).join('\n')) + expect(view.translation.length, view.id).toBeGreaterThan(0) + } + }) + + it('id 是块在树里的位置，行号 0 起，多行块去掉共同缩进', async () => { + const env = await views(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt')) + expect(byId(env, '6.1.0')).toMatchObject({ kind: 'return', startLine: 25, endLine: 25, code: 'return []' }) + expect(byId(env, '6.1')).toMatchObject({ kind: 'if', startLine: 24, code: 'if (!folder)' }) + expect(byId(env, '8.2.1')).toMatchObject({ kind: 'catch', startLine: 45, code: 'catch (err) {' }) + + const issue = await views(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt')) + expect(byId(issue, '3.4')).toMatchObject({ + kind: 'return', + startLine: 19, + endLine: 23, + code: 'return {\n locked: true,\n prerequisiteNumber: prereq.number,\n prerequisiteColumn: prereq.column,\n}', + }) + }) + + it('源码着色来自语法树：每个叶子一段，空白和标点归 plain', async () => { + const env = await views(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt')) + const tokens = byId(env, '6.3').codeTokens + expect(byId(env, '6.3').code).toBe('const uris = await workspace.findFiles(include, EXCLUDE_PATTERN)') + expect(tokens.slice(0, 7)).toEqual([ + { text: 'const', role: 'storage' }, + { text: ' ', role: 'plain' }, + { text: 'uris', role: 'variable' }, + { text: ' = ', role: 'plain' }, + { text: 'await', role: 'keyword' }, + { text: ' ', role: 'plain' }, + { text: 'workspace', role: 'variable' }, + ]) + expect(tokens).toContainEqual({ text: 'findFiles', role: 'function' }) + expect(byId(env, '0').codeTokens).toEqual([{ text: byId(env, '0').code, role: 'comment' }]) + + // 中文注释之后的行：语法树下标按字符串下标算，不按 UTF-8 字节 + const issue = await views(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt')) + expect(byId(issue, '3').codeTokens.slice(0, 5)).toEqual([ + { text: 'export', role: 'keyword' }, + { text: ' ', role: 'plain' }, + { text: 'function', role: 'storage' }, + { text: ' ', role: 'plain' }, + { text: 'evaluateLockedReason', role: 'function' }, + ]) + expect(byId(issue, '2').codeTokens).toEqual([{ text: byId(issue, '2').code, role: 'comment' }]) + + const sample = await views(py, 'python', pythonZhTranslator, readFixture('sample.py')) + const signature = byId(sample, '5').codeTokens + expect(signature).toContainEqual({ text: 'def', role: 'storage' }) + expect(signature).toContainEqual({ text: 'find_env_files', role: 'function' }) + expect(signature.filter(token => token.text === 'str')).toEqual([{ text: 'str', role: 'type' }, { text: 'str', role: 'type' }]) + }) + + it('\\r\\n 和制表符缩进：code 用 \\n 连接，去掉共同的制表符', async () => { + const source = 'function f() {\r\n\tif (a &&\r\n\t\tb) {\r\n\t\treturn 1\r\n\t}\r\n}\r\n' + const list = await views(ts, 'typescript', undefined, source) + const cond = byId(list, '0.0') + expect(cond).toMatchObject({ kind: 'if', startLine: 1, endLine: 2, code: 'if (a &&\n\tb) {', translation: [] }) + expect(cond.codeTokens.map(token => token.text).join('')).toBe(cond.code) + }) + + it('readBlockViews：解析、转视图、释放语法树', async () => { + const list = await readBlockViews(runtime, py, pythonZhTranslator, 'python', 'def f(x: int) -> bool:\n return not x\n') + expect(list).toHaveLength(1) + expect(list[0].translation.map(segment => segment.text).join('')).toBe('函数 f(x: 整数) -> 布尔') + expect(list[0].children[0].translation.map(segment => segment.text).join('')).toBe('返回 非 x') + }) +}) diff --git a/vscode/src/codeReader/blockView.ts b/vscode/src/codeReader/blockView.ts new file mode 100644 index 0000000..cf447db --- /dev/null +++ b/vscode/src/codeReader/blockView.ts @@ -0,0 +1,119 @@ +import type { BlockKind, CodeBlock, LanguageAdapter, Node, Segment, TokenRole, Translator } from './contract' +import type { TreeSitterRuntime } from './runtime' +import { commonIndent, splitLines } from './editText' + +/** 发给 webview 的一块（可序列化）。 */ +export interface BlockView { + /** 在树里的位置，例如 '6.1.0'。 */ + id: string + kind: BlockKind + /** header 的起止行，0 起。 */ + startLine: number + endLine: number + /** header 所在的整行源码，去掉共同缩进。 */ + code: string + /** 源码着色（拼起来等于 code）。 */ + codeTokens: Segment[] + translation: Segment[] + children: BlockView[] +} + +interface Line { start: number, end: number } + +/** 每行在源码里的起止下标（end 不含换行符）。 */ +function lineRanges(source: string): Line[] { + const lines: Line[] = [] + let start = 0 + for (const text of splitLines(source)) { + lines.push({ start, end: start + text.length }) + start += text.length + (source[start + text.length] === '\r' ? 2 : 1) + } + return lines +} + +function leavesIn(node: Node, from: number, to: number, out: Node[]): void { + if (node.endIndex <= from || node.startIndex >= to || node.startIndex === node.endIndex) + return + if (node.childCount === 0) { + out.push(node) + return + } + for (const child of node.children) { + if (child) + leavesIn(child, from, to, out) + } +} + +/** 按语法树叶子给 [startRow, endRow] 这几行着色：每个有词性的叶子一段，空白、标点等 plain 的连成一段。 */ +function colorLines(root: Node, source: string, lines: Line[], startRow: number, endRow: number, indent: number, tokenRole: (leaf: Node) => TokenRole): Segment[] { + const from = lines[startRow].start + const to = lines[endRow].end + const owner = new Int32Array(to - from).fill(-1) + const roles: TokenRole[] = [] + const leaves: Node[] = [] + leavesIn(root.descendantForIndex(from, to) ?? root, from, to, leaves) + leaves.forEach((leaf, i) => { + roles.push(tokenRole(leaf)) + for (let at = Math.max(leaf.startIndex, from); at < Math.min(leaf.endIndex, to); at++) + owner[at - from] = i + }) + + const segments: Segment[] = [] + let current = -2 + const put = (char: string, who: number): void => { + const prev = segments.at(-1) + const role = who < 0 ? 'plain' : roles[who] + if (prev && (who === current || (role === 'plain' && prev.role === 'plain'))) + prev.text += char + else + segments.push({ text: char, role }) + current = who + } + for (let row = startRow; row <= endRow; row++) { + const line = lines[row] + if (row > startRow) + put('\n', owner[line.start - 1 - from]) + for (let at = line.start + Math.min(indent, line.end - line.start); at < line.end; at++) + put(source[at], owner[at - from]) + } + return segments +} + +/** 代码块树 → 视图数据：header 整行源码去共同缩进、按语法树着色、调用翻译插件。 */ +export function toBlockViews(blocks: CodeBlock[], source: string, adapter: LanguageAdapter, translator: Translator | undefined): BlockView[] { + const lines = lineRanges(source) + const texts = splitLines(source) + const build = (block: CodeBlock, id: string): BlockView => { + const { startRow, endRow } = block.header + const rows = texts.slice(startRow, endRow + 1) + const indent = commonIndent(rows).length + return { + id, + kind: block.kind, + startLine: startRow, + endLine: endRow, + code: rows.map(row => row.slice(Math.min(indent, row.length))).join('\n'), + codeTokens: colorLines(block.node.tree.rootNode, source, lines, startRow, endRow, indent, adapter.tokenRole), + translation: translator ? translator.translate(block, source) : [], + children: block.children.map((child, i) => build(child, `${id}.${i}`)), + } + } + return blocks.map((block, i) => build(block, String(i))) +} + +/** 解析一份源码并转成视图数据；语法树用完即释放。 */ +export async function readBlockViews( + runtime: Pick, + adapter: LanguageAdapter, + translator: Translator | undefined, + languageId: string, + source: string, +): Promise { + const tree = await runtime.parse(adapter, languageId, source) + try { + return toBlockViews(adapter.toBlocks(tree.rootNode, source), source, adapter, translator) + } + finally { + tree.delete() + } +} diff --git a/vscode/src/codeReader/contract.ts b/vscode/src/codeReader/contract.ts new file mode 100644 index 0000000..5ef12bd --- /dev/null +++ b/vscode/src/codeReader/contract.ts @@ -0,0 +1,54 @@ +import type { Node } from '@vscode/tree-sitter-wasm' +import type { Disposable } from 'vscode' + +export type { Node } + +export type BlockKind + = | 'import' | 'function' | 'class' | 'type' | 'field' | 'const' | 'variable' + | 'if' | 'else' | 'loop' | 'try' | 'catch' | 'finally' | 'return' | 'statement' | 'comment' + +/** 源码里的一段，行号从 0 开始；下标是源码字符串下标（web-tree-sitter 的 startIndex / endIndex），end 为开区间。 */ +export interface SourceSpan { startIndex: number, endIndex: number, startRow: number, endRow: number } + +export interface CodeBlock { + kind: BlockKind + /** 这一块对应的语法树节点（整条语句）。 */ + node: Node + /** 块自己显示 / 编辑的那一段：复合语句只到主体开始之前，其余是整条语句。 */ + header: SourceSpan + children: CodeBlock[] +} + +/** 着色用的词性，源码着色和译文着色共用。 */ +export type TokenRole + = | 'keyword' // 流程关键字：import from export if else for return try catch await … + | 'storage' // 声明关键字与字面量关键字：const function async class interface new true false null … + | 'operator' // 被翻成文字的运算符：全等 与 或 非 + | 'variable' | 'function' | 'type' | 'string' | 'number' | 'literal' | 'comment' | 'plain' + +export interface Segment { text: string, role: TokenRole } + +/** 语言插件：语法树 → 代码块树。 */ +export interface LanguageAdapter { + id: string + /** 覆盖的 VS Code languageId，例如 ['typescript', 'typescriptreact', 'javascript', 'javascriptreact']。 */ + languageIds: string[] + /** 该 languageId 用的语法包 .wasm 的绝对路径（宿主负责加载，保证和解析器运行时是同一份）。 */ + grammarPath: (languageId: string) => string + toBlocks: (root: Node, source: string) => CodeBlock[] + /** 源码着色：给语法树叶子节点定词性。 */ + tokenRole: (leaf: Node) => TokenRole +} + +/** 翻译插件：代码块 → 译文。 */ +export interface Translator { + id: string // 'typescript.zh-template' + label: string // '中文直译' + languageIds: string[] + translate: (block: CodeBlock, source: string) => Segment[] +} + +export interface CodeReaderApi { + registerLanguage: (adapter: LanguageAdapter) => Disposable + registerTranslator: (translator: Translator) => Disposable +} diff --git a/vscode/src/codeReader/editText.test.ts b/vscode/src/codeReader/editText.test.ts new file mode 100644 index 0000000..424e9a3 --- /dev/null +++ b/vscode/src/codeReader/editText.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import { toBlockViews } from './blockView' +import { commonIndent, reindentBlock, replaceLines } from './editText' +import { createTypeScriptAdapter } from './languages/typescript' +import { readFixture, WASM_DIR, withBlocks } from './testSupport' + +describe('行替换', () => { + it('多行块替换后每个非空行补回原来的缩进，空行保持空', () => { + const source = [ + 'function f() {', + ' if (a)', + ' return 1', + '}', + ].join('\n') + const edited = replaceLines(source, 1, 2, 'if (a && b) {\n\n return 2\n}') + expect(edited).toBe([ + 'function f() {', + ' if (a && b) {', + '', + ' return 2', + ' }', + '}', + ].join('\n')) + }) + + it('保留 \\r\\n 换行', () => { + expect(replaceLines('a\r\n b\r\n c\r\n', 1, 1, 'x\ny')).toBe('a\r\n x\r\n y\r\n c\r\n') + }) + + it('共同缩进逐字符比，混用制表符和空格时只取相同的前缀', () => { + expect(commonIndent(['\t a', '', '\t b'])).toBe('\t ') + expect(commonIndent(['\ta', ' b'])).toBe('') + expect(reindentBlock([' x'], 'a\n b')).toBe(' a\n b') + }) + + it('把预览里的 code 原样写回，文件不变（去缩进与补缩进互逆）', async () => { + const adapter = createTypeScriptAdapter(WASM_DIR) + const source = readFixture('envLock.ts.txt') + const views = await withBlocks(adapter, 'typescript', source, blocks => toBlockViews(blocks, source, adapter, undefined)) + const all = views.flatMap(function flat(view): typeof views { + return [view, ...view.children.flatMap(flat)] + }) + for (const view of all) + expect(replaceLines(source, view.startLine, view.endLine, view.code), view.id).toBe(source) + }) +}) diff --git a/vscode/src/codeReader/editText.ts b/vscode/src/codeReader/editText.ts new file mode 100644 index 0000000..85934f8 --- /dev/null +++ b/vscode/src/codeReader/editText.ts @@ -0,0 +1,41 @@ +/** 按 \n 或 \r\n 切行。 */ +export function splitLines(text: string): string[] { + return text.split(/\r?\n/) +} + +/** 非空行共同的前导空白（空格 / 制表符按原样比，不换算）。 */ +export function commonIndent(lines: string[]): string { + let common: string | undefined + for (const line of lines) { + if (!line.trim()) + continue + const indent = line.slice(0, line.length - line.trimStart().length) + if (common === undefined) { + common = indent + continue + } + let i = 0 + while (i < common.length && i < indent.length && common[i] === indent[i]) + i++ + common = common.slice(0, i) + } + return common ?? '' +} + +/** + * 块内改完的文本（webview 里是去掉共同缩进的样子）→ 写回文件的文本： + * 每个非空行补回这几行原来的共同缩进，和 BlockView.code 去缩进的做法互逆。 + * @param originalLines 被替换的那几行原文 + */ +export function reindentBlock(originalLines: string[], text: string): string { + const indent = commonIndent(originalLines) + return splitLines(text).map(line => line.trim() ? indent + line : line).join('\n') +} + +/** 把 source 的 [startLine, endLine] 整行替换成 text（补回缩进），返回新全文；换行符沿用原文。 */ +export function replaceLines(source: string, startLine: number, endLine: number, text: string): string { + const eol = source.includes('\r\n') ? '\r\n' : '\n' + const lines = splitLines(source) + const replacement = splitLines(reindentBlock(lines.slice(startLine, endLine + 1), text)) + return [...lines.slice(0, startLine), ...replacement, ...lines.slice(endLine + 1)].join(eol) +} diff --git a/vscode/src/codeReader/fixtures/envLock.ts.txt b/vscode/src/codeReader/fixtures/envLock.ts.txt new file mode 100644 index 0000000..a08d2d2 --- /dev/null +++ b/vscode/src/codeReader/fixtures/envLock.ts.txt @@ -0,0 +1,61 @@ +/** + * .env file lock / unlock helpers. + * + * Recursively scans the main workspace folder for `.env*` files (skipping + * `node_modules`, `.git`, `.claude`) and chmods them to 0o444 (locked) or + * 0o644 (unlocked) in bulk. Backs the toolbar Lock toggle in `PanelHeader`. + * + * No persistent watcher — re-scan happens on every toggle, so newly created + * `.env` files are only picked up the next time the user clicks the button. + */ + +import { promises as fsp } from 'node:fs' +import { RelativePattern, workspace } from 'vscode' + +const ENV_PATTERN = '**/.env*' +const EXCLUDE_PATTERN = '{**/node_modules/**,**/.git/**,**/.claude/**}' + +/** + * Scan the workspace folder rooted at `workspaceRoot` for all `.env*` + * files, skipping `node_modules`, `.git`, and `.claude`. Returns absolute + * fs paths. Empty array if the workspace folder isn't currently open. + */ +export async function findEnvFiles(workspaceRoot: string): Promise { + const folder = workspace.workspaceFolders?.find(f => f.uri.fsPath === workspaceRoot) + if (!folder) + return [] + const include = new RelativePattern(folder, ENV_PATTERN) + const uris = await workspace.findFiles(include, EXCLUDE_PATTERN) + return uris.map(u => u.fsPath) +} + +export interface EnvLockResult { + total: number + ok: string[] + failed: { path: string, reason: string }[] +} + +async function chmodAll(workspaceRoot: string, mode: number): Promise { + const files = await findEnvFiles(workspaceRoot) + const result: EnvLockResult = { total: files.length, ok: [], failed: [] } + for (const f of files) { + try { + await fsp.chmod(f, mode) + result.ok.push(f) + } + catch (err) { + result.failed.push({ path: f, reason: err instanceof Error ? err.message : String(err) }) + } + } + return result +} + +/** Set every `.env*` file in the workspace to 0o444 (read-only). */ +export async function lockEnvFiles(workspaceRoot: string): Promise { + return chmodAll(workspaceRoot, 0o444) +} + +/** Restore every `.env*` file in the workspace to 0o644 (read-write). */ +export async function unlockEnvFiles(workspaceRoot: string): Promise { + return chmodAll(workspaceRoot, 0o644) +} diff --git a/vscode/src/codeReader/fixtures/issueLock.ts.txt b/vscode/src/codeReader/fixtures/issueLock.ts.txt new file mode 100644 index 0000000..4fcd409 --- /dev/null +++ b/vscode/src/codeReader/fixtures/issueLock.ts.txt @@ -0,0 +1,25 @@ +import type { Issue } from '../gitea/types' + +export interface LockedReason { + locked: boolean + prerequisiteNumber?: number + prerequisiteColumn?: string +} + +/** + * 纯判定：工单被未完成的前置挡住才算锁定。快照里找不到工单或前置一律放行， + * 与 webview 的 `isIssueLocked` 保持同一语义。 + */ +export function evaluateLockedReason(issues: Issue[], issueNumber: number): LockedReason { + const issue = issues.find(i => i.number === issueNumber) + if (!issue || issue.prerequisite === undefined) + return { locked: false } + const prereq = issues.find(i => i.number === issue.prerequisite) + if (!prereq || prereq.column === 'done') + return { locked: false } + return { + locked: true, + prerequisiteNumber: prereq.number, + prerequisiteColumn: prereq.column, + } +} diff --git a/vscode/src/codeReader/fixtures/sample.py b/vscode/src/codeReader/fixtures/sample.py new file mode 100644 index 0000000..cf747f2 --- /dev/null +++ b/vscode/src/codeReader/fixtures/sample.py @@ -0,0 +1,40 @@ +"""Scan a workspace for .env files.""" + +import os +from pathlib import Path as P + +MAX_DEPTH = 8 +skipped = [] + + +def find_env_files(root: str) -> list[str]: + """Return every .env file under root.""" + # 跳过备份文件 + found = [p for p in os.listdir(root) if p.startswith('.env') and not p.endswith('.bak')] + if not found: + return [] + elif len(found) > MAX_DEPTH: + found = found[:MAX_DEPTH] + else: + pass + for name in found: + if name in skipped: + continue + try: + with open(os.path.join(root, found[0])) as handle: + handle.read() + except OSError as err: + raise RuntimeError(str(err)) from err + finally: + skipped.clear() + return found + + +class EnvLockResult: + total: int + ok: list[str] = [] + failed: dict[str, str] = {} + label = 'env' + + def summary(self, verbose: bool = False) -> str: + return f'{self.total} files' if verbose is not None else '' diff --git a/vscode/src/codeReader/languages/python.test.ts b/vscode/src/codeReader/languages/python.test.ts new file mode 100644 index 0000000..f735db8 --- /dev/null +++ b/vscode/src/codeReader/languages/python.test.ts @@ -0,0 +1,88 @@ +import type { Outline } from '../testSupport' +import { describe, expect, it } from 'vitest' +import { outline, readFixture, WASM_DIR, withBlocks } from '../testSupport' +import { createPythonAdapter } from './python' + +const adapter = createPythonAdapter(WASM_DIR) + +function tree(source: string): Promise { + return withBlocks(adapter, 'python', source, outline) +} + +// fixtures/sample.py：行号 1 起 +const SAMPLE: Outline[] = [ + { at: 1, kind: 'comment' }, + { at: 3, kind: 'import' }, + { at: 4, kind: 'import' }, + { at: 6, kind: 'const' }, + { at: 7, kind: 'variable' }, + { at: 10, kind: 'function', kids: [ + { at: 11, kind: 'comment' }, + { at: 12, kind: 'comment' }, + { at: 13, kind: 'variable' }, + { at: 14, kind: 'if', kids: [{ at: 15, kind: 'return' }] }, + { at: 16, kind: 'else', kids: [{ at: 17, kind: 'variable' }] }, + { at: 18, kind: 'else', kids: [{ at: 19, kind: 'statement' }] }, + { at: 20, kind: 'loop', kids: [ + { at: 21, kind: 'if', kids: [{ at: 22, kind: 'statement' }] }, + ] }, + { at: 23, kind: 'try', kids: [ + { at: 24, kind: 'statement', kids: [{ at: 25, kind: 'statement' }] }, + ] }, + { at: 26, kind: 'catch', kids: [{ at: 27, kind: 'statement' }] }, + { at: 28, kind: 'finally', kids: [{ at: 29, kind: 'statement' }] }, + { at: 30, kind: 'return' }, + ] }, + { at: 33, kind: 'class', kids: [ + { at: 34, kind: 'field' }, + { at: 35, kind: 'field' }, + { at: 36, kind: 'field' }, + { at: 37, kind: 'variable' }, + { at: 39, kind: 'function', kids: [{ at: 40, kind: 'return' }] }, + ] }, +] + +describe('python 代码块树', () => { + it('样例覆盖 import / def / docstring / if-elif-else / for / try-except-finally / with / 推导式 / class 字段', async () => { + expect(await tree(readFixture('sample.py'))).toEqual(SAMPLE) + }) + + it('装饰器算进头部；冒号后、主体前的注释归子块；循环的 else 是兄弟块；match 的 case 是子块 if', async () => { + const source = [ + '@cache', + 'def load(key):', + ' # 先查缓存', + ' while key:', + ' key = key[1:]', + ' else:', + ' pass', + ' match key:', + ' case \'a\':', + ' return 1', + ' case _:', + ' return 2', + ].join('\n') + expect(await tree(source)).toEqual([ + { at: [1, 2], kind: 'function', kids: [ + { at: 3, kind: 'comment' }, + { at: 4, kind: 'loop', kids: [{ at: 5, kind: 'variable' }] }, + { at: 6, kind: 'else', kids: [{ at: 7, kind: 'statement' }] }, + { at: 8, kind: 'if', kids: [ + { at: 9, kind: 'if', kids: [{ at: 10, kind: 'return' }] }, + { at: 11, kind: 'if', kids: [{ at: 12, kind: 'return' }] }, + ] }, + ] }, + ]) + }) + + it('docstring 只认主体开头；不是开头的字符串表达式是语句', async () => { + const source = [ + 'def f():', + ' x = 1', + ' """not a docstring"""', + ].join('\n') + expect(await tree(source)).toEqual([ + { at: 1, kind: 'function', kids: [{ at: 2, kind: 'variable' }, { at: 3, kind: 'statement' }] }, + ]) + }) +}) diff --git a/vscode/src/codeReader/languages/python.ts b/vscode/src/codeReader/languages/python.ts new file mode 100644 index 0000000..8e4bce4 --- /dev/null +++ b/vscode/src/codeReader/languages/python.ts @@ -0,0 +1,217 @@ +import type { BlockKind, CodeBlock, LanguageAdapter, Node, TokenRole } from '../contract' +import * as path from 'node:path' +import { compoundBlock, leafBlock, listBlocks, named, sameNode } from './shared' + +/** 常量、类字段只认直接写在模块 / 类主体里的赋值。 */ +type Scope = 'module' | 'class' | 'other' + +const CONSTANT_NAME = /^[A-Z][A-Z0-9_]*$/ + +function statements(items: Node[], scope: Scope, afterRow: number, docstring: boolean): CodeBlock[] { + return listBlocks(items, afterRow, (item, first) => statementBlocks(item, scope, docstring && first), () => true) +} + +/** 冒号后面的主体：body / consequence 字段，except / finally 子句里是不带字段名的 block。 */ +function bodyOf(node: Node): Node | null { + const body = node.childForFieldName('body') ?? node.childForFieldName('consequence') + if (body) + return body + const blocks = named(node).filter(child => child.type === 'block') + return blocks.at(-1) ?? null +} + +function bodyBlock(kind: BlockKind, outer: Node, body: Node | null, scope: Scope, docstring = false): CodeBlock { + if (!body) + return leafBlock(kind, outer) + return compoundBlock(kind, outer, body.startIndex, named(body), (list, afterRow) => statements(list, scope, afterRow, docstring)) +} + +function statementBlocks(node: Node, scope: Scope, docstring: boolean): CodeBlock[] { + switch (node.type) { + case 'import_statement': + case 'import_from_statement': + case 'future_import_statement': + return [leafBlock('import', node)] + case 'function_definition': + case 'class_definition': + return [definitionBlock(node, node)] + case 'decorated_definition': { + const definition = node.childForFieldName('definition') + return [definition ? definitionBlock(definition, node) : leafBlock('statement', node)] + } + case 'if_statement': + return ifBlocks(node) + case 'for_statement': + case 'while_statement': + return loopBlocks(node) + case 'try_statement': + return tryBlocks(node) + case 'with_statement': + return [bodyBlock('statement', node, bodyOf(node), 'other')] + case 'match_statement': + return [bodyBlock('if', node, bodyOf(node), 'other')] + case 'case_clause': + return [bodyBlock('if', node, bodyOf(node), 'other')] + case 'return_statement': + return [leafBlock('return', node)] + case 'expression_statement': + return [leafBlock(expressionKind(node, scope, docstring), node)] + default: + return [leafBlock('statement', node)] + } +} + +/** function_definition / class_definition；outer 是带装饰器的整体时头部从装饰器开始。 */ +function definitionBlock(definition: Node, outer: Node): CodeBlock { + const isClass = definition.type === 'class_definition' + return bodyBlock(isClass ? 'class' : 'function', outer, definition.childForFieldName('body'), isClass ? 'class' : 'other', true) +} + +function isConstantTarget(target: Node | null): boolean { + if (!target) + return false + if (target.type === 'identifier') + return CONSTANT_NAME.test(target.text) + if (target.type === 'pattern_list' || target.type === 'tuple_pattern') { + const names = named(target) + return names.length > 0 && names.every(isConstantTarget) + } + return false +} + +function expressionKind(node: Node, scope: Scope, docstring: boolean): BlockKind { + const [expression, ...rest] = named(node) + if (docstring && rest.length === 0 && (expression?.type === 'string' || expression?.type === 'concatenated_string')) + return 'comment' + if (expression?.type !== 'assignment') + return 'statement' + if (scope === 'class' && expression.childForFieldName('type')) + return 'field' + if (scope === 'module' && isConstantTarget(expression.childForFieldName('left'))) + return 'const' + return 'variable' +} + +/** elif / else 展平成紧跟 if 的兄弟块。 */ +function ifBlocks(node: Node): CodeBlock[] { + const blocks = [bodyBlock('if', node, bodyOf(node), 'other')] + for (const alternative of node.childrenForFieldName('alternative')) { + if (alternative) + blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other')) + } + return blocks +} + +/** 循环的 else 是兄弟块 else。 */ +function loopBlocks(node: Node): CodeBlock[] { + const blocks = [bodyBlock('loop', node, bodyOf(node), 'other')] + const alternative = node.childForFieldName('alternative') + if (alternative) + blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other')) + return blocks +} + +const TRY_CLAUSES: Record = { + except_clause: 'catch', + except_group_clause: 'catch', + else_clause: 'else', + finally_clause: 'finally', +} + +function tryBlocks(node: Node): CodeBlock[] { + const blocks = [bodyBlock('try', node, node.childForFieldName('body'), 'other')] + for (const clause of named(node)) { + const kind = TRY_CLAUSES[clause.type] + if (kind) + blocks.push(bodyBlock(kind, clause, bodyOf(clause), 'other')) + } + return blocks +} + +// ─── 源码着色 ───────────────────────────────────────────────────────────────── + +const STORAGE_KEYWORDS = new Set(['def', 'class', 'async', 'lambda', 'global', 'nonlocal', 'del']) +const OPERATOR_KEYWORDS = new Set(['and', 'or', 'not', 'is']) +const COMPARISON = new Set(['comparison_operator', 'not in', 'is not']) +const LITERAL_KEYWORDS = new Set(['none', 'true', 'false']) + +function insideString(leaf: Node): boolean { + for (let node = leaf.parent; node; node = node.parent) { + if (node.type === 'interpolation') + return false + if (node.type === 'string') + return true + } + return false +} + +/** 注解里的名字：typed_parameter / 返回值 / 带注解赋值的 type 节点之内。 */ +export function insideAnnotation(leaf: Node): boolean { + for (let node = leaf.parent; node; node = node.parent) { + if (node.type === 'type') + return true + if (node.type === 'block' || node.type === 'module') + return false + } + return false +} + +function isCallee(node: Node): boolean { + const parent = node.parent + return !!parent && parent.type === 'call' && sameNode(parent.childForFieldName('function'), node) +} + +function identifierRole(leaf: Node): TokenRole { + const parent = leaf.parent + if (!parent) + return 'variable' + if (insideAnnotation(leaf)) + return 'type' + if (parent.type === 'function_definition' && sameNode(parent.childForFieldName('name'), leaf)) + return 'function' + if (parent.type === 'class_definition' && sameNode(parent.childForFieldName('name'), leaf)) + return 'type' + if (parent.type === 'decorator' || isCallee(leaf)) + return 'function' + if (parent.type === 'attribute' && sameNode(parent.childForFieldName('attribute'), leaf) && isCallee(parent)) + return 'function' + return 'variable' +} + +export function pythonTokenRole(leaf: Node): TokenRole { + if (leaf.type === 'comment') + return 'comment' + if (insideString(leaf)) + return 'string' + if (!leaf.isNamed) { + if (!/^[a-z]+$/.test(leaf.type)) + return 'plain' + if (STORAGE_KEYWORDS.has(leaf.type)) + return 'storage' + if (OPERATOR_KEYWORDS.has(leaf.type) || (leaf.type === 'in' && COMPARISON.has(leaf.parent?.type ?? ''))) + return 'operator' + return 'keyword' + } + if (LITERAL_KEYWORDS.has(leaf.type)) + return 'storage' + switch (leaf.type) { + case 'integer': + case 'float': + return 'number' + case 'identifier': + return identifierRole(leaf) + default: + return 'plain' + } +} + +/** Python 语言插件。@param wasmDir 语法包 .wasm 所在目录 */ +export function createPythonAdapter(wasmDir: string): LanguageAdapter { + return { + id: 'python', + languageIds: ['python'], + grammarPath: () => path.join(wasmDir, 'tree-sitter-python.wasm'), + toBlocks: root => statements(named(root), 'module', -1, true), + tokenRole: pythonTokenRole, + } +} diff --git a/vscode/src/codeReader/languages/shared.ts b/vscode/src/codeReader/languages/shared.ts new file mode 100644 index 0000000..e84be62 --- /dev/null +++ b/vscode/src/codeReader/languages/shared.ts @@ -0,0 +1,122 @@ +import type { BlockKind, CodeBlock, Node, SourceSpan } from '../contract' + +/** 语言插件共用的建树工具：取子节点、算头部、把一串语句切成块。 */ + +export function named(node: Node | null | undefined): Node[] { + return node ? node.namedChildren.filter((child): child is Node => child !== null) : [] +} + +export function children(node: Node): Node[] { + return node.children.filter((child): child is Node => child !== null) +} + +export function nodeSpan(node: Node): SourceSpan { + return { + startIndex: node.startIndex, + endIndex: node.endIndex, + startRow: node.startPosition.row, + endRow: node.endPosition.row, + } +} + +export function leafBlock(kind: BlockKind, node: Node): CodeBlock { + return { kind, node, header: nodeSpan(node), children: [] } +} + +/** 头部往前扩到 from（装饰器）。 */ +export function extendHeader(block: CodeBlock, from: Node): CodeBlock { + return { ...block, header: { ...block.header, startIndex: from.startIndex, startRow: from.startPosition.row } } +} + +function lastLeafBefore(node: Node, limit: number): Node | undefined { + for (let i = node.childCount - 1; i >= 0; i--) { + const child = node.child(i) + if (!child || child.startIndex >= limit || child.type === 'comment' || child.startIndex === child.endIndex) + continue + if (child.childCount === 0) + return child + const found = lastLeafBefore(child, limit) + if (found) + return found + } + return undefined +} + +/** 复合语句的头部：从 outer 开头到主体开始之前最后一个非注释的词为止。 */ +export function headerSpan(outer: Node, bodyStart: number): SourceSpan { + const last = lastLeafBefore(outer, bodyStart) + if (!last) + return nodeSpan(outer) + return { + startIndex: outer.startIndex, + endIndex: last.endIndex, + startRow: outer.startPosition.row, + endRow: last.endPosition.row, + } +} + +/** node 里完全落在 [from, to) 之间的注释。 */ +function commentsIn(node: Node, from: number, to: number): Node[] { + if (node.endIndex <= from || node.startIndex >= to) + return [] + if (node.type === 'comment') + return node.startIndex >= from && node.endIndex <= to ? [node] : [] + return children(node).flatMap(child => commentsIn(child, from, to)) +} + +export type ListBlocks = (items: Node[], afterRow: number) => CodeBlock[] + +/** + * 复合语句块：头部到 bodyStart 之前，子块来自 items。 + * 头部和主体之间的注释（例如 Python 冒号后、主体前的注释）也归子块。 + */ +export function compoundBlock(kind: BlockKind, outer: Node, bodyStart: number, items: Node[], list: ListBlocks): CodeBlock { + const header = headerSpan(outer, bodyStart) + const between = commentsIn(outer, header.endIndex, bodyStart) + return { kind, node: outer, header, children: list([...between, ...items], header.endRow) } +} + +/** + * 一串语句 → 块。注释单独成块，相邻行的行注释合成一块； + * 和前一条语句（或头部）同一行的注释是行尾注释，跟着那一行显示，不单独成块。 + * afterRow 是这串语句之前的最后一行（头部结尾行），模块顶层传 -1； + * toBlocks 的 first 表示它是这串里第一条非注释语句（Python 认 docstring 用）。 + */ +export function listBlocks( + items: Node[], + afterRow: number, + toBlocks: (item: Node, first: boolean) => CodeBlock[], + isLineComment: (comment: Node) => boolean, +): CodeBlock[] { + const out: CodeBlock[] = [] + let lastRow = afterRow + let lineRun: CodeBlock | undefined + let first = true + for (const item of items) { + if (item.type === 'comment') { + if (item.startPosition.row === lastRow) + continue + const line = isLineComment(item) + if (line && lineRun && item.startPosition.row === lineRun.header.endRow + 1) { + lineRun.header = { ...lineRun.header, endIndex: item.endIndex, endRow: item.endPosition.row } + } + else { + const block = leafBlock('comment', item) + out.push(block) + lineRun = line ? block : undefined + } + } + else { + out.push(...toBlocks(item, first)) + first = false + lineRun = undefined + } + lastRow = item.endPosition.row + } + return out +} + +/** 同一个节点：web-tree-sitter 每次取子节点都会新建对象，只能按位置和类型比。 */ +export function sameNode(a: Node | null | undefined, b: Node): boolean { + return !!a && a.startIndex === b.startIndex && a.endIndex === b.endIndex && a.type === b.type +} diff --git a/vscode/src/codeReader/languages/typescript.test.ts b/vscode/src/codeReader/languages/typescript.test.ts new file mode 100644 index 0000000..f502c4a --- /dev/null +++ b/vscode/src/codeReader/languages/typescript.test.ts @@ -0,0 +1,227 @@ +import type { Outline } from '../testSupport' +import { describe, expect, it } from 'vitest' +import { outline, readFixture, WASM_DIR, withBlocks } from '../testSupport' +import { createTypeScriptAdapter } from './typescript' + +const adapter = createTypeScriptAdapter(WASM_DIR) + +function tree(source: string, languageId = 'typescript'): Promise { + return withBlocks(adapter, languageId, source, outline) +} + +// 与原型 docs/prototype/code-reader/figma-plugin/code.js 的 CODE_TREE 一致 +const ENV_LOCK: Outline[] = [ + { at: [1, 10], kind: 'comment' }, + { at: 12, kind: 'import' }, + { at: 13, kind: 'import' }, + { at: 15, kind: 'const' }, + { at: 16, kind: 'const' }, + { at: [18, 22], kind: 'comment' }, + { at: 23, kind: 'function', kids: [ + { at: 24, kind: 'variable' }, + { at: 25, kind: 'if', kids: [ + { at: 26, kind: 'return' }, + ] }, + { at: 27, kind: 'variable' }, + { at: 28, kind: 'variable' }, + { at: 29, kind: 'return' }, + ] }, + { at: 32, kind: 'type', kids: [ + { at: 33, kind: 'field' }, + { at: 34, kind: 'field' }, + { at: 35, kind: 'field' }, + ] }, + { at: 38, kind: 'function', kids: [ + { at: 39, kind: 'variable' }, + { at: 40, kind: 'variable' }, + { at: 41, kind: 'loop', kids: [ + { at: 42, kind: 'try', kids: [ + { at: 43, kind: 'statement' }, + { at: 44, kind: 'statement' }, + ] }, + { at: 46, kind: 'catch', kids: [ + { at: 47, kind: 'statement' }, + ] }, + ] }, + { at: 50, kind: 'return' }, + ] }, + { at: 53, kind: 'comment' }, + { at: 54, kind: 'function', kids: [ + { at: 55, kind: 'return' }, + ] }, + { at: 58, kind: 'comment' }, + { at: 59, kind: 'function', kids: [ + { at: 60, kind: 'return' }, + ] }, +] + +const ISSUE_LOCK: Outline[] = [ + { at: 1, kind: 'import' }, + { at: 3, kind: 'type', kids: [ + { at: 4, kind: 'field' }, + { at: 5, kind: 'field' }, + { at: 6, kind: 'field' }, + ] }, + { at: [9, 12], kind: 'comment' }, + { at: 13, kind: 'function', kids: [ + { at: 14, kind: 'variable' }, + { at: 15, kind: 'if', kids: [ + { at: 16, kind: 'return' }, + ] }, + { at: 17, kind: 'variable' }, + { at: 18, kind: 'if', kids: [ + { at: 19, kind: 'return' }, + ] }, + { at: [20, 24], kind: 'return' }, + ] }, +] + +describe('typeScript 代码块树', () => { + it('envLock.ts 与原型一致', async () => { + expect(await tree(readFixture('envLock.ts.txt'))).toEqual(ENV_LOCK) + }) + + it('issueLock.ts 与原型一致', async () => { + expect(await tree(readFixture('issueLock.ts.txt'))).toEqual(ISSUE_LOCK) + }) + + it('else / else if 展平成兄弟块，else if 的子块来自它自己的主体', async () => { + const source = [ + 'if (a) {', + ' x()', + '} else if (b) {', + ' y()', + '} else', + ' z()', + ].join('\n') + expect(await tree(source)).toEqual([ + { at: 1, kind: 'if', kids: [{ at: 2, kind: 'statement' }] }, + { at: 3, kind: 'else', kids: [{ at: 4, kind: 'statement' }] }, + { at: 5, kind: 'else', kids: [{ at: 6, kind: 'statement' }] }, + ]) + }) + + it('try / catch / finally 是兄弟块；switch 的 case / default 是子块 if', async () => { + const source = [ + 'try {', + ' a()', + '}', + 'catch {', + ' b()', + '}', + 'finally {', + ' c()', + '}', + 'switch (x) {', + ' case 1:', + ' foo()', + ' break', + ' default:', + ' bar()', + '}', + ].join('\n') + expect(await tree(source)).toEqual([ + { at: 1, kind: 'try', kids: [{ at: 2, kind: 'statement' }] }, + { at: 4, kind: 'catch', kids: [{ at: 5, kind: 'statement' }] }, + { at: 7, kind: 'finally', kids: [{ at: 8, kind: 'statement' }] }, + { at: 10, kind: 'if', kids: [ + { at: 11, kind: 'if', kids: [{ at: 12, kind: 'statement' }, { at: 13, kind: 'statement' }] }, + { at: 14, kind: 'if', kids: [{ at: 15, kind: 'statement' }] }, + ] }, + ]) + }) + + it('类：方法 → function、字段 → field、注释 → comment，装饰器算进成员头部', async () => { + const source = [ + 'export class Store extends Base {', + ' // 缓存', + ' private readonly cache = new Map()', + ' @memo()', + ' get size(): number {', + ' return this.cache.size', + ' }', + ' handle = () => {', + ' this.cache.clear()', + ' }', + '}', + ].join('\n') + expect(await tree(source)).toEqual([ + { at: 1, kind: 'class', kids: [ + { at: 2, kind: 'comment' }, + { at: 3, kind: 'field' }, + { at: [4, 5], kind: 'function', kids: [{ at: 6, kind: 'return' }] }, + { at: 8, kind: 'function', kids: [{ at: 9, kind: 'statement' }] }, + ] }, + ]) + }) + + it('enum 成员 → field；type 别名 → type；多个循环 → loop', async () => { + const source = [ + 'enum Color {', + ' Red = 1,', + ' Blue,', + '}', + 'type Pair = [string, number]', + 'for (let i = 0; i < 3; i++) {', + ' while (busy) wait()', + ' do {', + ' step()', + ' } while (more)', + '}', + ].join('\n') + expect(await tree(source)).toEqual([ + { at: 1, kind: 'type', kids: [{ at: 2, kind: 'field' }, { at: 3, kind: 'field' }] }, + { at: 5, kind: 'type' }, + { at: 6, kind: 'loop', kids: [ + { at: 7, kind: 'loop', kids: [{ at: 7, kind: 'statement' }] }, + { at: 8, kind: 'loop', kids: [{ at: 9, kind: 'statement' }] }, + ] }, + ]) + }) + + it('函数值的变量 → function；回调参数带语句块主体的表达式语句 → statement 带子块', async () => { + const source = [ + 'const App = () => {', + ' useEffect(() => {', + ' load()', + ' }, [])', + ' let n = 0', + ' return null', + '}', + 'let total = 0', + 'describe(\'x\', () => {', + ' it(\'y\', async () => {', + ' await run()', + ' })', + '})', + ].join('\n') + expect(await tree(source, 'typescriptreact')).toEqual([ + { at: 1, kind: 'function', kids: [ + { at: 2, kind: 'statement', kids: [{ at: 3, kind: 'statement' }] }, + { at: 5, kind: 'variable' }, + { at: 6, kind: 'return' }, + ] }, + { at: 8, kind: 'variable' }, + { at: 9, kind: 'statement', kids: [ + { at: 10, kind: 'statement', kids: [{ at: 11, kind: 'statement' }] }, + ] }, + ]) + }) + + it('相邻行的行注释合成一块，空行隔开的分开；行尾注释不单独成块', async () => { + const source = [ + '// a', + '// b', + '', + '// c', + 'const x = 1 // 行尾', + 'foo() // 行尾', + ].join('\n') + expect(await tree(source, 'javascript')).toEqual([ + { at: [1, 2], kind: 'comment' }, + { at: 4, kind: 'comment' }, + { at: 5, kind: 'const' }, + { at: 6, kind: 'statement' }, + ]) + }) +}) diff --git a/vscode/src/codeReader/languages/typescript.ts b/vscode/src/codeReader/languages/typescript.ts new file mode 100644 index 0000000..ed5caf0 --- /dev/null +++ b/vscode/src/codeReader/languages/typescript.ts @@ -0,0 +1,328 @@ +import type { BlockKind, CodeBlock, LanguageAdapter, Node, TokenRole } from '../contract' +import * as path from 'node:path' +import { compoundBlock, extendHeader, leafBlock, listBlocks, named, sameNode } from './shared' + +const GRAMMARS: Record = { + typescript: 'tree-sitter-typescript.wasm', + typescriptreact: 'tree-sitter-tsx.wasm', + javascript: 'tree-sitter-javascript.wasm', + javascriptreact: 'tree-sitter-javascript.wasm', +} + +const FUNCTION_VALUES = new Set(['arrow_function', 'function_expression', 'function', 'generator_function']) +const LOOPS = new Set(['for_statement', 'for_in_statement', 'while_statement', 'do_statement']) +const CLASSES = new Set(['class_declaration', 'abstract_class_declaration', 'class']) + +function isLineComment(comment: Node): boolean { + return comment.text.startsWith('//') +} + +/** 带语句块主体的函数值（箭头函数 / 函数表达式）的主体。 */ +function functionBody(value: Node | null | undefined): Node | null { + if (!value || !FUNCTION_VALUES.has(value.type)) + return null + const body = value.childForFieldName('body') + return body?.type === 'statement_block' ? body : null +} + +/** 调用参数里第一个带语句块主体的函数的主体：useEffect(() => { … })、describe('x', () => { … })。 */ +function callbackBody(expression: Node | null | undefined): Node | null { + const call = expression?.type === 'await_expression' ? expression.firstNamedChild : expression + if (call?.type !== 'call_expression' && call?.type !== 'new_expression') + return null + for (const arg of named(call.childForFieldName('arguments'))) { + const body = functionBody(arg) + if (body) + return body + } + return null +} + +function statements(items: Node[], top: boolean, afterRow: number): CodeBlock[] { + return listBlocks(items, afterRow, item => statementBlocks(item, top), isLineComment) +} + +/** 主体是语句块时子块来自它的语句，否则主体本身就是唯一的子语句（`if (x) return`）。 */ +function bodyBlock(kind: BlockKind, outer: Node, body: Node | null, top = false): CodeBlock { + if (!body) + return leafBlock(kind, outer) + const items = body.type === 'statement_block' ? named(body) : [body] + return compoundBlock(kind, outer, body.startIndex, items, (list, afterRow) => statements(list, top, afterRow)) +} + +function statementBlocks(node: Node, top: boolean): CodeBlock[] { + switch (node.type) { + case 'import_statement': + return [leafBlock('import', node)] + case 'export_statement': + return exportBlocks(node, top) + case 'if_statement': + return ifBlocks(node, node, 'if') + case 'try_statement': + return tryBlocks(node) + case 'switch_statement': + return [switchBlock(node)] + case 'switch_case': + case 'switch_default': + return [caseBlock(node)] + case 'return_statement': + return [leafBlock('return', node)] + case 'expression_statement': + return [bodyBlock('statement', node, callbackBody(node.firstNamedChild))] + case 'labeled_statement': { + const [first, ...rest] = statementBlocks(node.childForFieldName('body') ?? node, top) + return first ? [{ ...extendHeader(first, node), node }, ...rest] : [] + } + case 'statement_block': + return statements(named(node), top, node.startPosition.row) + case 'empty_statement': + return [] + default: + if (LOOPS.has(node.type)) + return [bodyBlock('loop', node, node.childForFieldName('body'))] + return declarationBlocks(node, node, top) + } +} + +/** export 包着声明时按声明定种类，块节点取整个 export_statement（头部包含 export）。 */ +function exportBlocks(node: Node, top: boolean): CodeBlock[] { + const declaration = node.childForFieldName('declaration') + if (declaration) + return declarationBlocks(declaration, node, top) + const value = node.childForFieldName('value') + if (value && CLASSES.has(value.type)) + return [classBlock(value, node)] + const body = functionBody(value) + return [body ? bodyBlock('function', node, body) : leafBlock('statement', node)] +} + +function declarationBlocks(declaration: Node, outer: Node, top: boolean): CodeBlock[] { + switch (declaration.type) { + case 'function_declaration': + case 'generator_function_declaration': + return [bodyBlock('function', outer, declaration.childForFieldName('body'))] + case 'function_signature': + return [leafBlock('function', outer)] + case 'lexical_declaration': + case 'variable_declaration': + return [variableBlock(declaration, outer, top)] + case 'interface_declaration': + case 'enum_declaration': + return [membersBlock(declaration, outer)] + case 'type_alias_declaration': + return [leafBlock('type', outer)] + default: { + if (CLASSES.has(declaration.type)) + return [classBlock(declaration, outer)] + const body = declaration.childForFieldName('body') + return [body?.type === 'statement_block' ? bodyBlock('statement', outer, body) : leafBlock('statement', outer)] + } + } +} + +/** 唯一声明值是带语句块主体的函数 → function；否则模块顶层的 const → const，其余 → variable。 */ +function variableBlock(declaration: Node, outer: Node, top: boolean): CodeBlock { + const declarators = named(declaration).filter(child => child.type === 'variable_declarator') + const value = declarators.length === 1 ? declarators[0].childForFieldName('value') : null + const body = functionBody(value) + if (body) + return bodyBlock('function', outer, body) + const isConst = declaration.type === 'lexical_declaration' && declaration.childForFieldName('kind')?.type === 'const' + return bodyBlock(top && isConst ? 'const' : 'variable', outer, callbackBody(value)) +} + +/** if 的 else / else if 展平成紧跟其后的兄弟块。 */ +function ifBlocks(ifNode: Node, outer: Node, kind: 'if' | 'else'): CodeBlock[] { + const blocks = [bodyBlock(kind, outer, ifNode.childForFieldName('consequence'))] + const alternative = ifNode.childForFieldName('alternative') + if (alternative) { + const inner = named(alternative).find(child => child.type !== 'comment') ?? null + if (inner?.type === 'if_statement') + blocks.push(...ifBlocks(inner, alternative, 'else')) + else + blocks.push(bodyBlock('else', alternative, inner)) + } + return blocks +} + +function tryBlocks(node: Node): CodeBlock[] { + const blocks = [bodyBlock('try', node, node.childForFieldName('body'))] + const handler = node.childForFieldName('handler') + if (handler) + blocks.push(bodyBlock('catch', handler, handler.childForFieldName('body'))) + const finalizer = node.childForFieldName('finalizer') + if (finalizer) + blocks.push(bodyBlock('finally', finalizer, finalizer.childForFieldName('body'))) + return blocks +} + +function switchBlock(node: Node): CodeBlock { + const body = node.childForFieldName('body') + if (!body) + return leafBlock('if', node) + return compoundBlock('if', node, body.startIndex, named(body), (list, afterRow) => statements(list, false, afterRow)) +} + +/** case / default：头部到冒号为止，子块来自它下面的语句。 */ +function caseBlock(node: Node): CodeBlock { + const [first] = node.childrenForFieldName('body') + const bodyStart = first?.startIndex ?? node.endIndex + const items = named(node).filter(child => child.startIndex >= bodyStart) + return compoundBlock('if', node, bodyStart, items, (list, afterRow) => statements(list, false, afterRow)) +} + +function classBlock(declaration: Node, outer: Node): CodeBlock { + const body = declaration.childForFieldName('body') + if (!body) + return leafBlock('class', outer) + return compoundBlock('class', outer, body.startIndex, named(body), (list, afterRow) => { + let decorators: Node | undefined + return listBlocks(list, afterRow, (member) => { + if (member.type === 'decorator') { + decorators ??= member + return [] + } + const blocks = memberBlocks(member) + const from = decorators + decorators = undefined + return from ? blocks.map((block, i) => i === 0 ? extendHeader(block, from) : block) : blocks + }, isLineComment) + }) +} + +function memberBlocks(member: Node): CodeBlock[] { + switch (member.type) { + case 'method_definition': + return [bodyBlock('function', member, member.childForFieldName('body'))] + case 'method_signature': + case 'abstract_method_signature': + return [leafBlock('function', member)] + case 'class_static_block': + return [bodyBlock('statement', member, member.childForFieldName('body'))] + default: { + const body = functionBody(member.childForFieldName('value')) + return [body ? bodyBlock('function', member, body) : leafBlock('field', member)] + } + } +} + +/** interface / enum：成员（属性、方法签名、枚举项）→ field。 */ +function membersBlock(declaration: Node, outer: Node): CodeBlock { + const body = declaration.childForFieldName('body') + if (!body) + return leafBlock('type', outer) + return compoundBlock('type', outer, body.startIndex, named(body), (list, afterRow) => + listBlocks(list, afterRow, member => [leafBlock('field', member)], isLineComment)) +} + +// ─── 源码着色 ───────────────────────────────────────────────────────────────── + +/** 流程关键字；其余字母关键字都算声明关键字（storage）。 */ +const FLOW_KEYWORDS = new Set([ + 'import', + 'from', + 'as', + 'export', + 'default', + 'if', + 'else', + 'for', + 'of', + 'in', + 'while', + 'do', + 'switch', + 'case', + 'break', + 'continue', + 'return', + 'throw', + 'try', + 'catch', + 'finally', + 'await', + 'yield', +]) +const LITERAL_KEYWORDS = new Set(['true', 'false', 'null', 'undefined', 'this', 'super']) +const STRING_NODES = new Set(['string', 'template_string', 'regex']) +const FUNCTION_NAMED = new Set(['function_declaration', 'generator_function_declaration', 'function_expression', 'function', 'generator_function', 'method_definition', 'method_signature', 'abstract_method_signature', 'function_signature']) + +function insideString(leaf: Node): boolean { + for (let node = leaf.parent; node; node = node.parent) { + if (node.type === 'template_substitution') + return false + if (STRING_NODES.has(node.type)) + return true + } + return false +} + +function isCallee(node: Node): boolean { + const parent = node.parent + return !!parent && parent.type === 'call_expression' && sameNode(parent.childForFieldName('function'), node) +} + +function nameRole(leaf: Node): TokenRole { + const parent = leaf.parent + if (!parent) + return 'variable' + if (FUNCTION_NAMED.has(parent.type) && sameNode(parent.childForFieldName('name'), leaf)) + return 'function' + if (isCallee(leaf)) + return 'function' + if (parent.type === 'member_expression' && sameNode(parent.childForFieldName('property'), leaf) && isCallee(parent)) + return 'function' + if (parent.type === 'new_expression') + return 'type' + if (parent.type === 'variable_declarator' && sameNode(parent.childForFieldName('name'), leaf) && functionBody(parent.childForFieldName('value'))) + return 'function' + return 'variable' +} + +export function typescriptTokenRole(leaf: Node): TokenRole { + if (leaf.type === 'comment' || leaf.type === 'hash_bang_line') + return 'comment' + if (insideString(leaf)) + return 'string' + if (!leaf.isNamed) { + if (leaf.parent?.type === 'predefined_type') + return 'type' + if (/^[a-z]+$/.test(leaf.type)) + return FLOW_KEYWORDS.has(leaf.type) ? 'keyword' : 'storage' + return 'plain' + } + if (LITERAL_KEYWORDS.has(leaf.type)) + return 'storage' + switch (leaf.type) { + case 'number': + return 'number' + case 'type_identifier': + return 'type' + case 'identifier': + case 'property_identifier': + case 'private_property_identifier': + return nameRole(leaf) + case 'shorthand_property_identifier': + case 'shorthand_property_identifier_pattern': + case 'statement_identifier': + return 'variable' + default: + return 'plain' + } +} + +/** TypeScript / TSX / JavaScript / JSX 语言插件。@param wasmDir 语法包 .wasm 所在目录 */ +export function createTypeScriptAdapter(wasmDir: string): LanguageAdapter { + return { + id: 'typescript', + languageIds: Object.keys(GRAMMARS), + grammarPath: (languageId) => { + const file = GRAMMARS[languageId] + if (!file) + throw new Error(`TypeScript 语言插件不支持 ${languageId}`) + return path.join(wasmDir, file) + }, + toBlocks: root => statements(named(root), true, -1), + tokenRole: typescriptTokenRole, + } +} diff --git a/vscode/src/codeReader/registry.test.ts b/vscode/src/codeReader/registry.test.ts new file mode 100644 index 0000000..c720710 --- /dev/null +++ b/vscode/src/codeReader/registry.test.ts @@ -0,0 +1,88 @@ +import type { Memento } from 'vscode' +import type { LanguageAdapter, Translator } from './contract' +import { describe, expect, it } from 'vitest' +import { CodeReaderRegistry, TRANSLATOR_CHOICE_KEY } from './registry' + +function fakeMemento(store: Record = {}): Pick { + return { + get: (key: string): T | undefined => store[key] as T | undefined, + update: async (key: string, value: unknown) => { + store[key] = value + }, + } as Pick +} + +function adapter(id: string, languageIds: string[]): LanguageAdapter { + return { id, languageIds, grammarPath: () => '', toBlocks: () => [], tokenRole: () => 'plain' } +} + +function translator(id: string, languageIds: string[]): Translator { + return { id, label: id, languageIds, translate: () => [] } +} + +describe('codeReaderRegistry', () => { + it('注册 / 注销语言插件后 languageIds 跟着变，并触发变化事件', () => { + const registry = new CodeReaderRegistry(fakeMemento()) + let changes = 0 + registry.onDidChange(() => { + changes++ + }) + const tsReg = registry.registerLanguage(adapter('typescript', ['typescript', 'javascript'])) + const pyReg = registry.registerLanguage(adapter('python', ['python'])) + expect(registry.languageIds()).toEqual(['javascript', 'python', 'typescript']) + expect(registry.languageFor('python')?.id).toBe('python') + + tsReg.dispose() + tsReg.dispose() + expect(registry.languageIds()).toEqual(['python']) + expect(registry.languageFor('typescript')).toBeUndefined() + expect(changes).toBe(3) + + pyReg.dispose() + expect(registry.languageIds()).toEqual([]) + }) + + it('同一语言多个语言插件时后注册的生效，注销后退回先前的', () => { + const registry = new CodeReaderRegistry(fakeMemento()) + registry.registerLanguage(adapter('builtin', ['python'])) + const override = registry.registerLanguage(adapter('external', ['python'])) + expect(registry.languageFor('python')?.id).toBe('external') + override.dispose() + expect(registry.languageFor('python')?.id).toBe('builtin') + }) + + it('可选翻译按语言过滤；没选过时用第一个，按 globalState 记住选择', async () => { + const store: Record = {} + const registry = new CodeReaderRegistry(fakeMemento(store)) + registry.registerTranslator(translator('ts.zh', ['typescript'])) + const ai = registry.registerTranslator(translator('ts.ai', ['typescript'])) + registry.registerTranslator(translator('py.zh', ['python'])) + + expect(registry.translatorsFor('typescript').map(t => t.id)).toEqual(['ts.zh', 'ts.ai']) + expect(registry.translatorsFor('rust')).toEqual([]) + expect(registry.selectedTranslator('typescript')?.id).toBe('ts.zh') + + await registry.selectTranslator('typescript', 'ts.ai') + await registry.selectTranslator('python', 'py.zh') + expect(store[TRANSLATOR_CHOICE_KEY]).toEqual({ typescript: 'ts.ai', python: 'py.zh' }) + expect(registry.selectedTranslator('typescript')?.id).toBe('ts.ai') + expect(new CodeReaderRegistry(fakeMemento(store)).selectedTranslator('typescript')).toBeUndefined() + + ai.dispose() + expect(registry.translatorsFor('typescript').map(t => t.id)).toEqual(['ts.zh']) + expect(registry.selectedTranslator('typescript')?.id).toBe('ts.zh') + }) + + it('变化事件可退订，也能放进 disposables', () => { + const registry = new CodeReaderRegistry(fakeMemento()) + let changes = 0 + const bag: { dispose: () => unknown }[] = [] + registry.onDidChange(() => { + changes++ + }, undefined, bag) + registry.registerTranslator(translator('a', ['python'])) + bag.forEach(d => d.dispose()) + registry.registerTranslator(translator('b', ['python'])) + expect(changes).toBe(1) + }) +}) diff --git a/vscode/src/codeReader/registry.ts b/vscode/src/codeReader/registry.ts new file mode 100644 index 0000000..57825a5 --- /dev/null +++ b/vscode/src/codeReader/registry.ts @@ -0,0 +1,86 @@ +import type { Disposable, Event, Memento } from 'vscode' +import type { CodeReaderApi, LanguageAdapter, Translator } from './contract' + +/** globalState 里按 languageId 记选中的翻译插件 id：Record。 */ +export const TRANSLATOR_CHOICE_KEY = 'superpowers.codeReader.translatorByLanguage' + +interface Entry { value: T } + +/** + * 语言插件、翻译插件的注册表。内置实现和其他扩展都通过它注册（它本身就是对外的 CodeReaderApi）。 + * 不依赖 vscode 运行时：Disposable 和事件都是结构兼容的普通对象，便于单元测试。 + */ +export class CodeReaderRegistry implements CodeReaderApi { + private readonly adapters: Entry[] = [] + private readonly translators: Entry[] = [] + private readonly listeners = new Set<() => void>() + + constructor(private readonly state: Pick) {} + + /** 注册 / 注销任何插件后触发。 */ + readonly onDidChange: Event = (listener, thisArgs?, disposables?) => { + const bound = (): void => { + listener.call(thisArgs, undefined) + } + this.listeners.add(bound) + const disposable = { dispose: () => this.listeners.delete(bound) } + disposables?.push(disposable) + return disposable + } + + registerLanguage(adapter: LanguageAdapter): Disposable { + return this.add(this.adapters, adapter) + } + + registerTranslator(translator: Translator): Disposable { + return this.add(this.translators, translator) + } + + /** 当前语言插件覆盖的全部 languageId（去重、排序），给 when 子句用。 */ + languageIds(): string[] { + return [...new Set(this.adapters.flatMap(entry => entry.value.languageIds))].sort() + } + + /** 某 languageId 的语言插件；多个插件覆盖同一语言时后注册的生效。 */ + languageFor(languageId: string): LanguageAdapter | undefined { + return [...this.adapters].reverse().find(entry => entry.value.languageIds.includes(languageId))?.value + } + + /** 某 languageId 可选的翻译插件，按注册顺序。 */ + translatorsFor(languageId: string): Translator[] { + return this.translators.map(entry => entry.value).filter(translator => translator.languageIds.includes(languageId)) + } + + /** 选中的翻译：globalState 里记的那个还注册着就用它，否则用第一个。 */ + selectedTranslator(languageId: string): Translator | undefined { + const candidates = this.translatorsFor(languageId) + const chosen = this.state.get>(TRANSLATOR_CHOICE_KEY)?.[languageId] + return candidates.find(translator => translator.id === chosen) ?? candidates[0] + } + + /** 按 languageId 记住选中的翻译。 */ + async selectTranslator(languageId: string, translatorId: string): Promise { + const choices = this.state.get>(TRANSLATOR_CHOICE_KEY) ?? {} + await this.state.update(TRANSLATOR_CHOICE_KEY, { ...choices, [languageId]: translatorId }) + } + + private add(list: Entry[], value: T): Disposable { + const entry = { value } + list.push(entry) + this.fire() + return { + dispose: () => { + const index = list.indexOf(entry) + if (index < 0) + return + list.splice(index, 1) + this.fire() + }, + } + } + + private fire(): void { + for (const listener of [...this.listeners]) + listener() + } +} diff --git a/vscode/src/codeReader/runtime.ts b/vscode/src/codeReader/runtime.ts new file mode 100644 index 0000000..16e1140 --- /dev/null +++ b/vscode/src/codeReader/runtime.ts @@ -0,0 +1,50 @@ +import type { Tree } from '@vscode/tree-sitter-wasm' +import type { LanguageAdapter } from './contract' +import * as path from 'node:path' +import { Language, Parser } from '@vscode/tree-sitter-wasm' + +/** 打进扩展包里的 wasm 目录：dist/index.cjs 旁边的 dist/tree-sitter（esbuild 构建时复制）。只在打包产物里成立。 */ +export function bundledWasmDir(): string { + return path.join(__dirname, 'tree-sitter') +} + +// web-tree-sitter 的运行时是进程级单例：Parser.init 只能有效调用一次，后来的 wasm 目录不再生效 +let runtimeReady: Promise | undefined + +function initRuntime(wasmDir: string): Promise { + runtimeReady ??= Parser.init({ locateFile: (file: string) => path.join(wasmDir, file) }) + .catch((err: unknown) => { + runtimeReady = undefined + throw err + }) + return runtimeReady +} + +/** 解析运行时：初始化一次 web-tree-sitter，语法包按 wasm 路径懒加载并缓存。 */ +export class TreeSitterRuntime { + private readonly parsers = new Map>() + + /** @param wasmDir 放 tree-sitter.wasm 的目录 */ + constructor(private readonly wasmDir: string) {} + + /** 返回的语法树占着 wasm 内存，用完必须 `tree.delete()`。 */ + async parse(adapter: LanguageAdapter, languageId: string, source: string): Promise { + const parser = await this.parserFor(adapter.grammarPath(languageId)) + const tree = parser.parse(source) + if (!tree) + throw new Error(`tree-sitter 没能解析 ${languageId} 源码`) + return tree + } + + private parserFor(grammarPath: string): Promise { + let parser = this.parsers.get(grammarPath) + if (!parser) { + parser = initRuntime(this.wasmDir) + .then(() => Language.load(grammarPath)) + .then(language => new Parser().setLanguage(language)) + parser.catch(() => this.parsers.delete(grammarPath)) + this.parsers.set(grammarPath, parser) + } + return parser + } +} diff --git a/vscode/src/codeReader/testSupport.ts b/vscode/src/codeReader/testSupport.ts new file mode 100644 index 0000000..4c2cc14 --- /dev/null +++ b/vscode/src/codeReader/testSupport.ts @@ -0,0 +1,45 @@ +import type { BlockKind, CodeBlock, LanguageAdapter } from './contract' +import { readFileSync } from 'node:fs' +import * as path from 'node:path' +import { TreeSitterRuntime } from './runtime' + +/** 单元测试直接用依赖包里的 wasm，不走打包产物。 */ +export const WASM_DIR = path.resolve(__dirname, '../../node_modules/@vscode/tree-sitter-wasm/wasm') + +export const runtime = new TreeSitterRuntime(WASM_DIR) + +export function readFixture(name: string): string { + return readFileSync(path.join(__dirname, 'fixtures', name), 'utf8') +} + +/** 解析 source，把代码块树交给 use；语法树随后释放。 */ +export async function withBlocks(adapter: LanguageAdapter, languageId: string, source: string, use: (blocks: CodeBlock[]) => T): Promise { + const tree = await runtime.parse(adapter, languageId, source) + try { + return use(adapter.toBlocks(tree.rootNode, source)) + } + finally { + tree.delete() + } +} + +/** 和原型 CODE_TREE 同样的写法：at 是 1 起的行号（单行，或 [起, 止]）。 */ +export interface Outline { at: number | [number, number], kind: BlockKind, kids?: Outline[] } + +export function outline(blocks: CodeBlock[]): Outline[] { + return blocks.map((block) => { + const from = block.header.startRow + 1 + const to = block.header.endRow + 1 + const item: Outline = { at: from === to ? from : [from, to], kind: block.kind } + if (block.children.length > 0) + item.kids = outline(block.children) + return item + }) +} + +export function walk(blocks: CodeBlock[], visit: (block: CodeBlock) => void): void { + for (const block of blocks) { + visit(block) + walk(block.children, visit) + } +} diff --git a/vscode/src/codeReader/translators/pythonZh.ts b/vscode/src/codeReader/translators/pythonZh.ts new file mode 100644 index 0000000..5adfc92 --- /dev/null +++ b/vscode/src/codeReader/translators/pythonZh.ts @@ -0,0 +1,77 @@ +import type { Node, Translator } from '../contract' +import { insideAnnotation, pythonTokenRole } from '../languages/python' +import { createZhTemplateTranslator } from './zhTemplate' + +function stringContent(node: Node): string { + if (node.type === 'string_content') + return node.text + return node.children.map(child => child ? stringContent(child) : '').join('') +} + +/** `#` 注释去掉井号；docstring 只取引号里的内容。 */ +function commentText(node: Node): string { + return node.type === 'comment' ? node.text.replace(/^#+\s?/, '') : stringContent(node) +} + +export const pythonZhTranslator: Translator = createZhTemplateTranslator({ + id: 'python.zh-template', + label: '中文直译', + languageIds: ['python'], + tokenRole: pythonTokenRole, + keywords: { + 'import': '导入', + 'from': '来自', + 'as': '作为', + 'def': '函数', + 'async': '异步', + 'await': '等待', + 'class': '类', + 'return': '返回', + 'if': '如果', + 'elif': '否则如果', + 'else': '否则', + 'for': '遍历', + 'in': { zh: '在', byParent: { for_statement: '取自', for_in_clause: '取自' } }, + 'while': '当', + 'try': '尝试', + 'except': '捕获', + 'finally': '最终', + 'raise': '抛出', + 'with': '使用', + 'yield': '产出', + 'lambda': '匿名函数', + 'pass': '跳过', + 'break': '跳出', + 'continue': '继续', + 'global': '全局', + 'nonlocal': '非局部', + 'del': '删除', + 'assert': '断言', + 'not': '非', + 'and': '与', + 'or': '或', + 'is': '是', + 'not in': '不在', + 'is not': '不是', + 'match': '匹配', + 'case': '情况', + }, + literals: { none: '空', true: '真', false: '假' }, + builtinTypes: { + str: '字符串', + int: '整数', + float: '浮点数', + bool: '布尔', + list: '列表', + dict: '字典', + set: '集合', + tuple: '元组', + bytes: '字节串', + }, + isTypePosition: leaf => leaf.type === 'identifier' && leaf.parent?.type !== 'attribute' && insideAnnotation(leaf), + operators: { '==': '等于', '!=': '不等于' }, + operatorParents: ['comparison_operator'], + emptyLiterals: { list: '空列表', dictionary: '空字典' }, + stringNodes: ['string'], + commentText, +}) diff --git a/vscode/src/codeReader/translators/typescriptZh.ts b/vscode/src/codeReader/translators/typescriptZh.ts new file mode 100644 index 0000000..79e7043 --- /dev/null +++ b/vscode/src/codeReader/translators/typescriptZh.ts @@ -0,0 +1,90 @@ +import type { Node, Translator } from '../contract' +import { typescriptTokenRole } from '../languages/typescript' +import { createZhTemplateTranslator } from './zhTemplate' + +function commentText(node: Node): string { + const text = node.text + if (text.startsWith('//')) + return text.replace(/^\/{2,}\s?/, '') + return text + .replace(/^\/\*+/, '') + .replace(/\*+\/$/, '') + .split('\n') + .map(line => line.replace(/^\s*\*(?!\/)\s?/, '')) + .join('\n') +} + +export const typescriptZhTranslator: Translator = createZhTemplateTranslator({ + id: 'typescript.zh-template', + label: '中文直译', + languageIds: ['typescript', 'typescriptreact', 'javascript', 'javascriptreact'], + tokenRole: typescriptTokenRole, + keywords: { + // 流程 + import: '导入', + from: '来自', + as: '作为', + export: '公共', + default: '默认', + if: '如果', + else: '否则', + for: '遍历', + of: '取自', + in: '在', + while: '当', + do: '执行', + switch: '分支', + case: '情况', + break: '跳出', + continue: '继续', + return: '返回', + throw: '抛出', + try: '尝试', + catch: '捕获', + finally: '最终', + await: '等待', + yield: '产出', + // 声明 + const: '常量', + let: '变量', + var: '变量', + function: '函数', + async: '异步', + class: '类', + extends: '继承', + implements: '实现', + interface: '接口', + type: '类型', + enum: '枚举', + new: '新建', + instanceof: '属于', + typeof: '类型为', + keyof: '键名', + readonly: '只读', + public: '公共', + private: '私有', + protected: '受保护', + static: '静态', + abstract: '抽象', + delete: '删除', + }, + literals: { true: '真', false: '假', null: '空', undefined: '未定义' }, + builtinTypes: { + string: '字符串', + number: '数值', + boolean: '布尔', + any: '任意', + unknown: '未知', + never: '永不', + void: '无', + object: '对象', + bigint: '大整数', + symbol: '符号', + }, + isTypePosition: leaf => leaf.parent?.type === 'predefined_type', + operators: { '===': '全等', '!==': '不全等', '==': '等于', '!=': '不等于', '&&': '与', '||': '或', '!': '非' }, + operatorParents: ['binary_expression', 'unary_expression'], + emptyLiterals: { array: '空数组' }, + stringNodes: ['string', 'template_string', 'regex'], + commentText, +}) diff --git a/vscode/src/codeReader/translators/zhTemplate.test.ts b/vscode/src/codeReader/translators/zhTemplate.test.ts new file mode 100644 index 0000000..004e659 --- /dev/null +++ b/vscode/src/codeReader/translators/zhTemplate.test.ts @@ -0,0 +1,179 @@ +import type { LanguageAdapter, Segment, Translator } from '../contract' +import { describe, expect, it } from 'vitest' +import { createPythonAdapter } from '../languages/python' +import { createTypeScriptAdapter } from '../languages/typescript' +import { readFixture, walk, WASM_DIR, withBlocks } from '../testSupport' +import { pythonZhTranslator } from './pythonZh' +import { typescriptZhTranslator } from './typescriptZh' + +const ts = createTypeScriptAdapter(WASM_DIR) +const py = createPythonAdapter(WASM_DIR) + +interface Translated { kind: string, segments: Segment[] } + +/** 每块的译文，按 header 起始行（1 起）索引；同一行有多块时取外层那块。 */ +function translateAll(adapter: LanguageAdapter, languageId: string, translator: Translator, source: string): Promise> { + return withBlocks(adapter, languageId, source, (blocks) => { + const byLine = new Map() + walk(blocks, (block) => { + const line = block.header.startRow + 1 + if (!byLine.has(line)) + byLine.set(line, { kind: block.kind, segments: translator.translate(block, source) }) + }) + return byLine + }) +} + +function text(segments: Segment[] | undefined): string { + return (segments ?? []).map(segment => segment.text).join('') +} + +function texts(map: Map): Record { + return Object.fromEntries([...map].map(([line, { segments }]) => [line, text(segments)])) +} + +const CJK = /\p{Script=Han}/u +const CJK_ONLY = /^\p{Script=Han}+$/u +const WORD_CHAR = /[\p{Script=Han}\w]/u + +/** 被翻的词单独成段，而且不和前后的字母、数字、汉字粘在一起。 */ +function expectWordsApart(map: Map): void { + for (const [line, { kind, segments }] of map) { + if (kind === 'comment') + continue + const full = text(segments) + let at = 0 + for (const segment of segments) { + if (segment.role !== 'string' && CJK.test(segment.text)) { + expect(segment.text, `L${line} ${full}`).toMatch(CJK_ONLY) + for (const neighbor of [full[at - 1], full[at + segment.text.length]]) + expect(neighbor ?? ' ', `L${line} "${full}" 里的「${segment.text}」`).not.toMatch(WORD_CHAR) + } + at += segment.text.length + } + } +} + +describe('typeScript 中文直译', () => { + it('spec 样例逐字一致', async () => { + const env = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt')) + const issue = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt')) + expect(text(env.get(23)?.segments)).toBe('公共 异步 函数 findEnvFiles(workspaceRoot: 字符串): Promise<字符串[]>') + expect(text(env.get(38)?.segments)).toBe('异步 函数 chmodAll(workspaceRoot: 字符串, mode: 数值): Promise') + expect(text(env.get(12)?.segments)).toBe('导入 { promises 作为 fsp } 来自 \'node:fs\'') + expect(text(env.get(25)?.segments)).toBe('如果 (非 folder)') + expect(text(env.get(26)?.segments)).toBe('返回 空数组') + expect(text(env.get(41)?.segments)).toBe('遍历 (常量 f 取自 files)') + expect(text(env.get(35)?.segments)).toBe('failed: { path: 字符串, reason: 字符串 }[]') + expect(text(env.get(53)?.segments)).toBe('Set every `.env*` file in the workspace to 0o444 (read-only).') + expect(text(issue.get(14)?.segments)).toBe('常量 issue = issues.find(i => i.number 全等 issueNumber)') + expect(text(issue.get(20)?.segments)).toBe('返回 { locked: 真, prerequisiteNumber: prereq.number, prerequisiteColumn: prereq.column }') + }) + + it('其余块：结构照抄，复合语句去掉结尾的 {，注释去掉注释符号', async () => { + const env = texts(await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt'))) + const issue = texts(await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt'))) + expect(env[24]).toBe('常量 folder = workspace.workspaceFolders?.find(f => f.uri.fsPath 全等 workspaceRoot)') + expect(env[28]).toBe('常量 uris = 等待 workspace.findFiles(include, EXCLUDE_PATTERN)') + expect(env[32]).toBe('公共 接口 EnvLockResult') + expect(env[42]).toBe('尝试') + expect(env[46]).toBe('捕获 (err)') + expect(env[47]).toBe('result.failed.push({ path: f, reason: err 属于 Error ? err.message : String(err) })') + expect(env[1]).toBe('.env file lock / unlock helpers. Recursively scans the main workspace folder for `.env*` files (skipping `node_modules`, `.git`, `.claude`) and chmods them to 0o444 (locked) or 0o644 (unlocked) in bulk. Backs the toolbar Lock toggle in `PanelHeader`. No persistent watcher — re-scan happens on every toggle, so newly created `.env` files are only picked up the next time the user clicks the button.') + expect(issue[1]).toBe('导入 类型 { Issue } 来自 \'../gitea/types\'') + expect(issue[5]).toBe('prerequisiteNumber?: 数值') + expect(issue[9]).toBe('纯判定：工单被未完成的前置挡住才算锁定。快照里找不到工单或前置一律放行， 与 webview 的 `isIssueLocked` 保持同一语义。') + expect(issue[15]).toBe('如果 (非 issue 或 issue.prerequisite 全等 未定义)') + }) + + it('每个被翻的词单独成段，着色取语法树词性', async () => { + const env = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt')) + const issue = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt')) + expect(env.get(23)?.segments.filter(segment => segment.role !== 'plain').slice(0, 4)).toEqual([ + { text: '公共', role: 'keyword' }, + { text: '异步', role: 'storage' }, + { text: '函数', role: 'storage' }, + { text: 'findEnvFiles', role: 'function' }, + ]) + expect(issue.get(14)?.segments).toContainEqual({ text: '全等', role: 'operator' }) + expect(env.get(26)?.segments).toEqual([{ text: '返回', role: 'keyword' }, { text: ' ', role: 'plain' }, { text: '空数组', role: 'literal' }]) + expectWordsApart(env) + expectWordsApart(issue) + }) + + it('属性名、对象键、非空断言不被误翻', async () => { + const source = [ + 'const o = { if: 1, return: i.number, string: x!.type, new: a.default }', + 'let n = a !== b && !c || d == e != f', + // eslint-disable-next-line no-template-curly-in-string -- 被翻译的源码文本里本来就有模板字符串 + 'const s: string | void = typeof v === \'string\' ? `${v}` : \'number\'', + ].join('\n') + const map = texts(await translateAll(ts, 'typescript', typescriptZhTranslator, source)) + expect(map[1]).toBe('常量 o = { if: 1, return: i.number, string: x!.type, new: a.default }') + expect(map[2]).toBe('变量 n = a 不全等 b 与 非 c 或 d 等于 e 不等于 f') + // eslint-disable-next-line no-template-curly-in-string -- 模板字符串原样照抄进译文 + expect(map[3]).toBe('常量 s: 字符串 | 无 = 类型为 v 全等 \'string\' ? `${v}` : \'number\'') + }) + + it('多行合成一行、去掉右括号前的尾逗号、行中注释不进译文', async () => { + const source = [ + 'call(', + ' first, // 第一个', + ' [],', + ')', + ].join('\n') + expect(texts(await translateAll(ts, 'typescript', typescriptZhTranslator, source))[1]).toBe('call( first, 空数组 )') + }) +}) + +describe('python 中文直译', () => { + it('样例全文逐块一致', async () => { + const map = await translateAll(py, 'python', pythonZhTranslator, readFixture('sample.py')) + expect(texts(map)).toEqual({ + 1: 'Scan a workspace for .env files.', + 3: '导入 os', + 4: '来自 pathlib 导入 Path 作为 P', + 6: 'MAX_DEPTH = 8', + 7: 'skipped = 空列表', + 10: '函数 find_env_files(root: 字符串) -> 列表[字符串]', + 11: 'Return every .env file under root.', + 12: '跳过备份文件', + 13: 'found = [p 遍历 p 取自 os.listdir(root) 如果 p.startswith(\'.env\') 与 非 p.endswith(\'.bak\')]', + 14: '如果 非 found', + 15: '返回 空列表', + 16: '否则如果 len(found) > MAX_DEPTH', + 17: 'found = found[:MAX_DEPTH]', + 18: '否则', + 19: '跳过', + 20: '遍历 name 取自 found', + 21: '如果 name 在 skipped', + 22: '继续', + 23: '尝试', + 24: '使用 open(os.path.join(root, found[0])) 作为 handle', + 25: 'handle.read()', + 26: '捕获 OSError 作为 err', + 27: '抛出 RuntimeError(str(err)) 来自 err', + 28: '最终', + 29: 'skipped.clear()', + 30: '返回 found', + 33: '类 EnvLockResult', + 34: 'total: 整数', + 35: 'ok: 列表[字符串] = 空列表', + 36: 'failed: 字典[字符串, 字符串] = 空字典', + 37: 'label = \'env\'', + 39: '函数 summary(self, verbose: 布尔 = 假) -> 字符串', + 40: '返回 f\'{self.total} files\' 如果 verbose 不是 空 否则 \'\'', + }) + expectWordsApart(map) + }) + + it('内置类型只在注解里翻，in 按位置区分取自 / 在', async () => { + const source = [ + 'def pick(items: dict[str, int], list: set) -> tuple:', + ' return [str(k) for k in items if k not in list]', + ].join('\n') + const map = texts(await translateAll(py, 'python', pythonZhTranslator, source)) + expect(map[1]).toBe('函数 pick(items: 字典[字符串, 整数], list: 集合) -> 元组') + expect(map[2]).toBe('返回 [str(k) 遍历 k 取自 items 如果 k 不在 list]') + }) +}) diff --git a/vscode/src/codeReader/translators/zhTemplate.ts b/vscode/src/codeReader/translators/zhTemplate.ts new file mode 100644 index 0000000..c325427 --- /dev/null +++ b/vscode/src/codeReader/translators/zhTemplate.ts @@ -0,0 +1,169 @@ +import type { CodeBlock, Node, Segment, SourceSpan, TokenRole, Translator } from '../contract' + +/** 按父节点类型区分译法的关键字，例如 Python 的 in：for 头部里「取自」，比较里「在」。 */ +export interface ContextualWord { zh: string, byParent: Record } + +/** 一种语言的中文直译表。引擎只替换表里的三类东西，其余原样照抄。 */ +export interface ZhTable { + id: string + label: string + languageIds: string[] + /** 未翻译的词按源码着色的词性上色；关键字译文的词性也取它。 */ + tokenRole: (leaf: Node) => TokenRole + /** 关键字：按语法树里匿名节点的类型查（Python 的 `not in` 这类组合关键字本身也是一个匿名节点）。 */ + keywords: Record + /** 字面量关键字：具名的叶子节点（true / None …），按节点类型查。 */ + literals: Record + /** 内置类型：只在 isTypePosition 成立的叶子上按原文查。 */ + builtinTypes: Record + isTypePosition: (leaf: Node) => boolean + /** 翻成文字的运算符：只认父节点在 operatorParents 里的运算符节点（TS 的非空断言 `x!` 不算）。 */ + operators: Record + operatorParents: string[] + /** 空字面量（没有具名子节点）整个换成一个词：空数组 / 空列表 / 空字典。 */ + emptyLiterals: Record + /** 整段原样照抄、按字符串着色的节点。 */ + stringNodes: string[] + /** 注释（或 docstring）节点去掉注释符号后的原文，可以多行。 */ + commentText: (node: Node) => string +} + +interface Token { + start: number + end: number + text: string + role: TokenRole + translated: boolean + /** 源码里的原文，用来认出逗号、括号。 */ + raw: string +} + +const CLOSING = new Set([')', ']', '}']) +const HEADER_OPENERS = new Set(['{', ':']) + +function firstLeaf(node: Node): Node { + const first = node.child(0) + return first ? firstLeaf(first) : node +} + +function commentsWithin(node: Node, span: SourceSpan): Node[] { + if (node.endIndex <= span.startIndex || node.startIndex >= span.endIndex) + return [] + if (node.type === 'comment') + return [node] + return node.children.flatMap(child => child ? commentsWithin(child, span) : []) +} + +/** 下一个要输出的词的原文（跳过注释）。 */ +function nextRaw(tokens: Token[], i: number): string { + for (let j = i + 1; j < tokens.length; j++) { + if (tokens[j].text !== '') + return tokens[j].raw + } + return '' +} + +function oneLine(text: string): string { + return text.replace(/\s*\n\s*/g, ' ') +} + +export function createZhTemplateTranslator(table: ZhTable): Translator { + const operatorParents = new Set(table.operatorParents) + const stringNodes = new Set(table.stringNodes) + + /** 能整个换成中文的节点 → 译文和词性。 */ + function translateNode(node: Node): { text: string, role: TokenRole } | undefined { + if (node.childCount === 0 && table.isTypePosition(node)) { + const type = table.builtinTypes[node.text] + if (type) + return { text: type, role: 'type' } + } + const parent = node.parent + if (!node.isNamed) { + if (parent && operatorParents.has(parent.type) && table.operators[node.type]) + return { text: table.operators[node.type], role: 'operator' } + const word = table.keywords[node.type] + if (word !== undefined) { + const text = typeof word === 'string' ? word : (parent && word.byParent[parent.type]) || word.zh + return { text, role: table.tokenRole(firstLeaf(node)) } + } + return undefined + } + if (node.childCount === 0 && table.literals[node.type]) + return { text: table.literals[node.type], role: table.tokenRole(node) } + if (table.emptyLiterals[node.type] && node.namedChildCount === 0) + return { text: table.emptyLiterals[node.type], role: 'literal' } + return undefined + } + + function collect(node: Node, span: SourceSpan, out: Token[]): void { + if (node.endIndex <= span.startIndex || node.startIndex >= span.endIndex || node.startIndex === node.endIndex) + return + const base = { start: node.startIndex, end: node.endIndex } + if (node.type === 'comment') { + out.push({ ...base, raw: node.text, text: '', role: 'comment', translated: false }) + return + } + const translated = translateNode(node) + if (translated) { + out.push({ ...base, raw: node.text, ...translated, translated: true }) + return + } + if (node.childCount === 0 || stringNodes.has(node.type)) { + const role = stringNodes.has(node.type) ? 'string' : table.tokenRole(node) + out.push({ ...base, raw: node.text, text: oneLine(node.text), role, translated: false }) + return + } + for (const child of node.children) { + if (child) + collect(child, span, out) + } + } + + function translateComment(block: CodeBlock): Segment[] { + const { startIndex, endIndex } = block.header + const root = block.node.tree.rootNode + const nodes = block.node.type === 'comment' ? commentsWithin(root.descendantForIndex(startIndex, endIndex) ?? root, block.header) : [block.node] + const text = nodes.map(table.commentText).join('\n').split('\n').map(line => line.trim()).filter(Boolean).join(' ') + return text ? [{ text, role: 'comment' }] : [] + } + + function translate(block: CodeBlock, source: string): Segment[] { + if (block.kind === 'comment') + return translateComment(block) + const tokens: Token[] = [] + collect(block.node, block.header, tokens) + const compound = block.children.length > 0 || block.header.endIndex < block.node.endIndex + const last = tokens.at(-1) + if (compound && last && !last.translated && HEADER_OPENERS.has(last.raw)) + tokens.pop() + + const segments: Segment[] = [] + const push = (text: string, role: TokenRole): void => { + const prev = segments.at(-1) + if (role === 'plain' && prev?.role === 'plain') + prev.text += text + else + segments.push({ text, role }) + } + let gap = false + let prevEnd = block.header.startIndex + tokens.forEach((token, i) => { + gap ||= /\s/.test(source.slice(prevEnd, token.start)) + prevEnd = token.end + const trailingComma = token.raw === ',' && !token.translated && CLOSING.has(nextRaw(tokens, i)) + if (token.text === '' || trailingComma) { + gap ||= token.role === 'comment' + return + } + if (gap && segments.length > 0) + push(' ', 'plain') + push(token.text, token.role) + // 「非」是个词，后面补一个空格，不和操作数粘在一起 + gap = token.translated && token.raw === '!' + }) + return segments + } + + return { id: table.id, label: table.label, languageIds: table.languageIds, translate } +}