✨ feat(vscode): 代码阅读的解析与模板直译核心
This commit is contained in:
cruldra committed
1 parent
af56985d9a
commit
4d34b8c715
24 files changed
+2294
No files matched your search
@@ -0,0 +1,111 @@
|
||||
import type { BlockView } from './blockView'
|
||||
import type { LanguageAdapter, Translator } from './contract'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { readBlockViews, toBlockViews } from './blockView'
|
||||
import { createPythonAdapter } from './languages/python'
|
||||
import { createTypeScriptAdapter } from './languages/typescript'
|
||||
import { readFixture, runtime, WASM_DIR, withBlocks } from './testSupport'
|
||||
import { pythonZhTranslator } from './translators/pythonZh'
|
||||
import { typescriptZhTranslator } from './translators/typescriptZh'
|
||||
|
||||
const ts = createTypeScriptAdapter(WASM_DIR)
|
||||
const py = createPythonAdapter(WASM_DIR)
|
||||
|
||||
function views(adapter: LanguageAdapter, languageId: string, translator: Translator | undefined, source: string): Promise<BlockView[]> {
|
||||
return withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, source, adapter, translator))
|
||||
}
|
||||
|
||||
function flatten(list: BlockView[]): BlockView[] {
|
||||
return list.flatMap(view => [view, ...flatten(view.children)])
|
||||
}
|
||||
|
||||
function byId(list: BlockView[], id: string): BlockView {
|
||||
const found = flatten(list).find(view => view.id === id)
|
||||
if (!found)
|
||||
throw new Error(`没有块 ${id}`)
|
||||
return found
|
||||
}
|
||||
|
||||
const SAMPLES: [LanguageAdapter, string, Translator, string][] = [
|
||||
[ts, 'typescript', typescriptZhTranslator, 'envLock.ts.txt'],
|
||||
[ts, 'typescript', typescriptZhTranslator, 'issueLock.ts.txt'],
|
||||
[py, 'python', pythonZhTranslator, 'sample.py'],
|
||||
]
|
||||
|
||||
describe('blockView', () => {
|
||||
it.each(SAMPLES)('%#: codeTokens 拼起来等于 code，code 是 header 整行去共同缩进', async (adapter, languageId, translator, fixture) => {
|
||||
const source = readFixture(fixture)
|
||||
const lines = source.split('\n')
|
||||
for (const view of flatten(await views(adapter, languageId, translator, source))) {
|
||||
expect(view.codeTokens.map(token => token.text).join(''), view.id).toBe(view.code)
|
||||
const rows = lines.slice(view.startLine, view.endLine + 1)
|
||||
const indent = Math.min(...rows.filter(row => row.trim()).map(row => row.length - row.trimStart().length))
|
||||
expect(view.code, view.id).toBe(rows.map(row => row.slice(indent)).join('\n'))
|
||||
expect(view.translation.length, view.id).toBeGreaterThan(0)
|
||||
}
|
||||
})
|
||||
|
||||
it('id 是块在树里的位置，行号 0 起，多行块去掉共同缩进', async () => {
|
||||
const env = await views(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt'))
|
||||
expect(byId(env, '6.1.0')).toMatchObject({ kind: 'return', startLine: 25, endLine: 25, code: 'return []' })
|
||||
expect(byId(env, '6.1')).toMatchObject({ kind: 'if', startLine: 24, code: 'if (!folder)' })
|
||||
expect(byId(env, '8.2.1')).toMatchObject({ kind: 'catch', startLine: 45, code: 'catch (err) {' })
|
||||
|
||||
const issue = await views(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt'))
|
||||
expect(byId(issue, '3.4')).toMatchObject({
|
||||
kind: 'return',
|
||||
startLine: 19,
|
||||
endLine: 23,
|
||||
code: 'return {\n locked: true,\n prerequisiteNumber: prereq.number,\n prerequisiteColumn: prereq.column,\n}',
|
||||
})
|
||||
})
|
||||
|
||||
it('源码着色来自语法树：每个叶子一段，空白和标点归 plain', async () => {
|
||||
const env = await views(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt'))
|
||||
const tokens = byId(env, '6.3').codeTokens
|
||||
expect(byId(env, '6.3').code).toBe('const uris = await workspace.findFiles(include, EXCLUDE_PATTERN)')
|
||||
expect(tokens.slice(0, 7)).toEqual([
|
||||
{ text: 'const', role: 'storage' },
|
||||
{ text: ' ', role: 'plain' },
|
||||
{ text: 'uris', role: 'variable' },
|
||||
{ text: ' = ', role: 'plain' },
|
||||
{ text: 'await', role: 'keyword' },
|
||||
{ text: ' ', role: 'plain' },
|
||||
{ text: 'workspace', role: 'variable' },
|
||||
])
|
||||
expect(tokens).toContainEqual({ text: 'findFiles', role: 'function' })
|
||||
expect(byId(env, '0').codeTokens).toEqual([{ text: byId(env, '0').code, role: 'comment' }])
|
||||
|
||||
// 中文注释之后的行：语法树下标按字符串下标算，不按 UTF-8 字节
|
||||
const issue = await views(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt'))
|
||||
expect(byId(issue, '3').codeTokens.slice(0, 5)).toEqual([
|
||||
{ text: 'export', role: 'keyword' },
|
||||
{ text: ' ', role: 'plain' },
|
||||
{ text: 'function', role: 'storage' },
|
||||
{ text: ' ', role: 'plain' },
|
||||
{ text: 'evaluateLockedReason', role: 'function' },
|
||||
])
|
||||
expect(byId(issue, '2').codeTokens).toEqual([{ text: byId(issue, '2').code, role: 'comment' }])
|
||||
|
||||
const sample = await views(py, 'python', pythonZhTranslator, readFixture('sample.py'))
|
||||
const signature = byId(sample, '5').codeTokens
|
||||
expect(signature).toContainEqual({ text: 'def', role: 'storage' })
|
||||
expect(signature).toContainEqual({ text: 'find_env_files', role: 'function' })
|
||||
expect(signature.filter(token => token.text === 'str')).toEqual([{ text: 'str', role: 'type' }, { text: 'str', role: 'type' }])
|
||||
})
|
||||
|
||||
it('\\r\\n 和制表符缩进：code 用 \\n 连接，去掉共同的制表符', async () => {
|
||||
const source = 'function f() {\r\n\tif (a &&\r\n\t\tb) {\r\n\t\treturn 1\r\n\t}\r\n}\r\n'
|
||||
const list = await views(ts, 'typescript', undefined, source)
|
||||
const cond = byId(list, '0.0')
|
||||
expect(cond).toMatchObject({ kind: 'if', startLine: 1, endLine: 2, code: 'if (a &&\n\tb) {', translation: [] })
|
||||
expect(cond.codeTokens.map(token => token.text).join('')).toBe(cond.code)
|
||||
})
|
||||
|
||||
it('readBlockViews：解析、转视图、释放语法树', async () => {
|
||||
const list = await readBlockViews(runtime, py, pythonZhTranslator, 'python', 'def f(x: int) -> bool:\n return not x\n')
|
||||
expect(list).toHaveLength(1)
|
||||
expect(list[0].translation.map(segment => segment.text).join('')).toBe('函数 f(x: 整数) -> 布尔')
|
||||
expect(list[0].children[0].translation.map(segment => segment.text).join('')).toBe('返回 非 x')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,119 @@
|
||||
import type { BlockKind, CodeBlock, LanguageAdapter, Node, Segment, TokenRole, Translator } from './contract'
|
||||
import type { TreeSitterRuntime } from './runtime'
|
||||
import { commonIndent, splitLines } from './editText'
|
||||
|
||||
/** 发给 webview 的一块（可序列化）。 */
|
||||
export interface BlockView {
|
||||
/** 在树里的位置，例如 '6.1.0'。 */
|
||||
id: string
|
||||
kind: BlockKind
|
||||
/** header 的起止行，0 起。 */
|
||||
startLine: number
|
||||
endLine: number
|
||||
/** header 所在的整行源码，去掉共同缩进。 */
|
||||
code: string
|
||||
/** 源码着色（拼起来等于 code）。 */
|
||||
codeTokens: Segment[]
|
||||
translation: Segment[]
|
||||
children: BlockView[]
|
||||
}
|
||||
|
||||
interface Line { start: number, end: number }
|
||||
|
||||
/** 每行在源码里的起止下标（end 不含换行符）。 */
|
||||
function lineRanges(source: string): Line[] {
|
||||
const lines: Line[] = []
|
||||
let start = 0
|
||||
for (const text of splitLines(source)) {
|
||||
lines.push({ start, end: start + text.length })
|
||||
start += text.length + (source[start + text.length] === '\r' ? 2 : 1)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
function leavesIn(node: Node, from: number, to: number, out: Node[]): void {
|
||||
if (node.endIndex <= from || node.startIndex >= to || node.startIndex === node.endIndex)
|
||||
return
|
||||
if (node.childCount === 0) {
|
||||
out.push(node)
|
||||
return
|
||||
}
|
||||
for (const child of node.children) {
|
||||
if (child)
|
||||
leavesIn(child, from, to, out)
|
||||
}
|
||||
}
|
||||
|
||||
/** 按语法树叶子给 [startRow, endRow] 这几行着色：每个有词性的叶子一段，空白、标点等 plain 的连成一段。 */
|
||||
function colorLines(root: Node, source: string, lines: Line[], startRow: number, endRow: number, indent: number, tokenRole: (leaf: Node) => TokenRole): Segment[] {
|
||||
const from = lines[startRow].start
|
||||
const to = lines[endRow].end
|
||||
const owner = new Int32Array(to - from).fill(-1)
|
||||
const roles: TokenRole[] = []
|
||||
const leaves: Node[] = []
|
||||
leavesIn(root.descendantForIndex(from, to) ?? root, from, to, leaves)
|
||||
leaves.forEach((leaf, i) => {
|
||||
roles.push(tokenRole(leaf))
|
||||
for (let at = Math.max(leaf.startIndex, from); at < Math.min(leaf.endIndex, to); at++)
|
||||
owner[at - from] = i
|
||||
})
|
||||
|
||||
const segments: Segment[] = []
|
||||
let current = -2
|
||||
const put = (char: string, who: number): void => {
|
||||
const prev = segments.at(-1)
|
||||
const role = who < 0 ? 'plain' : roles[who]
|
||||
if (prev && (who === current || (role === 'plain' && prev.role === 'plain')))
|
||||
prev.text += char
|
||||
else
|
||||
segments.push({ text: char, role })
|
||||
current = who
|
||||
}
|
||||
for (let row = startRow; row <= endRow; row++) {
|
||||
const line = lines[row]
|
||||
if (row > startRow)
|
||||
put('\n', owner[line.start - 1 - from])
|
||||
for (let at = line.start + Math.min(indent, line.end - line.start); at < line.end; at++)
|
||||
put(source[at], owner[at - from])
|
||||
}
|
||||
return segments
|
||||
}
|
||||
|
||||
/** 代码块树 → 视图数据：header 整行源码去共同缩进、按语法树着色、调用翻译插件。 */
|
||||
export function toBlockViews(blocks: CodeBlock[], source: string, adapter: LanguageAdapter, translator: Translator | undefined): BlockView[] {
|
||||
const lines = lineRanges(source)
|
||||
const texts = splitLines(source)
|
||||
const build = (block: CodeBlock, id: string): BlockView => {
|
||||
const { startRow, endRow } = block.header
|
||||
const rows = texts.slice(startRow, endRow + 1)
|
||||
const indent = commonIndent(rows).length
|
||||
return {
|
||||
id,
|
||||
kind: block.kind,
|
||||
startLine: startRow,
|
||||
endLine: endRow,
|
||||
code: rows.map(row => row.slice(Math.min(indent, row.length))).join('\n'),
|
||||
codeTokens: colorLines(block.node.tree.rootNode, source, lines, startRow, endRow, indent, adapter.tokenRole),
|
||||
translation: translator ? translator.translate(block, source) : [],
|
||||
children: block.children.map((child, i) => build(child, `${id}.${i}`)),
|
||||
}
|
||||
}
|
||||
return blocks.map((block, i) => build(block, String(i)))
|
||||
}
|
||||
|
||||
/** 解析一份源码并转成视图数据；语法树用完即释放。 */
|
||||
export async function readBlockViews(
|
||||
runtime: Pick<TreeSitterRuntime, 'parse'>,
|
||||
adapter: LanguageAdapter,
|
||||
translator: Translator | undefined,
|
||||
languageId: string,
|
||||
source: string,
|
||||
): Promise<BlockView[]> {
|
||||
const tree = await runtime.parse(adapter, languageId, source)
|
||||
try {
|
||||
return toBlockViews(adapter.toBlocks(tree.rootNode, source), source, adapter, translator)
|
||||
}
|
||||
finally {
|
||||
tree.delete()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
import type { Node } from '@vscode/tree-sitter-wasm'
|
||||
import type { Disposable } from 'vscode'
|
||||
|
||||
export type { Node }
|
||||
|
||||
export type BlockKind
|
||||
= | 'import' | 'function' | 'class' | 'type' | 'field' | 'const' | 'variable'
|
||||
| 'if' | 'else' | 'loop' | 'try' | 'catch' | 'finally' | 'return' | 'statement' | 'comment'
|
||||
|
||||
/** 源码里的一段，行号从 0 开始；下标是源码字符串下标（web-tree-sitter 的 startIndex / endIndex），end 为开区间。 */
|
||||
export interface SourceSpan { startIndex: number, endIndex: number, startRow: number, endRow: number }
|
||||
|
||||
export interface CodeBlock {
|
||||
kind: BlockKind
|
||||
/** 这一块对应的语法树节点（整条语句）。 */
|
||||
node: Node
|
||||
/** 块自己显示 / 编辑的那一段：复合语句只到主体开始之前，其余是整条语句。 */
|
||||
header: SourceSpan
|
||||
children: CodeBlock[]
|
||||
}
|
||||
|
||||
/** 着色用的词性，源码着色和译文着色共用。 */
|
||||
export type TokenRole
|
||||
= | 'keyword' // 流程关键字：import from export if else for return try catch await …
|
||||
| 'storage' // 声明关键字与字面量关键字：const function async class interface new true false null …
|
||||
| 'operator' // 被翻成文字的运算符：全等 与 或 非
|
||||
| 'variable' | 'function' | 'type' | 'string' | 'number' | 'literal' | 'comment' | 'plain'
|
||||
|
||||
export interface Segment { text: string, role: TokenRole }
|
||||
|
||||
/** 语言插件：语法树 → 代码块树。 */
|
||||
export interface LanguageAdapter {
|
||||
id: string
|
||||
/** 覆盖的 VS Code languageId，例如 ['typescript', 'typescriptreact', 'javascript', 'javascriptreact']。 */
|
||||
languageIds: string[]
|
||||
/** 该 languageId 用的语法包 .wasm 的绝对路径（宿主负责加载，保证和解析器运行时是同一份）。 */
|
||||
grammarPath: (languageId: string) => string
|
||||
toBlocks: (root: Node, source: string) => CodeBlock[]
|
||||
/** 源码着色：给语法树叶子节点定词性。 */
|
||||
tokenRole: (leaf: Node) => TokenRole
|
||||
}
|
||||
|
||||
/** 翻译插件：代码块 → 译文。 */
|
||||
export interface Translator {
|
||||
id: string // 'typescript.zh-template'
|
||||
label: string // '中文直译'
|
||||
languageIds: string[]
|
||||
translate: (block: CodeBlock, source: string) => Segment[]
|
||||
}
|
||||
|
||||
export interface CodeReaderApi {
|
||||
registerLanguage: (adapter: LanguageAdapter) => Disposable
|
||||
registerTranslator: (translator: Translator) => Disposable
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { toBlockViews } from './blockView'
|
||||
import { commonIndent, reindentBlock, replaceLines } from './editText'
|
||||
import { createTypeScriptAdapter } from './languages/typescript'
|
||||
import { readFixture, WASM_DIR, withBlocks } from './testSupport'
|
||||
|
||||
describe('行替换', () => {
|
||||
it('多行块替换后每个非空行补回原来的缩进，空行保持空', () => {
|
||||
const source = [
|
||||
'function f() {',
|
||||
' if (a)',
|
||||
' return 1',
|
||||
'}',
|
||||
].join('\n')
|
||||
const edited = replaceLines(source, 1, 2, 'if (a && b) {\n\n return 2\n}')
|
||||
expect(edited).toBe([
|
||||
'function f() {',
|
||||
' if (a && b) {',
|
||||
'',
|
||||
' return 2',
|
||||
' }',
|
||||
'}',
|
||||
].join('\n'))
|
||||
})
|
||||
|
||||
it('保留 \\r\\n 换行', () => {
|
||||
expect(replaceLines('a\r\n b\r\n c\r\n', 1, 1, 'x\ny')).toBe('a\r\n x\r\n y\r\n c\r\n')
|
||||
})
|
||||
|
||||
it('共同缩进逐字符比，混用制表符和空格时只取相同的前缀', () => {
|
||||
expect(commonIndent(['\t a', '', '\t b'])).toBe('\t ')
|
||||
expect(commonIndent(['\ta', ' b'])).toBe('')
|
||||
expect(reindentBlock([' x'], 'a\n b')).toBe(' a\n b')
|
||||
})
|
||||
|
||||
it('把预览里的 code 原样写回，文件不变（去缩进与补缩进互逆）', async () => {
|
||||
const adapter = createTypeScriptAdapter(WASM_DIR)
|
||||
const source = readFixture('envLock.ts.txt')
|
||||
const views = await withBlocks(adapter, 'typescript', source, blocks => toBlockViews(blocks, source, adapter, undefined))
|
||||
const all = views.flatMap(function flat(view): typeof views {
|
||||
return [view, ...view.children.flatMap(flat)]
|
||||
})
|
||||
for (const view of all)
|
||||
expect(replaceLines(source, view.startLine, view.endLine, view.code), view.id).toBe(source)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
/** 按 \n 或 \r\n 切行。 */
|
||||
export function splitLines(text: string): string[] {
|
||||
return text.split(/\r?\n/)
|
||||
}
|
||||
|
||||
/** 非空行共同的前导空白（空格 / 制表符按原样比，不换算）。 */
|
||||
export function commonIndent(lines: string[]): string {
|
||||
let common: string | undefined
|
||||
for (const line of lines) {
|
||||
if (!line.trim())
|
||||
continue
|
||||
const indent = line.slice(0, line.length - line.trimStart().length)
|
||||
if (common === undefined) {
|
||||
common = indent
|
||||
continue
|
||||
}
|
||||
let i = 0
|
||||
while (i < common.length && i < indent.length && common[i] === indent[i])
|
||||
i++
|
||||
common = common.slice(0, i)
|
||||
}
|
||||
return common ?? ''
|
||||
}
|
||||
|
||||
/**
|
||||
* 块内改完的文本（webview 里是去掉共同缩进的样子）→ 写回文件的文本：
|
||||
* 每个非空行补回这几行原来的共同缩进，和 BlockView.code 去缩进的做法互逆。
|
||||
* @param originalLines 被替换的那几行原文
|
||||
*/
|
||||
export function reindentBlock(originalLines: string[], text: string): string {
|
||||
const indent = commonIndent(originalLines)
|
||||
return splitLines(text).map(line => line.trim() ? indent + line : line).join('\n')
|
||||
}
|
||||
|
||||
/** 把 source 的 [startLine, endLine] 整行替换成 text（补回缩进），返回新全文；换行符沿用原文。 */
|
||||
export function replaceLines(source: string, startLine: number, endLine: number, text: string): string {
|
||||
const eol = source.includes('\r\n') ? '\r\n' : '\n'
|
||||
const lines = splitLines(source)
|
||||
const replacement = splitLines(reindentBlock(lines.slice(startLine, endLine + 1), text))
|
||||
return [...lines.slice(0, startLine), ...replacement, ...lines.slice(endLine + 1)].join(eol)
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* .env file lock / unlock helpers.
|
||||
*
|
||||
* Recursively scans the main workspace folder for `.env*` files (skipping
|
||||
* `node_modules`, `.git`, `.claude`) and chmods them to 0o444 (locked) or
|
||||
* 0o644 (unlocked) in bulk. Backs the toolbar Lock toggle in `PanelHeader`.
|
||||
*
|
||||
* No persistent watcher — re-scan happens on every toggle, so newly created
|
||||
* `.env` files are only picked up the next time the user clicks the button.
|
||||
*/
|
||||
|
||||
import { promises as fsp } from 'node:fs'
|
||||
import { RelativePattern, workspace } from 'vscode'
|
||||
|
||||
const ENV_PATTERN = '**/.env*'
|
||||
const EXCLUDE_PATTERN = '{**/node_modules/**,**/.git/**,**/.claude/**}'
|
||||
|
||||
/**
|
||||
* Scan the workspace folder rooted at `workspaceRoot` for all `.env*`
|
||||
* files, skipping `node_modules`, `.git`, and `.claude`. Returns absolute
|
||||
* fs paths. Empty array if the workspace folder isn't currently open.
|
||||
*/
|
||||
export async function findEnvFiles(workspaceRoot: string): Promise<string[]> {
|
||||
const folder = workspace.workspaceFolders?.find(f => f.uri.fsPath === workspaceRoot)
|
||||
if (!folder)
|
||||
return []
|
||||
const include = new RelativePattern(folder, ENV_PATTERN)
|
||||
const uris = await workspace.findFiles(include, EXCLUDE_PATTERN)
|
||||
return uris.map(u => u.fsPath)
|
||||
}
|
||||
|
||||
export interface EnvLockResult {
|
||||
total: number
|
||||
ok: string[]
|
||||
failed: { path: string, reason: string }[]
|
||||
}
|
||||
|
||||
async function chmodAll(workspaceRoot: string, mode: number): Promise<EnvLockResult> {
|
||||
const files = await findEnvFiles(workspaceRoot)
|
||||
const result: EnvLockResult = { total: files.length, ok: [], failed: [] }
|
||||
for (const f of files) {
|
||||
try {
|
||||
await fsp.chmod(f, mode)
|
||||
result.ok.push(f)
|
||||
}
|
||||
catch (err) {
|
||||
result.failed.push({ path: f, reason: err instanceof Error ? err.message : String(err) })
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Set every `.env*` file in the workspace to 0o444 (read-only). */
|
||||
export async function lockEnvFiles(workspaceRoot: string): Promise<EnvLockResult> {
|
||||
return chmodAll(workspaceRoot, 0o444)
|
||||
}
|
||||
|
||||
/** Restore every `.env*` file in the workspace to 0o644 (read-write). */
|
||||
export async function unlockEnvFiles(workspaceRoot: string): Promise<EnvLockResult> {
|
||||
return chmodAll(workspaceRoot, 0o644)
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import type { Issue } from '../gitea/types'
|
||||
|
||||
export interface LockedReason {
|
||||
locked: boolean
|
||||
prerequisiteNumber?: number
|
||||
prerequisiteColumn?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 纯判定：工单被未完成的前置挡住才算锁定。快照里找不到工单或前置一律放行，
|
||||
* 与 webview 的 `isIssueLocked` 保持同一语义。
|
||||
*/
|
||||
export function evaluateLockedReason(issues: Issue[], issueNumber: number): LockedReason {
|
||||
const issue = issues.find(i => i.number === issueNumber)
|
||||
if (!issue || issue.prerequisite === undefined)
|
||||
return { locked: false }
|
||||
const prereq = issues.find(i => i.number === issue.prerequisite)
|
||||
if (!prereq || prereq.column === 'done')
|
||||
return { locked: false }
|
||||
return {
|
||||
locked: true,
|
||||
prerequisiteNumber: prereq.number,
|
||||
prerequisiteColumn: prereq.column,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Scan a workspace for .env files."""
|
||||
|
||||
import os
|
||||
from pathlib import Path as P
|
||||
|
||||
MAX_DEPTH = 8
|
||||
skipped = []
|
||||
|
||||
|
||||
def find_env_files(root: str) -> list[str]:
|
||||
"""Return every .env file under root."""
|
||||
# 跳过备份文件
|
||||
found = [p for p in os.listdir(root) if p.startswith('.env') and not p.endswith('.bak')]
|
||||
if not found:
|
||||
return []
|
||||
elif len(found) > MAX_DEPTH:
|
||||
found = found[:MAX_DEPTH]
|
||||
else:
|
||||
pass
|
||||
for name in found:
|
||||
if name in skipped:
|
||||
continue
|
||||
try:
|
||||
with open(os.path.join(root, found[0])) as handle:
|
||||
handle.read()
|
||||
except OSError as err:
|
||||
raise RuntimeError(str(err)) from err
|
||||
finally:
|
||||
skipped.clear()
|
||||
return found
|
||||
|
||||
|
||||
class EnvLockResult:
|
||||
total: int
|
||||
ok: list[str] = []
|
||||
failed: dict[str, str] = {}
|
||||
label = 'env'
|
||||
|
||||
def summary(self, verbose: bool = False) -> str:
|
||||
return f'{self.total} files' if verbose is not None else ''
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { Outline } from '../testSupport'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { outline, readFixture, WASM_DIR, withBlocks } from '../testSupport'
|
||||
import { createPythonAdapter } from './python'
|
||||
|
||||
const adapter = createPythonAdapter(WASM_DIR)
|
||||
|
||||
function tree(source: string): Promise<Outline[]> {
|
||||
return withBlocks(adapter, 'python', source, outline)
|
||||
}
|
||||
|
||||
// fixtures/sample.py：行号 1 起
|
||||
const SAMPLE: Outline[] = [
|
||||
{ at: 1, kind: 'comment' },
|
||||
{ at: 3, kind: 'import' },
|
||||
{ at: 4, kind: 'import' },
|
||||
{ at: 6, kind: 'const' },
|
||||
{ at: 7, kind: 'variable' },
|
||||
{ at: 10, kind: 'function', kids: [
|
||||
{ at: 11, kind: 'comment' },
|
||||
{ at: 12, kind: 'comment' },
|
||||
{ at: 13, kind: 'variable' },
|
||||
{ at: 14, kind: 'if', kids: [{ at: 15, kind: 'return' }] },
|
||||
{ at: 16, kind: 'else', kids: [{ at: 17, kind: 'variable' }] },
|
||||
{ at: 18, kind: 'else', kids: [{ at: 19, kind: 'statement' }] },
|
||||
{ at: 20, kind: 'loop', kids: [
|
||||
{ at: 21, kind: 'if', kids: [{ at: 22, kind: 'statement' }] },
|
||||
] },
|
||||
{ at: 23, kind: 'try', kids: [
|
||||
{ at: 24, kind: 'statement', kids: [{ at: 25, kind: 'statement' }] },
|
||||
] },
|
||||
{ at: 26, kind: 'catch', kids: [{ at: 27, kind: 'statement' }] },
|
||||
{ at: 28, kind: 'finally', kids: [{ at: 29, kind: 'statement' }] },
|
||||
{ at: 30, kind: 'return' },
|
||||
] },
|
||||
{ at: 33, kind: 'class', kids: [
|
||||
{ at: 34, kind: 'field' },
|
||||
{ at: 35, kind: 'field' },
|
||||
{ at: 36, kind: 'field' },
|
||||
{ at: 37, kind: 'variable' },
|
||||
{ at: 39, kind: 'function', kids: [{ at: 40, kind: 'return' }] },
|
||||
] },
|
||||
]
|
||||
|
||||
describe('python 代码块树', () => {
|
||||
it('样例覆盖 import / def / docstring / if-elif-else / for / try-except-finally / with / 推导式 / class 字段', async () => {
|
||||
expect(await tree(readFixture('sample.py'))).toEqual(SAMPLE)
|
||||
})
|
||||
|
||||
it('装饰器算进头部；冒号后、主体前的注释归子块；循环的 else 是兄弟块；match 的 case 是子块 if', async () => {
|
||||
const source = [
|
||||
'@cache',
|
||||
'def load(key):',
|
||||
' # 先查缓存',
|
||||
' while key:',
|
||||
' key = key[1:]',
|
||||
' else:',
|
||||
' pass',
|
||||
' match key:',
|
||||
' case \'a\':',
|
||||
' return 1',
|
||||
' case _:',
|
||||
' return 2',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: [1, 2], kind: 'function', kids: [
|
||||
{ at: 3, kind: 'comment' },
|
||||
{ at: 4, kind: 'loop', kids: [{ at: 5, kind: 'variable' }] },
|
||||
{ at: 6, kind: 'else', kids: [{ at: 7, kind: 'statement' }] },
|
||||
{ at: 8, kind: 'if', kids: [
|
||||
{ at: 9, kind: 'if', kids: [{ at: 10, kind: 'return' }] },
|
||||
{ at: 11, kind: 'if', kids: [{ at: 12, kind: 'return' }] },
|
||||
] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('docstring 只认主体开头；不是开头的字符串表达式是语句', async () => {
|
||||
const source = [
|
||||
'def f():',
|
||||
' x = 1',
|
||||
' """not a docstring"""',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'function', kids: [{ at: 2, kind: 'variable' }, { at: 3, kind: 'statement' }] },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,217 @@
|
||||
import type { BlockKind, CodeBlock, LanguageAdapter, Node, TokenRole } from '../contract'
|
||||
import * as path from 'node:path'
|
||||
import { compoundBlock, leafBlock, listBlocks, named, sameNode } from './shared'
|
||||
|
||||
/** 常量、类字段只认直接写在模块 / 类主体里的赋值。 */
|
||||
type Scope = 'module' | 'class' | 'other'
|
||||
|
||||
const CONSTANT_NAME = /^[A-Z][A-Z0-9_]*$/
|
||||
|
||||
function statements(items: Node[], scope: Scope, afterRow: number, docstring: boolean): CodeBlock[] {
|
||||
return listBlocks(items, afterRow, (item, first) => statementBlocks(item, scope, docstring && first), () => true)
|
||||
}
|
||||
|
||||
/** 冒号后面的主体：body / consequence 字段，except / finally 子句里是不带字段名的 block。 */
|
||||
function bodyOf(node: Node): Node | null {
|
||||
const body = node.childForFieldName('body') ?? node.childForFieldName('consequence')
|
||||
if (body)
|
||||
return body
|
||||
const blocks = named(node).filter(child => child.type === 'block')
|
||||
return blocks.at(-1) ?? null
|
||||
}
|
||||
|
||||
function bodyBlock(kind: BlockKind, outer: Node, body: Node | null, scope: Scope, docstring = false): CodeBlock {
|
||||
if (!body)
|
||||
return leafBlock(kind, outer)
|
||||
return compoundBlock(kind, outer, body.startIndex, named(body), (list, afterRow) => statements(list, scope, afterRow, docstring))
|
||||
}
|
||||
|
||||
function statementBlocks(node: Node, scope: Scope, docstring: boolean): CodeBlock[] {
|
||||
switch (node.type) {
|
||||
case 'import_statement':
|
||||
case 'import_from_statement':
|
||||
case 'future_import_statement':
|
||||
return [leafBlock('import', node)]
|
||||
case 'function_definition':
|
||||
case 'class_definition':
|
||||
return [definitionBlock(node, node)]
|
||||
case 'decorated_definition': {
|
||||
const definition = node.childForFieldName('definition')
|
||||
return [definition ? definitionBlock(definition, node) : leafBlock('statement', node)]
|
||||
}
|
||||
case 'if_statement':
|
||||
return ifBlocks(node)
|
||||
case 'for_statement':
|
||||
case 'while_statement':
|
||||
return loopBlocks(node)
|
||||
case 'try_statement':
|
||||
return tryBlocks(node)
|
||||
case 'with_statement':
|
||||
return [bodyBlock('statement', node, bodyOf(node), 'other')]
|
||||
case 'match_statement':
|
||||
return [bodyBlock('if', node, bodyOf(node), 'other')]
|
||||
case 'case_clause':
|
||||
return [bodyBlock('if', node, bodyOf(node), 'other')]
|
||||
case 'return_statement':
|
||||
return [leafBlock('return', node)]
|
||||
case 'expression_statement':
|
||||
return [leafBlock(expressionKind(node, scope, docstring), node)]
|
||||
default:
|
||||
return [leafBlock('statement', node)]
|
||||
}
|
||||
}
|
||||
|
||||
/** function_definition / class_definition；outer 是带装饰器的整体时头部从装饰器开始。 */
|
||||
function definitionBlock(definition: Node, outer: Node): CodeBlock {
|
||||
const isClass = definition.type === 'class_definition'
|
||||
return bodyBlock(isClass ? 'class' : 'function', outer, definition.childForFieldName('body'), isClass ? 'class' : 'other', true)
|
||||
}
|
||||
|
||||
function isConstantTarget(target: Node | null): boolean {
|
||||
if (!target)
|
||||
return false
|
||||
if (target.type === 'identifier')
|
||||
return CONSTANT_NAME.test(target.text)
|
||||
if (target.type === 'pattern_list' || target.type === 'tuple_pattern') {
|
||||
const names = named(target)
|
||||
return names.length > 0 && names.every(isConstantTarget)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function expressionKind(node: Node, scope: Scope, docstring: boolean): BlockKind {
|
||||
const [expression, ...rest] = named(node)
|
||||
if (docstring && rest.length === 0 && (expression?.type === 'string' || expression?.type === 'concatenated_string'))
|
||||
return 'comment'
|
||||
if (expression?.type !== 'assignment')
|
||||
return 'statement'
|
||||
if (scope === 'class' && expression.childForFieldName('type'))
|
||||
return 'field'
|
||||
if (scope === 'module' && isConstantTarget(expression.childForFieldName('left')))
|
||||
return 'const'
|
||||
return 'variable'
|
||||
}
|
||||
|
||||
/** elif / else 展平成紧跟 if 的兄弟块。 */
|
||||
function ifBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('if', node, bodyOf(node), 'other')]
|
||||
for (const alternative of node.childrenForFieldName('alternative')) {
|
||||
if (alternative)
|
||||
blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other'))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
/** 循环的 else 是兄弟块 else。 */
|
||||
function loopBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('loop', node, bodyOf(node), 'other')]
|
||||
const alternative = node.childForFieldName('alternative')
|
||||
if (alternative)
|
||||
blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other'))
|
||||
return blocks
|
||||
}
|
||||
|
||||
const TRY_CLAUSES: Record<string, BlockKind> = {
|
||||
except_clause: 'catch',
|
||||
except_group_clause: 'catch',
|
||||
else_clause: 'else',
|
||||
finally_clause: 'finally',
|
||||
}
|
||||
|
||||
function tryBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('try', node, node.childForFieldName('body'), 'other')]
|
||||
for (const clause of named(node)) {
|
||||
const kind = TRY_CLAUSES[clause.type]
|
||||
if (kind)
|
||||
blocks.push(bodyBlock(kind, clause, bodyOf(clause), 'other'))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
// ─── 源码着色 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const STORAGE_KEYWORDS = new Set(['def', 'class', 'async', 'lambda', 'global', 'nonlocal', 'del'])
|
||||
const OPERATOR_KEYWORDS = new Set(['and', 'or', 'not', 'is'])
|
||||
const COMPARISON = new Set(['comparison_operator', 'not in', 'is not'])
|
||||
const LITERAL_KEYWORDS = new Set(['none', 'true', 'false'])
|
||||
|
||||
function insideString(leaf: Node): boolean {
|
||||
for (let node = leaf.parent; node; node = node.parent) {
|
||||
if (node.type === 'interpolation')
|
||||
return false
|
||||
if (node.type === 'string')
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 注解里的名字：typed_parameter / 返回值 / 带注解赋值的 type 节点之内。 */
|
||||
export function insideAnnotation(leaf: Node): boolean {
|
||||
for (let node = leaf.parent; node; node = node.parent) {
|
||||
if (node.type === 'type')
|
||||
return true
|
||||
if (node.type === 'block' || node.type === 'module')
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function isCallee(node: Node): boolean {
|
||||
const parent = node.parent
|
||||
return !!parent && parent.type === 'call' && sameNode(parent.childForFieldName('function'), node)
|
||||
}
|
||||
|
||||
function identifierRole(leaf: Node): TokenRole {
|
||||
const parent = leaf.parent
|
||||
if (!parent)
|
||||
return 'variable'
|
||||
if (insideAnnotation(leaf))
|
||||
return 'type'
|
||||
if (parent.type === 'function_definition' && sameNode(parent.childForFieldName('name'), leaf))
|
||||
return 'function'
|
||||
if (parent.type === 'class_definition' && sameNode(parent.childForFieldName('name'), leaf))
|
||||
return 'type'
|
||||
if (parent.type === 'decorator' || isCallee(leaf))
|
||||
return 'function'
|
||||
if (parent.type === 'attribute' && sameNode(parent.childForFieldName('attribute'), leaf) && isCallee(parent))
|
||||
return 'function'
|
||||
return 'variable'
|
||||
}
|
||||
|
||||
export function pythonTokenRole(leaf: Node): TokenRole {
|
||||
if (leaf.type === 'comment')
|
||||
return 'comment'
|
||||
if (insideString(leaf))
|
||||
return 'string'
|
||||
if (!leaf.isNamed) {
|
||||
if (!/^[a-z]+$/.test(leaf.type))
|
||||
return 'plain'
|
||||
if (STORAGE_KEYWORDS.has(leaf.type))
|
||||
return 'storage'
|
||||
if (OPERATOR_KEYWORDS.has(leaf.type) || (leaf.type === 'in' && COMPARISON.has(leaf.parent?.type ?? '')))
|
||||
return 'operator'
|
||||
return 'keyword'
|
||||
}
|
||||
if (LITERAL_KEYWORDS.has(leaf.type))
|
||||
return 'storage'
|
||||
switch (leaf.type) {
|
||||
case 'integer':
|
||||
case 'float':
|
||||
return 'number'
|
||||
case 'identifier':
|
||||
return identifierRole(leaf)
|
||||
default:
|
||||
return 'plain'
|
||||
}
|
||||
}
|
||||
|
||||
/** Python 语言插件。@param wasmDir 语法包 .wasm 所在目录 */
|
||||
export function createPythonAdapter(wasmDir: string): LanguageAdapter {
|
||||
return {
|
||||
id: 'python',
|
||||
languageIds: ['python'],
|
||||
grammarPath: () => path.join(wasmDir, 'tree-sitter-python.wasm'),
|
||||
toBlocks: root => statements(named(root), 'module', -1, true),
|
||||
tokenRole: pythonTokenRole,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import type { BlockKind, CodeBlock, Node, SourceSpan } from '../contract'
|
||||
|
||||
/** 语言插件共用的建树工具：取子节点、算头部、把一串语句切成块。 */
|
||||
|
||||
export function named(node: Node | null | undefined): Node[] {
|
||||
return node ? node.namedChildren.filter((child): child is Node => child !== null) : []
|
||||
}
|
||||
|
||||
export function children(node: Node): Node[] {
|
||||
return node.children.filter((child): child is Node => child !== null)
|
||||
}
|
||||
|
||||
export function nodeSpan(node: Node): SourceSpan {
|
||||
return {
|
||||
startIndex: node.startIndex,
|
||||
endIndex: node.endIndex,
|
||||
startRow: node.startPosition.row,
|
||||
endRow: node.endPosition.row,
|
||||
}
|
||||
}
|
||||
|
||||
export function leafBlock(kind: BlockKind, node: Node): CodeBlock {
|
||||
return { kind, node, header: nodeSpan(node), children: [] }
|
||||
}
|
||||
|
||||
/** 头部往前扩到 from（装饰器）。 */
|
||||
export function extendHeader(block: CodeBlock, from: Node): CodeBlock {
|
||||
return { ...block, header: { ...block.header, startIndex: from.startIndex, startRow: from.startPosition.row } }
|
||||
}
|
||||
|
||||
function lastLeafBefore(node: Node, limit: number): Node | undefined {
|
||||
for (let i = node.childCount - 1; i >= 0; i--) {
|
||||
const child = node.child(i)
|
||||
if (!child || child.startIndex >= limit || child.type === 'comment' || child.startIndex === child.endIndex)
|
||||
continue
|
||||
if (child.childCount === 0)
|
||||
return child
|
||||
const found = lastLeafBefore(child, limit)
|
||||
if (found)
|
||||
return found
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** 复合语句的头部：从 outer 开头到主体开始之前最后一个非注释的词为止。 */
|
||||
export function headerSpan(outer: Node, bodyStart: number): SourceSpan {
|
||||
const last = lastLeafBefore(outer, bodyStart)
|
||||
if (!last)
|
||||
return nodeSpan(outer)
|
||||
return {
|
||||
startIndex: outer.startIndex,
|
||||
endIndex: last.endIndex,
|
||||
startRow: outer.startPosition.row,
|
||||
endRow: last.endPosition.row,
|
||||
}
|
||||
}
|
||||
|
||||
/** node 里完全落在 [from, to) 之间的注释。 */
|
||||
function commentsIn(node: Node, from: number, to: number): Node[] {
|
||||
if (node.endIndex <= from || node.startIndex >= to)
|
||||
return []
|
||||
if (node.type === 'comment')
|
||||
return node.startIndex >= from && node.endIndex <= to ? [node] : []
|
||||
return children(node).flatMap(child => commentsIn(child, from, to))
|
||||
}
|
||||
|
||||
export type ListBlocks = (items: Node[], afterRow: number) => CodeBlock[]
|
||||
|
||||
/**
|
||||
* 复合语句块：头部到 bodyStart 之前，子块来自 items。
|
||||
* 头部和主体之间的注释（例如 Python 冒号后、主体前的注释）也归子块。
|
||||
*/
|
||||
export function compoundBlock(kind: BlockKind, outer: Node, bodyStart: number, items: Node[], list: ListBlocks): CodeBlock {
|
||||
const header = headerSpan(outer, bodyStart)
|
||||
const between = commentsIn(outer, header.endIndex, bodyStart)
|
||||
return { kind, node: outer, header, children: list([...between, ...items], header.endRow) }
|
||||
}
|
||||
|
||||
/**
|
||||
* 一串语句 → 块。注释单独成块，相邻行的行注释合成一块；
|
||||
* 和前一条语句（或头部）同一行的注释是行尾注释，跟着那一行显示，不单独成块。
|
||||
* afterRow 是这串语句之前的最后一行（头部结尾行），模块顶层传 -1；
|
||||
* toBlocks 的 first 表示它是这串里第一条非注释语句（Python 认 docstring 用）。
|
||||
*/
|
||||
export function listBlocks(
|
||||
items: Node[],
|
||||
afterRow: number,
|
||||
toBlocks: (item: Node, first: boolean) => CodeBlock[],
|
||||
isLineComment: (comment: Node) => boolean,
|
||||
): CodeBlock[] {
|
||||
const out: CodeBlock[] = []
|
||||
let lastRow = afterRow
|
||||
let lineRun: CodeBlock | undefined
|
||||
let first = true
|
||||
for (const item of items) {
|
||||
if (item.type === 'comment') {
|
||||
if (item.startPosition.row === lastRow)
|
||||
continue
|
||||
const line = isLineComment(item)
|
||||
if (line && lineRun && item.startPosition.row === lineRun.header.endRow + 1) {
|
||||
lineRun.header = { ...lineRun.header, endIndex: item.endIndex, endRow: item.endPosition.row }
|
||||
}
|
||||
else {
|
||||
const block = leafBlock('comment', item)
|
||||
out.push(block)
|
||||
lineRun = line ? block : undefined
|
||||
}
|
||||
}
|
||||
else {
|
||||
out.push(...toBlocks(item, first))
|
||||
first = false
|
||||
lineRun = undefined
|
||||
}
|
||||
lastRow = item.endPosition.row
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** 同一个节点：web-tree-sitter 每次取子节点都会新建对象，只能按位置和类型比。 */
|
||||
export function sameNode(a: Node | null | undefined, b: Node): boolean {
|
||||
return !!a && a.startIndex === b.startIndex && a.endIndex === b.endIndex && a.type === b.type
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import type { Outline } from '../testSupport'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { outline, readFixture, WASM_DIR, withBlocks } from '../testSupport'
|
||||
import { createTypeScriptAdapter } from './typescript'
|
||||
|
||||
const adapter = createTypeScriptAdapter(WASM_DIR)
|
||||
|
||||
function tree(source: string, languageId = 'typescript'): Promise<Outline[]> {
|
||||
return withBlocks(adapter, languageId, source, outline)
|
||||
}
|
||||
|
||||
// 与原型 docs/prototype/code-reader/figma-plugin/code.js 的 CODE_TREE 一致
|
||||
const ENV_LOCK: Outline[] = [
|
||||
{ at: [1, 10], kind: 'comment' },
|
||||
{ at: 12, kind: 'import' },
|
||||
{ at: 13, kind: 'import' },
|
||||
{ at: 15, kind: 'const' },
|
||||
{ at: 16, kind: 'const' },
|
||||
{ at: [18, 22], kind: 'comment' },
|
||||
{ at: 23, kind: 'function', kids: [
|
||||
{ at: 24, kind: 'variable' },
|
||||
{ at: 25, kind: 'if', kids: [
|
||||
{ at: 26, kind: 'return' },
|
||||
] },
|
||||
{ at: 27, kind: 'variable' },
|
||||
{ at: 28, kind: 'variable' },
|
||||
{ at: 29, kind: 'return' },
|
||||
] },
|
||||
{ at: 32, kind: 'type', kids: [
|
||||
{ at: 33, kind: 'field' },
|
||||
{ at: 34, kind: 'field' },
|
||||
{ at: 35, kind: 'field' },
|
||||
] },
|
||||
{ at: 38, kind: 'function', kids: [
|
||||
{ at: 39, kind: 'variable' },
|
||||
{ at: 40, kind: 'variable' },
|
||||
{ at: 41, kind: 'loop', kids: [
|
||||
{ at: 42, kind: 'try', kids: [
|
||||
{ at: 43, kind: 'statement' },
|
||||
{ at: 44, kind: 'statement' },
|
||||
] },
|
||||
{ at: 46, kind: 'catch', kids: [
|
||||
{ at: 47, kind: 'statement' },
|
||||
] },
|
||||
] },
|
||||
{ at: 50, kind: 'return' },
|
||||
] },
|
||||
{ at: 53, kind: 'comment' },
|
||||
{ at: 54, kind: 'function', kids: [
|
||||
{ at: 55, kind: 'return' },
|
||||
] },
|
||||
{ at: 58, kind: 'comment' },
|
||||
{ at: 59, kind: 'function', kids: [
|
||||
{ at: 60, kind: 'return' },
|
||||
] },
|
||||
]
|
||||
|
||||
const ISSUE_LOCK: Outline[] = [
|
||||
{ at: 1, kind: 'import' },
|
||||
{ at: 3, kind: 'type', kids: [
|
||||
{ at: 4, kind: 'field' },
|
||||
{ at: 5, kind: 'field' },
|
||||
{ at: 6, kind: 'field' },
|
||||
] },
|
||||
{ at: [9, 12], kind: 'comment' },
|
||||
{ at: 13, kind: 'function', kids: [
|
||||
{ at: 14, kind: 'variable' },
|
||||
{ at: 15, kind: 'if', kids: [
|
||||
{ at: 16, kind: 'return' },
|
||||
] },
|
||||
{ at: 17, kind: 'variable' },
|
||||
{ at: 18, kind: 'if', kids: [
|
||||
{ at: 19, kind: 'return' },
|
||||
] },
|
||||
{ at: [20, 24], kind: 'return' },
|
||||
] },
|
||||
]
|
||||
|
||||
describe('typeScript 代码块树', () => {
|
||||
it('envLock.ts 与原型一致', async () => {
|
||||
expect(await tree(readFixture('envLock.ts.txt'))).toEqual(ENV_LOCK)
|
||||
})
|
||||
|
||||
it('issueLock.ts 与原型一致', async () => {
|
||||
expect(await tree(readFixture('issueLock.ts.txt'))).toEqual(ISSUE_LOCK)
|
||||
})
|
||||
|
||||
it('else / else if 展平成兄弟块，else if 的子块来自它自己的主体', async () => {
|
||||
const source = [
|
||||
'if (a) {',
|
||||
' x()',
|
||||
'} else if (b) {',
|
||||
' y()',
|
||||
'} else',
|
||||
' z()',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'if', kids: [{ at: 2, kind: 'statement' }] },
|
||||
{ at: 3, kind: 'else', kids: [{ at: 4, kind: 'statement' }] },
|
||||
{ at: 5, kind: 'else', kids: [{ at: 6, kind: 'statement' }] },
|
||||
])
|
||||
})
|
||||
|
||||
it('try / catch / finally 是兄弟块；switch 的 case / default 是子块 if', async () => {
|
||||
const source = [
|
||||
'try {',
|
||||
' a()',
|
||||
'}',
|
||||
'catch {',
|
||||
' b()',
|
||||
'}',
|
||||
'finally {',
|
||||
' c()',
|
||||
'}',
|
||||
'switch (x) {',
|
||||
' case 1:',
|
||||
' foo()',
|
||||
' break',
|
||||
' default:',
|
||||
' bar()',
|
||||
'}',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'try', kids: [{ at: 2, kind: 'statement' }] },
|
||||
{ at: 4, kind: 'catch', kids: [{ at: 5, kind: 'statement' }] },
|
||||
{ at: 7, kind: 'finally', kids: [{ at: 8, kind: 'statement' }] },
|
||||
{ at: 10, kind: 'if', kids: [
|
||||
{ at: 11, kind: 'if', kids: [{ at: 12, kind: 'statement' }, { at: 13, kind: 'statement' }] },
|
||||
{ at: 14, kind: 'if', kids: [{ at: 15, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('类：方法 → function、字段 → field、注释 → comment，装饰器算进成员头部', async () => {
|
||||
const source = [
|
||||
'export class Store extends Base {',
|
||||
' // 缓存',
|
||||
' private readonly cache = new Map()',
|
||||
' @memo()',
|
||||
' get size(): number {',
|
||||
' return this.cache.size',
|
||||
' }',
|
||||
' handle = () => {',
|
||||
' this.cache.clear()',
|
||||
' }',
|
||||
'}',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'class', kids: [
|
||||
{ at: 2, kind: 'comment' },
|
||||
{ at: 3, kind: 'field' },
|
||||
{ at: [4, 5], kind: 'function', kids: [{ at: 6, kind: 'return' }] },
|
||||
{ at: 8, kind: 'function', kids: [{ at: 9, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('enum 成员 → field；type 别名 → type；多个循环 → loop', async () => {
|
||||
const source = [
|
||||
'enum Color {',
|
||||
' Red = 1,',
|
||||
' Blue,',
|
||||
'}',
|
||||
'type Pair = [string, number]',
|
||||
'for (let i = 0; i < 3; i++) {',
|
||||
' while (busy) wait()',
|
||||
' do {',
|
||||
' step()',
|
||||
' } while (more)',
|
||||
'}',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'type', kids: [{ at: 2, kind: 'field' }, { at: 3, kind: 'field' }] },
|
||||
{ at: 5, kind: 'type' },
|
||||
{ at: 6, kind: 'loop', kids: [
|
||||
{ at: 7, kind: 'loop', kids: [{ at: 7, kind: 'statement' }] },
|
||||
{ at: 8, kind: 'loop', kids: [{ at: 9, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('函数值的变量 → function；回调参数带语句块主体的表达式语句 → statement 带子块', async () => {
|
||||
const source = [
|
||||
'const App = () => {',
|
||||
' useEffect(() => {',
|
||||
' load()',
|
||||
' }, [])',
|
||||
' let n = 0',
|
||||
' return null',
|
||||
'}',
|
||||
'let total = 0',
|
||||
'describe(\'x\', () => {',
|
||||
' it(\'y\', async () => {',
|
||||
' await run()',
|
||||
' })',
|
||||
'})',
|
||||
].join('\n')
|
||||
expect(await tree(source, 'typescriptreact')).toEqual([
|
||||
{ at: 1, kind: 'function', kids: [
|
||||
{ at: 2, kind: 'statement', kids: [{ at: 3, kind: 'statement' }] },
|
||||
{ at: 5, kind: 'variable' },
|
||||
{ at: 6, kind: 'return' },
|
||||
] },
|
||||
{ at: 8, kind: 'variable' },
|
||||
{ at: 9, kind: 'statement', kids: [
|
||||
{ at: 10, kind: 'statement', kids: [{ at: 11, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('相邻行的行注释合成一块，空行隔开的分开；行尾注释不单独成块', async () => {
|
||||
const source = [
|
||||
'// a',
|
||||
'// b',
|
||||
'',
|
||||
'// c',
|
||||
'const x = 1 // 行尾',
|
||||
'foo() // 行尾',
|
||||
].join('\n')
|
||||
expect(await tree(source, 'javascript')).toEqual([
|
||||
{ at: [1, 2], kind: 'comment' },
|
||||
{ at: 4, kind: 'comment' },
|
||||
{ at: 5, kind: 'const' },
|
||||
{ at: 6, kind: 'statement' },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,328 @@
|
||||
import type { BlockKind, CodeBlock, LanguageAdapter, Node, TokenRole } from '../contract'
|
||||
import * as path from 'node:path'
|
||||
import { compoundBlock, extendHeader, leafBlock, listBlocks, named, sameNode } from './shared'
|
||||
|
||||
const GRAMMARS: Record<string, string> = {
|
||||
typescript: 'tree-sitter-typescript.wasm',
|
||||
typescriptreact: 'tree-sitter-tsx.wasm',
|
||||
javascript: 'tree-sitter-javascript.wasm',
|
||||
javascriptreact: 'tree-sitter-javascript.wasm',
|
||||
}
|
||||
|
||||
const FUNCTION_VALUES = new Set(['arrow_function', 'function_expression', 'function', 'generator_function'])
|
||||
const LOOPS = new Set(['for_statement', 'for_in_statement', 'while_statement', 'do_statement'])
|
||||
const CLASSES = new Set(['class_declaration', 'abstract_class_declaration', 'class'])
|
||||
|
||||
function isLineComment(comment: Node): boolean {
|
||||
return comment.text.startsWith('//')
|
||||
}
|
||||
|
||||
/** 带语句块主体的函数值（箭头函数 / 函数表达式）的主体。 */
|
||||
function functionBody(value: Node | null | undefined): Node | null {
|
||||
if (!value || !FUNCTION_VALUES.has(value.type))
|
||||
return null
|
||||
const body = value.childForFieldName('body')
|
||||
return body?.type === 'statement_block' ? body : null
|
||||
}
|
||||
|
||||
/** 调用参数里第一个带语句块主体的函数的主体：useEffect(() => { … })、describe('x', () => { … })。 */
|
||||
function callbackBody(expression: Node | null | undefined): Node | null {
|
||||
const call = expression?.type === 'await_expression' ? expression.firstNamedChild : expression
|
||||
if (call?.type !== 'call_expression' && call?.type !== 'new_expression')
|
||||
return null
|
||||
for (const arg of named(call.childForFieldName('arguments'))) {
|
||||
const body = functionBody(arg)
|
||||
if (body)
|
||||
return body
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function statements(items: Node[], top: boolean, afterRow: number): CodeBlock[] {
|
||||
return listBlocks(items, afterRow, item => statementBlocks(item, top), isLineComment)
|
||||
}
|
||||
|
||||
/** 主体是语句块时子块来自它的语句，否则主体本身就是唯一的子语句（`if (x) return`）。 */
|
||||
function bodyBlock(kind: BlockKind, outer: Node, body: Node | null, top = false): CodeBlock {
|
||||
if (!body)
|
||||
return leafBlock(kind, outer)
|
||||
const items = body.type === 'statement_block' ? named(body) : [body]
|
||||
return compoundBlock(kind, outer, body.startIndex, items, (list, afterRow) => statements(list, top, afterRow))
|
||||
}
|
||||
|
||||
function statementBlocks(node: Node, top: boolean): CodeBlock[] {
|
||||
switch (node.type) {
|
||||
case 'import_statement':
|
||||
return [leafBlock('import', node)]
|
||||
case 'export_statement':
|
||||
return exportBlocks(node, top)
|
||||
case 'if_statement':
|
||||
return ifBlocks(node, node, 'if')
|
||||
case 'try_statement':
|
||||
return tryBlocks(node)
|
||||
case 'switch_statement':
|
||||
return [switchBlock(node)]
|
||||
case 'switch_case':
|
||||
case 'switch_default':
|
||||
return [caseBlock(node)]
|
||||
case 'return_statement':
|
||||
return [leafBlock('return', node)]
|
||||
case 'expression_statement':
|
||||
return [bodyBlock('statement', node, callbackBody(node.firstNamedChild))]
|
||||
case 'labeled_statement': {
|
||||
const [first, ...rest] = statementBlocks(node.childForFieldName('body') ?? node, top)
|
||||
return first ? [{ ...extendHeader(first, node), node }, ...rest] : []
|
||||
}
|
||||
case 'statement_block':
|
||||
return statements(named(node), top, node.startPosition.row)
|
||||
case 'empty_statement':
|
||||
return []
|
||||
default:
|
||||
if (LOOPS.has(node.type))
|
||||
return [bodyBlock('loop', node, node.childForFieldName('body'))]
|
||||
return declarationBlocks(node, node, top)
|
||||
}
|
||||
}
|
||||
|
||||
/** export 包着声明时按声明定种类，块节点取整个 export_statement（头部包含 export）。 */
|
||||
function exportBlocks(node: Node, top: boolean): CodeBlock[] {
|
||||
const declaration = node.childForFieldName('declaration')
|
||||
if (declaration)
|
||||
return declarationBlocks(declaration, node, top)
|
||||
const value = node.childForFieldName('value')
|
||||
if (value && CLASSES.has(value.type))
|
||||
return [classBlock(value, node)]
|
||||
const body = functionBody(value)
|
||||
return [body ? bodyBlock('function', node, body) : leafBlock('statement', node)]
|
||||
}
|
||||
|
||||
function declarationBlocks(declaration: Node, outer: Node, top: boolean): CodeBlock[] {
|
||||
switch (declaration.type) {
|
||||
case 'function_declaration':
|
||||
case 'generator_function_declaration':
|
||||
return [bodyBlock('function', outer, declaration.childForFieldName('body'))]
|
||||
case 'function_signature':
|
||||
return [leafBlock('function', outer)]
|
||||
case 'lexical_declaration':
|
||||
case 'variable_declaration':
|
||||
return [variableBlock(declaration, outer, top)]
|
||||
case 'interface_declaration':
|
||||
case 'enum_declaration':
|
||||
return [membersBlock(declaration, outer)]
|
||||
case 'type_alias_declaration':
|
||||
return [leafBlock('type', outer)]
|
||||
default: {
|
||||
if (CLASSES.has(declaration.type))
|
||||
return [classBlock(declaration, outer)]
|
||||
const body = declaration.childForFieldName('body')
|
||||
return [body?.type === 'statement_block' ? bodyBlock('statement', outer, body) : leafBlock('statement', outer)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 唯一声明值是带语句块主体的函数 → function；否则模块顶层的 const → const，其余 → variable。 */
|
||||
function variableBlock(declaration: Node, outer: Node, top: boolean): CodeBlock {
|
||||
const declarators = named(declaration).filter(child => child.type === 'variable_declarator')
|
||||
const value = declarators.length === 1 ? declarators[0].childForFieldName('value') : null
|
||||
const body = functionBody(value)
|
||||
if (body)
|
||||
return bodyBlock('function', outer, body)
|
||||
const isConst = declaration.type === 'lexical_declaration' && declaration.childForFieldName('kind')?.type === 'const'
|
||||
return bodyBlock(top && isConst ? 'const' : 'variable', outer, callbackBody(value))
|
||||
}
|
||||
|
||||
/** if 的 else / else if 展平成紧跟其后的兄弟块。 */
|
||||
function ifBlocks(ifNode: Node, outer: Node, kind: 'if' | 'else'): CodeBlock[] {
|
||||
const blocks = [bodyBlock(kind, outer, ifNode.childForFieldName('consequence'))]
|
||||
const alternative = ifNode.childForFieldName('alternative')
|
||||
if (alternative) {
|
||||
const inner = named(alternative).find(child => child.type !== 'comment') ?? null
|
||||
if (inner?.type === 'if_statement')
|
||||
blocks.push(...ifBlocks(inner, alternative, 'else'))
|
||||
else
|
||||
blocks.push(bodyBlock('else', alternative, inner))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
function tryBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('try', node, node.childForFieldName('body'))]
|
||||
const handler = node.childForFieldName('handler')
|
||||
if (handler)
|
||||
blocks.push(bodyBlock('catch', handler, handler.childForFieldName('body')))
|
||||
const finalizer = node.childForFieldName('finalizer')
|
||||
if (finalizer)
|
||||
blocks.push(bodyBlock('finally', finalizer, finalizer.childForFieldName('body')))
|
||||
return blocks
|
||||
}
|
||||
|
||||
function switchBlock(node: Node): CodeBlock {
|
||||
const body = node.childForFieldName('body')
|
||||
if (!body)
|
||||
return leafBlock('if', node)
|
||||
return compoundBlock('if', node, body.startIndex, named(body), (list, afterRow) => statements(list, false, afterRow))
|
||||
}
|
||||
|
||||
/** case / default：头部到冒号为止，子块来自它下面的语句。 */
|
||||
function caseBlock(node: Node): CodeBlock {
|
||||
const [first] = node.childrenForFieldName('body')
|
||||
const bodyStart = first?.startIndex ?? node.endIndex
|
||||
const items = named(node).filter(child => child.startIndex >= bodyStart)
|
||||
return compoundBlock('if', node, bodyStart, items, (list, afterRow) => statements(list, false, afterRow))
|
||||
}
|
||||
|
||||
function classBlock(declaration: Node, outer: Node): CodeBlock {
|
||||
const body = declaration.childForFieldName('body')
|
||||
if (!body)
|
||||
return leafBlock('class', outer)
|
||||
return compoundBlock('class', outer, body.startIndex, named(body), (list, afterRow) => {
|
||||
let decorators: Node | undefined
|
||||
return listBlocks(list, afterRow, (member) => {
|
||||
if (member.type === 'decorator') {
|
||||
decorators ??= member
|
||||
return []
|
||||
}
|
||||
const blocks = memberBlocks(member)
|
||||
const from = decorators
|
||||
decorators = undefined
|
||||
return from ? blocks.map((block, i) => i === 0 ? extendHeader(block, from) : block) : blocks
|
||||
}, isLineComment)
|
||||
})
|
||||
}
|
||||
|
||||
function memberBlocks(member: Node): CodeBlock[] {
|
||||
switch (member.type) {
|
||||
case 'method_definition':
|
||||
return [bodyBlock('function', member, member.childForFieldName('body'))]
|
||||
case 'method_signature':
|
||||
case 'abstract_method_signature':
|
||||
return [leafBlock('function', member)]
|
||||
case 'class_static_block':
|
||||
return [bodyBlock('statement', member, member.childForFieldName('body'))]
|
||||
default: {
|
||||
const body = functionBody(member.childForFieldName('value'))
|
||||
return [body ? bodyBlock('function', member, body) : leafBlock('field', member)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** interface / enum：成员（属性、方法签名、枚举项）→ field。 */
|
||||
function membersBlock(declaration: Node, outer: Node): CodeBlock {
|
||||
const body = declaration.childForFieldName('body')
|
||||
if (!body)
|
||||
return leafBlock('type', outer)
|
||||
return compoundBlock('type', outer, body.startIndex, named(body), (list, afterRow) =>
|
||||
listBlocks(list, afterRow, member => [leafBlock('field', member)], isLineComment))
|
||||
}
|
||||
|
||||
// ─── 源码着色 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** 流程关键字；其余字母关键字都算声明关键字（storage）。 */
|
||||
const FLOW_KEYWORDS = new Set([
|
||||
'import',
|
||||
'from',
|
||||
'as',
|
||||
'export',
|
||||
'default',
|
||||
'if',
|
||||
'else',
|
||||
'for',
|
||||
'of',
|
||||
'in',
|
||||
'while',
|
||||
'do',
|
||||
'switch',
|
||||
'case',
|
||||
'break',
|
||||
'continue',
|
||||
'return',
|
||||
'throw',
|
||||
'try',
|
||||
'catch',
|
||||
'finally',
|
||||
'await',
|
||||
'yield',
|
||||
])
|
||||
const LITERAL_KEYWORDS = new Set(['true', 'false', 'null', 'undefined', 'this', 'super'])
|
||||
const STRING_NODES = new Set(['string', 'template_string', 'regex'])
|
||||
const FUNCTION_NAMED = new Set(['function_declaration', 'generator_function_declaration', 'function_expression', 'function', 'generator_function', 'method_definition', 'method_signature', 'abstract_method_signature', 'function_signature'])
|
||||
|
||||
function insideString(leaf: Node): boolean {
|
||||
for (let node = leaf.parent; node; node = node.parent) {
|
||||
if (node.type === 'template_substitution')
|
||||
return false
|
||||
if (STRING_NODES.has(node.type))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function isCallee(node: Node): boolean {
|
||||
const parent = node.parent
|
||||
return !!parent && parent.type === 'call_expression' && sameNode(parent.childForFieldName('function'), node)
|
||||
}
|
||||
|
||||
function nameRole(leaf: Node): TokenRole {
|
||||
const parent = leaf.parent
|
||||
if (!parent)
|
||||
return 'variable'
|
||||
if (FUNCTION_NAMED.has(parent.type) && sameNode(parent.childForFieldName('name'), leaf))
|
||||
return 'function'
|
||||
if (isCallee(leaf))
|
||||
return 'function'
|
||||
if (parent.type === 'member_expression' && sameNode(parent.childForFieldName('property'), leaf) && isCallee(parent))
|
||||
return 'function'
|
||||
if (parent.type === 'new_expression')
|
||||
return 'type'
|
||||
if (parent.type === 'variable_declarator' && sameNode(parent.childForFieldName('name'), leaf) && functionBody(parent.childForFieldName('value')))
|
||||
return 'function'
|
||||
return 'variable'
|
||||
}
|
||||
|
||||
export function typescriptTokenRole(leaf: Node): TokenRole {
|
||||
if (leaf.type === 'comment' || leaf.type === 'hash_bang_line')
|
||||
return 'comment'
|
||||
if (insideString(leaf))
|
||||
return 'string'
|
||||
if (!leaf.isNamed) {
|
||||
if (leaf.parent?.type === 'predefined_type')
|
||||
return 'type'
|
||||
if (/^[a-z]+$/.test(leaf.type))
|
||||
return FLOW_KEYWORDS.has(leaf.type) ? 'keyword' : 'storage'
|
||||
return 'plain'
|
||||
}
|
||||
if (LITERAL_KEYWORDS.has(leaf.type))
|
||||
return 'storage'
|
||||
switch (leaf.type) {
|
||||
case 'number':
|
||||
return 'number'
|
||||
case 'type_identifier':
|
||||
return 'type'
|
||||
case 'identifier':
|
||||
case 'property_identifier':
|
||||
case 'private_property_identifier':
|
||||
return nameRole(leaf)
|
||||
case 'shorthand_property_identifier':
|
||||
case 'shorthand_property_identifier_pattern':
|
||||
case 'statement_identifier':
|
||||
return 'variable'
|
||||
default:
|
||||
return 'plain'
|
||||
}
|
||||
}
|
||||
|
||||
/** TypeScript / TSX / JavaScript / JSX 语言插件。@param wasmDir 语法包 .wasm 所在目录 */
|
||||
export function createTypeScriptAdapter(wasmDir: string): LanguageAdapter {
|
||||
return {
|
||||
id: 'typescript',
|
||||
languageIds: Object.keys(GRAMMARS),
|
||||
grammarPath: (languageId) => {
|
||||
const file = GRAMMARS[languageId]
|
||||
if (!file)
|
||||
throw new Error(`TypeScript 语言插件不支持 ${languageId}`)
|
||||
return path.join(wasmDir, file)
|
||||
},
|
||||
toBlocks: root => statements(named(root), true, -1),
|
||||
tokenRole: typescriptTokenRole,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import type { Memento } from 'vscode'
|
||||
import type { LanguageAdapter, Translator } from './contract'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { CodeReaderRegistry, TRANSLATOR_CHOICE_KEY } from './registry'
|
||||
|
||||
function fakeMemento(store: Record<string, unknown> = {}): Pick<Memento, 'get' | 'update'> {
|
||||
return {
|
||||
get: <T>(key: string): T | undefined => store[key] as T | undefined,
|
||||
update: async (key: string, value: unknown) => {
|
||||
store[key] = value
|
||||
},
|
||||
} as Pick<Memento, 'get' | 'update'>
|
||||
}
|
||||
|
||||
function adapter(id: string, languageIds: string[]): LanguageAdapter {
|
||||
return { id, languageIds, grammarPath: () => '', toBlocks: () => [], tokenRole: () => 'plain' }
|
||||
}
|
||||
|
||||
function translator(id: string, languageIds: string[]): Translator {
|
||||
return { id, label: id, languageIds, translate: () => [] }
|
||||
}
|
||||
|
||||
describe('codeReaderRegistry', () => {
|
||||
it('注册 / 注销语言插件后 languageIds 跟着变，并触发变化事件', () => {
|
||||
const registry = new CodeReaderRegistry(fakeMemento())
|
||||
let changes = 0
|
||||
registry.onDidChange(() => {
|
||||
changes++
|
||||
})
|
||||
const tsReg = registry.registerLanguage(adapter('typescript', ['typescript', 'javascript']))
|
||||
const pyReg = registry.registerLanguage(adapter('python', ['python']))
|
||||
expect(registry.languageIds()).toEqual(['javascript', 'python', 'typescript'])
|
||||
expect(registry.languageFor('python')?.id).toBe('python')
|
||||
|
||||
tsReg.dispose()
|
||||
tsReg.dispose()
|
||||
expect(registry.languageIds()).toEqual(['python'])
|
||||
expect(registry.languageFor('typescript')).toBeUndefined()
|
||||
expect(changes).toBe(3)
|
||||
|
||||
pyReg.dispose()
|
||||
expect(registry.languageIds()).toEqual([])
|
||||
})
|
||||
|
||||
it('同一语言多个语言插件时后注册的生效，注销后退回先前的', () => {
|
||||
const registry = new CodeReaderRegistry(fakeMemento())
|
||||
registry.registerLanguage(adapter('builtin', ['python']))
|
||||
const override = registry.registerLanguage(adapter('external', ['python']))
|
||||
expect(registry.languageFor('python')?.id).toBe('external')
|
||||
override.dispose()
|
||||
expect(registry.languageFor('python')?.id).toBe('builtin')
|
||||
})
|
||||
|
||||
it('可选翻译按语言过滤；没选过时用第一个，按 globalState 记住选择', async () => {
|
||||
const store: Record<string, unknown> = {}
|
||||
const registry = new CodeReaderRegistry(fakeMemento(store))
|
||||
registry.registerTranslator(translator('ts.zh', ['typescript']))
|
||||
const ai = registry.registerTranslator(translator('ts.ai', ['typescript']))
|
||||
registry.registerTranslator(translator('py.zh', ['python']))
|
||||
|
||||
expect(registry.translatorsFor('typescript').map(t => t.id)).toEqual(['ts.zh', 'ts.ai'])
|
||||
expect(registry.translatorsFor('rust')).toEqual([])
|
||||
expect(registry.selectedTranslator('typescript')?.id).toBe('ts.zh')
|
||||
|
||||
await registry.selectTranslator('typescript', 'ts.ai')
|
||||
await registry.selectTranslator('python', 'py.zh')
|
||||
expect(store[TRANSLATOR_CHOICE_KEY]).toEqual({ typescript: 'ts.ai', python: 'py.zh' })
|
||||
expect(registry.selectedTranslator('typescript')?.id).toBe('ts.ai')
|
||||
expect(new CodeReaderRegistry(fakeMemento(store)).selectedTranslator('typescript')).toBeUndefined()
|
||||
|
||||
ai.dispose()
|
||||
expect(registry.translatorsFor('typescript').map(t => t.id)).toEqual(['ts.zh'])
|
||||
expect(registry.selectedTranslator('typescript')?.id).toBe('ts.zh')
|
||||
})
|
||||
|
||||
it('变化事件可退订，也能放进 disposables', () => {
|
||||
const registry = new CodeReaderRegistry(fakeMemento())
|
||||
let changes = 0
|
||||
const bag: { dispose: () => unknown }[] = []
|
||||
registry.onDidChange(() => {
|
||||
changes++
|
||||
}, undefined, bag)
|
||||
registry.registerTranslator(translator('a', ['python']))
|
||||
bag.forEach(d => d.dispose())
|
||||
registry.registerTranslator(translator('b', ['python']))
|
||||
expect(changes).toBe(1)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
import type { Disposable, Event, Memento } from 'vscode'
|
||||
import type { CodeReaderApi, LanguageAdapter, Translator } from './contract'
|
||||
|
||||
/** globalState 里按 languageId 记选中的翻译插件 id：Record<languageId, translatorId>。 */
|
||||
export const TRANSLATOR_CHOICE_KEY = 'superpowers.codeReader.translatorByLanguage'
|
||||
|
||||
interface Entry<T> { value: T }
|
||||
|
||||
/**
|
||||
* 语言插件、翻译插件的注册表。内置实现和其他扩展都通过它注册（它本身就是对外的 CodeReaderApi）。
|
||||
* 不依赖 vscode 运行时：Disposable 和事件都是结构兼容的普通对象，便于单元测试。
|
||||
*/
|
||||
export class CodeReaderRegistry implements CodeReaderApi {
|
||||
private readonly adapters: Entry<LanguageAdapter>[] = []
|
||||
private readonly translators: Entry<Translator>[] = []
|
||||
private readonly listeners = new Set<() => void>()
|
||||
|
||||
constructor(private readonly state: Pick<Memento, 'get' | 'update'>) {}
|
||||
|
||||
/** 注册 / 注销任何插件后触发。 */
|
||||
readonly onDidChange: Event<void> = (listener, thisArgs?, disposables?) => {
|
||||
const bound = (): void => {
|
||||
listener.call(thisArgs, undefined)
|
||||
}
|
||||
this.listeners.add(bound)
|
||||
const disposable = { dispose: () => this.listeners.delete(bound) }
|
||||
disposables?.push(disposable)
|
||||
return disposable
|
||||
}
|
||||
|
||||
registerLanguage(adapter: LanguageAdapter): Disposable {
|
||||
return this.add(this.adapters, adapter)
|
||||
}
|
||||
|
||||
registerTranslator(translator: Translator): Disposable {
|
||||
return this.add(this.translators, translator)
|
||||
}
|
||||
|
||||
/** 当前语言插件覆盖的全部 languageId（去重、排序），给 when 子句用。 */
|
||||
languageIds(): string[] {
|
||||
return [...new Set(this.adapters.flatMap(entry => entry.value.languageIds))].sort()
|
||||
}
|
||||
|
||||
/** 某 languageId 的语言插件；多个插件覆盖同一语言时后注册的生效。 */
|
||||
languageFor(languageId: string): LanguageAdapter | undefined {
|
||||
return [...this.adapters].reverse().find(entry => entry.value.languageIds.includes(languageId))?.value
|
||||
}
|
||||
|
||||
/** 某 languageId 可选的翻译插件，按注册顺序。 */
|
||||
translatorsFor(languageId: string): Translator[] {
|
||||
return this.translators.map(entry => entry.value).filter(translator => translator.languageIds.includes(languageId))
|
||||
}
|
||||
|
||||
/** 选中的翻译：globalState 里记的那个还注册着就用它，否则用第一个。 */
|
||||
selectedTranslator(languageId: string): Translator | undefined {
|
||||
const candidates = this.translatorsFor(languageId)
|
||||
const chosen = this.state.get<Record<string, string>>(TRANSLATOR_CHOICE_KEY)?.[languageId]
|
||||
return candidates.find(translator => translator.id === chosen) ?? candidates[0]
|
||||
}
|
||||
|
||||
/** 按 languageId 记住选中的翻译。 */
|
||||
async selectTranslator(languageId: string, translatorId: string): Promise<void> {
|
||||
const choices = this.state.get<Record<string, string>>(TRANSLATOR_CHOICE_KEY) ?? {}
|
||||
await this.state.update(TRANSLATOR_CHOICE_KEY, { ...choices, [languageId]: translatorId })
|
||||
}
|
||||
|
||||
private add<T>(list: Entry<T>[], value: T): Disposable {
|
||||
const entry = { value }
|
||||
list.push(entry)
|
||||
this.fire()
|
||||
return {
|
||||
dispose: () => {
|
||||
const index = list.indexOf(entry)
|
||||
if (index < 0)
|
||||
return
|
||||
list.splice(index, 1)
|
||||
this.fire()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
private fire(): void {
|
||||
for (const listener of [...this.listeners])
|
||||
listener()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import type { Tree } from '@vscode/tree-sitter-wasm'
|
||||
import type { LanguageAdapter } from './contract'
|
||||
import * as path from 'node:path'
|
||||
import { Language, Parser } from '@vscode/tree-sitter-wasm'
|
||||
|
||||
/** 打进扩展包里的 wasm 目录：dist/index.cjs 旁边的 dist/tree-sitter（esbuild 构建时复制）。只在打包产物里成立。 */
|
||||
export function bundledWasmDir(): string {
|
||||
return path.join(__dirname, 'tree-sitter')
|
||||
}
|
||||
|
||||
// web-tree-sitter 的运行时是进程级单例：Parser.init 只能有效调用一次，后来的 wasm 目录不再生效
|
||||
let runtimeReady: Promise<void> | undefined
|
||||
|
||||
function initRuntime(wasmDir: string): Promise<void> {
|
||||
runtimeReady ??= Parser.init({ locateFile: (file: string) => path.join(wasmDir, file) })
|
||||
.catch((err: unknown) => {
|
||||
runtimeReady = undefined
|
||||
throw err
|
||||
})
|
||||
return runtimeReady
|
||||
}
|
||||
|
||||
/** 解析运行时：初始化一次 web-tree-sitter，语法包按 wasm 路径懒加载并缓存。 */
|
||||
export class TreeSitterRuntime {
|
||||
private readonly parsers = new Map<string, Promise<Parser>>()
|
||||
|
||||
/** @param wasmDir 放 tree-sitter.wasm 的目录 */
|
||||
constructor(private readonly wasmDir: string) {}
|
||||
|
||||
/** 返回的语法树占着 wasm 内存，用完必须 `tree.delete()`。 */
|
||||
async parse(adapter: LanguageAdapter, languageId: string, source: string): Promise<Tree> {
|
||||
const parser = await this.parserFor(adapter.grammarPath(languageId))
|
||||
const tree = parser.parse(source)
|
||||
if (!tree)
|
||||
throw new Error(`tree-sitter 没能解析 ${languageId} 源码`)
|
||||
return tree
|
||||
}
|
||||
|
||||
private parserFor(grammarPath: string): Promise<Parser> {
|
||||
let parser = this.parsers.get(grammarPath)
|
||||
if (!parser) {
|
||||
parser = initRuntime(this.wasmDir)
|
||||
.then(() => Language.load(grammarPath))
|
||||
.then(language => new Parser().setLanguage(language))
|
||||
parser.catch(() => this.parsers.delete(grammarPath))
|
||||
this.parsers.set(grammarPath, parser)
|
||||
}
|
||||
return parser
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import type { BlockKind, CodeBlock, LanguageAdapter } from './contract'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import * as path from 'node:path'
|
||||
import { TreeSitterRuntime } from './runtime'
|
||||
|
||||
/** 单元测试直接用依赖包里的 wasm，不走打包产物。 */
|
||||
export const WASM_DIR = path.resolve(__dirname, '../../node_modules/@vscode/tree-sitter-wasm/wasm')
|
||||
|
||||
export const runtime = new TreeSitterRuntime(WASM_DIR)
|
||||
|
||||
export function readFixture(name: string): string {
|
||||
return readFileSync(path.join(__dirname, 'fixtures', name), 'utf8')
|
||||
}
|
||||
|
||||
/** 解析 source，把代码块树交给 use；语法树随后释放。 */
|
||||
export async function withBlocks<T>(adapter: LanguageAdapter, languageId: string, source: string, use: (blocks: CodeBlock[]) => T): Promise<T> {
|
||||
const tree = await runtime.parse(adapter, languageId, source)
|
||||
try {
|
||||
return use(adapter.toBlocks(tree.rootNode, source))
|
||||
}
|
||||
finally {
|
||||
tree.delete()
|
||||
}
|
||||
}
|
||||
|
||||
/** 和原型 CODE_TREE 同样的写法：at 是 1 起的行号（单行，或 [起, 止]）。 */
|
||||
export interface Outline { at: number | [number, number], kind: BlockKind, kids?: Outline[] }
|
||||
|
||||
export function outline(blocks: CodeBlock[]): Outline[] {
|
||||
return blocks.map((block) => {
|
||||
const from = block.header.startRow + 1
|
||||
const to = block.header.endRow + 1
|
||||
const item: Outline = { at: from === to ? from : [from, to], kind: block.kind }
|
||||
if (block.children.length > 0)
|
||||
item.kids = outline(block.children)
|
||||
return item
|
||||
})
|
||||
}
|
||||
|
||||
export function walk(blocks: CodeBlock[], visit: (block: CodeBlock) => void): void {
|
||||
for (const block of blocks) {
|
||||
visit(block)
|
||||
walk(block.children, visit)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { Node, Translator } from '../contract'
|
||||
import { insideAnnotation, pythonTokenRole } from '../languages/python'
|
||||
import { createZhTemplateTranslator } from './zhTemplate'
|
||||
|
||||
function stringContent(node: Node): string {
|
||||
if (node.type === 'string_content')
|
||||
return node.text
|
||||
return node.children.map(child => child ? stringContent(child) : '').join('')
|
||||
}
|
||||
|
||||
/** `#` 注释去掉井号；docstring 只取引号里的内容。 */
|
||||
function commentText(node: Node): string {
|
||||
return node.type === 'comment' ? node.text.replace(/^#+\s?/, '') : stringContent(node)
|
||||
}
|
||||
|
||||
export const pythonZhTranslator: Translator = createZhTemplateTranslator({
|
||||
id: 'python.zh-template',
|
||||
label: '中文直译',
|
||||
languageIds: ['python'],
|
||||
tokenRole: pythonTokenRole,
|
||||
keywords: {
|
||||
'import': '导入',
|
||||
'from': '来自',
|
||||
'as': '作为',
|
||||
'def': '函数',
|
||||
'async': '异步',
|
||||
'await': '等待',
|
||||
'class': '类',
|
||||
'return': '返回',
|
||||
'if': '如果',
|
||||
'elif': '否则如果',
|
||||
'else': '否则',
|
||||
'for': '遍历',
|
||||
'in': { zh: '在', byParent: { for_statement: '取自', for_in_clause: '取自' } },
|
||||
'while': '当',
|
||||
'try': '尝试',
|
||||
'except': '捕获',
|
||||
'finally': '最终',
|
||||
'raise': '抛出',
|
||||
'with': '使用',
|
||||
'yield': '产出',
|
||||
'lambda': '匿名函数',
|
||||
'pass': '跳过',
|
||||
'break': '跳出',
|
||||
'continue': '继续',
|
||||
'global': '全局',
|
||||
'nonlocal': '非局部',
|
||||
'del': '删除',
|
||||
'assert': '断言',
|
||||
'not': '非',
|
||||
'and': '与',
|
||||
'or': '或',
|
||||
'is': '是',
|
||||
'not in': '不在',
|
||||
'is not': '不是',
|
||||
'match': '匹配',
|
||||
'case': '情况',
|
||||
},
|
||||
literals: { none: '空', true: '真', false: '假' },
|
||||
builtinTypes: {
|
||||
str: '字符串',
|
||||
int: '整数',
|
||||
float: '浮点数',
|
||||
bool: '布尔',
|
||||
list: '列表',
|
||||
dict: '字典',
|
||||
set: '集合',
|
||||
tuple: '元组',
|
||||
bytes: '字节串',
|
||||
},
|
||||
isTypePosition: leaf => leaf.type === 'identifier' && leaf.parent?.type !== 'attribute' && insideAnnotation(leaf),
|
||||
operators: { '==': '等于', '!=': '不等于' },
|
||||
operatorParents: ['comparison_operator'],
|
||||
emptyLiterals: { list: '空列表', dictionary: '空字典' },
|
||||
stringNodes: ['string'],
|
||||
commentText,
|
||||
})
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { Node, Translator } from '../contract'
|
||||
import { typescriptTokenRole } from '../languages/typescript'
|
||||
import { createZhTemplateTranslator } from './zhTemplate'
|
||||
|
||||
function commentText(node: Node): string {
|
||||
const text = node.text
|
||||
if (text.startsWith('//'))
|
||||
return text.replace(/^\/{2,}\s?/, '')
|
||||
return text
|
||||
.replace(/^\/\*+/, '')
|
||||
.replace(/\*+\/$/, '')
|
||||
.split('\n')
|
||||
.map(line => line.replace(/^\s*\*(?!\/)\s?/, ''))
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
export const typescriptZhTranslator: Translator = createZhTemplateTranslator({
|
||||
id: 'typescript.zh-template',
|
||||
label: '中文直译',
|
||||
languageIds: ['typescript', 'typescriptreact', 'javascript', 'javascriptreact'],
|
||||
tokenRole: typescriptTokenRole,
|
||||
keywords: {
|
||||
// 流程
|
||||
import: '导入',
|
||||
from: '来自',
|
||||
as: '作为',
|
||||
export: '公共',
|
||||
default: '默认',
|
||||
if: '如果',
|
||||
else: '否则',
|
||||
for: '遍历',
|
||||
of: '取自',
|
||||
in: '在',
|
||||
while: '当',
|
||||
do: '执行',
|
||||
switch: '分支',
|
||||
case: '情况',
|
||||
break: '跳出',
|
||||
continue: '继续',
|
||||
return: '返回',
|
||||
throw: '抛出',
|
||||
try: '尝试',
|
||||
catch: '捕获',
|
||||
finally: '最终',
|
||||
await: '等待',
|
||||
yield: '产出',
|
||||
// 声明
|
||||
const: '常量',
|
||||
let: '变量',
|
||||
var: '变量',
|
||||
function: '函数',
|
||||
async: '异步',
|
||||
class: '类',
|
||||
extends: '继承',
|
||||
implements: '实现',
|
||||
interface: '接口',
|
||||
type: '类型',
|
||||
enum: '枚举',
|
||||
new: '新建',
|
||||
instanceof: '属于',
|
||||
typeof: '类型为',
|
||||
keyof: '键名',
|
||||
readonly: '只读',
|
||||
public: '公共',
|
||||
private: '私有',
|
||||
protected: '受保护',
|
||||
static: '静态',
|
||||
abstract: '抽象',
|
||||
delete: '删除',
|
||||
},
|
||||
literals: { true: '真', false: '假', null: '空', undefined: '未定义' },
|
||||
builtinTypes: {
|
||||
string: '字符串',
|
||||
number: '数值',
|
||||
boolean: '布尔',
|
||||
any: '任意',
|
||||
unknown: '未知',
|
||||
never: '永不',
|
||||
void: '无',
|
||||
object: '对象',
|
||||
bigint: '大整数',
|
||||
symbol: '符号',
|
||||
},
|
||||
isTypePosition: leaf => leaf.parent?.type === 'predefined_type',
|
||||
operators: { '===': '全等', '!==': '不全等', '==': '等于', '!=': '不等于', '&&': '与', '||': '或', '!': '非' },
|
||||
operatorParents: ['binary_expression', 'unary_expression'],
|
||||
emptyLiterals: { array: '空数组' },
|
||||
stringNodes: ['string', 'template_string', 'regex'],
|
||||
commentText,
|
||||
})
|
||||
@@ -0,0 +1,179 @@
|
||||
import type { LanguageAdapter, Segment, Translator } from '../contract'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createPythonAdapter } from '../languages/python'
|
||||
import { createTypeScriptAdapter } from '../languages/typescript'
|
||||
import { readFixture, walk, WASM_DIR, withBlocks } from '../testSupport'
|
||||
import { pythonZhTranslator } from './pythonZh'
|
||||
import { typescriptZhTranslator } from './typescriptZh'
|
||||
|
||||
const ts = createTypeScriptAdapter(WASM_DIR)
|
||||
const py = createPythonAdapter(WASM_DIR)
|
||||
|
||||
interface Translated { kind: string, segments: Segment[] }
|
||||
|
||||
/** 每块的译文，按 header 起始行（1 起）索引；同一行有多块时取外层那块。 */
|
||||
function translateAll(adapter: LanguageAdapter, languageId: string, translator: Translator, source: string): Promise<Map<number, Translated>> {
|
||||
return withBlocks(adapter, languageId, source, (blocks) => {
|
||||
const byLine = new Map<number, Translated>()
|
||||
walk(blocks, (block) => {
|
||||
const line = block.header.startRow + 1
|
||||
if (!byLine.has(line))
|
||||
byLine.set(line, { kind: block.kind, segments: translator.translate(block, source) })
|
||||
})
|
||||
return byLine
|
||||
})
|
||||
}
|
||||
|
||||
function text(segments: Segment[] | undefined): string {
|
||||
return (segments ?? []).map(segment => segment.text).join('')
|
||||
}
|
||||
|
||||
function texts(map: Map<number, Translated>): Record<number, string> {
|
||||
return Object.fromEntries([...map].map(([line, { segments }]) => [line, text(segments)]))
|
||||
}
|
||||
|
||||
const CJK = /\p{Script=Han}/u
|
||||
const CJK_ONLY = /^\p{Script=Han}+$/u
|
||||
const WORD_CHAR = /[\p{Script=Han}\w]/u
|
||||
|
||||
/** 被翻的词单独成段，而且不和前后的字母、数字、汉字粘在一起。 */
|
||||
function expectWordsApart(map: Map<number, Translated>): void {
|
||||
for (const [line, { kind, segments }] of map) {
|
||||
if (kind === 'comment')
|
||||
continue
|
||||
const full = text(segments)
|
||||
let at = 0
|
||||
for (const segment of segments) {
|
||||
if (segment.role !== 'string' && CJK.test(segment.text)) {
|
||||
expect(segment.text, `L${line} ${full}`).toMatch(CJK_ONLY)
|
||||
for (const neighbor of [full[at - 1], full[at + segment.text.length]])
|
||||
expect(neighbor ?? ' ', `L${line} "${full}" 里的「${segment.text}」`).not.toMatch(WORD_CHAR)
|
||||
}
|
||||
at += segment.text.length
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('typeScript 中文直译', () => {
|
||||
it('spec 样例逐字一致', async () => {
|
||||
const env = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt'))
|
||||
const issue = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt'))
|
||||
expect(text(env.get(23)?.segments)).toBe('公共 异步 函数 findEnvFiles(workspaceRoot: 字符串): Promise<字符串[]>')
|
||||
expect(text(env.get(38)?.segments)).toBe('异步 函数 chmodAll(workspaceRoot: 字符串, mode: 数值): Promise<EnvLockResult>')
|
||||
expect(text(env.get(12)?.segments)).toBe('导入 { promises 作为 fsp } 来自 \'node:fs\'')
|
||||
expect(text(env.get(25)?.segments)).toBe('如果 (非 folder)')
|
||||
expect(text(env.get(26)?.segments)).toBe('返回 空数组')
|
||||
expect(text(env.get(41)?.segments)).toBe('遍历 (常量 f 取自 files)')
|
||||
expect(text(env.get(35)?.segments)).toBe('failed: { path: 字符串, reason: 字符串 }[]')
|
||||
expect(text(env.get(53)?.segments)).toBe('Set every `.env*` file in the workspace to 0o444 (read-only).')
|
||||
expect(text(issue.get(14)?.segments)).toBe('常量 issue = issues.find(i => i.number 全等 issueNumber)')
|
||||
expect(text(issue.get(20)?.segments)).toBe('返回 { locked: 真, prerequisiteNumber: prereq.number, prerequisiteColumn: prereq.column }')
|
||||
})
|
||||
|
||||
it('其余块：结构照抄，复合语句去掉结尾的 {，注释去掉注释符号', async () => {
|
||||
const env = texts(await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt')))
|
||||
const issue = texts(await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt')))
|
||||
expect(env[24]).toBe('常量 folder = workspace.workspaceFolders?.find(f => f.uri.fsPath 全等 workspaceRoot)')
|
||||
expect(env[28]).toBe('常量 uris = 等待 workspace.findFiles(include, EXCLUDE_PATTERN)')
|
||||
expect(env[32]).toBe('公共 接口 EnvLockResult')
|
||||
expect(env[42]).toBe('尝试')
|
||||
expect(env[46]).toBe('捕获 (err)')
|
||||
expect(env[47]).toBe('result.failed.push({ path: f, reason: err 属于 Error ? err.message : String(err) })')
|
||||
expect(env[1]).toBe('.env file lock / unlock helpers. Recursively scans the main workspace folder for `.env*` files (skipping `node_modules`, `.git`, `.claude`) and chmods them to 0o444 (locked) or 0o644 (unlocked) in bulk. Backs the toolbar Lock toggle in `PanelHeader`. No persistent watcher — re-scan happens on every toggle, so newly created `.env` files are only picked up the next time the user clicks the button.')
|
||||
expect(issue[1]).toBe('导入 类型 { Issue } 来自 \'../gitea/types\'')
|
||||
expect(issue[5]).toBe('prerequisiteNumber?: 数值')
|
||||
expect(issue[9]).toBe('纯判定：工单被未完成的前置挡住才算锁定。快照里找不到工单或前置一律放行， 与 webview 的 `isIssueLocked` 保持同一语义。')
|
||||
expect(issue[15]).toBe('如果 (非 issue 或 issue.prerequisite 全等 未定义)')
|
||||
})
|
||||
|
||||
it('每个被翻的词单独成段，着色取语法树词性', async () => {
|
||||
const env = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('envLock.ts.txt'))
|
||||
const issue = await translateAll(ts, 'typescript', typescriptZhTranslator, readFixture('issueLock.ts.txt'))
|
||||
expect(env.get(23)?.segments.filter(segment => segment.role !== 'plain').slice(0, 4)).toEqual([
|
||||
{ text: '公共', role: 'keyword' },
|
||||
{ text: '异步', role: 'storage' },
|
||||
{ text: '函数', role: 'storage' },
|
||||
{ text: 'findEnvFiles', role: 'function' },
|
||||
])
|
||||
expect(issue.get(14)?.segments).toContainEqual({ text: '全等', role: 'operator' })
|
||||
expect(env.get(26)?.segments).toEqual([{ text: '返回', role: 'keyword' }, { text: ' ', role: 'plain' }, { text: '空数组', role: 'literal' }])
|
||||
expectWordsApart(env)
|
||||
expectWordsApart(issue)
|
||||
})
|
||||
|
||||
it('属性名、对象键、非空断言不被误翻', async () => {
|
||||
const source = [
|
||||
'const o = { if: 1, return: i.number, string: x!.type, new: a.default }',
|
||||
'let n = a !== b && !c || d == e != f',
|
||||
// eslint-disable-next-line no-template-curly-in-string -- 被翻译的源码文本里本来就有模板字符串
|
||||
'const s: string | void = typeof v === \'string\' ? `${v}` : \'number\'',
|
||||
].join('\n')
|
||||
const map = texts(await translateAll(ts, 'typescript', typescriptZhTranslator, source))
|
||||
expect(map[1]).toBe('常量 o = { if: 1, return: i.number, string: x!.type, new: a.default }')
|
||||
expect(map[2]).toBe('变量 n = a 不全等 b 与 非 c 或 d 等于 e 不等于 f')
|
||||
// eslint-disable-next-line no-template-curly-in-string -- 模板字符串原样照抄进译文
|
||||
expect(map[3]).toBe('常量 s: 字符串 | 无 = 类型为 v 全等 \'string\' ? `${v}` : \'number\'')
|
||||
})
|
||||
|
||||
it('多行合成一行、去掉右括号前的尾逗号、行中注释不进译文', async () => {
|
||||
const source = [
|
||||
'call(',
|
||||
' first, // 第一个',
|
||||
' [],',
|
||||
')',
|
||||
].join('\n')
|
||||
expect(texts(await translateAll(ts, 'typescript', typescriptZhTranslator, source))[1]).toBe('call( first, 空数组 )')
|
||||
})
|
||||
})
|
||||
|
||||
describe('python 中文直译', () => {
|
||||
it('样例全文逐块一致', async () => {
|
||||
const map = await translateAll(py, 'python', pythonZhTranslator, readFixture('sample.py'))
|
||||
expect(texts(map)).toEqual({
|
||||
1: 'Scan a workspace for .env files.',
|
||||
3: '导入 os',
|
||||
4: '来自 pathlib 导入 Path 作为 P',
|
||||
6: 'MAX_DEPTH = 8',
|
||||
7: 'skipped = 空列表',
|
||||
10: '函数 find_env_files(root: 字符串) -> 列表[字符串]',
|
||||
11: 'Return every .env file under root.',
|
||||
12: '跳过备份文件',
|
||||
13: 'found = [p 遍历 p 取自 os.listdir(root) 如果 p.startswith(\'.env\') 与 非 p.endswith(\'.bak\')]',
|
||||
14: '如果 非 found',
|
||||
15: '返回 空列表',
|
||||
16: '否则如果 len(found) > MAX_DEPTH',
|
||||
17: 'found = found[:MAX_DEPTH]',
|
||||
18: '否则',
|
||||
19: '跳过',
|
||||
20: '遍历 name 取自 found',
|
||||
21: '如果 name 在 skipped',
|
||||
22: '继续',
|
||||
23: '尝试',
|
||||
24: '使用 open(os.path.join(root, found[0])) 作为 handle',
|
||||
25: 'handle.read()',
|
||||
26: '捕获 OSError 作为 err',
|
||||
27: '抛出 RuntimeError(str(err)) 来自 err',
|
||||
28: '最终',
|
||||
29: 'skipped.clear()',
|
||||
30: '返回 found',
|
||||
33: '类 EnvLockResult',
|
||||
34: 'total: 整数',
|
||||
35: 'ok: 列表[字符串] = 空列表',
|
||||
36: 'failed: 字典[字符串, 字符串] = 空字典',
|
||||
37: 'label = \'env\'',
|
||||
39: '函数 summary(self, verbose: 布尔 = 假) -> 字符串',
|
||||
40: '返回 f\'{self.total} files\' 如果 verbose 不是 空 否则 \'\'',
|
||||
})
|
||||
expectWordsApart(map)
|
||||
})
|
||||
|
||||
it('内置类型只在注解里翻，in 按位置区分取自 / 在', async () => {
|
||||
const source = [
|
||||
'def pick(items: dict[str, int], list: set) -> tuple:',
|
||||
' return [str(k) for k in items if k not in list]',
|
||||
].join('\n')
|
||||
const map = texts(await translateAll(py, 'python', pythonZhTranslator, source))
|
||||
expect(map[1]).toBe('函数 pick(items: 字典[字符串, 整数], list: 集合) -> 元组')
|
||||
expect(map[2]).toBe('返回 [str(k) 遍历 k 取自 items 如果 k 不在 list]')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,169 @@
|
||||
import type { CodeBlock, Node, Segment, SourceSpan, TokenRole, Translator } from '../contract'
|
||||
|
||||
/** 按父节点类型区分译法的关键字，例如 Python 的 in：for 头部里「取自」，比较里「在」。 */
|
||||
export interface ContextualWord { zh: string, byParent: Record<string, string> }
|
||||
|
||||
/** 一种语言的中文直译表。引擎只替换表里的三类东西，其余原样照抄。 */
|
||||
export interface ZhTable {
|
||||
id: string
|
||||
label: string
|
||||
languageIds: string[]
|
||||
/** 未翻译的词按源码着色的词性上色；关键字译文的词性也取它。 */
|
||||
tokenRole: (leaf: Node) => TokenRole
|
||||
/** 关键字：按语法树里匿名节点的类型查（Python 的 `not in` 这类组合关键字本身也是一个匿名节点）。 */
|
||||
keywords: Record<string, string | ContextualWord>
|
||||
/** 字面量关键字：具名的叶子节点（true / None …），按节点类型查。 */
|
||||
literals: Record<string, string>
|
||||
/** 内置类型：只在 isTypePosition 成立的叶子上按原文查。 */
|
||||
builtinTypes: Record<string, string>
|
||||
isTypePosition: (leaf: Node) => boolean
|
||||
/** 翻成文字的运算符：只认父节点在 operatorParents 里的运算符节点（TS 的非空断言 `x!` 不算）。 */
|
||||
operators: Record<string, string>
|
||||
operatorParents: string[]
|
||||
/** 空字面量（没有具名子节点）整个换成一个词：空数组 / 空列表 / 空字典。 */
|
||||
emptyLiterals: Record<string, string>
|
||||
/** 整段原样照抄、按字符串着色的节点。 */
|
||||
stringNodes: string[]
|
||||
/** 注释（或 docstring）节点去掉注释符号后的原文，可以多行。 */
|
||||
commentText: (node: Node) => string
|
||||
}
|
||||
|
||||
interface Token {
|
||||
start: number
|
||||
end: number
|
||||
text: string
|
||||
role: TokenRole
|
||||
translated: boolean
|
||||
/** 源码里的原文，用来认出逗号、括号。 */
|
||||
raw: string
|
||||
}
|
||||
|
||||
const CLOSING = new Set([')', ']', '}'])
|
||||
const HEADER_OPENERS = new Set(['{', ':'])
|
||||
|
||||
function firstLeaf(node: Node): Node {
|
||||
const first = node.child(0)
|
||||
return first ? firstLeaf(first) : node
|
||||
}
|
||||
|
||||
function commentsWithin(node: Node, span: SourceSpan): Node[] {
|
||||
if (node.endIndex <= span.startIndex || node.startIndex >= span.endIndex)
|
||||
return []
|
||||
if (node.type === 'comment')
|
||||
return [node]
|
||||
return node.children.flatMap(child => child ? commentsWithin(child, span) : [])
|
||||
}
|
||||
|
||||
/** 下一个要输出的词的原文（跳过注释）。 */
|
||||
function nextRaw(tokens: Token[], i: number): string {
|
||||
for (let j = i + 1; j < tokens.length; j++) {
|
||||
if (tokens[j].text !== '')
|
||||
return tokens[j].raw
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function oneLine(text: string): string {
|
||||
return text.replace(/\s*\n\s*/g, ' ')
|
||||
}
|
||||
|
||||
export function createZhTemplateTranslator(table: ZhTable): Translator {
|
||||
const operatorParents = new Set(table.operatorParents)
|
||||
const stringNodes = new Set(table.stringNodes)
|
||||
|
||||
/** 能整个换成中文的节点 → 译文和词性。 */
|
||||
function translateNode(node: Node): { text: string, role: TokenRole } | undefined {
|
||||
if (node.childCount === 0 && table.isTypePosition(node)) {
|
||||
const type = table.builtinTypes[node.text]
|
||||
if (type)
|
||||
return { text: type, role: 'type' }
|
||||
}
|
||||
const parent = node.parent
|
||||
if (!node.isNamed) {
|
||||
if (parent && operatorParents.has(parent.type) && table.operators[node.type])
|
||||
return { text: table.operators[node.type], role: 'operator' }
|
||||
const word = table.keywords[node.type]
|
||||
if (word !== undefined) {
|
||||
const text = typeof word === 'string' ? word : (parent && word.byParent[parent.type]) || word.zh
|
||||
return { text, role: table.tokenRole(firstLeaf(node)) }
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
if (node.childCount === 0 && table.literals[node.type])
|
||||
return { text: table.literals[node.type], role: table.tokenRole(node) }
|
||||
if (table.emptyLiterals[node.type] && node.namedChildCount === 0)
|
||||
return { text: table.emptyLiterals[node.type], role: 'literal' }
|
||||
return undefined
|
||||
}
|
||||
|
||||
function collect(node: Node, span: SourceSpan, out: Token[]): void {
|
||||
if (node.endIndex <= span.startIndex || node.startIndex >= span.endIndex || node.startIndex === node.endIndex)
|
||||
return
|
||||
const base = { start: node.startIndex, end: node.endIndex }
|
||||
if (node.type === 'comment') {
|
||||
out.push({ ...base, raw: node.text, text: '', role: 'comment', translated: false })
|
||||
return
|
||||
}
|
||||
const translated = translateNode(node)
|
||||
if (translated) {
|
||||
out.push({ ...base, raw: node.text, ...translated, translated: true })
|
||||
return
|
||||
}
|
||||
if (node.childCount === 0 || stringNodes.has(node.type)) {
|
||||
const role = stringNodes.has(node.type) ? 'string' : table.tokenRole(node)
|
||||
out.push({ ...base, raw: node.text, text: oneLine(node.text), role, translated: false })
|
||||
return
|
||||
}
|
||||
for (const child of node.children) {
|
||||
if (child)
|
||||
collect(child, span, out)
|
||||
}
|
||||
}
|
||||
|
||||
function translateComment(block: CodeBlock): Segment[] {
|
||||
const { startIndex, endIndex } = block.header
|
||||
const root = block.node.tree.rootNode
|
||||
const nodes = block.node.type === 'comment' ? commentsWithin(root.descendantForIndex(startIndex, endIndex) ?? root, block.header) : [block.node]
|
||||
const text = nodes.map(table.commentText).join('\n').split('\n').map(line => line.trim()).filter(Boolean).join(' ')
|
||||
return text ? [{ text, role: 'comment' }] : []
|
||||
}
|
||||
|
||||
function translate(block: CodeBlock, source: string): Segment[] {
|
||||
if (block.kind === 'comment')
|
||||
return translateComment(block)
|
||||
const tokens: Token[] = []
|
||||
collect(block.node, block.header, tokens)
|
||||
const compound = block.children.length > 0 || block.header.endIndex < block.node.endIndex
|
||||
const last = tokens.at(-1)
|
||||
if (compound && last && !last.translated && HEADER_OPENERS.has(last.raw))
|
||||
tokens.pop()
|
||||
|
||||
const segments: Segment[] = []
|
||||
const push = (text: string, role: TokenRole): void => {
|
||||
const prev = segments.at(-1)
|
||||
if (role === 'plain' && prev?.role === 'plain')
|
||||
prev.text += text
|
||||
else
|
||||
segments.push({ text, role })
|
||||
}
|
||||
let gap = false
|
||||
let prevEnd = block.header.startIndex
|
||||
tokens.forEach((token, i) => {
|
||||
gap ||= /\s/.test(source.slice(prevEnd, token.start))
|
||||
prevEnd = token.end
|
||||
const trailingComma = token.raw === ',' && !token.translated && CLOSING.has(nextRaw(tokens, i))
|
||||
if (token.text === '' || trailingComma) {
|
||||
gap ||= token.role === 'comment'
|
||||
return
|
||||
}
|
||||
if (gap && segments.length > 0)
|
||||
push(' ', 'plain')
|
||||
push(token.text, token.role)
|
||||
// 「非」是个词，后面补一个空格，不和操作数粘在一起
|
||||
gap = token.translated && token.raw === '!'
|
||||
})
|
||||
return segments
|
||||
}
|
||||
|
||||
return { id: table.id, label: table.label, languageIds: table.languageIds, translate }
|
||||
}
|
||||
Reference in new issue
Block a user