✨ feat(vscode): 代码阅读的解析与模板直译核心
This commit is contained in:
cruldra committed
1 parent
af56985d9a
commit
4d34b8c715
24 files changed
+2294
No files matched your search
@@ -0,0 +1,88 @@
|
||||
import type { Outline } from '../testSupport'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { outline, readFixture, WASM_DIR, withBlocks } from '../testSupport'
|
||||
import { createPythonAdapter } from './python'
|
||||
|
||||
const adapter = createPythonAdapter(WASM_DIR)
|
||||
|
||||
function tree(source: string): Promise<Outline[]> {
|
||||
return withBlocks(adapter, 'python', source, outline)
|
||||
}
|
||||
|
||||
// fixtures/sample.py：行号 1 起
|
||||
const SAMPLE: Outline[] = [
|
||||
{ at: 1, kind: 'comment' },
|
||||
{ at: 3, kind: 'import' },
|
||||
{ at: 4, kind: 'import' },
|
||||
{ at: 6, kind: 'const' },
|
||||
{ at: 7, kind: 'variable' },
|
||||
{ at: 10, kind: 'function', kids: [
|
||||
{ at: 11, kind: 'comment' },
|
||||
{ at: 12, kind: 'comment' },
|
||||
{ at: 13, kind: 'variable' },
|
||||
{ at: 14, kind: 'if', kids: [{ at: 15, kind: 'return' }] },
|
||||
{ at: 16, kind: 'else', kids: [{ at: 17, kind: 'variable' }] },
|
||||
{ at: 18, kind: 'else', kids: [{ at: 19, kind: 'statement' }] },
|
||||
{ at: 20, kind: 'loop', kids: [
|
||||
{ at: 21, kind: 'if', kids: [{ at: 22, kind: 'statement' }] },
|
||||
] },
|
||||
{ at: 23, kind: 'try', kids: [
|
||||
{ at: 24, kind: 'statement', kids: [{ at: 25, kind: 'statement' }] },
|
||||
] },
|
||||
{ at: 26, kind: 'catch', kids: [{ at: 27, kind: 'statement' }] },
|
||||
{ at: 28, kind: 'finally', kids: [{ at: 29, kind: 'statement' }] },
|
||||
{ at: 30, kind: 'return' },
|
||||
] },
|
||||
{ at: 33, kind: 'class', kids: [
|
||||
{ at: 34, kind: 'field' },
|
||||
{ at: 35, kind: 'field' },
|
||||
{ at: 36, kind: 'field' },
|
||||
{ at: 37, kind: 'variable' },
|
||||
{ at: 39, kind: 'function', kids: [{ at: 40, kind: 'return' }] },
|
||||
] },
|
||||
]
|
||||
|
||||
describe('python 代码块树', () => {
|
||||
it('样例覆盖 import / def / docstring / if-elif-else / for / try-except-finally / with / 推导式 / class 字段', async () => {
|
||||
expect(await tree(readFixture('sample.py'))).toEqual(SAMPLE)
|
||||
})
|
||||
|
||||
it('装饰器算进头部；冒号后、主体前的注释归子块；循环的 else 是兄弟块；match 的 case 是子块 if', async () => {
|
||||
const source = [
|
||||
'@cache',
|
||||
'def load(key):',
|
||||
' # 先查缓存',
|
||||
' while key:',
|
||||
' key = key[1:]',
|
||||
' else:',
|
||||
' pass',
|
||||
' match key:',
|
||||
' case \'a\':',
|
||||
' return 1',
|
||||
' case _:',
|
||||
' return 2',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: [1, 2], kind: 'function', kids: [
|
||||
{ at: 3, kind: 'comment' },
|
||||
{ at: 4, kind: 'loop', kids: [{ at: 5, kind: 'variable' }] },
|
||||
{ at: 6, kind: 'else', kids: [{ at: 7, kind: 'statement' }] },
|
||||
{ at: 8, kind: 'if', kids: [
|
||||
{ at: 9, kind: 'if', kids: [{ at: 10, kind: 'return' }] },
|
||||
{ at: 11, kind: 'if', kids: [{ at: 12, kind: 'return' }] },
|
||||
] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('docstring 只认主体开头；不是开头的字符串表达式是语句', async () => {
|
||||
const source = [
|
||||
'def f():',
|
||||
' x = 1',
|
||||
' """not a docstring"""',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'function', kids: [{ at: 2, kind: 'variable' }, { at: 3, kind: 'statement' }] },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,217 @@
|
||||
import type { BlockKind, CodeBlock, LanguageAdapter, Node, TokenRole } from '../contract'
|
||||
import * as path from 'node:path'
|
||||
import { compoundBlock, leafBlock, listBlocks, named, sameNode } from './shared'
|
||||
|
||||
/** 常量、类字段只认直接写在模块 / 类主体里的赋值。 */
|
||||
type Scope = 'module' | 'class' | 'other'
|
||||
|
||||
const CONSTANT_NAME = /^[A-Z][A-Z0-9_]*$/
|
||||
|
||||
function statements(items: Node[], scope: Scope, afterRow: number, docstring: boolean): CodeBlock[] {
|
||||
return listBlocks(items, afterRow, (item, first) => statementBlocks(item, scope, docstring && first), () => true)
|
||||
}
|
||||
|
||||
/** 冒号后面的主体：body / consequence 字段，except / finally 子句里是不带字段名的 block。 */
|
||||
function bodyOf(node: Node): Node | null {
|
||||
const body = node.childForFieldName('body') ?? node.childForFieldName('consequence')
|
||||
if (body)
|
||||
return body
|
||||
const blocks = named(node).filter(child => child.type === 'block')
|
||||
return blocks.at(-1) ?? null
|
||||
}
|
||||
|
||||
function bodyBlock(kind: BlockKind, outer: Node, body: Node | null, scope: Scope, docstring = false): CodeBlock {
|
||||
if (!body)
|
||||
return leafBlock(kind, outer)
|
||||
return compoundBlock(kind, outer, body.startIndex, named(body), (list, afterRow) => statements(list, scope, afterRow, docstring))
|
||||
}
|
||||
|
||||
function statementBlocks(node: Node, scope: Scope, docstring: boolean): CodeBlock[] {
|
||||
switch (node.type) {
|
||||
case 'import_statement':
|
||||
case 'import_from_statement':
|
||||
case 'future_import_statement':
|
||||
return [leafBlock('import', node)]
|
||||
case 'function_definition':
|
||||
case 'class_definition':
|
||||
return [definitionBlock(node, node)]
|
||||
case 'decorated_definition': {
|
||||
const definition = node.childForFieldName('definition')
|
||||
return [definition ? definitionBlock(definition, node) : leafBlock('statement', node)]
|
||||
}
|
||||
case 'if_statement':
|
||||
return ifBlocks(node)
|
||||
case 'for_statement':
|
||||
case 'while_statement':
|
||||
return loopBlocks(node)
|
||||
case 'try_statement':
|
||||
return tryBlocks(node)
|
||||
case 'with_statement':
|
||||
return [bodyBlock('statement', node, bodyOf(node), 'other')]
|
||||
case 'match_statement':
|
||||
return [bodyBlock('if', node, bodyOf(node), 'other')]
|
||||
case 'case_clause':
|
||||
return [bodyBlock('if', node, bodyOf(node), 'other')]
|
||||
case 'return_statement':
|
||||
return [leafBlock('return', node)]
|
||||
case 'expression_statement':
|
||||
return [leafBlock(expressionKind(node, scope, docstring), node)]
|
||||
default:
|
||||
return [leafBlock('statement', node)]
|
||||
}
|
||||
}
|
||||
|
||||
/** function_definition / class_definition；outer 是带装饰器的整体时头部从装饰器开始。 */
|
||||
function definitionBlock(definition: Node, outer: Node): CodeBlock {
|
||||
const isClass = definition.type === 'class_definition'
|
||||
return bodyBlock(isClass ? 'class' : 'function', outer, definition.childForFieldName('body'), isClass ? 'class' : 'other', true)
|
||||
}
|
||||
|
||||
function isConstantTarget(target: Node | null): boolean {
|
||||
if (!target)
|
||||
return false
|
||||
if (target.type === 'identifier')
|
||||
return CONSTANT_NAME.test(target.text)
|
||||
if (target.type === 'pattern_list' || target.type === 'tuple_pattern') {
|
||||
const names = named(target)
|
||||
return names.length > 0 && names.every(isConstantTarget)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function expressionKind(node: Node, scope: Scope, docstring: boolean): BlockKind {
|
||||
const [expression, ...rest] = named(node)
|
||||
if (docstring && rest.length === 0 && (expression?.type === 'string' || expression?.type === 'concatenated_string'))
|
||||
return 'comment'
|
||||
if (expression?.type !== 'assignment')
|
||||
return 'statement'
|
||||
if (scope === 'class' && expression.childForFieldName('type'))
|
||||
return 'field'
|
||||
if (scope === 'module' && isConstantTarget(expression.childForFieldName('left')))
|
||||
return 'const'
|
||||
return 'variable'
|
||||
}
|
||||
|
||||
/** elif / else 展平成紧跟 if 的兄弟块。 */
|
||||
function ifBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('if', node, bodyOf(node), 'other')]
|
||||
for (const alternative of node.childrenForFieldName('alternative')) {
|
||||
if (alternative)
|
||||
blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other'))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
/** 循环的 else 是兄弟块 else。 */
|
||||
function loopBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('loop', node, bodyOf(node), 'other')]
|
||||
const alternative = node.childForFieldName('alternative')
|
||||
if (alternative)
|
||||
blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other'))
|
||||
return blocks
|
||||
}
|
||||
|
||||
const TRY_CLAUSES: Record<string, BlockKind> = {
|
||||
except_clause: 'catch',
|
||||
except_group_clause: 'catch',
|
||||
else_clause: 'else',
|
||||
finally_clause: 'finally',
|
||||
}
|
||||
|
||||
function tryBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('try', node, node.childForFieldName('body'), 'other')]
|
||||
for (const clause of named(node)) {
|
||||
const kind = TRY_CLAUSES[clause.type]
|
||||
if (kind)
|
||||
blocks.push(bodyBlock(kind, clause, bodyOf(clause), 'other'))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
// ─── 源码着色 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
const STORAGE_KEYWORDS = new Set(['def', 'class', 'async', 'lambda', 'global', 'nonlocal', 'del'])
|
||||
const OPERATOR_KEYWORDS = new Set(['and', 'or', 'not', 'is'])
|
||||
const COMPARISON = new Set(['comparison_operator', 'not in', 'is not'])
|
||||
const LITERAL_KEYWORDS = new Set(['none', 'true', 'false'])
|
||||
|
||||
function insideString(leaf: Node): boolean {
|
||||
for (let node = leaf.parent; node; node = node.parent) {
|
||||
if (node.type === 'interpolation')
|
||||
return false
|
||||
if (node.type === 'string')
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/** 注解里的名字：typed_parameter / 返回值 / 带注解赋值的 type 节点之内。 */
|
||||
export function insideAnnotation(leaf: Node): boolean {
|
||||
for (let node = leaf.parent; node; node = node.parent) {
|
||||
if (node.type === 'type')
|
||||
return true
|
||||
if (node.type === 'block' || node.type === 'module')
|
||||
return false
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function isCallee(node: Node): boolean {
|
||||
const parent = node.parent
|
||||
return !!parent && parent.type === 'call' && sameNode(parent.childForFieldName('function'), node)
|
||||
}
|
||||
|
||||
function identifierRole(leaf: Node): TokenRole {
|
||||
const parent = leaf.parent
|
||||
if (!parent)
|
||||
return 'variable'
|
||||
if (insideAnnotation(leaf))
|
||||
return 'type'
|
||||
if (parent.type === 'function_definition' && sameNode(parent.childForFieldName('name'), leaf))
|
||||
return 'function'
|
||||
if (parent.type === 'class_definition' && sameNode(parent.childForFieldName('name'), leaf))
|
||||
return 'type'
|
||||
if (parent.type === 'decorator' || isCallee(leaf))
|
||||
return 'function'
|
||||
if (parent.type === 'attribute' && sameNode(parent.childForFieldName('attribute'), leaf) && isCallee(parent))
|
||||
return 'function'
|
||||
return 'variable'
|
||||
}
|
||||
|
||||
export function pythonTokenRole(leaf: Node): TokenRole {
|
||||
if (leaf.type === 'comment')
|
||||
return 'comment'
|
||||
if (insideString(leaf))
|
||||
return 'string'
|
||||
if (!leaf.isNamed) {
|
||||
if (!/^[a-z]+$/.test(leaf.type))
|
||||
return 'plain'
|
||||
if (STORAGE_KEYWORDS.has(leaf.type))
|
||||
return 'storage'
|
||||
if (OPERATOR_KEYWORDS.has(leaf.type) || (leaf.type === 'in' && COMPARISON.has(leaf.parent?.type ?? '')))
|
||||
return 'operator'
|
||||
return 'keyword'
|
||||
}
|
||||
if (LITERAL_KEYWORDS.has(leaf.type))
|
||||
return 'storage'
|
||||
switch (leaf.type) {
|
||||
case 'integer':
|
||||
case 'float':
|
||||
return 'number'
|
||||
case 'identifier':
|
||||
return identifierRole(leaf)
|
||||
default:
|
||||
return 'plain'
|
||||
}
|
||||
}
|
||||
|
||||
/** Python 语言插件。@param wasmDir 语法包 .wasm 所在目录 */
|
||||
export function createPythonAdapter(wasmDir: string): LanguageAdapter {
|
||||
return {
|
||||
id: 'python',
|
||||
languageIds: ['python'],
|
||||
grammarPath: () => path.join(wasmDir, 'tree-sitter-python.wasm'),
|
||||
toBlocks: root => statements(named(root), 'module', -1, true),
|
||||
tokenRole: pythonTokenRole,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
import type { BlockKind, CodeBlock, Node, SourceSpan } from '../contract'
|
||||
|
||||
/** 语言插件共用的建树工具：取子节点、算头部、把一串语句切成块。 */
|
||||
|
||||
export function named(node: Node | null | undefined): Node[] {
|
||||
return node ? node.namedChildren.filter((child): child is Node => child !== null) : []
|
||||
}
|
||||
|
||||
export function children(node: Node): Node[] {
|
||||
return node.children.filter((child): child is Node => child !== null)
|
||||
}
|
||||
|
||||
export function nodeSpan(node: Node): SourceSpan {
|
||||
return {
|
||||
startIndex: node.startIndex,
|
||||
endIndex: node.endIndex,
|
||||
startRow: node.startPosition.row,
|
||||
endRow: node.endPosition.row,
|
||||
}
|
||||
}
|
||||
|
||||
export function leafBlock(kind: BlockKind, node: Node): CodeBlock {
|
||||
return { kind, node, header: nodeSpan(node), children: [] }
|
||||
}
|
||||
|
||||
/** 头部往前扩到 from（装饰器）。 */
|
||||
export function extendHeader(block: CodeBlock, from: Node): CodeBlock {
|
||||
return { ...block, header: { ...block.header, startIndex: from.startIndex, startRow: from.startPosition.row } }
|
||||
}
|
||||
|
||||
function lastLeafBefore(node: Node, limit: number): Node | undefined {
|
||||
for (let i = node.childCount - 1; i >= 0; i--) {
|
||||
const child = node.child(i)
|
||||
if (!child || child.startIndex >= limit || child.type === 'comment' || child.startIndex === child.endIndex)
|
||||
continue
|
||||
if (child.childCount === 0)
|
||||
return child
|
||||
const found = lastLeafBefore(child, limit)
|
||||
if (found)
|
||||
return found
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
/** 复合语句的头部：从 outer 开头到主体开始之前最后一个非注释的词为止。 */
|
||||
export function headerSpan(outer: Node, bodyStart: number): SourceSpan {
|
||||
const last = lastLeafBefore(outer, bodyStart)
|
||||
if (!last)
|
||||
return nodeSpan(outer)
|
||||
return {
|
||||
startIndex: outer.startIndex,
|
||||
endIndex: last.endIndex,
|
||||
startRow: outer.startPosition.row,
|
||||
endRow: last.endPosition.row,
|
||||
}
|
||||
}
|
||||
|
||||
/** node 里完全落在 [from, to) 之间的注释。 */
|
||||
function commentsIn(node: Node, from: number, to: number): Node[] {
|
||||
if (node.endIndex <= from || node.startIndex >= to)
|
||||
return []
|
||||
if (node.type === 'comment')
|
||||
return node.startIndex >= from && node.endIndex <= to ? [node] : []
|
||||
return children(node).flatMap(child => commentsIn(child, from, to))
|
||||
}
|
||||
|
||||
export type ListBlocks = (items: Node[], afterRow: number) => CodeBlock[]
|
||||
|
||||
/**
|
||||
* 复合语句块：头部到 bodyStart 之前，子块来自 items。
|
||||
* 头部和主体之间的注释（例如 Python 冒号后、主体前的注释）也归子块。
|
||||
*/
|
||||
export function compoundBlock(kind: BlockKind, outer: Node, bodyStart: number, items: Node[], list: ListBlocks): CodeBlock {
|
||||
const header = headerSpan(outer, bodyStart)
|
||||
const between = commentsIn(outer, header.endIndex, bodyStart)
|
||||
return { kind, node: outer, header, children: list([...between, ...items], header.endRow) }
|
||||
}
|
||||
|
||||
/**
|
||||
* 一串语句 → 块。注释单独成块，相邻行的行注释合成一块；
|
||||
* 和前一条语句（或头部）同一行的注释是行尾注释，跟着那一行显示，不单独成块。
|
||||
* afterRow 是这串语句之前的最后一行（头部结尾行），模块顶层传 -1；
|
||||
* toBlocks 的 first 表示它是这串里第一条非注释语句（Python 认 docstring 用）。
|
||||
*/
|
||||
export function listBlocks(
|
||||
items: Node[],
|
||||
afterRow: number,
|
||||
toBlocks: (item: Node, first: boolean) => CodeBlock[],
|
||||
isLineComment: (comment: Node) => boolean,
|
||||
): CodeBlock[] {
|
||||
const out: CodeBlock[] = []
|
||||
let lastRow = afterRow
|
||||
let lineRun: CodeBlock | undefined
|
||||
let first = true
|
||||
for (const item of items) {
|
||||
if (item.type === 'comment') {
|
||||
if (item.startPosition.row === lastRow)
|
||||
continue
|
||||
const line = isLineComment(item)
|
||||
if (line && lineRun && item.startPosition.row === lineRun.header.endRow + 1) {
|
||||
lineRun.header = { ...lineRun.header, endIndex: item.endIndex, endRow: item.endPosition.row }
|
||||
}
|
||||
else {
|
||||
const block = leafBlock('comment', item)
|
||||
out.push(block)
|
||||
lineRun = line ? block : undefined
|
||||
}
|
||||
}
|
||||
else {
|
||||
out.push(...toBlocks(item, first))
|
||||
first = false
|
||||
lineRun = undefined
|
||||
}
|
||||
lastRow = item.endPosition.row
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** 同一个节点：web-tree-sitter 每次取子节点都会新建对象，只能按位置和类型比。 */
|
||||
export function sameNode(a: Node | null | undefined, b: Node): boolean {
|
||||
return !!a && a.startIndex === b.startIndex && a.endIndex === b.endIndex && a.type === b.type
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
import type { Outline } from '../testSupport'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { outline, readFixture, WASM_DIR, withBlocks } from '../testSupport'
|
||||
import { createTypeScriptAdapter } from './typescript'
|
||||
|
||||
const adapter = createTypeScriptAdapter(WASM_DIR)
|
||||
|
||||
function tree(source: string, languageId = 'typescript'): Promise<Outline[]> {
|
||||
return withBlocks(adapter, languageId, source, outline)
|
||||
}
|
||||
|
||||
// 与原型 docs/prototype/code-reader/figma-plugin/code.js 的 CODE_TREE 一致
|
||||
const ENV_LOCK: Outline[] = [
|
||||
{ at: [1, 10], kind: 'comment' },
|
||||
{ at: 12, kind: 'import' },
|
||||
{ at: 13, kind: 'import' },
|
||||
{ at: 15, kind: 'const' },
|
||||
{ at: 16, kind: 'const' },
|
||||
{ at: [18, 22], kind: 'comment' },
|
||||
{ at: 23, kind: 'function', kids: [
|
||||
{ at: 24, kind: 'variable' },
|
||||
{ at: 25, kind: 'if', kids: [
|
||||
{ at: 26, kind: 'return' },
|
||||
] },
|
||||
{ at: 27, kind: 'variable' },
|
||||
{ at: 28, kind: 'variable' },
|
||||
{ at: 29, kind: 'return' },
|
||||
] },
|
||||
{ at: 32, kind: 'type', kids: [
|
||||
{ at: 33, kind: 'field' },
|
||||
{ at: 34, kind: 'field' },
|
||||
{ at: 35, kind: 'field' },
|
||||
] },
|
||||
{ at: 38, kind: 'function', kids: [
|
||||
{ at: 39, kind: 'variable' },
|
||||
{ at: 40, kind: 'variable' },
|
||||
{ at: 41, kind: 'loop', kids: [
|
||||
{ at: 42, kind: 'try', kids: [
|
||||
{ at: 43, kind: 'statement' },
|
||||
{ at: 44, kind: 'statement' },
|
||||
] },
|
||||
{ at: 46, kind: 'catch', kids: [
|
||||
{ at: 47, kind: 'statement' },
|
||||
] },
|
||||
] },
|
||||
{ at: 50, kind: 'return' },
|
||||
] },
|
||||
{ at: 53, kind: 'comment' },
|
||||
{ at: 54, kind: 'function', kids: [
|
||||
{ at: 55, kind: 'return' },
|
||||
] },
|
||||
{ at: 58, kind: 'comment' },
|
||||
{ at: 59, kind: 'function', kids: [
|
||||
{ at: 60, kind: 'return' },
|
||||
] },
|
||||
]
|
||||
|
||||
const ISSUE_LOCK: Outline[] = [
|
||||
{ at: 1, kind: 'import' },
|
||||
{ at: 3, kind: 'type', kids: [
|
||||
{ at: 4, kind: 'field' },
|
||||
{ at: 5, kind: 'field' },
|
||||
{ at: 6, kind: 'field' },
|
||||
] },
|
||||
{ at: [9, 12], kind: 'comment' },
|
||||
{ at: 13, kind: 'function', kids: [
|
||||
{ at: 14, kind: 'variable' },
|
||||
{ at: 15, kind: 'if', kids: [
|
||||
{ at: 16, kind: 'return' },
|
||||
] },
|
||||
{ at: 17, kind: 'variable' },
|
||||
{ at: 18, kind: 'if', kids: [
|
||||
{ at: 19, kind: 'return' },
|
||||
] },
|
||||
{ at: [20, 24], kind: 'return' },
|
||||
] },
|
||||
]
|
||||
|
||||
describe('typeScript 代码块树', () => {
|
||||
it('envLock.ts 与原型一致', async () => {
|
||||
expect(await tree(readFixture('envLock.ts.txt'))).toEqual(ENV_LOCK)
|
||||
})
|
||||
|
||||
it('issueLock.ts 与原型一致', async () => {
|
||||
expect(await tree(readFixture('issueLock.ts.txt'))).toEqual(ISSUE_LOCK)
|
||||
})
|
||||
|
||||
it('else / else if 展平成兄弟块，else if 的子块来自它自己的主体', async () => {
|
||||
const source = [
|
||||
'if (a) {',
|
||||
' x()',
|
||||
'} else if (b) {',
|
||||
' y()',
|
||||
'} else',
|
||||
' z()',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'if', kids: [{ at: 2, kind: 'statement' }] },
|
||||
{ at: 3, kind: 'else', kids: [{ at: 4, kind: 'statement' }] },
|
||||
{ at: 5, kind: 'else', kids: [{ at: 6, kind: 'statement' }] },
|
||||
])
|
||||
})
|
||||
|
||||
it('try / catch / finally 是兄弟块；switch 的 case / default 是子块 if', async () => {
|
||||
const source = [
|
||||
'try {',
|
||||
' a()',
|
||||
'}',
|
||||
'catch {',
|
||||
' b()',
|
||||
'}',
|
||||
'finally {',
|
||||
' c()',
|
||||
'}',
|
||||
'switch (x) {',
|
||||
' case 1:',
|
||||
' foo()',
|
||||
' break',
|
||||
' default:',
|
||||
' bar()',
|
||||
'}',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'try', kids: [{ at: 2, kind: 'statement' }] },
|
||||
{ at: 4, kind: 'catch', kids: [{ at: 5, kind: 'statement' }] },
|
||||
{ at: 7, kind: 'finally', kids: [{ at: 8, kind: 'statement' }] },
|
||||
{ at: 10, kind: 'if', kids: [
|
||||
{ at: 11, kind: 'if', kids: [{ at: 12, kind: 'statement' }, { at: 13, kind: 'statement' }] },
|
||||
{ at: 14, kind: 'if', kids: [{ at: 15, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('类：方法 → function、字段 → field、注释 → comment，装饰器算进成员头部', async () => {
|
||||
const source = [
|
||||
'export class Store extends Base {',
|
||||
' // 缓存',
|
||||
' private readonly cache = new Map()',
|
||||
' @memo()',
|
||||
' get size(): number {',
|
||||
' return this.cache.size',
|
||||
' }',
|
||||
' handle = () => {',
|
||||
' this.cache.clear()',
|
||||
' }',
|
||||
'}',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'class', kids: [
|
||||
{ at: 2, kind: 'comment' },
|
||||
{ at: 3, kind: 'field' },
|
||||
{ at: [4, 5], kind: 'function', kids: [{ at: 6, kind: 'return' }] },
|
||||
{ at: 8, kind: 'function', kids: [{ at: 9, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('enum 成员 → field；type 别名 → type；多个循环 → loop', async () => {
|
||||
const source = [
|
||||
'enum Color {',
|
||||
' Red = 1,',
|
||||
' Blue,',
|
||||
'}',
|
||||
'type Pair = [string, number]',
|
||||
'for (let i = 0; i < 3; i++) {',
|
||||
' while (busy) wait()',
|
||||
' do {',
|
||||
' step()',
|
||||
' } while (more)',
|
||||
'}',
|
||||
].join('\n')
|
||||
expect(await tree(source)).toEqual([
|
||||
{ at: 1, kind: 'type', kids: [{ at: 2, kind: 'field' }, { at: 3, kind: 'field' }] },
|
||||
{ at: 5, kind: 'type' },
|
||||
{ at: 6, kind: 'loop', kids: [
|
||||
{ at: 7, kind: 'loop', kids: [{ at: 7, kind: 'statement' }] },
|
||||
{ at: 8, kind: 'loop', kids: [{ at: 9, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('函数值的变量 → function；回调参数带语句块主体的表达式语句 → statement 带子块', async () => {
|
||||
const source = [
|
||||
'const App = () => {',
|
||||
' useEffect(() => {',
|
||||
' load()',
|
||||
' }, [])',
|
||||
' let n = 0',
|
||||
' return null',
|
||||
'}',
|
||||
'let total = 0',
|
||||
'describe(\'x\', () => {',
|
||||
' it(\'y\', async () => {',
|
||||
' await run()',
|
||||
' })',
|
||||
'})',
|
||||
].join('\n')
|
||||
expect(await tree(source, 'typescriptreact')).toEqual([
|
||||
{ at: 1, kind: 'function', kids: [
|
||||
{ at: 2, kind: 'statement', kids: [{ at: 3, kind: 'statement' }] },
|
||||
{ at: 5, kind: 'variable' },
|
||||
{ at: 6, kind: 'return' },
|
||||
] },
|
||||
{ at: 8, kind: 'variable' },
|
||||
{ at: 9, kind: 'statement', kids: [
|
||||
{ at: 10, kind: 'statement', kids: [{ at: 11, kind: 'statement' }] },
|
||||
] },
|
||||
])
|
||||
})
|
||||
|
||||
it('相邻行的行注释合成一块，空行隔开的分开；行尾注释不单独成块', async () => {
|
||||
const source = [
|
||||
'// a',
|
||||
'// b',
|
||||
'',
|
||||
'// c',
|
||||
'const x = 1 // 行尾',
|
||||
'foo() // 行尾',
|
||||
].join('\n')
|
||||
expect(await tree(source, 'javascript')).toEqual([
|
||||
{ at: [1, 2], kind: 'comment' },
|
||||
{ at: 4, kind: 'comment' },
|
||||
{ at: 5, kind: 'const' },
|
||||
{ at: 6, kind: 'statement' },
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,328 @@
|
||||
import type { BlockKind, CodeBlock, LanguageAdapter, Node, TokenRole } from '../contract'
|
||||
import * as path from 'node:path'
|
||||
import { compoundBlock, extendHeader, leafBlock, listBlocks, named, sameNode } from './shared'
|
||||
|
||||
const GRAMMARS: Record<string, string> = {
|
||||
typescript: 'tree-sitter-typescript.wasm',
|
||||
typescriptreact: 'tree-sitter-tsx.wasm',
|
||||
javascript: 'tree-sitter-javascript.wasm',
|
||||
javascriptreact: 'tree-sitter-javascript.wasm',
|
||||
}
|
||||
|
||||
const FUNCTION_VALUES = new Set(['arrow_function', 'function_expression', 'function', 'generator_function'])
|
||||
const LOOPS = new Set(['for_statement', 'for_in_statement', 'while_statement', 'do_statement'])
|
||||
const CLASSES = new Set(['class_declaration', 'abstract_class_declaration', 'class'])
|
||||
|
||||
function isLineComment(comment: Node): boolean {
|
||||
return comment.text.startsWith('//')
|
||||
}
|
||||
|
||||
/** 带语句块主体的函数值（箭头函数 / 函数表达式）的主体。 */
|
||||
function functionBody(value: Node | null | undefined): Node | null {
|
||||
if (!value || !FUNCTION_VALUES.has(value.type))
|
||||
return null
|
||||
const body = value.childForFieldName('body')
|
||||
return body?.type === 'statement_block' ? body : null
|
||||
}
|
||||
|
||||
/** 调用参数里第一个带语句块主体的函数的主体：useEffect(() => { … })、describe('x', () => { … })。 */
|
||||
function callbackBody(expression: Node | null | undefined): Node | null {
|
||||
const call = expression?.type === 'await_expression' ? expression.firstNamedChild : expression
|
||||
if (call?.type !== 'call_expression' && call?.type !== 'new_expression')
|
||||
return null
|
||||
for (const arg of named(call.childForFieldName('arguments'))) {
|
||||
const body = functionBody(arg)
|
||||
if (body)
|
||||
return body
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
function statements(items: Node[], top: boolean, afterRow: number): CodeBlock[] {
|
||||
return listBlocks(items, afterRow, item => statementBlocks(item, top), isLineComment)
|
||||
}
|
||||
|
||||
/** 主体是语句块时子块来自它的语句，否则主体本身就是唯一的子语句（`if (x) return`）。 */
|
||||
function bodyBlock(kind: BlockKind, outer: Node, body: Node | null, top = false): CodeBlock {
|
||||
if (!body)
|
||||
return leafBlock(kind, outer)
|
||||
const items = body.type === 'statement_block' ? named(body) : [body]
|
||||
return compoundBlock(kind, outer, body.startIndex, items, (list, afterRow) => statements(list, top, afterRow))
|
||||
}
|
||||
|
||||
function statementBlocks(node: Node, top: boolean): CodeBlock[] {
|
||||
switch (node.type) {
|
||||
case 'import_statement':
|
||||
return [leafBlock('import', node)]
|
||||
case 'export_statement':
|
||||
return exportBlocks(node, top)
|
||||
case 'if_statement':
|
||||
return ifBlocks(node, node, 'if')
|
||||
case 'try_statement':
|
||||
return tryBlocks(node)
|
||||
case 'switch_statement':
|
||||
return [switchBlock(node)]
|
||||
case 'switch_case':
|
||||
case 'switch_default':
|
||||
return [caseBlock(node)]
|
||||
case 'return_statement':
|
||||
return [leafBlock('return', node)]
|
||||
case 'expression_statement':
|
||||
return [bodyBlock('statement', node, callbackBody(node.firstNamedChild))]
|
||||
case 'labeled_statement': {
|
||||
const [first, ...rest] = statementBlocks(node.childForFieldName('body') ?? node, top)
|
||||
return first ? [{ ...extendHeader(first, node), node }, ...rest] : []
|
||||
}
|
||||
case 'statement_block':
|
||||
return statements(named(node), top, node.startPosition.row)
|
||||
case 'empty_statement':
|
||||
return []
|
||||
default:
|
||||
if (LOOPS.has(node.type))
|
||||
return [bodyBlock('loop', node, node.childForFieldName('body'))]
|
||||
return declarationBlocks(node, node, top)
|
||||
}
|
||||
}
|
||||
|
||||
/** export 包着声明时按声明定种类，块节点取整个 export_statement（头部包含 export）。 */
|
||||
function exportBlocks(node: Node, top: boolean): CodeBlock[] {
|
||||
const declaration = node.childForFieldName('declaration')
|
||||
if (declaration)
|
||||
return declarationBlocks(declaration, node, top)
|
||||
const value = node.childForFieldName('value')
|
||||
if (value && CLASSES.has(value.type))
|
||||
return [classBlock(value, node)]
|
||||
const body = functionBody(value)
|
||||
return [body ? bodyBlock('function', node, body) : leafBlock('statement', node)]
|
||||
}
|
||||
|
||||
function declarationBlocks(declaration: Node, outer: Node, top: boolean): CodeBlock[] {
|
||||
switch (declaration.type) {
|
||||
case 'function_declaration':
|
||||
case 'generator_function_declaration':
|
||||
return [bodyBlock('function', outer, declaration.childForFieldName('body'))]
|
||||
case 'function_signature':
|
||||
return [leafBlock('function', outer)]
|
||||
case 'lexical_declaration':
|
||||
case 'variable_declaration':
|
||||
return [variableBlock(declaration, outer, top)]
|
||||
case 'interface_declaration':
|
||||
case 'enum_declaration':
|
||||
return [membersBlock(declaration, outer)]
|
||||
case 'type_alias_declaration':
|
||||
return [leafBlock('type', outer)]
|
||||
default: {
|
||||
if (CLASSES.has(declaration.type))
|
||||
return [classBlock(declaration, outer)]
|
||||
const body = declaration.childForFieldName('body')
|
||||
return [body?.type === 'statement_block' ? bodyBlock('statement', outer, body) : leafBlock('statement', outer)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 唯一声明值是带语句块主体的函数 → function；否则模块顶层的 const → const，其余 → variable。 */
|
||||
function variableBlock(declaration: Node, outer: Node, top: boolean): CodeBlock {
|
||||
const declarators = named(declaration).filter(child => child.type === 'variable_declarator')
|
||||
const value = declarators.length === 1 ? declarators[0].childForFieldName('value') : null
|
||||
const body = functionBody(value)
|
||||
if (body)
|
||||
return bodyBlock('function', outer, body)
|
||||
const isConst = declaration.type === 'lexical_declaration' && declaration.childForFieldName('kind')?.type === 'const'
|
||||
return bodyBlock(top && isConst ? 'const' : 'variable', outer, callbackBody(value))
|
||||
}
|
||||
|
||||
/** if 的 else / else if 展平成紧跟其后的兄弟块。 */
|
||||
function ifBlocks(ifNode: Node, outer: Node, kind: 'if' | 'else'): CodeBlock[] {
|
||||
const blocks = [bodyBlock(kind, outer, ifNode.childForFieldName('consequence'))]
|
||||
const alternative = ifNode.childForFieldName('alternative')
|
||||
if (alternative) {
|
||||
const inner = named(alternative).find(child => child.type !== 'comment') ?? null
|
||||
if (inner?.type === 'if_statement')
|
||||
blocks.push(...ifBlocks(inner, alternative, 'else'))
|
||||
else
|
||||
blocks.push(bodyBlock('else', alternative, inner))
|
||||
}
|
||||
return blocks
|
||||
}
|
||||
|
||||
function tryBlocks(node: Node): CodeBlock[] {
|
||||
const blocks = [bodyBlock('try', node, node.childForFieldName('body'))]
|
||||
const handler = node.childForFieldName('handler')
|
||||
if (handler)
|
||||
blocks.push(bodyBlock('catch', handler, handler.childForFieldName('body')))
|
||||
const finalizer = node.childForFieldName('finalizer')
|
||||
if (finalizer)
|
||||
blocks.push(bodyBlock('finally', finalizer, finalizer.childForFieldName('body')))
|
||||
return blocks
|
||||
}
|
||||
|
||||
function switchBlock(node: Node): CodeBlock {
|
||||
const body = node.childForFieldName('body')
|
||||
if (!body)
|
||||
return leafBlock('if', node)
|
||||
return compoundBlock('if', node, body.startIndex, named(body), (list, afterRow) => statements(list, false, afterRow))
|
||||
}
|
||||
|
||||
/** case / default：头部到冒号为止，子块来自它下面的语句。 */
|
||||
function caseBlock(node: Node): CodeBlock {
|
||||
const [first] = node.childrenForFieldName('body')
|
||||
const bodyStart = first?.startIndex ?? node.endIndex
|
||||
const items = named(node).filter(child => child.startIndex >= bodyStart)
|
||||
return compoundBlock('if', node, bodyStart, items, (list, afterRow) => statements(list, false, afterRow))
|
||||
}
|
||||
|
||||
function classBlock(declaration: Node, outer: Node): CodeBlock {
|
||||
const body = declaration.childForFieldName('body')
|
||||
if (!body)
|
||||
return leafBlock('class', outer)
|
||||
return compoundBlock('class', outer, body.startIndex, named(body), (list, afterRow) => {
|
||||
let decorators: Node | undefined
|
||||
return listBlocks(list, afterRow, (member) => {
|
||||
if (member.type === 'decorator') {
|
||||
decorators ??= member
|
||||
return []
|
||||
}
|
||||
const blocks = memberBlocks(member)
|
||||
const from = decorators
|
||||
decorators = undefined
|
||||
return from ? blocks.map((block, i) => i === 0 ? extendHeader(block, from) : block) : blocks
|
||||
}, isLineComment)
|
||||
})
|
||||
}
|
||||
|
||||
function memberBlocks(member: Node): CodeBlock[] {
|
||||
switch (member.type) {
|
||||
case 'method_definition':
|
||||
return [bodyBlock('function', member, member.childForFieldName('body'))]
|
||||
case 'method_signature':
|
||||
case 'abstract_method_signature':
|
||||
return [leafBlock('function', member)]
|
||||
case 'class_static_block':
|
||||
return [bodyBlock('statement', member, member.childForFieldName('body'))]
|
||||
default: {
|
||||
const body = functionBody(member.childForFieldName('value'))
|
||||
return [body ? bodyBlock('function', member, body) : leafBlock('field', member)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** interface / enum：成员（属性、方法签名、枚举项）→ field。 */
|
||||
function membersBlock(declaration: Node, outer: Node): CodeBlock {
|
||||
const body = declaration.childForFieldName('body')
|
||||
if (!body)
|
||||
return leafBlock('type', outer)
|
||||
return compoundBlock('type', outer, body.startIndex, named(body), (list, afterRow) =>
|
||||
listBlocks(list, afterRow, member => [leafBlock('field', member)], isLineComment))
|
||||
}
|
||||
|
||||
// ─── 源码着色 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/** 流程关键字；其余字母关键字都算声明关键字（storage）。 */
|
||||
const FLOW_KEYWORDS = new Set([
|
||||
'import',
|
||||
'from',
|
||||
'as',
|
||||
'export',
|
||||
'default',
|
||||
'if',
|
||||
'else',
|
||||
'for',
|
||||
'of',
|
||||
'in',
|
||||
'while',
|
||||
'do',
|
||||
'switch',
|
||||
'case',
|
||||
'break',
|
||||
'continue',
|
||||
'return',
|
||||
'throw',
|
||||
'try',
|
||||
'catch',
|
||||
'finally',
|
||||
'await',
|
||||
'yield',
|
||||
])
|
||||
const LITERAL_KEYWORDS = new Set(['true', 'false', 'null', 'undefined', 'this', 'super'])
|
||||
const STRING_NODES = new Set(['string', 'template_string', 'regex'])
|
||||
const FUNCTION_NAMED = new Set(['function_declaration', 'generator_function_declaration', 'function_expression', 'function', 'generator_function', 'method_definition', 'method_signature', 'abstract_method_signature', 'function_signature'])
|
||||
|
||||
function insideString(leaf: Node): boolean {
|
||||
for (let node = leaf.parent; node; node = node.parent) {
|
||||
if (node.type === 'template_substitution')
|
||||
return false
|
||||
if (STRING_NODES.has(node.type))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function isCallee(node: Node): boolean {
|
||||
const parent = node.parent
|
||||
return !!parent && parent.type === 'call_expression' && sameNode(parent.childForFieldName('function'), node)
|
||||
}
|
||||
|
||||
function nameRole(leaf: Node): TokenRole {
|
||||
const parent = leaf.parent
|
||||
if (!parent)
|
||||
return 'variable'
|
||||
if (FUNCTION_NAMED.has(parent.type) && sameNode(parent.childForFieldName('name'), leaf))
|
||||
return 'function'
|
||||
if (isCallee(leaf))
|
||||
return 'function'
|
||||
if (parent.type === 'member_expression' && sameNode(parent.childForFieldName('property'), leaf) && isCallee(parent))
|
||||
return 'function'
|
||||
if (parent.type === 'new_expression')
|
||||
return 'type'
|
||||
if (parent.type === 'variable_declarator' && sameNode(parent.childForFieldName('name'), leaf) && functionBody(parent.childForFieldName('value')))
|
||||
return 'function'
|
||||
return 'variable'
|
||||
}
|
||||
|
||||
export function typescriptTokenRole(leaf: Node): TokenRole {
|
||||
if (leaf.type === 'comment' || leaf.type === 'hash_bang_line')
|
||||
return 'comment'
|
||||
if (insideString(leaf))
|
||||
return 'string'
|
||||
if (!leaf.isNamed) {
|
||||
if (leaf.parent?.type === 'predefined_type')
|
||||
return 'type'
|
||||
if (/^[a-z]+$/.test(leaf.type))
|
||||
return FLOW_KEYWORDS.has(leaf.type) ? 'keyword' : 'storage'
|
||||
return 'plain'
|
||||
}
|
||||
if (LITERAL_KEYWORDS.has(leaf.type))
|
||||
return 'storage'
|
||||
switch (leaf.type) {
|
||||
case 'number':
|
||||
return 'number'
|
||||
case 'type_identifier':
|
||||
return 'type'
|
||||
case 'identifier':
|
||||
case 'property_identifier':
|
||||
case 'private_property_identifier':
|
||||
return nameRole(leaf)
|
||||
case 'shorthand_property_identifier':
|
||||
case 'shorthand_property_identifier_pattern':
|
||||
case 'statement_identifier':
|
||||
return 'variable'
|
||||
default:
|
||||
return 'plain'
|
||||
}
|
||||
}
|
||||
|
||||
/** TypeScript / TSX / JavaScript / JSX 语言插件。@param wasmDir 语法包 .wasm 所在目录 */
|
||||
export function createTypeScriptAdapter(wasmDir: string): LanguageAdapter {
|
||||
return {
|
||||
id: 'typescript',
|
||||
languageIds: Object.keys(GRAMMARS),
|
||||
grammarPath: (languageId) => {
|
||||
const file = GRAMMARS[languageId]
|
||||
if (!file)
|
||||
throw new Error(`TypeScript 语言插件不支持 ${languageId}`)
|
||||
return path.join(wasmDir, file)
|
||||
},
|
||||
toBlocks: root => statements(named(root), true, -1),
|
||||
tokenRole: typescriptTokenRole,
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user