import type { Access, BlockShape, CallArgument, CallFacts, ClassFacts, CodeBlock, CommentBlock, EntryFacts, FunctionFacts, ImportFacts, ImportItem, IterationClause, IterationFacts, IterationResult, LanguageAdapter, Node, ParameterFacts, TextKind, TokenRole } from '../contract' import type { CommentRules, ExceptionRules } from './shared' import * as path from 'node:path' import { pythonNameReader } from './pythonNames' import { attachComments, children, compoundShape, leafBlock, leafShape, listBlocks, named, sameNode, syntaxChildren, withExceptions } from './shared' /** 常量、类字段只认直接写在模块 / 类主体里的赋值;这两处的声明也才有访问级别。 */ type Scope = 'module' | 'class' | 'other' const CONSTANT_NAME = /^[A-Z][A-Z0-9_]*$/ const PARAMETER_SEPARATORS = new Set(['keyword_separator', 'positional_separator']) const PATTERNS = new Set(['pattern_list', 'tuple_pattern', 'list_pattern']) /** 字符串开头(string_start):普通字符串的前缀只有 r / u 或没有;f、t、b 开头的不是。 */ const STRING_PREFIX = /^[ru]?["']/i /** PEP 263 写编码的注释:`# -*- coding: utf-8 -*-`、`# vim: set fileencoding=utf-8 :`。 */ const CODING = /^#.*?coding[:=]\s*[-\w.]+/ /** 文件头两行里写给解释器看的注释:第一行的 `#!`,头两行里写编码的。 */ function isPragma(comment: Node): boolean { const row = comment.startPosition.row return (row === 0 && comment.text.startsWith('#!')) || (row <= 1 && CODING.test(comment.text)) } // 只有 # 行注释;函数、类的说明是主体开头的 docstring,不是写在前面的注释。写给解释器的那两行不和后面的注释连成一块 const COMMENTS: CommentRules = { lineStyle: comment => isPragma(comment) ? '#!' : '#', isDoc: () => false, documents: new Set() } /** 讲整个文件的注释:模块开头的 docstring(第一条语句,注释不算),写给解释器的那两行。 */ function aboutModule(comment: CommentBlock): boolean { const { node } = comment if (node.type === 'comment') return isPragma(node) return node.parent?.type === 'module' && sameNode(named(node.parent).find(child => child.type !== 'comment'), node) } function statements(items: Node[], scope: Scope, afterRow: number): CodeBlock[] { const docs = attributeDocs(items, scope) const attached = new Set([...docs.values()].map(statement => statement.startIndex)) return listBlocks(items, afterRow, (item) => { if (attached.has(item.startIndex)) return [] const docStatement = docs.get(item.startIndex) return statementBlocks(item, scope, docStatement && docstringIn(docStatement)) }, COMMENTS) } /** * 属性 docstring(PEP 257):模块、类主体里紧跟在一条赋值后面的字符串是这个变量的说明。返回赋值语句的起始下标 → 那条字符串语句。 * 同 JSDoc 挂到下一条语句:中间可以隔空行和赋值的行尾注释,隔着别的注释、语句就不算。 */ function attributeDocs(items: Node[], scope: Scope): Map { const docs = new Map() if (scope === 'other') return docs items.forEach((item, i) => { if (!assignmentIn(item)) return let next = i + 1 while (items[next]?.type === 'comment' && items[next].startPosition.row === item.endPosition.row) next++ const statement = items[next] if (statement && docstringIn(statement)) docs.set(item.startIndex, statement) }) return docs } /** 冒号后面的主体:body / consequence 字段,except / finally 子句里是不带字段名的 block。 */ function bodyOf(node: Node): Node | null { const body = node.childForFieldName('body') ?? node.childForFieldName('consequence') if (body) return body const blocks = named(node).filter(child => child.type === 'block') return blocks.at(-1) ?? null } function bodyShape(outer: Node, body: Node | null, scope: Scope): BlockShape { if (!body) return leafShape(outer) return compoundShape(outer, body.startIndex, named(body), (list, afterRow) => statements(list, scope, afterRow)) } function bodyBlock(kind: TextKind, outer: Node, body: Node | null, scope: Scope): CodeBlock { return { kind, ...bodyShape(outer, body, scope) } } /** doc 是紧跟在这条赋值后面的属性 docstring(见 attributeDocs)。 */ function statementBlocks(node: Node, scope: Scope, doc: Node | undefined): CodeBlock[] { switch (node.type) { case 'import_statement': case 'import_from_statement': case 'future_import_statement': return [{ kind: 'import', ...leafShape(node), facts: importFacts(node) }] case 'function_definition': case 'class_definition': return [definitionBlock(node, node, scope)] case 'decorated_definition': { const definition = node.childForFieldName('definition') return [definition ? definitionBlock(definition, node, scope) : leafBlock('statement', node)] } case 'if_statement': return ifBlocks(node) case 'for_statement': case 'while_statement': return loopBlocks(node) case 'try_statement': return tryBlocks(node) case 'with_statement': return [bodyBlock('statement', node, bodyOf(node), 'other')] case 'match_statement': return [bodyBlock('if', node, bodyOf(node), 'other')] case 'case_clause': return [bodyBlock('if', node, bodyOf(node), 'other')] case 'return_statement': return [leafBlock('return', node)] case 'expression_statement': return [expressionBlock(node, scope, doc)] default: return [leafBlock('statement', node)] } } /** function_definition / class_definition;outer 是带装饰器的整体时头部从装饰器开始。 */ function definitionBlock(definition: Node, outer: Node, scope: Scope): CodeBlock { const body = definition.childForFieldName('body') const doc = docstringOf(body) const inner: Scope = definition.type === 'class_definition' ? 'class' : 'other' const shape = body && doc ? documentedShape(outer, body, doc, inner) : bodyShape(outer, body, inner) if (definition.type === 'class_definition') return { kind: 'class', ...shape, facts: classFacts(definition, outer, scope, doc) } return { kind: 'function', ...shape, facts: functionFacts(definition, outer, scope, doc) } } /** 主体开头的 docstring:第一条语句(注释不算)是单独一个字符串表达式时,返回那个字符串节点。 */ function docstringOf(body: Node | null): Node | undefined { const first = named(body).find(child => child.type !== 'comment') return first && docstringIn(first) } /** * 语句是单独一个字符串字面量(docstring 的写法,隐式拼接的几段也算)时返回那个字符串节点。 * f-string、t-string、字节串不算:Python 不把它们当 docstring。 */ function docstringIn(statement: Node): Node | undefined { if (statement.type !== 'expression_statement') return undefined const [expression, ...rest] = named(statement) if (!expression || rest.length > 0) return undefined const parts = expression.type === 'concatenated_string' ? named(expression) : [expression] return parts.every(isTextString) ? expression : undefined } /** 普通字符串字面量:前缀只有 r / u(不分大小写)或没有。 */ function isTextString(node: Node): boolean { const start = node.type === 'string' ? node.firstChild : null return start?.type === 'string_start' && STRING_PREFIX.test(start.text) } /** 函数、类的 docstring 是它自己的说明:算进头部(连同它前面的注释),子块从它后面开始。 */ function documentedShape(outer: Node, body: Node, doc: Node, scope: Scope): BlockShape { const rest = named(body).filter(item => item.startIndex >= doc.endIndex) return compoundShape(outer, doc.endIndex, rest, (list, afterRow) => statements(list, scope, afterRow)) } /** 单下划线开头的名字是私有(`_x`、`__x`),前后双下划线的特殊方法(`__init__`)是公共。 */ function nameAccess(name: string): Access { const dunder = name.length > 4 && name.startsWith('__') && name.endsWith('__') return name.startsWith('_') && !dunder ? 'private' : 'public' } /** 赋值目标里的名字;目标是属性、下标(`self.x = …`、`d[k] = …`)时没有。 */ function targetNames(target: Node): Node[] | undefined { if (target.type === 'identifier') return [target] if (!PATTERNS.has(target.type) && target.type !== 'list_splat_pattern') return undefined const names: Node[] = [] for (const part of syntaxChildren(target)) { const found = targetNames(part) if (!found) return undefined names.push(...found) } return names } /** 模块、类主体里的声明才有访问级别;解构赋值里的名字全是私有才算私有。 */ function scopeAccess(target: Node, scope: Scope): Access | undefined { const names = scope === 'other' ? undefined : targetNames(target) if (!names?.length) return undefined return names.every(name => nameAccess(name.text) === 'private') ? 'private' : 'public' } function functionFacts(definition: Node, outer: Node, scope: Scope, doc: Node | undefined): FunctionFacts { const name = definition.childForFieldName('name') return { name: name ?? undefined, doc, access: name ? scopeAccess(name, scope) : undefined, async: children(definition).some(child => child.type === 'async'), decorators: decoratorsOf(outer), parameters: syntaxChildren(definition.childForFieldName('parameters')) .filter(param => !PARAMETER_SEPARATORS.has(param.type)) .map(parameterFacts), returnType: definition.childForFieldName('return_type') ?? undefined, } } /** 父类是类名后面括号里的参数列表,照写的顺序一项一项取,`metaclass=…` 这类关键字参数也在里面。 */ function classFacts(definition: Node, outer: Node, scope: Scope, doc: Node | undefined): ClassFacts { const name = definition.childForFieldName('name') return { name: name ?? undefined, doc, access: name ? scopeAccess(name, scope) : undefined, decorators: decoratorsOf(outer), // Python 3.12 的 `class G[T]:` typeParameters: syntaxChildren(definition.childForFieldName('type_parameters')), bases: syntaxChildren(definition.childForFieldName('superclasses')).map(value => ({ value })), implements: [], } } /** 装饰器在 decorated_definition 里,和被装饰的定义是兄弟节点。 */ function decoratorsOf(outer: Node): Node[] { return named(outer).filter(child => child.type === 'decorator') } /** 带注解的参数(typed_parameter)语法没给名字起字段名,名字是第一个具名子节点(可能是 `*args`)。 */ function parameterFacts(param: Node): ParameterFacts { switch (param.type) { case 'typed_parameter': return { name: syntaxChildren(param)[0] ?? param, optional: false, type: param.childForFieldName('type') ?? undefined } case 'default_parameter': case 'typed_default_parameter': return { name: param.childForFieldName('name') ?? param, optional: false, type: param.childForFieldName('type') ?? undefined, default: param.childForFieldName('value') ?? undefined, } default: return { name: param, optional: false } } } /** * import a.b as c, d:每个模块一个来源,导入项是整个模块; * from m import a as b / from m import *:来源是 m(可以是相对的 `.models`),future 导入的来源是 `__future__` 这个词本身。 */ function importFacts(node: Node): ImportFacts { const names = node.childrenForFieldName('name').filter((name): name is Node => name !== null) const alias = (name: Node): Node | undefined => name.type === 'aliased_import' ? name.childForFieldName('alias') ?? undefined : undefined const original = (name: Node): Node => name.type === 'aliased_import' ? name.childForFieldName('name') ?? name : name if (node.type === 'import_statement') { return { sources: names.map(original), items: names.map(name => ({ kind: 'module', module: original(name), alias: alias(name) })), } } const source = node.type === 'future_import_statement' ? children(node).find(child => child.type === '__future__') : node.childForFieldName('module_name') const items: ImportItem[] = names.map(name => ({ kind: 'named', name: original(name), alias: alias(name) })) if (named(node).some(child => child.type === 'wildcard_import')) items.push({ kind: 'namespace' }) return { sources: source ? [source] : [], items } } function isConstantTarget(target: Node | null): boolean { if (!target) return false if (target.type === 'identifier') return CONSTANT_NAME.test(target.text) if (target.type === 'pattern_list' || target.type === 'tuple_pattern') { const names = named(target) return names.length > 0 && names.every(isConstantTarget) } return false } /** * 赋值 → 字段 / 常量 / 变量,doc 是它的属性 docstring:进字段的 doc、常量和变量的 facts.doc,头部往下延到字符串结尾; * 单独一个字符串(模块开头的 docstring、写在语句中间当注释用的字符串)是注释;其余表达式是语句。 */ function expressionBlock(node: Node, scope: Scope, doc: Node | undefined): CodeBlock { if (docstringIn(node)) return leafBlock('comment', node) const expression = assignmentIn(node) const left = expression?.childForFieldName('left') if (!expression || !left) return leafBlock('statement', node) const leaf = leafShape(node) const shape = doc ? { ...leaf, header: { ...leaf.header, endIndex: doc.endIndex, endRow: doc.endPosition.row } } : leaf const type = expression.childForFieldName('type') if (scope === 'class' && type) return { kind: 'field', ...shape, doc } const value = expression.childForFieldName('right') return { kind: scope === 'module' && isConstantTarget(left) ? 'const' : 'variable', ...shape, facts: { access: scopeAccess(left, scope), doc, declarators: [{ name: left, type: type ?? undefined, value: value ?? undefined, entries: dictionaryEntries(value) }], }, } } /** 语句是一条赋值(带不带注解、有没有值)时返回那个 assignment 节点。 */ function assignmentIn(statement: Node): Node | undefined { const expression = statement.type === 'expression_statement' ? named(statement)[0] : undefined return expression?.type === 'assignment' ? expression : undefined } /** 字典字面量的各项:`键: 值`(值又是字典时连它的各项)、`**other`;值不是字典字面量时没有。 */ function dictionaryEntries(value: Node | null): EntryFacts[] | undefined { if (value?.type !== 'dictionary') return undefined return syntaxChildren(value).flatMap((item): EntryFacts[] => { if (item.type === 'dictionary_splat') { const [spread] = syntaxChildren(item) return spread ? [{ kind: 'spread', value: spread }] : [] } const key = item.childForFieldName('key') const itemValue = item.childForFieldName('value') if (item.type !== 'pair' || !key || !itemValue) return [] return [{ kind: 'pair', key, value: itemValue, entries: dictionaryEntries(itemValue) }] }) } const ITERATION_RESULTS: Record = { generator_expression: 'generator', list_comprehension: 'list', set_comprehension: 'set', dictionary_comprehension: 'dict', } /** * 推导式、生成器表达式的结构,别的节点没有。几种读不成遍历的写法也没有,照常翻: * `for x in a, b`(语法树认、Python 3 不认的多个来源),以及缺了变量、来源、条件或每一项的值(写到一半的代码)。 */ function pythonIteration(node: Node): IterationFacts | undefined { const result = ITERATION_RESULTS[node.type] const body = node.childForFieldName('body') if (!result || !body) return undefined const clauses: IterationClause[] = [] for (const child of syntaxChildren(node)) { if (child.type === 'for_in_clause') { const variables = child.childForFieldName('left') const sources = child.childrenForFieldName('right').filter(source => source !== null) if (!variables || sources.length !== 1) return undefined clauses.push({ kind: 'for', variables, source: sources[0], async: child.children.some(token => token?.type === 'async') }) } else if (child.type === 'if_clause') { const [condition] = syntaxChildren(child) if (!condition) return undefined clauses.push({ kind: 'if', condition }) } } if (clauses[0]?.kind !== 'for') return undefined const key = result === 'dict' ? body.childForFieldName('key') : undefined const value = result === 'dict' ? body.childForFieldName('value') : body if (!value || key === null) return undefined const facts: IterationFacts = { result, clauses, value, ...(key && { key }) } // 调用唯一的参数是生成器表达式时只写一对括号(f(x for x in y)),语法树把这对括号算给生成器 const call = node.parent const open = node.child(0) const close = node.child(node.childCount - 1) if (result === 'generator' && call?.type === 'call' && sameNode(call.childForFieldName('arguments'), node) && open && close) facts.sharedParentheses = { open, close } return facts } /** elif / else 展平成紧跟 if 的兄弟块。 */ /** 调用的实参:name=value 按名字传,*x、**x 展开,别的按位置传;唯一的实参是生成器表达式(f(x for x in y))时就是它。 */ function callArgument(arg: Node): CallArgument | undefined { switch (arg.type) { case 'keyword_argument': { const name = arg.childForFieldName('name') const value = arg.childForFieldName('value') return name && value ? { kind: 'keyword', name, value } : undefined } case 'list_splat': case 'dictionary_splat': { const [value] = syntaxChildren(arg) if (!value) return undefined return { kind: arg.type === 'list_splat' ? 'spread' : 'keywordSpread', value } } default: return { kind: 'positional', value: arg } } } /** 调用的结构:新建对象和调用函数写法一样,不给 construct。被调用的是名字或点分名字时 name 是最后那个名字。 */ function pythonCall(node: Node): CallFacts | undefined { const callee = node.type === 'call' ? node.childForFieldName('function') : null const list = node.childForFieldName('arguments') if (!callee || !list) return undefined const name = callee.type === 'identifier' ? callee : callee.type === 'attribute' ? callee.childForFieldName('attribute') : null const args = list.type === 'generator_expression' ? [list] : syntaxChildren(list) return { callee, ...(name && { name }), argumentList: list, arguments: args.map(callArgument).filter(arg => arg !== undefined), } } function ifBlocks(node: Node): CodeBlock[] { const blocks = [bodyBlock('if', node, bodyOf(node), 'other')] for (const alternative of node.childrenForFieldName('alternative')) { if (alternative) blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other')) } return blocks } /** 循环的 else 是兄弟块 else。 */ function loopBlocks(node: Node): CodeBlock[] { const blocks = [bodyBlock('loop', node, bodyOf(node), 'other')] const alternative = node.childForFieldName('alternative') if (alternative) blocks.push(bodyBlock('else', alternative, bodyOf(alternative), 'other')) return blocks } const TRY_CLAUSES: Record = { except_clause: 'catch', except_group_clause: 'catch', else_clause: 'else', finally_clause: 'finally', } function tryBlocks(node: Node): CodeBlock[] { const blocks = [bodyBlock('try', node, node.childForFieldName('body'), 'other')] for (const clause of named(node)) { const kind = TRY_CLAUSES[clause.type] if (kind) blocks.push(bodyBlock(kind, clause, bodyOf(clause), 'other')) } return blocks } // ─── 异常类 ─────────────────────────────────────────────────────────────────── /** Python 自带的异常类(builtins 里的,含各种 Warning)。 */ const BUILTIN_EXCEPTIONS = new Set([ 'BaseException', 'BaseExceptionGroup', 'GeneratorExit', 'KeyboardInterrupt', 'SystemExit', 'Exception', 'ArithmeticError', 'FloatingPointError', 'OverflowError', 'ZeroDivisionError', 'AssertionError', 'AttributeError', 'BufferError', 'EOFError', 'ExceptionGroup', 'ImportError', 'ModuleNotFoundError', 'LookupError', 'IndexError', 'KeyError', 'MemoryError', 'NameError', 'UnboundLocalError', 'OSError', 'EnvironmentError', 'IOError', 'WindowsError', 'BlockingIOError', 'ChildProcessError', 'ConnectionError', 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError', 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError', 'PermissionError', 'ProcessLookupError', 'TimeoutError', 'ReferenceError', 'RuntimeError', 'NotImplementedError', 'PythonFinalizationError', 'RecursionError', 'StopAsyncIteration', 'StopIteration', 'SyntaxError', 'IndentationError', 'TabError', 'SystemError', 'TypeError', 'ValueError', 'UnicodeError', 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeTranslateError', 'Warning', 'BytesWarning', 'DeprecationWarning', 'EncodingWarning', 'FutureWarning', 'ImportWarning', 'PendingDeprecationWarning', 'ResourceWarning', 'RuntimeWarning', 'SyntaxWarning', 'UnicodeWarning', 'UserWarning', ]) /** PEP 8 的命名习惯:异常类名以 Error 结尾(别的文件里的 `AppException`、`*Warning` 也常见)。 */ const EXCEPTION_NAME = /(?:Error|Exception|Warning)$/ /** 父类写的类名:`X`、`pkg.X` 取 X,泛型 `Base[T]` 取 Base 的;关键字参数、`*bases`、调用没有。 */ function className(node: Node | null): string | undefined { switch (node?.type) { case 'identifier': return node.text case 'attribute': return node.childForFieldName('attribute')?.text case 'subscript': return className(node.childForFieldName('value')) default: return undefined } } /** * 类属性:类主体里给一个名字赋值的语句(`x = 1`、`x: int = 1`、只写注解的 `x: int`);连着赋值(`a = b = 1`)、解构赋值不算。 * 构造函数是 `__init__`,参数去掉第一个(self;`*args` 这类不是 self 的不去)。 */ const PYTHON_EXCEPTIONS: ExceptionRules = { baseName: base => className(base.value), isExceptionName: name => BUILTIN_EXCEPTIONS.has(name) || EXCEPTION_NAME.test(name), field: (block) => { const assignment = block.kind === 'field' || block.kind === 'variable' ? assignmentIn(block.node) : undefined const name = assignment?.childForFieldName('left') const value = assignment?.childForFieldName('right') if (!assignment || name?.type !== 'identifier' || value?.type === 'assignment') return undefined return { name, type: assignment.childForFieldName('type') ?? undefined, value: value ?? undefined } }, constructorParameters: (block) => { if (block.kind !== 'function' || block.facts.name?.text !== '__init__') return undefined const [receiver, ...rest] = block.facts.parameters return receiver?.name.type === 'identifier' ? rest : block.facts.parameters }, } // ─── 源码着色 ───────────────────────────────────────────────────────────────── const STORAGE_KEYWORDS = new Set(['def', 'class', 'async', 'lambda', 'global', 'nonlocal', 'del']) const OPERATOR_KEYWORDS = new Set(['and', 'or', 'not', 'is']) const COMPARISON = new Set(['comparison_operator', 'not in', 'is not']) const LITERAL_KEYWORDS = new Set(['none', 'true', 'false']) function insideString(leaf: Node): boolean { for (let node = leaf.parent; node; node = node.parent) { if (node.type === 'interpolation') return false if (node.type === 'string') return true } return false } /** 注解里的名字:typed_parameter / 返回值 / 带注解赋值的 type 节点之内。 */ function insideAnnotation(leaf: Node): boolean { for (let node = leaf.parent; node; node = node.parent) { if (node.type === 'type') return true if (node.type === 'block' || node.type === 'module') return false } return false } function isCallee(node: Node): boolean { const parent = node.parent return !!parent && parent.type === 'call' && sameNode(parent.childForFieldName('function'), node) } function identifierRole(leaf: Node): TokenRole { const parent = leaf.parent if (!parent) return 'variable' if (insideAnnotation(leaf)) return 'type' if (parent.type === 'function_definition' && sameNode(parent.childForFieldName('name'), leaf)) return 'function' if (parent.type === 'class_definition' && sameNode(parent.childForFieldName('name'), leaf)) return 'type' // 父类:类名后面括号里直接写的名字 if (parent.type === 'argument_list' && parent.parent?.type === 'class_definition') return 'type' if (parent.type === 'decorator' || isCallee(leaf)) return 'function' if (parent.type === 'attribute' && sameNode(parent.childForFieldName('attribute'), leaf) && isCallee(parent)) return 'function' return 'variable' } function pythonTokenRole(leaf: Node): TokenRole { if (leaf.type === 'comment') return 'comment' if (insideString(leaf)) return 'string' if (!leaf.isNamed) { if (!/^[a-z]+$/.test(leaf.type)) return 'plain' if (STORAGE_KEYWORDS.has(leaf.type)) return 'storage' if (OPERATOR_KEYWORDS.has(leaf.type) || (leaf.type === 'in' && COMPARISON.has(leaf.parent?.type ?? ''))) return 'operator' return 'keyword' } if (LITERAL_KEYWORDS.has(leaf.type)) return 'storage' switch (leaf.type) { case 'integer': case 'float': return 'number' case 'identifier': return identifierRole(leaf) default: return 'plain' } } /** Python 语言插件。@param wasmDir 语法包 .wasm 所在目录 */ export function createPythonAdapter(wasmDir: string): LanguageAdapter { return { id: 'python', languageIds: ['python'], grammarPath: () => path.join(wasmDir, 'tree-sitter-python.wasm'), toBlocks: root => withExceptions(attachComments(statements(named(root), 'module', -1), aboutModule), PYTHON_EXCEPTIONS), tokenRole: pythonTokenRole, lineComment: '#', iteration: pythonIteration, call: pythonCall, displayNames: pythonNameReader, } }