import type { IterationFacts, Node } from '../contract' import type { Outline } from '../testSupport' import { describe, expect, it } from 'vitest' import { factsByLine, nodeFacts, nodesToText, outline, readFixture, runtime, walk, WASM_DIR, withBlocks } from '../testSupport' import { createPythonAdapter } from './python' const adapter = createPythonAdapter(WASM_DIR) function tree(source: string): Promise { return withBlocks(adapter, 'python', source, outline) } function facts(source: string): Promise> { return withBlocks(adapter, 'python', source, factsByLine) } // fixtures/sample.py:行号 1 起 const SAMPLE: Outline[] = [ { at: 1, kind: 'comment' }, { at: 3, kind: 'import' }, { at: 4, kind: 'import' }, { at: 6, kind: 'const' }, { at: 7, kind: 'variable' }, { at: [10, 11], kind: 'function', kids: [ { at: 13, kind: 'variable', notes: [12] }, { at: 14, kind: 'if', kids: [{ at: 15, kind: 'return' }] }, { at: 16, kind: 'else', kids: [{ at: 17, kind: 'variable' }] }, { at: 18, kind: 'else', kids: [{ at: 19, kind: 'statement' }] }, { at: 20, kind: 'loop', kids: [ { at: 21, kind: 'if', kids: [{ at: 22, kind: 'statement' }] }, ] }, { at: 23, kind: 'try', kids: [ { at: 24, kind: 'statement', kids: [{ at: 25, kind: 'statement' }] }, ] }, { at: 26, kind: 'catch', kids: [{ at: 27, kind: 'statement' }] }, { at: 28, kind: 'finally', kids: [{ at: 29, kind: 'statement' }] }, { at: 30, kind: 'return' }, ] }, { at: 33, kind: 'class', kids: [ { at: 34, kind: 'field' }, { at: 35, kind: 'field' }, { at: 36, kind: 'field' }, { at: 37, kind: 'variable' }, { at: 39, kind: 'function', kids: [{ at: 40, kind: 'return' }] }, ] }, ] describe('python 代码块树', () => { it('样例覆盖 import / def / docstring / if-elif-else / for / try-except-finally / with / 推导式 / class 字段', async () => { expect(await tree(readFixture('sample.py'))).toEqual(SAMPLE) }) it('装饰器算进头部;冒号后、主体前的注释挂到主体第一条语句上;循环的 else 是兄弟块;match 的 case 是子块 if', async () => { const source = [ '@cache', 'def load(key):', ' # 先查缓存', ' while key:', ' key = key[1:]', ' else:', ' pass', ' match key:', ' case \'a\':', ' return 1', ' case _:', ' return 2', ].join('\n') expect(await tree(source)).toEqual([ { at: [1, 2], kind: 'function', kids: [ { at: 4, kind: 'loop', notes: [3], kids: [{ at: 5, kind: 'variable' }] }, { at: 6, kind: 'else', kids: [{ at: 7, kind: 'statement' }] }, { at: 8, kind: 'if', kids: [ { at: 9, kind: 'if', kids: [{ at: 10, kind: 'return' }] }, { at: 11, kind: 'if', kids: [{ at: 12, kind: 'return' }] }, ] }, ] }, ]) }) it('docstring 只认主体开头;不是开头、也没挂到赋值上的字符串是注释,挂到后面那条语句上,f-string、字节串照旧是语句', async () => { const source = [ 'def f():', ' x = 1', ' """not a docstring"""', ' "a" \'b\'', ' f"{x}"', ' b"x"', ' "a" f"b"', ].join('\n') expect(await tree(source)).toEqual([ { at: 1, kind: 'function', kids: [ { at: 2, kind: 'variable' }, { at: 5, kind: 'statement', notes: [3, 4] }, { at: 6, kind: 'statement' }, { at: 7, kind: 'statement' }, ] }, ]) expect(await tree('def f(): f"not a docstring"')).toEqual([{ at: 1, kind: 'function', kids: [{ at: 1, kind: 'statement' }] }]) }) }) describe('python 属性 docstring', () => { /** 字段(一行译文的块)的说明,按 header 起始行(1 起)索引。 */ function fieldDocs(source: string): Promise> { return withBlocks(adapter, 'python', source, (blocks) => { const byLine: Record = {} walk(blocks, (block) => { if (block.kind === 'field' && block.doc) byLine[block.header.startRow + 1] = nodesToText(block.doc) }) return byLine }) } it('attributeDocs.py:模块、类主体里紧跟在赋值后面的字符串并进那块,头部延到字符串结尾;隔着注释、不跟在赋值后面的是注释,挂到讲的块上', async () => { expect(await tree(readFixture('attributeDocs.py'))).toEqual([ { at: 1, kind: 'comment' }, { at: [3, 4], kind: 'const' }, { at: [6, 8], kind: 'variable' }, { at: [11, 12], kind: 'class', kids: [ { at: [14, 15], kind: 'variable' }, { at: [17, 19], kind: 'field', notes: [21, 22] }, ] }, { at: 25, kind: 'function', kids: [ { at: 26, kind: 'variable' }, { at: 31, kind: 'statement', notes: [[27, 30]] }, { at: 32, kind: 'statement' }, { at: 33, kind: 'return' }, ] }, ]) }) it('进常量、变量的 facts.doc 和字段的 doc,是那个字符串节点;函数体里的赋值没有', async () => { const source = readFixture('attributeDocs.py') const found = await facts(source) expect(found[3]).toMatchObject({ doc: '"""一 deployment 一案例的唯一约束名; 入库侧按它认"重复入库", 故与建表同源。"""' }) expect(found[6]).toMatchObject({ doc: '"""隔着空行、赋值行尾有注释也算。"""' }) expect(found[14]).toMatchObject({ doc: '"""正在分析源 deployment."""' }) expect(found[26]).not.toHaveProperty('doc') expect(await fieldDocs(source)).toEqual({ 17: '"""案例标题,\n 用户可编辑."""' }) }) it('只挂到紧跟着的下一条语句:两个字符串只有第一个是说明;没有值的注解、拼接的字符串也算;条件分支里的赋值不算', async () => { const source = [ 'class K:', ' a: int', ' "A" "B"', ' "又一个"', 'if DEBUG:', ' X = 1', ' """不算"""', ].join('\n') expect(await tree(source)).toEqual([ { at: 1, kind: 'class', kids: [{ at: [2, 3], kind: 'field', notes: [4] }] }, { at: 5, kind: 'if', kids: [{ at: 6, kind: 'variable', notes: [7] }] }, ]) expect(await fieldDocs(source)).toEqual({ 2: '"A" "B"' }) }) }) describe('python 函数、类的 docstring', () => { const source = [ 'class K:', ' """类说明"""', ' def run(self): # 行尾', ' # 先说一句', ' """跑一遍。', '', ' 细节。', ' """ # 行尾', ' # 真正的注释', ' return 1', 'def g(): "单行"', ].join('\n') it('算进函数、类头部、不单独成块,它前面的注释也在头部里;后面的注释挂到后面那条语句上', async () => { expect(await tree(source)).toEqual([ { at: [1, 2], kind: 'class', kids: [ { at: [3, 8], kind: 'function', kids: [ { at: 10, kind: 'return', notes: [9] }, ] }, ] }, { at: 11, kind: 'function' }, ]) }) it('进 facts.doc:是那个字符串节点;没有 docstring 的函数没有;模块开头的 docstring 照旧是注释块', async () => { const found = await facts(source) expect(found[1]).toMatchObject({ name: 'K', doc: '"""类说明"""' }) expect(found[3]).toMatchObject({ name: 'run', doc: '"""跑一遍。\n\n 细节。\n """' }) expect(await tree('"""模块说明"""\nclass K:\n pass')).toEqual([{ at: 1, kind: 'comment' }, { at: 2, kind: 'class', kids: [{ at: 3, kind: 'statement' }] }]) expect(found[11]).toMatchObject({ name: 'g', doc: '"单行"' }) expect(await facts('def f():\n x = 1\n """not a docstring"""')).not.toHaveProperty('1.doc') }) }) describe('python 属性表信息(facts)', () => { it('sample.py:import / from import、模块和类里的赋值、函数和方法', async () => { const sample = await facts(readFixture('sample.py')) expect(sample[3]).toEqual({ sources: ['os'], items: [{ kind: 'module', module: 'os' }] }) expect(sample[4]).toEqual({ sources: ['pathlib'], items: [{ kind: 'named', name: 'Path', alias: 'P' }] }) expect(sample[6]).toEqual({ access: 'public', declarators: [{ name: 'MAX_DEPTH', value: '8' }] }) expect(sample[10]).toEqual({ name: 'find_env_files', doc: '"""Return every .env file under root."""', access: 'public', async: false, decorators: [], parameters: [{ name: 'root', optional: false, type: 'str' }], returnType: 'list[str]', }) expect(sample[17]).toEqual({ declarators: [{ name: 'found', value: 'found[:MAX_DEPTH]' }] }) expect(sample[37]).toEqual({ access: 'public', declarators: [{ name: 'label', value: '\'env\'' }] }) expect(sample[39]).toMatchObject({ name: 'summary', access: 'public', parameters: [{ name: 'self', optional: false }, { name: 'verbose', optional: false, type: 'bool', default: 'False' }], returnType: 'str', }) }) it('routes.py:future 导入、多行括号导入、带装饰器的 async def、await 赋值', async () => { const routes = await facts(readFixture('routes.py')) expect(routes[6]).toEqual({ sources: ['__future__'], items: [{ kind: 'named', name: 'annotations' }] }) expect(routes[19]).toEqual({ sources: ['.schemas'], items: [ { kind: 'named', name: 'UserAvatarUploadResponse' }, { kind: 'named', name: 'UserDetailResponse' }, { kind: 'named', name: 'UserUpdateRequest' }, ], }) expect(routes[29]).toEqual({ name: 'get_user', doc: '"""获取用户详情。\n\n 管理员可查看任意用户,普通用户只能查看自己。\n """', access: 'public', async: true, decorators: ['@router.get("/{user_id}", response_model=UserDetailResponse)'], parameters: [ { name: 'user_id', optional: false, type: 'Int4Path' }, { name: 'db', optional: false, type: 'AsyncSession', default: 'Depends(get_db)' }, { name: 'current_user', optional: false, type: 'User', default: 'Depends(get_current_user)' }, ], returnType: 'UserDetailResponse', }) expect(routes[118]).toEqual({ declarators: [{ name: 'payload', value: 'await file.read()' }] }) }) it('导入:import a.b as c, d 每个模块一个来源;from . import、from m import *', async () => { const source = [ 'import os.path as osp, sys', 'from . import x', 'from m import *', ].join('\n') expect(await facts(source)).toEqual({ 1: { sources: ['os.path', 'sys'], items: [{ kind: 'module', module: 'os.path', alias: 'osp' }, { kind: 'module', module: 'sys' }] }, 2: { sources: ['.'], items: [{ kind: 'named', name: 'x' }] }, 3: { sources: ['m'], items: [{ kind: 'namespace' }] }, }) }) it('形参:*args / **kwargs 带前缀,* 和 / 分隔符不算参数', async () => { const found = await facts('async def f(a, /, b: int, *args: int, c=1, d: str = \'x\', **kw) -> None:\n pass') expect(found[1]).toEqual({ name: 'f', access: 'public', async: true, decorators: [], parameters: [ { name: 'a', optional: false }, { name: 'b', optional: false, type: 'int' }, { name: '*args', optional: false, type: 'int' }, { name: 'c', optional: false, default: '1' }, { name: 'd', optional: false, type: 'str', default: '\'x\'' }, { name: '**kw', optional: false }, ], returnType: 'None', }) }) it('访问级别:单下划线开头私有、__dunder__ 公共;解构全是私有名才私有;函数体里的没有', async () => { const source = [ '_cache = {}', '__all__ = [\'f\']', 'a, _b = 1, 2', '_c, _d = 1, 2', 'total: int', 'class K:', ' def __init__(self):', ' self.x = 1', ' def _helper(self):', ' def inner():', ' pass', ].join('\n') const found = await facts(source) expect(found[1]).toMatchObject({ access: 'private' }) expect(found[2]).toMatchObject({ access: 'public' }) expect(found[3]).toEqual({ access: 'public', declarators: [{ name: 'a, _b', value: '1, 2' }] }) expect(found[4]).toMatchObject({ access: 'private' }) expect(found[5]).toEqual({ access: 'public', declarators: [{ name: 'total', type: 'int' }] }) expect(found[7]).toMatchObject({ name: '__init__', access: 'public' }) expect(found[8]).toEqual({ declarators: [{ name: 'self.x', value: '1' }] }) expect(found[9]).toMatchObject({ name: '_helper', access: 'private' }) expect(found[10]).not.toHaveProperty('access') }) it('声明的值照源码给,字面量的各项不在 facts 里(见 LanguageAdapter.collection)', async () => { const found = await facts('EMPTY = {}\nITEMS = [1]\nclass K:\n labels = {"x": 1}') expect(found[1]).toEqual({ access: 'public', declarators: [{ name: 'EMPTY', value: '{}' }] }) expect(found[2]).toEqual({ access: 'public', declarators: [{ name: 'ITEMS', value: '[1]' }] }) expect(found[4]).toEqual({ access: 'public', declarators: [{ name: 'labels', value: '{"x": 1}' }] }) }) }) describe('python 字面量的各项', () => { const collections = (source: string): Promise => nodeFacts(adapter, 'python', source, node => adapter.collection?.(node)) it('字典:字符串键、名字键、**展开,注释不算一项;值里的字典另有自己的各项,空字典没有项', async () => { const source = [ '_KEYS = {', ' "published": "contests:publish",', ' key: 1,', ' # 注释不算一项', ' "nested": {"a": {}},', ' **other,', '}', ].join('\n') expect(await collections(source)).toEqual([ { kind: 'map', entries: [ { kind: 'pair', key: '"published"', value: '"contests:publish"' }, { kind: 'pair', key: 'key', value: '1' }, { kind: 'pair', key: '"nested"', value: '{"a": {}}' }, { kind: 'spread', value: 'other' }, ] }, { kind: 'map', entries: [{ kind: 'pair', key: '"a"', value: '{}' }] }, { kind: 'map', entries: [] }, ]) }) it('列表、元组、集合:各项照源码顺序,*x 是展开,注释不算一项;值里的字面量另有自己的各项,空列表没有项', async () => { const source = [ 'filters = [', ' Filter(field="user_id"),', ' # 注释不算一项', ' *rest,', ' (a, b),', ' {1},', ' [],', ']', ].join('\n') expect(await collections(source)).toEqual([ { kind: 'sequence', elements: [ { kind: 'item', value: 'Filter(field="user_id")' }, { kind: 'spread', value: 'rest' }, { kind: 'item', value: '(a, b)' }, { kind: 'item', value: '{1}' }, { kind: 'item', value: '[]' }, ] }, { kind: 'sequence', elements: [{ kind: 'item', value: 'a' }, { kind: 'item', value: 'b' }] }, { kind: 'sequence', elements: [{ kind: 'item', value: '1' }] }, { kind: 'sequence', elements: [] }, ]) }) it('推导式、括号里的一个式子、调用的实参列表、别的值不是', async () => { expect(await collections('x = [a for a in b]\ny = (a)\nf(a, b)\nz = a, b')).toEqual([]) }) }) describe('python 类', () => { it('classes.py:docstring(连同它前面的注释)算进类头部,类里的类、函数里的类照常成块', async () => { expect(await tree(readFixture('classes.py'))).toEqual([ { at: 1, kind: 'comment' }, { at: 3, kind: 'import' }, { at: [6, 10], kind: 'class', kids: [ { at: 12, kind: 'field' }, { at: 14, kind: 'function', kids: [{ at: 15, kind: 'variable' }] }, ] }, { at: [18, 20], kind: 'class', kids: [{ at: 21, kind: 'variable' }] }, { at: [24, 25], kind: 'class', kids: [ { at: 26, kind: 'class', kids: [{ at: 27, kind: 'variable' }] }, { at: 29, kind: 'class', kids: [{ at: 30, kind: 'statement' }] }, ] }, { at: 33, kind: 'class', kids: [{ at: 34, kind: 'statement' }] }, { at: 37, kind: 'function', kids: [ { at: [38, 39], kind: 'class' }, { at: 40, kind: 'return' }, ] }, ]) }) it('classes.py:名称、说明、访问级别(同函数的下划线规则,类里的类也按它,函数里的没有)、装饰器、父类(参数列表照写的顺序,关键字参数也算)', async () => { const found = await facts(readFixture('classes.py')) expect(found[6]).toEqual({ name: 'AppResponse', doc: '"""应用响应的基类。\n\n 带上 HTTP 状态码。\n """', access: 'public', decorators: [], typeParameters: [], bases: [{ value: 'BaseModel' }], implements: [], }) expect(found[18]).toMatchObject({ name: 'NotFound', doc: '"""找不到资源。"""', bases: [{ value: 'AppResponse' }] }) expect(found[24]).toEqual({ name: 'Meta', access: 'public', decorators: ['@dataclass(frozen=True)'], typeParameters: [], bases: [{ value: 'Base' }, { value: 'metaclass=ABCMeta' }], implements: [], }) expect(found[26]).toMatchObject({ name: 'Config', access: 'public', bases: [] }) expect(found[29]).toMatchObject({ name: '_Hidden', access: 'private' }) expect(found[33]).toMatchObject({ name: '_Private', access: 'private' }) expect(found[38]).toEqual({ name: 'Local', doc: '"""局部类。"""', decorators: [], typeParameters: [], bases: [], implements: [] }) }) it('没有「抽象」这个概念;Python 3.12 的泛型参数进 typeParameters;*bases、**kw 也是父类里的一项', async () => { expect((await facts('class G[T](Generic[T], *mixins, **kw):\n pass'))[1]).toEqual({ name: 'G', access: 'public', decorators: [], typeParameters: ['T'], bases: [{ value: 'Generic[T]' }, { value: '*mixins' }, { value: '**kw' }], implements: [], }) }) }) describe('python 写在表达式里的遍历(推导式)', () => { /** 源码里每个认得出的遍历(外层在前),节点换成源码。 */ async function iterations(source: string): Promise { const tree = await runtime.parse(adapter, 'python', source) try { const found: IterationFacts[] = [] const visit = (node: Node): void => { const facts = adapter.iteration?.(node) if (facts) found.push(facts) for (const child of node.children) { if (child) visit(child) } } visit(tree.rootNode) return found.map(nodesToText) } finally { tree.delete() } } it('调用唯一的参数是生成器:那对括号是调用的,记在 sharedParentheses;参数不止一个时生成器自己带括号,没有', async () => { expect(await iterations('"".join(secrets.choice(ALPHABET) for _ in range(length))')).toEqual([{ result: 'generator', clauses: [{ kind: 'for', variables: '_', source: 'range(length)', async: false }], value: 'secrets.choice(ALPHABET)', sharedParentheses: { open: '(', close: ')' }, }]) const source = 'f(a, (x for x in y))' expect(await iterations(source)).toEqual([{ result: 'generator', clauses: [{ kind: 'for', variables: 'x', source: 'y', async: false }], value: 'x', }]) }) it('列表带条件、字典分出键和值、集合的变量是解构', async () => { expect(await iterations('[u.id for u in users if u.is_active]')).toEqual([{ result: 'list', clauses: [{ kind: 'for', variables: 'u', source: 'users', async: false }, { kind: 'if', condition: 'u.is_active' }], value: 'u.id', }]) expect(await iterations('{u.id: u for u in users}')).toEqual([{ result: 'dict', clauses: [{ kind: 'for', variables: 'u', source: 'users', async: false }], key: 'u.id', value: 'u', }]) expect(await iterations('{n for n, _ in pairs}')).toEqual([{ result: 'set', clauses: [{ kind: 'for', variables: 'n, _', source: 'pairs', async: false }], value: 'n', }]) }) it('几节 for、if 照源码顺序;async for 标出异步', async () => { expect(await iterations('[c for row in grid if row for c in row if c]')).toEqual([{ result: 'list', clauses: [ { kind: 'for', variables: 'row', source: 'grid', async: false }, { kind: 'if', condition: 'row' }, { kind: 'for', variables: 'c', source: 'row', async: false }, { kind: 'if', condition: 'c' }, ], value: 'c', }]) expect(await iterations('[v async for v in stream() if v]')).toEqual([{ result: 'list', clauses: [{ kind: 'for', variables: 'v', source: 'stream()', async: true }, { kind: 'if', condition: 'v' }], value: 'v', }]) }) it('套着的遍历各自认,外层取的值就是里层整个遍历', async () => { expect(await iterations('[[y for y in row] for row in grid]')).toEqual([ { result: 'list', clauses: [{ kind: 'for', variables: 'row', source: 'grid', async: false }], value: '[y for y in row]' }, { result: 'list', clauses: [{ kind: 'for', variables: 'y', source: 'row', async: false }], value: 'y' }, ]) }) it('列表、字典、元组字面量和 for 语句不是', async () => { expect(await iterations('x = [1, 2]\ny = {"a": 1}\nz = (a, b)\nfor x in y:\n pass')).toEqual([]) }) }) describe('python 调用', () => { const calls = (source: string): Promise => nodeFacts(adapter, 'python', source, node => adapter.call?.(node)) it('实参照源码顺序:按位置、按名字、*x、**x;新建对象和调用写法一样,不给 construct;点分名字取最后一节', async () => { const source = 'user_service.update_user(\n db, # 会话\n user_id=user_id,\n *rest,\n **options,\n)' expect(await calls(source)).toEqual([{ callee: 'user_service.update_user', name: 'update_user', argumentList: '(\n db, # 会话\n user_id=user_id,\n *rest,\n **options,\n)', arguments: [ { kind: 'positional', value: 'db' }, { kind: 'keyword', name: 'user_id', value: 'user_id' }, { kind: 'spread', value: 'rest' }, { kind: 'keywordSpread', value: 'options' }, ], }]) }) it('唯一的实参是生成器表达式时就是它;被调用的不是名字时没有 name;装饰器、父类列表不是调用', async () => { expect(await calls('sum(x for x in xs)')).toEqual([{ callee: 'sum', name: 'sum', argumentList: '(x for x in xs)', arguments: [{ kind: 'positional', value: '(x for x in xs)' }] }]) expect(await calls('handlers[k]()')).toEqual([{ callee: 'handlers[k]', argumentList: '()', arguments: [] }]) expect(await calls('@cache\nclass A(Base):\n pass')).toEqual([]) }) }) describe('python 注释挂到它讲的块上', () => { it('同一层后面最近的那块(隔着空行也算);主体最后的挂到前面那块;主体里只有注释时挂到父块;文件最后的挂到最后一块', async () => { const source = [ '"""模块说明。"""', '', 'import os', '', '', '# 讲 load:隔着空行', 'def load(path):', ' """读出来。"""', ' # 先检查', ' if not path:', ' return None', ' return path', ' # 函数体最后的注释', '', '', 'class Empty:', ' """只有说明。"""', ' # 只有注释的主体', '', '', '# 文件最后的注释', ].join('\n') expect(await tree(source)).toEqual([ { at: 1, kind: 'comment' }, { at: 3, kind: 'import' }, { at: [7, 8], kind: 'function', notes: [6], kids: [ { at: 10, kind: 'if', notes: [9], kids: [{ at: 11, kind: 'return' }] }, { at: 12, kind: 'return', notes: [13] }, ] }, { at: [16, 17], kind: 'class', notes: [18, 21] }, ]) }) it('文件开头的注释和第一块代码隔着空行是文件的说明,紧挨着就挂到它上面;#! 和写编码的那行总是文件的', async () => { expect(await tree('# 文件的说明\n\nimport os\n')).toEqual([{ at: 1, kind: 'comment' }, { at: 3, kind: 'import' }]) expect(await tree('# 讲导入\nimport os\n')).toEqual([{ at: 2, kind: 'import', notes: [1] }]) expect(await tree('#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n# 讲导入\nimport os\n')).toEqual([ { at: [1, 2], kind: 'comment' }, { at: 4, kind: 'import', notes: [3] }, ]) }) it('luffy 的 acquisition/repository.py:导入后面隔着两个空行的注释挂到类上,是类卡片的「注释」一行,类的编辑范围连着这几行', async () => { const source = readFixture('acquisitionRepository.py') const [klass] = (await tree(source)).filter(block => block.kind === 'class') expect(klass).toMatchObject({ at: [19, 21], notes: [[13, 16]] }) }) })