✨ feat(vscode): 代码阅读认出代码里起的中文名(Annotated[…, Comment("…")]、@Comment 装饰器、行尾 # @Comment、docstring / JSDoc 的参数说明),预览里定义和用到它的各处都写中文名,悬停看原名,定义处后面淡淡写原名

This commit is contained in:
cruldra committed 2026-09-27 00:56:39 +08:00
1 parent 1b1299dc1f
commit 44c221ea2b
45 files changed
+1685 -128

No files matched your search

+1 -1
View File
@@ -13,7 +13,7 @@ const ts = createTypeScriptAdapter(WASM_DIR)
const py = createPythonAdapter(WASM_DIR)
function views(adapter: LanguageAdapter, languageId: string, source: string, translator?: Translator): Promise<BlockView[]> {
return withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, source, adapter, translator))
return withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, { source, language: adapter, markers: [] }, translator))
}
/** 每块的筛选属性,按 header 起始行(1 起)索引;同一行有几块取外层那块。 */
+2 -2
View File
@@ -13,7 +13,7 @@ const ts = createTypeScriptAdapter(WASM_DIR)
const py = createPythonAdapter(WASM_DIR)
function views(adapter: LanguageAdapter, languageId: string, translator: Translator | undefined, source: string): Promise<BlockView[]> {
return withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, source, adapter, translator))
return withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, { source, language: adapter, markers: [] }, translator))
}
function flatten(list: BlockView[]): BlockView[] {
@@ -245,7 +245,7 @@ describe('blockView', () => {
})
it('readBlockViews:解析、转视图、释放语法树', async () => {
const list = await readBlockViews(runtime, py, pythonZhTranslator, 'python', 'def f(x: int) -> bool:\n return not x\n')
const { blocks: list } = await readBlockViews(runtime, py, pythonZhTranslator, 'python', 'def f(x: int) -> bool:\n return not x\n')
expect(list).toHaveLength(1)
expect('properties' in list[0] && propertyLines(list[0].properties)).toEqual([
'名称: f',
+14 -6
View File
@@ -1,4 +1,4 @@
import type { BlockAttributes, CodeBlock, KindBadge, LanguageAdapter, LibraryRule, Markdown, Node, PreviewText, PropertyRow, RuleContext, Segment, SourceDocument, SourceRange, TextBlock, TokenRole, Translator } from './contract'
import type { BlockAttributes, CodeBlock, DisplayName, KindBadge, LanguageAdapter, LibraryRule, Markdown, Node, PreviewText, PropertyRow, RuleContext, Segment, SourceDocument, SourceRange, TextBlock, TokenRole, Translator } from './contract'
import type { TreeSitterRuntime } from './runtime'
import { blockAttributes } from './attributes'
import { commonIndent, splitLines } from './editText'
@@ -129,10 +129,10 @@ const NO_RULES: RuleRun = { rules: [], report: () => {} }
* 每块先问启用的库规则,认领了就换成规则给的替身(见 RuleBlock):规则的卡片,或认成的属性表块(交给翻译插件排,同语言插件报的这种块),
* 并进去的子块不再单独成块。
*/
export function toBlockViews(blocks: CodeBlock[], source: string, adapter: LanguageAdapter, translator: Translator | undefined, rules: RuleRun = NO_RULES): BlockView[] {
export function toBlockViews(blocks: CodeBlock[], document: SourceDocument, translator: Translator | undefined, rules: RuleRun = NO_RULES): BlockView[] {
const { source, language: adapter } = document
const lines = lineRanges(source)
const texts = splitLines(source)
const document: SourceDocument = { source, language: adapter }
const imports = importsIn(blocks)
const active = activeRules(rules.rules, imports)
const context: RuleContext = {
@@ -203,7 +203,13 @@ export function toBlockViews(blocks: CodeBlock[], source: string, adapter: Langu
return blocks.map((block, i) => build(block, String(i), ''))
}
/** 解析一份源码并转成视图数据;语法树用完即释放。 */
/** 一份源码的视图数据,和它里面起了中文名的定义(预览找用到它们的各处要用,见 displayNames.ts)。 */
export interface DocumentViews {
blocks: BlockView[]
displayNames: DisplayName[]
}
/** 解析一份源码并转成视图数据(中文名记号交给翻译插件去掉);语法树用完即释放。 */
export async function readBlockViews(
runtime: Pick<TreeSitterRuntime, 'parse'>,
adapter: LanguageAdapter,
@@ -211,10 +217,12 @@ export async function readBlockViews(
languageId: string,
source: string,
rules: RuleRun = NO_RULES,
): Promise<BlockView[]> {
): Promise<DocumentViews> {
const tree = await runtime.parse(adapter, languageId, source)
try {
return toBlockViews(adapter.toBlocks(tree.rootNode, source, languageId), source, adapter, translator, rules)
const { names, markers } = adapter.displayNames?.find(tree.rootNode, source, languageId) ?? { names: [], markers: [] }
const blocks = toBlockViews(adapter.toBlocks(tree.rootNode, source, languageId), { source, language: adapter, markers }, translator, rules)
return { blocks, displayNames: names }
}
finally {
tree.delete()
+31
View File
@@ -312,12 +312,43 @@ export interface LanguageAdapter {
tokenRole: (leaf: Node) => TokenRole
/** node 是写在表达式里的遍历(推导式、生成器表达式)时给出它的结构,别的节点没有;没有这种写法的语言不填。 */
iteration?: (node: Node) => IterationFacts | undefined
/** 认代码里给定义起的中文名(见「中文名」);不填就没有中文名。 */
displayNames?: DisplayNameReader
}
// ── 中文名(见「中文名」)──
/** 代码里给一个定义起的中文名:预览里这个定义和用到它的各处都显示中文名,悬停看原名。 */
export interface DisplayName {
/** 定义处那个名字(函数名、类名、参数名、变量名、字段名、枚举成员名……) */
name: SourceRange
/** 中文名 */
text: string
}
/** 一份源码里认出的中文名。 */
export interface DisplayNameFacts {
/** 起了中文名的定义,照源码顺序;一个定义几处都写了时语言插件定取哪个(明写的记号优先于说明文字) */
names: DisplayName[]
/** 写在注释里给代码阅读看的记号(`# @Comment 当前用户` 里的 `@Comment 当前用户`):宿主交给翻译插件,翻注释、说明时去掉 */
markers: SourceRange[]
}
/** 语言插件认中文名:写法(记号、说明文字里的哪一句)归语言插件;找用到它的各处、显示归宿主和预览。 */
export interface DisplayNameReader {
/** languageId → 工作区里这种语言的文件(glob,Python 是所有 .py 文件):宿主建工作区索引,认出别的文件里定义的中文名 */
files: Record<string, string>
/** 便宜的文字预筛:可能写了中文名的源码才解析 */
mentions: (source: string) => boolean
find: (root: Node, source: string, languageId: string) => DisplayNameFacts
}
/** 翻译插件翻一块时拿到的这份源码:源码字符串,和解析它的语言插件。 */
export interface SourceDocument {
source: string
language: LanguageAdapter
/** 写在注释里给代码阅读看的记号(中文名记号,见 DisplayNameFacts.markers):不是说明文字,翻注释、说明时去掉 */
markers: SourceRange[]
}
/** 筛选属性和值的显示名,筛选栏、种类标签用它;没给名字的显示 id。 */
+1 -1
View File
@@ -43,7 +43,7 @@ describe('行替换', () => {
[createPythonAdapter(WASM_DIR), 'python', 'classes.py'],
] as const)('%#: 把预览里的 code 原样写回,文件不变(去缩进与补缩进互逆,函数、类头部连着说明也一样)', async (adapter, languageId, fixture) => {
const source = readFixture(fixture)
const views = await withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, source, adapter, undefined))
const views = await withBlocks(adapter, languageId, source, blocks => toBlockViews(blocks, { source, language: adapter, markers: [] }, undefined))
const all = views.flatMap(function flat(view): typeof views {
return [view, ...view.children.flatMap(flat)]
})
@@ -12,7 +12,7 @@ const py = createPythonAdapter(WASM_DIR)
const SNAPSHOT = readFixture('celeryApp.py')
function views(source: string): Promise<BlockView[]> {
return withBlocks(py, 'python', source, blocks => toBlockViews(blocks, source, py, pythonZhTranslator, { rules: [celeryRule], report: () => {} }))
return withBlocks(py, 'python', source, blocks => toBlockViews(blocks, { source, language: py, markers: [] }, pythonZhTranslator, { rules: [celeryRule], report: () => {} }))
}
/** 记下问了哪边:规则(引用的键)还是语言扩展(下标)。 */
@@ -49,7 +49,7 @@ function doc(languageId: string, text: string, version = 3): { languageId: strin
describe('预览的 document 消息', () => {
it('受支持的语言:带文档版本、代码块、可选翻译和选中的翻译', async () => {
const message = await documentMessage(doc('typescript', 'if (!folder)\n return []\n'), registryWithTypeScript(), runtime, ignore)
const { message } = await documentMessage(doc('typescript', 'if (!folder)\n return []\n'), registryWithTypeScript(), runtime, ignore)
expect(message).toMatchObject({
type: 'document',
version: 3,
@@ -66,7 +66,7 @@ describe('预览的 document 消息', () => {
})
it('没有语言插件的语言:带 error,不解析', async () => {
const message = await documentMessage(doc('rust', 'fn main() {}'), registryWithTypeScript(), runtime, ignore)
const { message } = await documentMessage(doc('rust', 'fn main() {}'), registryWithTypeScript(), runtime, ignore)
expect(message.blocks).toEqual([])
expect(message.attributes).toEqual([])
expect(message.translators).toEqual([])
@@ -74,7 +74,7 @@ describe('预览的 document 消息', () => {
})
it('库规则认领的块:种类是规则的种类 id,筛选栏里叫规则给的名字', async () => {
const message = await documentMessage(doc('python', MODEL), registryWithPython(sqlalchemyRule), runtime, ignore)
const { message } = await documentMessage(doc('python', MODEL), registryWithPython(sqlalchemyRule), runtime, ignore)
expect(message.blocks.map(b => b.kind)).toEqual(['import', 'sqlalchemy.table'])
expect(message.attributes.find(attribute => attribute.id === 'kind')?.values).toEqual([
{ id: 'import', name: '导入', count: 1 },
@@ -93,7 +93,7 @@ describe('预览的 document 消息', () => {
},
}
const reports: string[] = []
const message = await documentMessage(doc('python', MODEL), registryWithPython(broken), runtime, text => reports.push(text))
const { message } = await documentMessage(doc('python', MODEL), registryWithPython(broken), runtime, text => reports.push(text))
expect(message.error).toBeUndefined()
expect(message.blocks.map(b => b.kind)).toEqual(['import', 'class'])
expect(reports).toEqual([1, 3, 4, 5].map((line, i) =>
@@ -102,7 +102,7 @@ describe('预览的 document 消息', () => {
it('解析失败:把原因放进 error', async () => {
const failing = { parse: async () => Promise.reject(new Error('语法包加载失败')) }
const message = await documentMessage(doc('typescript', 'const a = 1'), registryWithTypeScript(), failing, ignore)
const { message } = await documentMessage(doc('typescript', 'const a = 1'), registryWithTypeScript(), failing, ignore)
expect(message.blocks).toEqual([])
expect(message.error).toBe('语法包加载失败')
})
@@ -1,3 +1,4 @@
import type { DisplayName } from '../contract'
import type { CodeReaderRegistry } from '../registry'
import type { TreeSitterRuntime } from '../runtime'
import type { DocumentMessage } from './messages'
@@ -11,13 +12,19 @@ export interface DocumentSnapshot {
getText: () => string
}
/** 发给预览的文档,和这份源码里起了中文名的定义(扩展端接着找用到它们的各处)。 */
export interface DocumentRead {
message: DocumentMessage
displayNames: DisplayName[]
}
/** 当前文档 → 发给预览的 document 消息;语言不支持或解析失败时带 error,blocks 为空。库规则出错不算失败,交给 report 记下。 */
export async function documentMessage(
document: DocumentSnapshot,
registry: Pick<CodeReaderRegistry, 'languageFor' | 'translatorsFor' | 'selectedTranslator' | 'rulesFor'>,
runtime: Pick<TreeSitterRuntime, 'parse'>,
report: (message: string) => void,
): Promise<DocumentMessage> {
): Promise<DocumentRead> {
const { languageId, version } = document
const source = document.getText()
const translator = registry.selectedTranslator(languageId)
@@ -29,15 +36,16 @@ export async function documentMessage(
attributes: [],
translators: registry.translatorsFor(languageId).map(({ id, label }) => ({ id, label })),
translatorId: translator?.id ?? '',
names: [],
}
const adapter = registry.languageFor(languageId)
if (!adapter)
return { ...message, error: `代码阅读还不支持「${languageId}」这种语言` }
return { message: { ...message, error: `代码阅读还不支持「${languageId}」这种语言` }, displayNames: [] }
try {
const blocks = await readBlockViews(runtime, adapter, translator, languageId, source, { rules: registry.rulesFor(languageId), report })
return { ...message, blocks, attributes: fileAttributes(blocks, translator?.attributeNames) }
const { blocks, displayNames } = await readBlockViews(runtime, adapter, translator, languageId, source, { rules: registry.rulesFor(languageId), report })
return { message: { ...message, blocks, attributes: fileAttributes(blocks, translator?.attributeNames) }, displayNames }
}
catch (err) {
return { ...message, error: err instanceof Error ? err.message : String(err) }
return { message: { ...message, error: err instanceof Error ? err.message : String(err) }, displayNames: [] }
}
}
+6 -1
View File
@@ -5,11 +5,12 @@
import type { ClaudeImage } from '../../cc/spawnClaude'
import type { FileAttribute } from '../attributes'
import type { BlockView } from '../blockView'
import type { ShownName } from '../nameResolution'
import type { ProfileChoice } from '../qa/askProfiles'
import type { RunningAsk } from '../qa/asks'
import type { Excluded } from './blockFilter'
export type { Excluded }
export type { Excluded, ShownName }
export interface TranslatorOption { id: string, label: string }
@@ -45,6 +46,8 @@ export interface DocumentMessage {
error?: string
/** 新预览只在第一份 document 里带:从文本编辑器切过来时是它顶部那一行,跳到定义时是定义处 */
reveal?: Reveal
/** 这个版本眼下已知的中文名:查过的,或上一版查到的跟着改动挪了位置的;查完再发 displayNames 换掉 */
names: ShownName[]
}
/** 问过的一次(给预览看的):问题、回答(Markdown)、什么时候问的(ISO 时间)、用哪个 profile 答的。 */
@@ -86,6 +89,7 @@ export interface QaMessage {
* answer:askId 那个问题的回答长了(text 是到现在为止的全文)、或换了在做的事
* answerEnd:askId 那个问题答完了:done 带最终的回答和存下的那条的 id,failed 带给人看的原因,stopped 是被叫停
* moveOccurrence:跳到高亮着的名字的下一处(step 1)/ 上一处(step -1)出现,由这两条命令发给活动的预览
* displayNames:version 这份文档里显示成中文名的名字,查到一步发一步(先本文件定义的,再别的文件定义的);不是页面上这个版本的不理
*/
export type ExtensionToWebview
= | DocumentMessage
@@ -93,6 +97,7 @@ export type ExtensionToWebview
| { type: 'definitionProbed', id: number, found: boolean }
| { type: 'occurrencesFound', id: number, occurrences: Occurrence[] }
| { type: 'moveOccurrence', step: 1 | -1 }
| { type: 'displayNames', version: number, names: ShownName[] }
| { type: 'filter', excluded: Excluded }
| QaMessage
| ({ type: 'askProfiles' } & ProfileChoice)
+32 -4
View File
@@ -2,6 +2,7 @@ import type { CustomTextEditorProvider, Disposable, TextDocument, TextDocumentSh
import type { ClaudeImage } from '../../cc/spawnClaude'
import type { BlockView } from '../blockView'
import type { RuleLocation, RuleReference } from '../contract'
import type { IndexedName, NameSources } from '../nameResolution'
import type { CodeQa } from '../qa/codeQa'
import type { CodeReaderRegistry } from '../registry'
import type { TreeSitterRuntime } from '../runtime'
@@ -13,6 +14,7 @@ import { logger } from '../../logging/logger'
import { webviewHtml, webviewUiRoot } from '../../webview/html'
import { blockReplacement } from '../editText'
import { locate } from '../folderStore'
import { DocumentNames, resolveNames } from '../nameResolution'
import { askPrompt, followUpPrompt } from '../qa/prompt'
import { definitionAction } from './definitionAction'
import { findDefinitions, languageDefinitions, pickDefinition, ruleDefinitions } from './definitions'
@@ -46,6 +48,8 @@ export class PreviewEditorProvider implements CustomTextEditorProvider {
readonly resolveReference: (reference: RuleReference) => Promise<RuleLocation[]>,
/** 问 AI:各预览共用,同一份文件开在几处时问答、正在答的问题都一样 */
readonly qa: CodeQa,
/** 工作区里别的文件定义的、原名叫 name 的中文名(languageId 是查的这份文档的语言) */
readonly indexedNames: (languageId: string, name: string) => Promise<IndexedName[]>,
) {}
resolveCustomTextEditor(document: TextDocument, panel: WebviewPanel): void {
@@ -118,6 +122,8 @@ class Preview {
visibleLine: number | undefined
/** 最近一次发出去的块树和它的源码:问 AI 时按块的记号在里面找那块代码 */
private sent: { blocks: BlockView[], source: string } | undefined
/** 这份文档各版本显示成中文名的名字 */
private readonly names: DocumentNames
constructor(
private readonly document: TextDocument,
@@ -136,6 +142,10 @@ class Preview {
rule: async reference => ruleDefinitions(await owner.resolveReference(reference)),
language: offset => languageDefinitions(document.uri, document.positionAt(offset)),
})
this.names = new DocumentNames((version, names) => {
if (this.ready && version === document.version)
this.post({ type: 'displayNames', version, names })
})
this.disposables = [
panel.webview.onDidReceiveMessage((msg: WebviewToExtension) => this.handleMessage(msg)),
panel.onDidChangeViewState(() => {
@@ -146,8 +156,10 @@ class Preview {
owner.navigation.update(this, false)
}),
workspace.onDidChangeTextDocument((e) => {
if (e.document === document && e.contentChanges.length > 0)
this.schedulePush()
if (e.document !== document || e.contentChanges.length === 0)
return
this.names.edited(document.version, e.contentChanges.map(change => ({ offset: change.rangeOffset, length: change.rangeLength, text: change.text })))
this.schedulePush()
}),
// 页面没 ready 时收不到,等它 ready 再随第一份文档之前补发
filter.subscribe((excluded) => {
@@ -202,7 +214,7 @@ class Preview {
const request = ++this.latest
const { languageId, version } = this.document
const source = this.document.getText()
const message = await documentMessage({ languageId, version, getText: () => source }, this.registry, this.runtime, (text) => {
const { message, displayNames } = await documentMessage({ languageId, version, getText: () => source }, this.registry, this.runtime, (text) => {
logger.add({ level: 'warn', source: 'codeReader', message: `${this.document.uri.fsPath}:${text}` })
})
if (request !== this.latest || !this.ready)
@@ -211,7 +223,22 @@ class Preview {
this.sent = { blocks: message.blocks, source }
const reveal = this.pendingReveal
this.pendingReveal = undefined
this.post(reveal === undefined ? message : { ...message, reveal })
this.post({ ...message, names: this.names.known(version), ...(reveal && { reveal }) })
// 文档在解析期间又改了的,等下一次发文档再查
if (version === this.document.version)
this.names.resolve(version, report => resolveNames(displayNames, message.blocks, this.nameSources(), report))
}
/** 找中文名落在哪些名字上时问的:这份文档的语言扩展、工作区里的中文名索引。 */
private nameSources(): NameSources {
const { document } = this
return {
uri: document.uri.toString(),
occurrences: async offset => (await occurrencesOf(document, offset)).map(({ start, end }) => ({ start, end })),
definitions: async offset => (await languageDefinitions(document.uri, document.positionAt(offset)))
.map(({ uri, range }) => ({ uri: uri.toString(), line: range.start.line, character: range.start.character })),
indexed: name => this.owner.indexedNames(document.languageId, name),
}
}
/** 跳到高亮着的名字的下一处 / 上一处出现;页面不在就算了(页面重新加载时高亮已经没了)。 */
@@ -230,6 +257,7 @@ class Preview {
dispose(): void {
clearTimeout(this.timer)
this.names.dispose()
for (const disposable of this.disposables)
disposable.dispose()
}
+7 -1
View File
@@ -1,5 +1,6 @@
import type { ExtensionContext, Uri } from 'vscode'
import type { CodeReaderApi, RuleLocation, RuleReference } from './contract'
import type { IndexedName } from './nameResolution'
import { commands, window, workspace } from 'vscode'
import { logger } from '../logging/logger'
import { BlockFilter } from './editor/blockFilter'
@@ -11,6 +12,7 @@ import { FileTreeProvider } from './explorer/fileTree'
import { createPythonAdapter } from './languages/python'
import { createTypeScriptAdapter } from './languages/typescript'
import { resolveReference } from './libraryRules'
import { indexedNames } from './nameResolution'
import { registerNoteCommands } from './notes/commands'
import { FileNotes } from './notes/fileNotes'
import { CodeQa } from './qa/codeQa'
@@ -66,7 +68,11 @@ export function registerCodeReader(context: ExtensionContext): CodeReaderApi {
}, report))
const resolve = (reference: RuleReference): Promise<RuleLocation[]> => resolveReference(reference, registry.libraryRules(), indexes, report)
const qa = new CodeQa(context.globalState)
const previews = new PreviewEditorProvider(context.extensionUri, registry, runtime, new BlockFilter(context.workspaceState), resolve, qa)
const namesIn = async (languageId: string, name: string): Promise<IndexedName[]> => {
const adapter = registry.languageFor(languageId)
return adapter ? indexedNames((index, key) => indexes.find(index, key), adapter, name) : []
}
const previews = new PreviewEditorProvider(context.extensionUri, registry, runtime, new BlockFilter(context.workspaceState), resolve, qa, namesIn)
const notes = new FileNotes()
const files = new FileTreeProvider(OPEN_COMMAND, notes)
@@ -0,0 +1,74 @@
import type { DisplayName, DisplayNameFacts, Node, SourceRange } from '../contract'
/** 给定义起中文名的那个调用 / 装饰器 / 注释记号的名字:`Comment("当前用户")`、`@Comment("…")`、`# @Comment 当前用户`。 */
export const COMMENT_MARK = 'Comment'
/** 说明文字里的一句话最多几个字才当中文名 */
export const CLAUSE_LIMIT = 12
/** 说明文字的第一小句(到第一个 ,。,.;;:: 为止)去掉首尾空白;空的、超过 CLAUSE_LIMIT 个字的不当中文名。 */
export function firstClause(text: string): string | undefined {
const clause = text.split(/[,。,.;;::]/, 1)[0].trim()
return clause && [...clause].length <= CLAUSE_LIMIT ? clause : undefined
}
/** 行注释里的中文名记号 `@Comment 名字`(名字一直到注释结尾):名字,和记号从注释文字的第几个字开始。 */
export function commentTag(text: string): { text: string, start: number } | undefined {
const match = new RegExp(`@${COMMENT_MARK}[ \\t]+(\\S.*)$`).exec(text)
return match ? { text: match[1].trimEnd(), start: match.index } : undefined
}
/** 一个定义和语言插件从代码里读到的中文名。 */
export interface NameCandidate {
/** 定义处那个名字 */
name: Node
/** 写在代码里的记号给的:`Annotated[T, Comment("…")]`、`@Comment("…")` */
marked?: string
/** 说明文字里取的:docstring 的参数说明、JSDoc 的 @param */
described?: string
/** 行尾的 `@Comment` 记号能给它:参数、变量、字段、枚举成员、属性(函数、类这些用装饰器) */
trailing?: boolean
}
function rangeOf(node: Node): SourceRange {
return { startIndex: node.startIndex, endIndex: node.endIndex }
}
/** 注释前面同一行有代码:是写在行尾的注释。 */
function isTrailing(comment: Node, source: string): boolean {
const lineStart = source.lastIndexOf('\n', comment.startIndex - 1) + 1
return source.slice(lineStart, comment.startIndex).trim() !== ''
}
/**
* 汇总一份源码里的中文名。行尾注释里的记号(`x = 1 # @Comment 数量`)给那一行第一个能带行尾记号的定义(多行的参数表里就是那一行的参数),
* 记号都算 markers(翻注释、说明时去掉),不管这个定义最后取没取它。
* 每个定义取哪个:写在代码里的记号,其次行尾记号,最后才是说明文字。
*/
export function displayNameFacts(candidates: readonly NameCandidate[], lineComments: readonly Node[], source: string): DisplayNameFacts {
const byStart = new Map<number, NameCandidate & { tagged?: string }>()
for (const candidate of candidates) {
const known = byStart.get(candidate.name.startIndex)
byStart.set(candidate.name.startIndex, {
name: candidate.name,
marked: known?.marked ?? candidate.marked,
described: known?.described ?? candidate.described,
trailing: known?.trailing || candidate.trailing,
})
}
const ordered = [...byStart.values()].sort((a, b) => a.name.startIndex - b.name.startIndex)
const markers: SourceRange[] = []
for (const comment of lineComments) {
const tag = isTrailing(comment, source) ? commentTag(comment.text) : undefined
const owner = tag && ordered.find(candidate => candidate.trailing && candidate.name.startPosition.row === comment.startPosition.row)
if (!tag || !owner)
continue
owner.tagged ??= tag.text
markers.push({ startIndex: comment.startIndex + tag.start, endIndex: comment.endIndex })
}
const names = ordered.flatMap((candidate): DisplayName[] => {
const text = candidate.marked ?? candidate.tagged ?? candidate.described
return text ? [{ name: rangeOf(candidate.name), text }] : []
})
return { names, markers }
}
@@ -1,6 +1,7 @@
import type { Access, BlockShape, ClassFacts, CodeBlock, EntryFacts, FunctionFacts, ImportFacts, ImportItem, IterationClause, IterationFacts, IterationResult, LanguageAdapter, Node, ParameterFacts, TextKind, TokenRole } from '../contract'
import type { CommentRules } from './shared'
import * as path from 'node:path'
import { pythonNameReader } from './pythonNames'
import { children, compoundShape, leafBlock, leafShape, listBlocks, named, sameNode, syntaxChildren } from './shared'
/** 常量、类字段只认直接写在模块 / 类主体里的赋值;这两处的声明也才有访问级别。 */
@@ -486,5 +487,6 @@ export function createPythonAdapter(wasmDir: string): LanguageAdapter {
toBlocks: root => statements(named(root), 'module', -1),
tokenRole: pythonTokenRole,
iteration: pythonIteration,
displayNames: pythonNameReader,
}
}
@@ -0,0 +1,137 @@
import { describe, expect, it } from 'vitest'
import { runtime, WASM_DIR } from '../testSupport'
import { createPythonAdapter } from './python'
import { docstringParameters } from './pythonNames'
const py = createPythonAdapter(WASM_DIR)
/** 认出的中文名写成「原名 → 中文名」(原名前面加行号,1 起),记号写成源码原文。 */
async function names(source: string): Promise<{ names: string[], markers: string[] }> {
const tree = await runtime.parse(py, 'python', source)
try {
const facts = py.displayNames!.find(tree.rootNode, source, 'python')
const row = (index: number): number => source.slice(0, index).split('\n').length
return {
names: facts.names.map(({ name, text }) => `${row(name.startIndex)}:${source.slice(name.startIndex, name.endIndex)} → ${text}`),
markers: facts.markers.map(({ startIndex, endIndex }) => source.slice(startIndex, endIndex)),
}
}
finally {
tree.delete()
}
}
describe('python 中文名:写在代码里的记号', () => {
it('注解 Annotated[T, Comment("…")]:参数、变量、类字段;typing.Annotated、t.Comment 也算,调用的名字最后一节是 Comment 就算', async () => {
const source = [
'def get(current_user: Annotated[User, Depends(auth), Comment("当前用户")], db: typing.Annotated[Session, t.Comment("数据库")] = None):',
' count: Annotated[int, Comment("数量")] = 0',
'class Order:',
' total: Annotated[Decimal, Comment("总价")]',
' note: Annotated[str, Other("不是")]',
' tag: Annotated[str, Comment(f"{x}")]',
].join('\n')
expect((await names(source)).names).toEqual(['1:current_user → 当前用户', '1:db → 数据库', '2:count → 数量', '4:total → 总价'])
})
it('@Comment("…") 装饰器:函数、类', async () => {
const source = '@Comment("取用户")\n@router.get("/")\nasync def get_user():\n pass\n\n@Comment("订单")\nclass Order:\n pass\n\n@other("x")\ndef plain():\n pass\n'
expect((await names(source)).names).toEqual(['3:get_user → 取用户', '7:Order → 订单'])
})
it('行尾的 # @Comment 名字:多行参数表里的参数、变量、字段、枚举成员、self.x;记号从 @Comment 到行尾', async () => {
const source = [
'async def delete_item(',
' item_id: int,',
' current_user: User = Depends(get_current_user), # @Comment 当前用户',
' *args, # @Comment 其余参数',
'):',
' self.owner = current_user # 持有人 @Comment 拥有者',
'LIMIT = 10 # @Comment 上限',
'class Role(str, Enum):',
' ADMIN = "admin" # @Comment 管理员',
' name: str # @Comment 名称 ',
].join('\n')
const found = await names(source)
expect(found.names).toEqual(['3:current_user → 当前用户', '4:args → 其余参数', '6:owner → 拥有者', '7:LIMIT → 上限', '9:ADMIN → 管理员', '10:name → 名称'])
expect(found.markers).toEqual(['@Comment 当前用户', '@Comment 其余参数', '@Comment 拥有者', '@Comment 上限', '@Comment 管理员', '@Comment 名称 '])
})
it('单独成行的注释、行尾没有名字的记号、函数名那一行的记号不算(函数、类用装饰器);那一行第一个能带记号的定义拿到它', async () => {
const source = [
'# @Comment 单独一行',
'x = 1',
'print(x) # @Comment 没有定义',
'def f(a, b): # @Comment 甲',
' pass',
].join('\n')
const found = await names(source)
expect(found.names).toEqual(['4:a → 甲'])
expect(found.markers).toEqual(['@Comment 甲'])
})
})
describe('python 中文名:函数 docstring 里的参数说明(没有明写的记号时)', () => {
it('谷歌写法 Args: 一节、Sphinx 写法 :param:取第一小句,不超过 12 个字才算', async () => {
const source = [
'def create(user_id, amount, note, *items, **extra):',
' """建订单。',
'',
' Args:',
' user_id (int): 下单用户的编号,从 token 里取',
' amount: 金额。单位是分',
' note: 这是一段特别特别特别长的备注说明文字',
' 续行不算一项',
' *items: 商品; 可以有多个',
'',
' Returns:',
' extra: 不在 Args 里',
' """',
'',
'def update(order_id, status):',
' """:param order_id: 订单编号,必填',
' :param str status: 新状态',
' """',
].join('\n')
expect((await names(source)).names).toEqual([
'1:user_id → 下单用户的编号',
'1:amount → 金额',
'1:items → 商品',
'15:order_id → 订单编号',
'15:status → 新状态',
])
})
it('明写的记号优先:Annotated / 行尾记号盖过 docstring;同一个参数两种记号都写了取 Annotated,行尾那个照样算记号(显示说明时去掉)', async () => {
const source = [
'def get(',
' a: Annotated[int, Comment("甲")], # @Comment 乙',
' b: int, # @Comment 丙',
' c: int,',
'):',
' """Args:',
' a: 丁',
' b: 戊',
' c: 己',
' """',
].join('\n')
const found = await names(source)
expect(found.names).toEqual(['2:a → 甲', '3:b → 丙', '4:c → 己'])
expect(found.markers).toEqual(['@Comment 乙', '@Comment 丙'])
})
it('docstringParameters:名字去掉星号,第一个说明为准,段落结束就不再认', () => {
expect([...docstringParameters('Args:\n x: 甲\n x: 乙\nNotes:\n y: 丙\n')]).toEqual([['x', '甲']])
})
})
describe('python 中文名:预筛和工作区里的文件', () => {
it('写了 Comment、Args:、:param 的源码才解析;文件是所有 .py', () => {
const reader = py.displayNames!
expect(reader.files).toEqual({ python: '**/*.py' })
expect(reader.mentions('x = 1 # @Comment 数量')).toBe(true)
expect(reader.mentions('"""\nArgs:\n x: 甲\n"""')).toBe(true)
expect(reader.mentions(':param x: 甲')).toBe(true)
expect(reader.mentions('x = 1\n')).toBe(false)
})
})
@@ -0,0 +1,189 @@
import type { DisplayNameFacts, DisplayNameReader, Node } from '../contract'
import type { NameCandidate } from './displayNames'
import { COMMENT_MARK, displayNameFacts, firstClause } from './displayNames'
import { children, named } from './shared'
/** 名字的最后一节:`Comment`、`typing.Annotated` 的 `Annotated`。 */
function lastName(node: Node | null | undefined): string | undefined {
if (node?.type === 'identifier')
return node.text
return node?.type === 'attribute' ? node.childForFieldName('attribute')?.text : undefined
}
/** 普通字符串字面量的内容(不带引号、前缀);f-string 这类带插值的没有。 */
function plainString(node: Node | undefined): string | undefined {
if (node?.type !== 'string' || named(node).some(part => part.type === 'interpolation'))
return undefined
const start = named(node).find(part => part.type === 'string_start')
const end = named(node).find(part => part.type === 'string_end')
const text = start && end ? node.text.slice(start.endIndex - node.startIndex, end.startIndex - node.startIndex).trim() : ''
return text || undefined
}
/** `Comment("当前用户")`(调用的名字最后一节是 Comment,不管从哪导入、是不是本地定义的)的第一个参数。 */
function commentCall(node: Node | undefined): string | undefined {
if (node?.type !== 'call' || lastName(node.childForFieldName('function')) !== COMMENT_MARK)
return undefined
return plainString(named(node.childForFieldName('arguments'))[0])
}
/** 注解 `Annotated[T, …, Comment("…")]`(`typing.Annotated` 也算)里的中文名。 */
function annotatedName(annotation: Node | null): string | undefined {
const inner = named(annotation)[0]
let head: Node | undefined
let items: Node[] = []
if (inner?.type === 'generic_type') {
head = named(inner)[0]
items = named(named(inner).find(part => part.type === 'type_parameter')).map(item => item.type === 'type' ? named(item)[0] : item)
}
else if (inner?.type === 'subscript') {
head = inner.childForFieldName('value') ?? undefined
items = inner.childrenForFieldName('subscript').filter((item): item is Node => item !== null)
}
if (lastName(head) !== 'Annotated')
return undefined
return items.slice(1).map(commentCall).find(text => text !== undefined)
}
/** 装饰器 `@Comment("…")` 给的中文名。 */
function decoratorName(definition: Node): string | undefined {
const outer = definition.parent?.type === 'decorated_definition' ? definition.parent : undefined
return named(outer).filter(child => child.type === 'decorator').map(decorator => commentCall(named(decorator)[0])).find(text => text !== undefined)
}
/** 参数节点里定义的那个名字:`x`、`x: int`、`x=1`、`*args`、`**kw`;`*`、`/` 分隔符没有。 */
function parameterName(param: Node): Node | undefined {
switch (param.type) {
case 'identifier':
return param
case 'typed_parameter':
return parameterName(named(param)[0])
case 'default_parameter':
case 'typed_default_parameter':
return param.childForFieldName('name') ?? undefined
case 'list_splat_pattern':
case 'dictionary_splat_pattern':
return named(param).find(child => child.type === 'identifier')
default:
return undefined
}
}
/**
* docstring 里各参数的说明取第一小句:Google 写法 `Args:` 一节里的 `name (type): 说明`,Sphinx 写法 `:param name: 说明`
* (`:param type name:` 也算)。名字去掉 `*` / `**`。
*/
export function docstringParameters(doc: string): Map<string, string> {
const found = new Map<string, string>()
const put = (name: string, text: string): void => {
const clause = firstClause(text)
if (clause && !found.has(name))
found.set(name, clause)
}
const lines = doc.split('\n')
const indentOf = (line: string): number => line.length - line.trimStart().length
for (let i = 0; i < lines.length; i++) {
const sphinx = /^\s*:param\s+(?:[^\s:]+\s+)?\*{0,2}(\w+)\s*:(.*)$/.exec(lines[i])
if (sphinx) {
put(sphinx[1], sphinx[2])
continue
}
if (!/^\s*(?:Args|Arguments):\s*$/.test(lines[i]))
continue
const sectionIndent = indentOf(lines[i])
let entryIndent: number | undefined
for (i++; i < lines.length; i++) {
const line = lines[i]
if (line.trim() === '')
continue
const indent = indentOf(line)
if (indent <= sectionIndent) {
i--
break
}
entryIndent ??= indent
const entry = indent === entryIndent ? /^\s*\*{0,2}(\w+)(?:\s*\([^)]*\))?\s*:(.*)$/.exec(line) : null
if (entry)
put(entry[1], entry[2])
}
}
return found
}
/** 函数主体开头的 docstring 的原文。 */
function docstringOf(fn: Node): string | undefined {
const first = named(fn.childForFieldName('body')).find(child => child.type !== 'comment')
const [expression, ...rest] = first?.type === 'expression_statement' ? named(first) : []
return rest.length === 0 ? plainString(expression) : undefined
}
function functionCandidates(fn: Node): NameCandidate[] {
const name = fn.childForFieldName('name')
const doc = docstringOf(fn)
const described = doc === undefined ? new Map<string, string>() : docstringParameters(doc)
const params = named(fn.childForFieldName('parameters')).flatMap((param): NameCandidate[] => {
const paramName = parameterName(param)
if (!paramName)
return []
return [{ name: paramName, marked: annotatedName(param.childForFieldName('type')), described: described.get(paramName.text), trailing: true }]
})
return [...(name ? [{ name, marked: decoratorName(fn) }] : []), ...params]
}
/** 赋值里定义的名字:`x = …`、`x: T = …`、只写注解的 `x: T`、`self.x = …` 的 x、解构的各个名字。 */
function targetNames(target: Node | null): Node[] {
if (!target)
return []
if (target.type === 'identifier')
return [target]
if (target.type === 'attribute') {
const attribute = target.childForFieldName('attribute')
return attribute ? [attribute] : []
}
if (['pattern_list', 'tuple_pattern', 'list_pattern', 'list_splat_pattern'].includes(target.type))
return named(target).flatMap(targetNames)
return []
}
function assignmentCandidates(assignment: Node): NameCandidate[] {
const marked = annotatedName(assignment.childForFieldName('type'))
return targetNames(assignment.childForFieldName('left')).map(name => ({ name, marked, trailing: true }))
}
/**
* Python 的中文名:`Annotated[T, Comment("…")]`(参数、变量、类字段)、`@Comment("…")`(函数、类)、
* 定义那一行行尾的 `# @Comment 名字`(多行参数表里的参数、变量、字段、枚举成员),都没有时参数取函数 docstring 里它的说明。
*/
export function pythonDisplayNames(root: Node, source: string): DisplayNameFacts {
const candidates: NameCandidate[] = []
const comments: Node[] = []
const visit = (node: Node): void => {
switch (node.type) {
case 'comment':
comments.push(node)
return
case 'function_definition':
candidates.push(...functionCandidates(node))
break
case 'class_definition': {
const name = node.childForFieldName('name')
if (name)
candidates.push({ name, marked: decoratorName(node) })
break
}
case 'assignment':
candidates.push(...assignmentCandidates(node))
break
}
for (const child of children(node))
visit(child)
}
visit(root)
return displayNameFacts(candidates, comments, source)
}
export const pythonNameReader: DisplayNameReader = {
files: { python: '**/*.py' },
mentions: source => source.includes(COMMENT_MARK) || /\b(?:Args|Arguments):|:param\s/.test(source),
find: (root, source) => pythonDisplayNames(root, source),
}
@@ -2,6 +2,7 @@ import type { Access, BaseFacts, BlockShape, ClassFacts, CodeBlock, DeclaratorFa
import type { CommentRules } from './shared'
import * as path from 'node:path'
import { children, compoundShape, extendHeader, headerSpan, leafBlock, leafShape, listBlocks, named, nodeSpan, sameNode, syntaxChildren } from './shared'
import { typescriptNameReader } from './typescriptNames'
const GRAMMARS: Record<string, string> = {
typescript: 'tree-sitter-typescript.wasm',
@@ -585,5 +586,6 @@ export function createTypeScriptAdapter(wasmDir: string): LanguageAdapter {
},
toBlocks: (root, _source, languageId) => blockReader(TYPED_LANGUAGES.has(languageId))(root),
tokenRole: typescriptTokenRole,
displayNames: typescriptNameReader,
}
}
@@ -0,0 +1,145 @@
import { describe, expect, it } from 'vitest'
import { runtime, WASM_DIR } from '../testSupport'
import { createTypeScriptAdapter } from './typescript'
import { jsdocParameters } from './typescriptNames'
const ts = createTypeScriptAdapter(WASM_DIR)
/** 认出的中文名写成「行号:原名 → 中文名」(行号 1 起),记号写成源码原文。 */
async function names(source: string, languageId = 'typescript'): Promise<{ names: string[], markers: string[] }> {
const tree = await runtime.parse(ts, languageId, source)
try {
const facts = ts.displayNames!.find(tree.rootNode, source, languageId)
const row = (index: number): number => source.slice(0, index).split('\n').length
return {
names: facts.names.map(({ name, text }) => `${row(name.startIndex)}:${source.slice(name.startIndex, name.endIndex)} → ${text}`),
markers: facts.markers.map(({ startIndex, endIndex }) => source.slice(startIndex, endIndex)),
}
}
finally {
tree.delete()
}
}
describe('typescript 中文名:@Comment 装饰器', () => {
it('类(@d export class 也算)、方法、字段、构造函数和方法的参数;调用的名字最后一节是 Comment 就算', async () => {
const source = [
'@Comment(\'服务\')',
'export class UserService {',
' @Comment(\'仓库\') repo = new Repo()',
' constructor(@Comment(\'配置\') private readonly config: Config, other: number) {}',
' @decorators.Comment(`取用户`)',
' @Get()',
' find(@Comment("编号") id: string) {}',
' @Other(\'不是\') plain() {}',
'}',
].join('\n')
expect((await names(source)).names).toEqual(['2:UserService → 服务', '3:repo → 仓库', '4:config → 配置', '7:find → 取用户', '7:id → 编号'])
})
})
describe('typescript 中文名:行尾的 // @Comment 名字', () => {
it('参数、变量、枚举成员、类字段、接口属性;块注释、单独成行的注释不算;函数那一行给那一行的参数(函数、类用装饰器)', async () => {
const source = [
'function remove(',
' itemId: number, // @Comment 编号',
' user?: User, // 当前登录的 @Comment 当前用户',
') {}',
'const total = items.map(item => item.price) // @Comment 总价',
'enum Role {',
' ADMIN = \'admin\', // @Comment 管理员',
' USER, // @Comment 普通用户',
'}',
'class Cart {',
' count = 0 // @Comment 件数',
'}',
'interface Page {',
' size: number // @Comment 每页条数',
'}',
'let a = 1 /* @Comment 块注释不算 */',
'// @Comment 单独一行',
'let b = 2',
'function f(x: number) { // @Comment 函数那一行',
'}',
].join('\n')
const found = await names(source)
expect(found.names).toEqual([
'2:itemId → 编号',
'3:user → 当前用户',
'5:total → 总价',
'7:ADMIN → 管理员',
'8:USER → 普通用户',
'11:count → 件数',
'14:size → 每页条数',
'19:x → 函数那一行',
])
expect(found.markers).toEqual(['@Comment 编号', '@Comment 当前用户', '@Comment 总价', '@Comment 管理员', '@Comment 普通用户', '@Comment 件数', '@Comment 每页条数', '@Comment 函数那一行'])
})
})
describe('typescript 中文名:JSDoc 的 @param(没有明写的记号时)', () => {
it('export 的函数、方法(隔着装饰器)、值是箭头函数的变量、类字段;{类型}、短横、[可选] 都认,取第一小句、不超过 12 个字', async () => {
const source = [
'/**',
' * 删一项。',
' * @param {number} itemId - 要删的编号,从路径取',
' * @param [force=false] 强制删除',
' * @param opts.x 属性不算',
' * @param note 一段特别特别特别长的备注说明文字',
' */',
'export function remove(itemId: number, force = false, note = \'\') {}',
'class Api {',
' /** @param id 编号 */',
' @Get()',
' load(id: string) {}',
' /** @param e 事件 */',
' onClick = (e: Event) => {}',
'}',
'/** @param a 甲 */',
'const add = (a: number) => a',
'/** @param a 丁 */',
'const other = 1',
'function g(a: number) {}',
].join('\n')
expect((await names(source)).names).toEqual([
'8:itemId → 要删的编号',
'8:force → 强制删除',
'12:id → 编号',
'14:e → 事件',
'17:a → 甲',
])
})
it('明写的记号优先:装饰器、行尾记号盖过 JSDoc;JavaScript 的默认值参数也认', async () => {
const source = [
'/**',
' * @param a 甲',
' * @param b 乙',
' */',
'function f(',
' a, // @Comment 丙',
' b = 1,',
') {}',
].join('\n')
expect((await names(source, 'javascript')).names).toEqual(['6:a → 丙', '7:b → 乙'])
})
it('jsdocParameters:同名的取第一个', () => {
expect([...jsdocParameters('@param x 甲\n@param x 乙')]).toEqual([['x', '甲']])
})
})
describe('typescript 中文名:预筛和工作区里的文件', () => {
it('写了 Comment、@param 的源码才解析;四种方言各自的文件', () => {
const reader = ts.displayNames!
expect(reader.files).toEqual({
typescript: '**/*.{ts,mts,cts}',
typescriptreact: '**/*.tsx',
javascript: '**/*.{js,mjs,cjs}',
javascriptreact: '**/*.jsx',
})
expect(reader.mentions('const a = 1 // @Comment 甲')).toBe(true)
expect(reader.mentions('/** @param a 甲 */')).toBe(true)
expect(reader.mentions('const a = 1')).toBe(false)
})
})
@@ -0,0 +1,172 @@
import type { DisplayNameFacts, DisplayNameReader, Node } from '../contract'
import type { NameCandidate } from './displayNames'
import { COMMENT_MARK, displayNameFacts, firstClause } from './displayNames'
import { children, named } from './shared'
const DECLARED_FUNCTIONS = new Set(['function_declaration', 'generator_function_declaration', 'function_signature'])
const METHODS = new Set(['method_definition', 'method_signature', 'abstract_method_signature'])
const FUNCTION_VALUES = new Set(['function_expression', 'function', 'generator_function', 'arrow_function'])
const CLASSES = new Set(['class_declaration', 'abstract_class_declaration', 'class'])
const DECLARATIONS = new Set(['interface_declaration', 'type_alias_declaration', 'enum_declaration'])
const PROPERTIES = new Set(['public_field_definition', 'property_signature'])
const NAMES = new Set(['identifier', 'type_identifier', 'property_identifier', 'private_property_identifier'])
/** 字符串字面量的内容(不带引号);带 `${}` 的模板字符串没有。 */
function plainString(node: Node | undefined): string | undefined {
if (!node || (node.type !== 'string' && node.type !== 'template_string') || named(node).some(part => part.type === 'template_substitution'))
return undefined
return node.text.slice(1, -1).trim() || undefined
}
/** `@Comment('…')`(调用的名字最后一节是 Comment)给的中文名。 */
function decoratorName(decorators: readonly Node[]): string | undefined {
for (const decorator of decorators) {
const call = named(decorator)[0]
const callee = call?.type === 'call_expression' ? call.childForFieldName('function') : null
const last = callee?.type === 'member_expression' ? callee.childForFieldName('property') : callee
if (last?.text === COMMENT_MARK) {
const text = plainString(named(call?.childForFieldName('arguments'))[0])
if (text)
return text
}
}
return undefined
}
/** 节点自己带的装饰器(参数、字段、类声明)。 */
function ownDecorators(node: Node | null | undefined): Node[] {
return named(node).filter(child => child.type === 'decorator')
}
/** 类主体里排在成员前面的装饰器(方法的装饰器是它前面的兄弟节点)。 */
function precedingDecorators(member: Node): Node[] {
const found: Node[] = []
for (let prev = member.previousNamedSibling; prev?.type === 'decorator'; prev = prev.previousNamedSibling)
found.push(prev)
return found
}
/** 声明外面包着的 export 语句(没有就是它自己):JSDoc、`@d export class` 的装饰器挂在它上面。 */
function outerStatement(node: Node): Node {
return node.parent?.type === 'export_statement' ? node.parent : node
}
/** 紧贴在前面的 JSDoc(中间隔着装饰器也算)的原文,去掉开头结尾的注释符号和每行开头的星号。 */
function jsdocBefore(node: Node): string | undefined {
let prev = node.previousNamedSibling
while (prev?.type === 'decorator')
prev = prev.previousNamedSibling
if (prev?.type !== 'comment' || !prev.text.startsWith('/**') || prev.text.startsWith('/**/'))
return undefined
return prev.text.replace(/^\/\*+/, '').replace(/\*+\/$/, '').split('\n').map(line => line.replace(/^\s*\*(?!\/)\s?/, '')).join('\n')
}
/** JSDoc 里各参数 `@param [{类型}] 名字 [-] 说明` 的说明取第一小句;`[名字]`、`[名字=默认]` 也算,`opts.x` 这种属性不算。 */
export function jsdocParameters(doc: string): Map<string, string> {
const found = new Map<string, string>()
for (const line of doc.split('\n')) {
const at = line.indexOf('@param')
let rest = at < 0 ? '' : line.slice(at + '@param'.length).trimStart()
if (rest.startsWith('{'))
rest = rest.includes('}') ? rest.slice(rest.indexOf('}') + 1).trimStart() : ''
const token = rest.split(/\s/, 1)[0]
const name = token.replace(/^\[/, '').replace(/\]$/, '').split('=')[0]
const clause = /^[A-Z_$][\w$]*$/i.test(name) ? firstClause(rest.slice(token.length).trim().replace(/^-\s*/, '')) : undefined
if (clause && !found.has(name))
found.set(name, clause)
}
return found
}
/** 函数的 JSDoc 挂在哪:声明(连 export)、方法本身、值是这个函数的变量声明(连 export)或类字段。 */
function functionDoc(fn: Node): string | undefined {
if (METHODS.has(fn.type))
return jsdocBefore(fn)
if (DECLARED_FUNCTIONS.has(fn.type))
return jsdocBefore(outerStatement(fn))
const holder = fn.parent
if (holder?.type === 'public_field_definition')
return jsdocBefore(holder)
const declaration = holder?.type === 'variable_declarator' ? holder.parent : null
return declaration ? jsdocBefore(outerStatement(declaration)) : undefined
}
/** 参数节点里定义的那个名字:`x`、`x?: T`、`x = 1`(JavaScript)、不带括号的箭头函数参数;解构的不算。 */
function parameterName(param: Node): Node | undefined {
switch (param.type) {
case 'identifier':
return param
case 'required_parameter':
case 'optional_parameter': {
const pattern = param.childForFieldName('pattern')
return pattern?.type === 'identifier' ? pattern : undefined
}
case 'assignment_pattern': {
const left = param.childForFieldName('left')
return left?.type === 'identifier' ? left : undefined
}
default:
return undefined
}
}
function functionCandidates(fn: Node): NameCandidate[] {
const doc = functionDoc(fn)
const described = doc === undefined ? new Map<string, string>() : jsdocParameters(doc)
const single = fn.childForFieldName('parameter')
const params = (single ? [single] : named(fn.childForFieldName('parameters'))).flatMap((param): NameCandidate[] => {
const name = parameterName(param)
return name ? [{ name, marked: decoratorName(ownDecorators(param)), described: described.get(name.text), trailing: true }] : []
})
const name = FUNCTION_VALUES.has(fn.type) ? null : fn.childForFieldName('name')
const own = name ? [{ name, marked: METHODS.has(fn.type) ? decoratorName(precedingDecorators(fn)) : undefined }] : []
return [...own, ...params]
}
/**
* TypeScript / JavaScript 的中文名:`@Comment('…')` 装饰器(类、方法、字段、方法和构造函数的参数)、
* 定义那一行行尾的 `// @Comment 名字`(参数、变量、枚举成员、属性),都没有时参数取 JSDoc 里 `@param` 的说明。
*/
export function typescriptDisplayNames(root: Node, source: string): DisplayNameFacts {
const candidates: NameCandidate[] = []
const comments: Node[] = []
const nameOf = (node: Node): Node | undefined => {
const name = node.childForFieldName('name')
return name && NAMES.has(name.type) ? name : undefined
}
const visit = (node: Node): void => {
const name = nameOf(node)
if (node.type === 'comment') {
if (node.text.startsWith('//'))
comments.push(node)
return
}
if (DECLARED_FUNCTIONS.has(node.type) || METHODS.has(node.type) || FUNCTION_VALUES.has(node.type))
candidates.push(...functionCandidates(node))
else if (CLASSES.has(node.type) && name)
candidates.push({ name, marked: decoratorName([...ownDecorators(node), ...ownDecorators(outerStatement(node) === node ? undefined : node.parent)]) })
else if (PROPERTIES.has(node.type) && name)
candidates.push({ name, marked: decoratorName(ownDecorators(node)), trailing: true })
else if ((node.type === 'variable_declarator' || node.type === 'enum_assignment') && name)
candidates.push({ name, trailing: true })
else if (DECLARATIONS.has(node.type) && name)
candidates.push({ name })
else if (node.type === 'property_identifier' && node.parent?.type === 'enum_body')
candidates.push({ name: node, trailing: true })
for (const child of children(node))
visit(child)
}
visit(root)
return displayNameFacts(candidates, comments, source)
}
export const typescriptNameReader: DisplayNameReader = {
files: {
typescript: '**/*.{ts,mts,cts}',
typescriptreact: '**/*.tsx',
javascript: '**/*.{js,mjs,cjs}',
javascriptreact: '**/*.jsx',
},
mentions: source => source.includes(COMMENT_MARK) || source.includes('@param'),
find: (root, source) => typescriptDisplayNames(root, source),
}
+1 -1
View File
@@ -49,7 +49,7 @@ const throwing: LibraryRule = {
async function views(source: string, rules: LibraryRule[], reports: string[] = []): Promise<BlockView[]> {
const run: RuleRun = { rules, report: message => reports.push(message) }
return withBlocks(py, 'python', source, blocks => toBlockViews(blocks, source, py, pythonZhTranslator, run))
return withBlocks(py, 'python', source, blocks => toBlockViews(blocks, { source, language: py, markers: [] }, pythonZhTranslator, run))
}
function flatten(list: BlockView[]): BlockView[] {
@@ -0,0 +1,250 @@
import type { BlockView } from './blockView'
import type { DefinitionStart, IndexedName, NameSources, ShownName, TextRange } from './nameResolution'
import { describe, expect, it } from 'vitest'
import { readBlockViews } from './blockView'
import { createPythonAdapter } from './languages/python'
import { createTypeScriptAdapter } from './languages/typescript'
import { displayNameIndex, DocumentNames, indexedNames, renderedNames, resolveNames, shiftNames } from './nameResolution'
import { enumRule } from './rules/enum'
import { MemoryIndexHost, propertyLines, runtime, WASM_DIR } from './testSupport'
import { pythonZhTranslator } from './translators/pythonZh'
import { typescriptZhTranslator } from './translators/typescriptZh'
import { WorkspaceIndexes } from './workspaceIndex'
const py = createPythonAdapter(WASM_DIR)
const ts = createTypeScriptAdapter(WASM_DIR)
const ROUTES = [
'async def delete_item(',
' item_id: int,',
' current_user: User = Depends(get_current_user), # @Comment 当前用户',
'):',
' if current_user.role != UserRole.ADMIN:',
' raise Forbidden()',
'',
].join('\n')
function read(source: string): ReturnType<typeof readBlockViews> {
return readBlockViews(runtime, py, pythonZhTranslator, 'python', source, { rules: [enumRule], report: () => {} })
}
function allProperties(blocks: readonly BlockView[]): string[] {
return blocks.flatMap(block => [...('properties' in block ? propertyLines(block.properties) : []), ...allProperties(block.children)])
}
/** 源码里第 n 个 text 的下标 */
function at(source: string, text: string, n = 0): number {
let index = -1
for (let i = 0; i <= n; i++)
index = source.indexOf(text, index + 1)
return index
}
describe('中文名记号不是说明文字:宿主把语言插件认出的记号交给翻译插件去掉', () => {
it('python 枚举成员的行尾注释:只有记号时没有说明,记号前面的字照常是说明;readBlockViews 交出这份源码起了中文名的定义', async () => {
const source = [
'import enum',
'class Role(str, enum.Enum):',
' ADMIN = "admin" # 管理员账号 @Comment 管理员',
' MEMBER = "member" # @Comment 普通成员',
' GUEST = "guest" # 游客',
].join('\n')
const { blocks, displayNames } = await read(source)
expect(allProperties(blocks).filter(line => line.startsWith(' ') && line.includes('|'))).toEqual([
' ADMIN | "admin" | 管理员账号',
' MEMBER | "member" | ',
' GUEST | "guest" | 游客',
])
expect(displayNames.map(({ name, text }) => `${source.slice(name.startIndex, name.endIndex)} → ${text}`)).toEqual(['ADMIN → 管理员', 'MEMBER → 普通成员'])
})
it('typescript 枚举成员同样去掉;一个成员都没有说明时没有说明这一列', async () => {
const source = 'export enum Role {\n ADMIN = \'admin\', // @Comment 管理员\n USER = \'user\', // @Comment 普通用户\n}\n'
const { blocks } = await readBlockViews(runtime, ts, typescriptZhTranslator, 'typescript', source)
expect(allProperties(blocks).filter(line => line.startsWith(' '))).toEqual([' ADMIN | \'admin\'', ' USER | \'user\''])
})
})
/** 假的语言扩展:各处出现按名字文字认(测试里的名字都不重名),定义按表查。 */
function fakeSources(source: string, options: {
highlights?: Record<number, TextRange[]>
definitions?: Record<number, DefinitionStart[]>
indexed?: Record<string, IndexedName[]>
asked?: string[]
} = {}): NameSources {
return {
uri: 'file:///w/routes.py',
occurrences: async offset => options.highlights?.[offset] ?? [],
definitions: async (offset) => {
options.asked?.push(`definition ${offset}`)
return options.definitions?.[offset] ?? []
},
indexed: async (name) => {
options.asked?.push(`indexed ${name}`)
return options.indexed?.[name] ?? []
},
}
}
describe('中文名落到哪些名字上', () => {
it('renderedNames:一行译文、属性表的值和各层子行里带下标的名字,下标 → 原文', async () => {
const { blocks } = await read(ROUTES)
const rendered = renderedNames(blocks)
expect(rendered.get(at(ROUTES, 'current_user'))).toBe('current_user')
expect(rendered.get(at(ROUTES, 'current_user.role'))).toBe('current_user')
expect(rendered.get(at(ROUTES, 'ADMIN'))).toBe('ADMIN')
expect(rendered.get(at(ROUTES, 'delete_item'))).toBe('delete_item')
})
it('本文件定义的:定义处问各处出现,落在画出来的名字上的都换,定义处本身标 definition;长度对不上、没画出来的不换', async () => {
const { blocks, displayNames } = await read(ROUTES)
const definition = at(ROUTES, 'current_user')
const use = at(ROUTES, 'current_user.role')
const sources = fakeSources(ROUTES, { highlights: { [definition]: [{ start: definition, end: definition + 12 }, { start: use, end: use + 12 }, { start: 3, end: 5 }, { start: at(ROUTES, 'ADMIN'), end: at(ROUTES, 'ADMIN') + 2 }] } })
const reports: [ShownName[], boolean][] = []
const settled = await resolveNames(displayNames, blocks, sources, (names, final) => reports.push([names, final]))
expect(settled).toBe(true)
const expected = [
{ offset: definition, length: 12, name: '当前用户', definition: true },
{ offset: use, length: 12, name: '当前用户' },
]
expect(reports).toEqual([[expected, false], [expected, true]])
})
it('别的文件定义的:原名在索引里(别的文件)的名字才问定义,定义处正好是索引里那个定义才换', async () => {
const { blocks } = await read(ROUTES)
const admin = at(ROUTES, 'ADMIN')
const role = at(ROUTES, 'role')
const asked: string[] = []
const models = 'file:///w/models.py'
const sources = fakeSources(ROUTES, {
asked,
indexed: {
ADMIN: [{ location: { uri: models, start: { line: 6, character: 4 }, end: { line: 6, character: 9 } }, text: '管理员' }],
role: [{ location: { uri: models, start: { line: 11, character: 4 }, end: { line: 11, character: 8 } }, text: '角色' }],
User: [{ location: { uri: 'file:///w/routes.py', start: { line: 0, character: 0 }, end: { line: 0, character: 4 } }, text: '本文件的不算' }],
},
definitions: {
[admin]: [{ uri: models, line: 6, character: 4 }],
// 定义提供者说 role 在别处(同名的另一个定义)
[role]: [{ uri: models, line: 20, character: 4 }],
},
})
const reports: ShownName[][] = []
const settled = await resolveNames([], blocks, sources, names => reports.push(names))
expect(settled).toBe(true)
expect(reports.at(-1)).toEqual([{ offset: admin, length: 5, name: '管理员' }])
expect(asked.filter(line => line.startsWith('definition')).sort()).toEqual([`definition ${admin}`, `definition ${role}`].sort())
})
it('语言扩展还没起来:定义处自己都查不到出现,或要问的定义一个都没答,算没答完(过一会儿再查)', async () => {
const { blocks, displayNames } = await read(ROUTES)
expect(await resolveNames(displayNames, blocks, fakeSources(ROUTES), () => {})).toBe(false)
const admin = at(ROUTES, 'ADMIN')
const indexed = { ADMIN: [{ location: { uri: 'file:///w/models.py', start: { line: 6, character: 4 }, end: { line: 6, character: 9 } }, text: '管理员' }] }
expect(await resolveNames([], blocks, fakeSources(ROUTES, { indexed }), () => {})).toBe(false)
expect(await resolveNames([], blocks, fakeSources(ROUTES, { indexed, definitions: { [admin]: [{ uri: 'file:///w/other.py', line: 0, character: 0 }] } }), () => {})).toBe(true)
})
})
describe('中文名跟着文档版本走', () => {
const name = (offset: number, text = '甲', length = 3): ShownName => ({ offset, length, name: text })
it('shiftNames:改动在前面的往后挪,碰到名字的丢掉,后面的不动;紧贴在名字前面插字也往后挪', () => {
const names = [name(10), name(20), name(30)]
expect(shiftNames(names, [{ offset: 0, length: 0, text: 'xx' }])).toEqual([name(12), name(22), name(32)])
expect(shiftNames(names, [{ offset: 21, length: 1, text: '' }])).toEqual([name(10), name(29)])
expect(shiftNames(names, [{ offset: 33, length: 5, text: '' }, { offset: 10, length: 0, text: 'y' }])).toEqual([name(11), name(21), name(31)])
})
it('每个版本查一次;手上没有时查到一步交一步,有上一版挪过来的就等查完再换;查的时候文档又改了,查到的丢掉', async () => {
const delivered: string[] = []
const names = new DocumentNames((version, list) => delivered.push(`${version}:${list.map(item => item.offset).join(',')}`), [])
let runs = 0
let finish: (() => void) | undefined
const run = (list: ShownName[]) => async (report: (names: ShownName[], final: boolean) => void): Promise<boolean> => {
runs++
report(list.slice(0, 1), false)
await new Promise<void>((resolve) => {
finish = resolve
})
report(list, true)
return true
}
names.resolve(1, run([name(10), name(20)]))
names.resolve(1, run([]))
expect(runs).toBe(1)
finish?.()
await Promise.resolve()
await Promise.resolve()
expect(delivered).toEqual(['1:10', '1:10,20'])
expect(names.known(1)).toEqual([name(10), name(20)])
names.edited(2, [{ offset: 0, length: 0, text: 'x' }])
expect(names.known(2)).toEqual([name(11), name(21)])
expect(names.known(1)).toEqual([])
names.resolve(2, run([name(11), name(21), name(40)]))
// 版本 2 查到一半文档又改了:查到的丢掉,版本 3 先用挪过的
names.edited(3, [{ offset: 0, length: 1, text: '' }])
finish?.()
await Promise.resolve()
await Promise.resolve()
expect(delivered).toEqual(['1:10', '1:10,20'])
expect(names.known(3)).toEqual([name(10), name(20)])
})
it('没答完隔一会儿再查,按给的间隔、最多那么多次;答完了就不再查', async () => {
const scheduled: number[] = []
const pending: (() => void)[] = []
const names = new DocumentNames(() => {}, [5, 10], (task, ms) => {
scheduled.push(ms)
pending.push(task)
return setTimeout(() => {}, 0)
})
let runs = 0
names.resolve(1, async () => {
runs++
return false
})
for (let i = 0; i < 4; i++) {
await Promise.resolve()
pending.shift()?.()
}
await Promise.resolve()
expect(runs).toBe(3)
expect(scheduled).toEqual([5, 10])
names.dispose()
})
})
describe('工作区里的中文名索引', () => {
it('python:别的文件里起了中文名的定义按原名查,给出定义处名字的位置和中文名;没写中文名的文件不解析', async () => {
const host = new MemoryIndexHost({
'app/models.py': 'import enum\n\n\nclass UserRole(str, enum.Enum):\n """角色。"""\n\n ADMIN = "admin" # @Comment 管理员\n',
'app/plain.py': 'ADMIN = 1\n',
}, py)
const indexes = new WorkspaceIndexes(host)
const found = await indexedNames((index, key) => indexes.find(index, key), py, 'ADMIN')
expect(found).toEqual([{ location: { uri: 'file:///w/app/models.py', start: { line: 6, character: 4 }, end: { line: 6, character: 9 } }, text: '管理员' }])
expect(await indexedNames((index, key) => indexes.find(index, key), py, 'UserRole')).toEqual([])
expect(host.reads.sort()).toEqual(['app/models.py', 'app/plain.py'])
indexes.dispose()
})
it('一种 languageId 一份索引;语言插件不认中文名、不管这种文件时没有', () => {
expect(displayNameIndex(py, 'python')?.id).toBe('displayNames:python')
expect(displayNameIndex(ts, 'typescriptreact')?.include).toBe('**/*.tsx')
expect(displayNameIndex(py, 'typescript')).toBeUndefined()
const { displayNames: _, ...plain } = py
expect(displayNameIndex(plain, 'python')).toBeUndefined()
})
it('查到的项带着认的时候给的中文名,库规则的 lookup 照旧只给位置', async () => {
const host = new MemoryIndexHost({ 'm.py': 'LIMIT = 1 # @Comment 上限\n' }, py)
const indexes = new WorkspaceIndexes(host)
const index = displayNameIndex(py, 'python')!
expect((await indexes.find(index, 'LIMIT')).map(({ entry }) => entry.text)).toEqual(['上限'])
expect(await indexes.lookup(index, 'LIMIT')).toEqual([{ uri: 'file:///w/m.py', start: { line: 0, character: 0 }, end: { line: 0, character: 5 } }])
indexes.dispose()
})
})
+250
View File
@@ -0,0 +1,250 @@
import type { BlockView } from './blockView'
import type { DisplayName, IndexEntry, Inline, LanguageAdapter, PreviewText, PropertyItem, RuleLocation } from './contract'
import type { IndexedEntry, WorkspaceIndexOf } from './workspaceIndex'
/**
* 中文名落到预览里的哪些名字上(见「中文名」):语言插件只认出定义和它的中文名,
* 用到它的各处由语言扩展按语义认(同一文件问 document highlights,别的文件定义的问定义提供者),这里不碰 VS Code。
*/
/** 预览里显示成中文名的一个名字:源码下标 [offset, offset + length) 的名字显示成 name;definition 是定义处本身(名称格子、名称行后面淡淡写原名)。 */
export interface ShownName {
offset: number
length: number
name: string
definition?: true
}
/** 视图数据里照抄自源码、带下标的名字(一行译文、属性表的值、各层子行的格子、框里):下标 → 原文。 */
export function renderedNames(blocks: readonly BlockView[]): Map<number, string> {
const found = new Map<number, string>()
const inline = (pieces: readonly Inline[]): void => {
for (const piece of pieces) {
if (!('role' in piece))
inline(piece.segments)
else if (piece.offset !== undefined)
found.set(piece.offset, piece.text)
}
}
const text = (value: PreviewText): void => {
if (!('markdown' in value))
inline(value)
}
const items = (list: readonly PropertyItem[] | undefined): void => {
for (const item of list ?? []) {
item.cells.forEach(inline)
items(item.items)
}
}
const visit = (list: readonly BlockView[]): void => {
for (const block of list) {
if ('properties' in block) {
for (const row of block.properties) {
text(row.value)
items(row.items)
}
}
else {
text(block.translation)
}
visit(block.children)
}
}
visit(blocks)
return found
}
/** 源码里的一段 [start, end)。 */
export interface TextRange { start: number, end: number }
/** 定义处名字的起点(行列 0 起,列按 UTF-16)。 */
export interface DefinitionStart { uri: string, line: number, character: number }
/** 工作区索引里别的文件定义的一个中文名。 */
export interface IndexedName { location: RuleLocation, text: string }
/** 找各处要问的:这份文档的语言扩展、工作区索引。 */
export interface NameSources {
/** 这份文档的 uri */
uri: string
/** offset 处的名字在这份文档里的各处出现(document highlights) */
occurrences: (offset: number) => Promise<TextRange[]>
/** offset 处名字的定义(定义提供者) */
definitions: (offset: number) => Promise<DefinitionStart[]>
/** 工作区里定义的、原名叫 name 的中文名 */
indexed: (name: string) => Promise<IndexedName[]>
}
/**
* 找这份文档里显示成中文名的各处,分两步交出(先交本文件的,不等工作区索引建好;最后一次 final):
* 1. 本文件定义的:每个定义问一次它的各处出现,落在视图数据里带下标的名字上的显示成中文名,定义处本身标 definition;
* 2. 别的文件定义的:原名和工作区索引里某个中文名一样的名字才问定义提供者,定义处正是索引里那个定义才算。
* 返回语言扩展答没答(settled):定义处自己都查不到出现,或要问的定义一个都没答,是语言扩展还没起来,过一会儿再查。
*/
export async function resolveNames(
names: readonly DisplayName[],
blocks: readonly BlockView[],
sources: NameSources,
report: (shown: ShownName[], final: boolean) => void,
): Promise<boolean> {
const rendered = renderedNames(blocks)
const shown = new Map<number, ShownName>()
let settled = true
await Promise.all(names.map(async ({ name, text }) => {
const ranges = await sources.occurrences(name.startIndex)
if (!ranges.some(range => range.start === name.startIndex))
settled = false
for (const { start, end } of ranges) {
if (!shown.has(start) && rendered.get(start)?.length === end - start)
shown.set(start, { offset: start, length: end - start, name: text, ...(start === name.startIndex && { definition: true as const }) })
}
}))
report([...shown.values()], false)
const byName = new Map<string, number[]>()
for (const [offset, original] of rendered) {
if (!shown.has(offset))
byName.set(original, [...(byName.get(original) ?? []), offset])
}
let asked = 0
let answered = 0
await Promise.all([...byName].map(async ([original, offsets]) => {
const elsewhere = (await sources.indexed(original)).filter(entry => entry.location.uri !== sources.uri)
if (elsewhere.length === 0)
return
await Promise.all(offsets.map(async (offset) => {
asked++
const found = await sources.definitions(offset)
if (found.length > 0)
answered++
const entry = elsewhere.find(({ location }) => found.some(at => at.uri === location.uri && at.line === location.start.line && at.character === location.start.character))
if (entry)
shown.set(offset, { offset, length: original.length, name: entry.text })
}))
}))
if (asked > 0 && answered === 0)
settled = false
report([...shown.values()], true)
return settled
}
/** 中文名索引的一项:键是原名。 */
export interface NameEntry extends IndexEntry { text: string }
/** 一种语言的文件在整个工作区里的中文名索引(同一个 languageId 只建一份);语言插件不认中文名、不管这种文件时没有。 */
export function displayNameIndex(adapter: LanguageAdapter, languageId: string): WorkspaceIndexOf<NameEntry> | undefined {
const reader = adapter.displayNames
const include = reader?.files[languageId]
if (!reader || !include)
return undefined
return {
id: `displayNames:${languageId}`,
languageId,
include,
mentions: reader.mentions,
entries: (root, file) => reader.find(root, file.source, languageId).names.map(({ name, text }) => ({
key: file.source.slice(name.startIndex, name.endIndex),
name,
text,
})),
}
}
/** 这个语言插件管的各种文件里,原名叫 name 的中文名。 */
export async function indexedNames(
find: <E extends IndexEntry>(index: WorkspaceIndexOf<E>, key: string) => Promise<IndexedEntry<E>[]>,
adapter: LanguageAdapter,
name: string,
): Promise<IndexedName[]> {
const indexes = Object.keys(adapter.displayNames?.files ?? {}).flatMap(languageId => displayNameIndex(adapter, languageId) ?? [])
const found = await Promise.all(indexes.map(index => find(index, name)))
return found.flat().map(({ location, entry }) => ({ location, text: entry.text }))
}
/** 文档的一处改动(TextDocumentContentChangeEvent 的下标写法):从 offset 起 length 个字换成 text。 */
export interface TextChange { offset: number, length: number, text: string }
/**
* 中文名跟着改动挪位置:改动整个在名字前面的往后挪,整个在后面的不动,碰到名字的丢掉;一次的几处改动按顺序一个个套。
* 紧贴着名字插的字把名字改长了(`foo` → `xfoo`)也不会认错:页面只把中文名放在长度对得上的名字上。
*/
export function shiftNames(names: readonly ShownName[], changes: readonly TextChange[]): ShownName[] {
let current = [...names]
for (const { offset, length, text } of changes) {
const delta = text.length - length
current = current.flatMap((name) => {
if (offset + length <= name.offset)
return [{ ...name, offset: name.offset + delta }]
return offset >= name.offset + name.length ? [name] : []
})
}
return current
}
/** 语言扩展还没起来时隔多久再查(毫秒),查到最后一次还没答就算了 */
export const RETRY_DELAYS = [1000, 2000, 4000, 8000, 16000]
/**
* 一份打开着的文档的中文名:每个版本查一次(语言扩展还没起来时照 RETRY_DELAYS 隔一会儿再查),只交出眼下这个版本的;
* 文档改了,已知的中文名先跟着改动挪位置当新版本的用,新版本查完再换掉,免得每改一下预览里的中文名闪一下。
*/
export class DocumentNames {
private version: number | undefined
private names: ShownName[] = []
/** 眼下这个版本查过了(或正在查) */
private resolving: number | undefined
private timer: ReturnType<typeof setTimeout> | undefined
constructor(
private readonly deliver: (version: number, names: ShownName[]) => void,
private readonly delays: readonly number[] = RETRY_DELAYS,
private readonly schedule: (run: () => void, ms: number) => ReturnType<typeof setTimeout> = setTimeout,
) {}
/** version 这份文档已知的中文名(发文档时带上);不是眼下这个版本的没有。 */
known(version: number): ShownName[] {
return version === this.version ? this.names : []
}
/** 文档改成了 version:已知的中文名跟着改动挪位置,等新版本查完再换。 */
edited(version: number, changes: readonly TextChange[]): void {
clearTimeout(this.timer)
this.names = shiftNames(this.names, changes)
this.version = version
this.resolving = undefined
}
/**
* 发出去了 version 这份文档:查它的中文名;同一个版本只查一次(页面重新加载时照旧用查到的)。
* 手上还没有这个版本的中文名时查到一步交一步(先交本文件的),有(上一版挪过来的)就等查完一起换,免得少了的那些闪一下。
* 查的过程中文档又改了的,查到的丢掉。
*/
resolve(version: number, run: (report: (names: ShownName[], final: boolean) => void) => Promise<boolean>): void {
if (this.resolving === version)
return
clearTimeout(this.timer)
if (this.version !== version) {
this.version = version
this.names = []
}
this.resolving = version
const report = (names: ShownName[], final: boolean): void => {
if (this.resolving !== version || (!final && this.names.length > 0))
return
this.names = names
this.deliver(version, names)
}
const attempt = (tries: number): void => {
void run(report).then((settled) => {
if (!settled && this.resolving === version && tries < this.delays.length)
this.timer = this.schedule(() => attempt(tries + 1), this.delays[tries])
})
}
attempt(0)
}
dispose(): void {
clearTimeout(this.timer)
this.resolving = undefined
}
}
+1 -1
View File
@@ -8,7 +8,7 @@ import { BlockAnchors } from './anchor'
const py = createPythonAdapter(WASM_DIR)
function views(source: string): Promise<BlockView[]> {
return withBlocks(py, 'python', source, blocks => toBlockViews(blocks, source, py, undefined))
return withBlocks(py, 'python', source, blocks => toBlockViews(blocks, { source, language: py, markers: [] }, undefined))
}
function flatten(list: BlockView[]): BlockView[] {
+1 -1
View File
@@ -21,7 +21,7 @@ class ReferralRelation(Base):
`
async function subject(source: string, pick: (views: BlockView[]) => BlockView): Promise<AskSubject> {
const blocks = await withBlocks(py, 'python', source, list => toBlockViews(list, source, py, undefined))
const blocks = await withBlocks(py, 'python', source, list => toBlockViews(list, { source, language: py, markers: [] }, undefined))
return { path: 'app/models.py', languageId: 'python', source, blocks, anchor: pick(blocks).anchor }
}
+1 -1
View File
@@ -13,7 +13,7 @@ const SNAPSHOT = readFixture('celeryApp.py')
async function views(source: string, reports: string[] = []): Promise<BlockView[]> {
return withBlocks(py, 'python', source, blocks =>
toBlockViews(blocks, source, py, pythonZhTranslator, { rules: [celeryRule], report: message => reports.push(message) }))
toBlockViews(blocks, { source, language: py, markers: [] }, pythonZhTranslator, { rules: [celeryRule], report: message => reports.push(message) }))
}
function properties(view: BlockView | undefined): PropertyRow[] {
@@ -19,7 +19,7 @@ const RULES = [sqlalchemyRule, celeryRule, enumRule, dataclassRule]
async function views(source: string, reports: string[] = []): Promise<BlockView[]> {
return withBlocks(py, 'python', source, blocks =>
toBlockViews(blocks, source, py, pythonZhTranslator, { rules: RULES, report: message => reports.push(message) }))
toBlockViews(blocks, { source, language: py, markers: [] }, pythonZhTranslator, { rules: RULES, report: message => reports.push(message) }))
}
function properties(view: BlockView | undefined): PropertyRow[] {
+2 -2
View File
@@ -21,11 +21,11 @@ const RULES = [sqlalchemyRule, celeryRule, enumRule]
async function views(source: string, reports: string[] = []): Promise<BlockView[]> {
return withBlocks(py, 'python', source, blocks =>
toBlockViews(blocks, source, py, pythonZhTranslator, { rules: RULES, report: message => reports.push(message) }))
toBlockViews(blocks, { source, language: py, markers: [] }, pythonZhTranslator, { rules: RULES, report: message => reports.push(message) }))
}
function tsViews(source: string): Promise<BlockView[]> {
return withBlocks(ts, 'typescript', source, blocks => toBlockViews(blocks, source, ts, typescriptZhTranslator, { rules: RULES, report: () => {} }))
return withBlocks(ts, 'typescript', source, blocks => toBlockViews(blocks, { source, language: ts, markers: [] }, typescriptZhTranslator, { rules: RULES, report: () => {} }))
}
function properties(view: BlockView | undefined): PropertyRow[] {
@@ -13,7 +13,7 @@ const SNAPSHOT = readFixture('sqlalchemyModels.py')
async function views(source: string, reports: string[] = []): Promise<BlockView[]> {
return withBlocks(py, 'python', source, blocks =>
toBlockViews(blocks, source, py, pythonZhTranslator, { rules: [sqlalchemyRule], report: message => reports.push(message) }))
toBlockViews(blocks, { source, language: py, markers: [] }, pythonZhTranslator, { rules: [sqlalchemyRule], report: message => reports.push(message) }))
}
function flatten(list: BlockView[]): BlockView[] {
@@ -53,7 +53,7 @@ const translator = createZhTemplateTranslator({
function translateFirst(language: LanguageAdapter, source: string): Promise<string> {
return withBlocks(language, 'fake', source, ([block]) => {
const translation = translator.translate(block as TextBlock, { source, language })
const translation = translator.translate(block as TextBlock, { source, language, markers: [] })
return 'markdown' in translation ? translation.markdown : segmentText(translation)
})
}
@@ -19,8 +19,8 @@ function stringContent(node: Node): string {
}
/** `#` 注释去掉井号;docstring 只取引号里的内容。 */
function commentText(node: Node): string {
return node.type === 'comment' ? node.text.replace(/^#+\s?/, '') : stringContent(node)
function commentText(node: Node, text: string): string {
return node.type === 'comment' ? text.replace(/^#+\s?/, '') : stringContent(node)
}
export const pythonZhTranslator: Translator = createZhTemplateTranslator({
@@ -1,8 +1,7 @@
import type { Node, Translator } from '../contract'
import { createZhTemplateTranslator } from './zhTemplate'
function commentText(node: Node): string {
const text = node.text
function commentText(_node: Node, text: string): string {
if (text.startsWith('//'))
return text.replace(/^\/{2,}\s?/, '')
return text
@@ -22,7 +22,7 @@ function translateAll(adapter: LanguageAdapter, languageId: string, translator:
const line = block.header.startRow + 1
if (byLine.has(line) || 'facts' in block)
return
const translation = translator.translate(block, { source, language: adapter })
const translation = translator.translate(block, { source, language: adapter, markers: [] })
byLine.set(line, 'markdown' in translation
? { kind: block.kind, segments: [], markdown: translation.markdown }
: { kind: block.kind, segments: translation })
@@ -38,7 +38,7 @@ function describeAll(adapter: LanguageAdapter, languageId: string, translator: T
walk(blocks, (block) => {
const line = block.header.startRow + 1
if (!byLine.has(line) && 'facts' in block)
byLine.set(line, translator.describe(block, { source, language: adapter }))
byLine.set(line, translator.describe(block, { source, language: adapter, markers: [] }))
})
return byLine
})
@@ -1,4 +1,4 @@
import type { Access, AttributeNames, BaseFacts, BlockKind, ClassFacts, CommonAttribute, Defines, EntryFacts, EnumFacts, FactBlock, FunctionFacts, ImportFacts, ImportItem, Inline, IterationClause, IterationFacts, IterationResult, LanguageAdapter, Markdown, Node, PreviewText, PropertyItem, PropertyRow, Segment, SegmentGroup, SourceDocument, SourceSpan, TextBlock, TokenRole, Translator, VariableBlock } from '../contract'
import type { Access, AttributeNames, BaseFacts, BlockKind, ClassFacts, CommonAttribute, Defines, EntryFacts, EnumFacts, FactBlock, FunctionFacts, ImportFacts, ImportItem, Inline, IterationClause, IterationFacts, IterationResult, LanguageAdapter, Markdown, Node, PreviewText, PropertyItem, PropertyRow, Segment, SegmentGroup, SourceDocument, SourceRange, SourceSpan, TextBlock, TokenRole, Translator, VariableBlock } from '../contract'
import { commonIndent, splitLines } from '../editText'
/** 按所在语法节点区分译法的关键字,例如 Python 的 else:if 后「否则」,for / while 后「未中断时」。 */
@@ -44,8 +44,8 @@ export interface ZhTable {
mapLiterals: Record<string, string>
/** 整段原样照抄、按字符串着色的节点。 */
stringNodes: string[]
/** 注释(或 docstring)节点去掉注释符号后的原文,可以多行。 */
commentText: (node: Node) => string
/** 注释(或 docstring)节点去掉注释符号后的原文,可以多行;text 是节点的源码去掉代码阅读的记号(SourceDocument.markers)以后剩下的。 */
commentText: (node: Node, text: string) => string
/**
* 节点模板:节点类型 → 按中文语序重排的译文。命中的节点只输出模板里的译词和槽位,节点自己的关键字、标点不再输出;
* 槽位里的子节点照常递归翻译。模板引用的子节点缺了,或节点没整个落在要翻的这一段里,就不重排、照常翻。
@@ -95,6 +95,17 @@ function commentsWithin(node: Node, span: SourceSpan): Node[] {
return node.children.flatMap(child => child ? commentsWithin(child, span) : [])
}
/** 节点的源码去掉整个落在它里面的记号(中文名记号 `@Comment 当前用户` 不是说明文字)。 */
function withoutMarkers(node: Node, markers: readonly SourceRange[]): string {
let text = ''
let cursor = node.startIndex
for (const marker of markers.filter(m => m.startIndex >= node.startIndex && m.endIndex <= node.endIndex).sort((a, b) => a.startIndex - b.startIndex)) {
text += node.text.slice(cursor - node.startIndex, marker.startIndex - node.startIndex)
cursor = Math.max(cursor, marker.endIndex)
}
return text + node.text.slice(cursor - node.startIndex)
}
function isComment(token: Token): boolean {
return token.text === '' && !token.group
}
@@ -380,16 +391,21 @@ export function createZhTemplateTranslator(table: ZhTable): Translator {
return segments
}
function translateComment(block: TextBlock): Markdown {
/** 注释节点去掉注释符号和记号后的原文。 */
function commentOf(node: Node, { markers }: SourceDocument): string {
return table.commentText(node, withoutMarkers(node, markers))
}
function translateComment(block: TextBlock, document: SourceDocument): Markdown {
const { startIndex, endIndex } = block.header
const root = block.node.tree.rootNode
const nodes = block.node.type === 'comment' ? commentsWithin(root.descendantForIndex(startIndex, endIndex) ?? root, block.header) : [block.node]
return { markdown: docText(nodes.map(table.commentText)) }
return { markdown: docText(nodes.map(node => commentOf(node, document))) }
}
function translate(block: TextBlock, document: SourceDocument): PreviewText {
if (block.kind === 'comment')
return translateComment(block)
return translateComment(block, document)
const tokens = tokensIn(block.node, block.header, document)
const compound = block.children.length > 0 || block.header.endIndex < block.node.endIndex
const last = tokens.at(-1)
@@ -402,8 +418,8 @@ export function createZhTemplateTranslator(table: ZhTable): Translator {
return joinTokens(tokensIn(node, nodeSpan(node), document))
}
function doc(node: Node): Markdown | undefined {
const markdown = docText([table.commentText(node)])
function doc(node: Node, document: SourceDocument): Markdown | undefined {
const markdown = docText([commentOf(node, document)])
return markdown ? { markdown } : undefined
}
@@ -426,7 +442,7 @@ export function createZhTemplateTranslator(table: ZhTable): Translator {
// 点分模块名逐节照抄,每一节各自能跳到定义,和 VS Code 里一样
module: node => stringNodes.has(node.type) ? [segment(node.text, 'string', offsetOf(node, language))] : verbatim(node, document, 'type'),
name: (node, role) => [segment(node.text, role, offsetOf(node, language))],
doc,
doc: node => doc(node, document),
}
switch (block.kind) {
case 'function':
+29 -16
View File
@@ -1,9 +1,9 @@
import type { Tree } from '@vscode/tree-sitter-wasm'
import type { RuleLocation, RulePosition, RuleWorkspace, WorkspaceIndex } from './contract'
import type { IndexedFile, IndexEntry, Node, RuleLocation, RulePosition, RuleWorkspace, WorkspaceIndex } from './contract'
import * as path from 'node:path'
/**
* 库规则的工作区索引(WorkspaceIndex)的宿主部分:找文件、预筛、解析、缓存,跟着文件改动只重看改了的文件。
* 工作区索引(WorkspaceIndex:库规则的,和宿主自己认中文名的)的宿主部分:找文件、预筛、解析、缓存,跟着文件改动只重看改了的文件。
* 规则只说要什么;这里不碰 VS Code,工作区的能力由 IndexHost 给(VS Code 里是 workspaceFiles.ts,测试里是内存里的文件)。
*/
@@ -52,10 +52,18 @@ function positionAt(starts: number[], index: number): RulePosition {
return { line: low, character: index - starts[low] }
}
/** 认出的一项在哪:定义处名字的位置,连同认的时候给的这一项(宿主自己的索引在项里多带东西,例如中文名)。 */
export interface IndexedEntry<E extends IndexEntry = IndexEntry> { location: RuleLocation, entry: E }
/** 认出的项是 E 的索引。 */
export interface WorkspaceIndexOf<E extends IndexEntry> extends WorkspaceIndex {
entries: (root: Node, file: IndexedFile) => E[]
}
/** 一份看过的文件:认出的各项,和建索引时问过在不在的文件(它们新建、删掉时这份要重看)。 */
interface Indexed {
interface Indexed<E extends IndexEntry> {
file: WorkspaceFile
entries: { key: string, location: RuleLocation }[]
entries: IndexedEntry<E>[]
asked: Set<string>
}
@@ -64,25 +72,25 @@ function compareLocations(a: RuleLocation, b: RuleLocation): number {
}
/** 一个 WorkspaceIndex 建出来的索引:建好之前的查询、文件改动都排在建索引后面,一件件来。 */
class BuiltIndex {
class BuiltIndex<E extends IndexEntry> {
/** include 找到的全部文件(按 keyOf),IndexedFile.exists 查的就是它 */
private readonly files = new Map<string, WorkspaceFile>()
/** 预筛过了、解析过的文件 */
private readonly indexed = new Map<string, Indexed>()
private readonly indexed = new Map<string, Indexed<E>>()
private readonly watcher: { dispose: () => void }
private queue: Promise<void>
constructor(private readonly index: WorkspaceIndex, private readonly host: IndexHost) {
constructor(private readonly index: WorkspaceIndexOf<E>, private readonly host: IndexHost) {
// 先盯着再找文件:找文件期间的改动排在建好之后补上
this.watcher = host.watch(index.include, event => this.enqueue(() => this.apply(event)))
this.queue = this.guard(() => this.build())
}
async lookup(key: string): Promise<RuleLocation[]> {
async lookup(key: string): Promise<IndexedEntry<E>[]> {
await this.queue
return [...this.indexed.values()]
.flatMap(indexed => indexed.entries.filter(entry => entry.key === key).map(entry => entry.location))
.sort(compareLocations)
.flatMap(indexed => indexed.entries.filter(({ entry }) => entry.key === key))
.sort((a, b) => compareLocations(a.location, b.location))
}
dispose(): void {
@@ -134,9 +142,9 @@ class BuiltIndex {
return this.files.has(other)
}
const starts = lineStarts(source)
const entries = this.index.entries(tree.rootNode, { path: file.path, source, exists }).map(({ key: entryKey, name }) => ({
key: entryKey,
location: { uri: file.uri, start: positionAt(starts, name.startIndex), end: positionAt(starts, name.endIndex) },
const entries = this.index.entries(tree.rootNode, { path: file.path, source, exists }).map(entry => ({
entry,
location: { uri: file.uri, start: positionAt(starts, entry.name.startIndex), end: positionAt(starts, entry.name.endIndex) },
}))
this.indexed.set(key, { file, entries, asked })
}
@@ -186,12 +194,17 @@ class BuiltIndex {
/** 各库规则的工作区索引:第一次查某个索引时才建,同一个 id 只建一份。 */
export class WorkspaceIndexes implements RuleWorkspace {
private readonly built = new Map<string, BuiltIndex>()
private readonly built = new Map<string, BuiltIndex<IndexEntry>>()
constructor(private readonly host: IndexHost) {}
lookup(index: WorkspaceIndex, key: string): Promise<RuleLocation[]> {
let built = this.built.get(index.id)
async lookup(index: WorkspaceIndex, key: string): Promise<RuleLocation[]> {
return (await this.find(index, key)).map(({ location }) => location)
}
/** 同 lookup,连同认出的那一项(第一次查时建,同一个 id 只建一份)。 */
find<E extends IndexEntry>(index: WorkspaceIndexOf<E>, key: string): Promise<IndexedEntry<E>[]> {
let built = this.built.get(index.id) as BuiltIndex<E> | undefined
if (!built) {
built = new BuiltIndex(index, this.host)
this.built.set(index.id, built)