Files
xtype/src/xtype.swift
T
adminandClaude Code dd0d1b4e74 fix: 复用 URLSession 避免 webSocketTask 抛 NSException + 全抛点桥接
影响: XType 在 ~8 分钟级会再次 SIGSEGV(crash 4F17E81B,
      EXC_BAD_ACCESS 0xD,栈顶 webSocketTaskForRequest → objc_exception_throw
      → _Unwind_RaiseException);Swift 不接 ObjC NSException,任何抛 NSException
      的 ObjC API 调用都会让进程 SIGSEGV。架构脆弱性:installTap / webSocketTask
      都撞过,需要把剩余可能抛点也桥接。

改动:
- XunfeiEngine session 字段改为 private lazy var,与实例同生命周期
  (进程级,永不 invalidate,deinit 兜底);反复 invalidate/create 让
  self delegate 同时关联 invalidated 旧 session + 新 session,
  webSocketTask(with:) 抛 INVALIDATED_SESSION
- cleanup() 不再 invalidateAndCancel session,只 cancel 旧 wsTask
- ObjCBridge.h/.m 加通用 XTObjCRun(void (^)(void)) helper:
  包任意 Swift block,@try/@catch NSException,失败 NSLog reason 返回 1
- XunfeiEngine.prepare() 把 webSocketTask(with:) 用 XTObjCRun 包
  (belt-and-suspenders:lazy session 已避免 invalidated 状态污染,
  但加一层通用兜底)
- AppleEngine.prepare() 把 SFSpeechAudioBufferRecognitionRequest()
  init 用 XTObjCRun 包;同时把 tap 回调从 [req] capture 改成
  [weak self] + self?.request?.append(buf) 防 req 已 endAudio
  后还在 fire 回调
- AppleEngine.beginTask() 把 recognitionTask(with:) 用 XTObjCRun 包

设计: 不动 AVAudioEngine stored init(改造成本/收益不划算);
      后续新引入的 ObjC API 都需要走 XTObjCRun 兜底。

Co-Authored-By: Claude Code <[email protected]>
2026-10-08 12:08:27 +08:00

1026 lines
44 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// xtype: 长按鼠标左键 → 说话 → 松开自动粘贴到光标处
// 引擎可选: apple / xunfei / dual(apple+xunfei 并行,apple 先贴)
// 凭据(仅 xunfei 需要): 经 xop run --secret 注入到环境变量,源码/git/日志绝不出现
// LLM 清洗(可选): ~/ssw.ini 的 [llm-api] 段,key/base-url/model;无配置则跳过
// 权限: 麦克风 + 语音识别(apple) / 麦克风 + 辅助功能(全模式)
import AppKit
import ApplicationServices
import AVFoundation
import CoreGraphics
import CryptoKit
import Speech
// ---------- 配置 ----------
let holdMs: Double = 350 // 长按激活阈值
let movePx: CGFloat = 10 // 位移超过此值视为拖选,不激活
let maxRecSec: Double = 60 // 最长录音(到点直接贴当前已识别部分,不继续等)
let pollUs: UInt32 = 8_000 // 轮询周期 8ms (~125Hz)
let weChatPrefix = "com.tencent.xinWeChat"
let engineConfigPath = NSHomeDirectory() + "/.xtype-engine"
let llmIniPath = NSHomeDirectory() + "/ssw.ini"
// ---------- INI 读取(仅支持 section/key=value 简单格式) ----------
// 容忍 section/key 名里的全角破折号 `—` 与半角 `-` 等价(~/ssw.ini 历史上用 —)
func normINIKey(_ s: String) -> String {
s.replacingOccurrences(of: "—", with: "-").trimmingCharacters(in: .whitespaces)
}
func loadINI(_ path: String) -> [String: [String: String]] {
guard let raw = try? String(contentsOfFile: path, encoding: .utf8) else { return [:] }
var sections: [String: [String: String]] = [:]
var current: String?
for line in raw.split(whereSeparator: { $0 == "\n" || $0 == "\r" }) {
let s = line.trimmingCharacters(in: .whitespaces)
if s.isEmpty || s.hasPrefix("#") || s.hasPrefix(";") { continue }
if s.hasPrefix("[") && s.hasSuffix("]") {
current = normINIKey(String(s.dropFirst().dropLast()))
sections[current!] = [:]
} else if let cur = current, let eq = s.firstIndex(of: "=") {
let k = normINIKey(String(s[..<eq]))
let v = String(s[s.index(after: eq)...]).trimmingCharacters(in: .whitespaces)
sections[cur]?[k] = v
}
}
return sections
}
struct LLMConfig {
let key: String
let baseURL: String // 例如 https://api.openai.com/v1
let model: String // 例如 gpt-4o-mini
static func load() -> LLMConfig? {
// 优先读 env(xop run --secret 注入),env 没有再回退 ~/ssw.ini
let env = ProcessInfo.processInfo.environment
if let k = env["LLM_KEY"], !k.isEmpty,
let u = env["LLM_URL"], !u.isEmpty,
let m = env["LLM_MODEL"], !m.isEmpty {
log("llm: env LLM_KEY/LLM_URL/LLM_MODEL (xop 注入,不落盘)")
return LLMConfig(key: k, baseURL: u, model: m)
}
let s = loadINI(llmIniPath)["llm-api"] ?? [:]
guard let k = s["key"], !k.isEmpty,
let u = s["base-url"], !u.isEmpty,
let m = s["model"], !m.isEmpty else {
log("llm: env 缺失且 ~/ssw.ini [llm-api] 缺失或字段不全,跳过 LLM 清洗")
return nil
}
log("llm: 加载 [llm-api] model=\(m) base=\(u.prefix(30))…")
return LLMConfig(key: k, baseURL: u, model: m)
}
}
// 讯飞 IAT v2(参考 xfyun 开放平台"语音听写(流式版)WebAPI")
let xunfeiURL = "wss://iat-api.xfyun.cn/v2/iat"
let xunfeiHost = "iat-api.xfyun.cn"
setbuf(stdout, nil)
func ts() -> String { let f = DateFormatter(); f.dateFormat = "HH:mm:ss.SSS"; return f.string(from: Date()) }
func log(_ s: String) { print("[\(ts())] \(s)") }
// ---------- 共享状态(仅 poller 线程写) ----------
final class Box<T> { var v: T; init(_ v: T) { self.v = v } }
enum St { case idle; case press(origin: CGPoint, start: Date); case recording(origin: CGPoint, start: Date); case dead }
let state = Box<St>(.idle)
let maxMove = Box<CGFloat>(0)
let suppressDrag = Box<Bool>(false)
// ---------- ASR 引擎抽象 ----------
protocol ASREngine {
func prepare() // 鼠标按下瞬间: 预卷/连WS
func beginTask() // 长按激活: 开始识别
func cancel() // 普通点击抬起: 丢弃
func stop() -> String // 录音中抬起: 阻塞等最终结果(dual 模式下返回 apple 的)
func flushNow() -> String // force-stop / maxRecSec: 不等,直接拿当前累计文本
var readyForBegin: Bool { get } // prepare() 是否成功(凭据检查等)
}
func pickEngine() -> ASREngine {
let raw: String
if !FileManager.default.fileExists(atPath: engineConfigPath) {
// 首次启动: 写一份模板 + 提醒,默认 apple(无需凭据,保底能用)
let tmpl = """
# XType 引擎选择 — 取消注释任一行或填一个值,留空/注释全开 = apple 端侧
# apple Apple 端侧 SFSpeechRecognizer(无需凭据,zh-CN,流式,~1s)
# apple-llm apple + LLM 异步精修(凭据 xop 注入或 ~/ssw.ini [llm-api])
# xunfei 讯飞 IAT v2 WebSocket 流式(凭据 xop 注入;env 空则启动报错)
# dual apple + xunfei 并行 + LLM 精修(apple 先贴,讯飞覆盖,LLM 再覆盖)
#
# 凭据注入(可选): build 时 xop run --secret XUNFEI_APPID=xf_xt_APPID ...
# build 时 xop run --secret LLM_KEY=llm_key ...
#
apple
"""
try? tmpl.write(toFile: engineConfigPath, atomically: true, encoding: .utf8)
log("first launch: ~/.xtype-engine missing, wrote template (uncomment/select one of apple/apple-llm/xunfei/dual);defaulting to apple")
raw = "apple"
} else {
raw = (try? String(contentsOfFile: engineConfigPath, encoding: .utf8))?
.trimmingCharacters(in: .whitespacesAndNewlines) ?? "apple"
}
switch raw {
case "xunfei":
log("engine: xunfei (require XUNFEI_APPID/API_SECRET/API_KEY in env)")
return XunfeiEngine()
case "dual":
log("engine: dual (apple first, xunfei overwrite later, llm refine)")
return DualEngine()
case "apple-llm", "llm":
log("engine: apple + LLM refine (env LLM_KEY/LLM_URL/LLM_MODEL preferred, else ~/ssw.ini [llm-api];缺失 → silent fallback to apple)")
return AppleLLMEngine()
default:
log("engine: apple (on-device, no LLM)")
return AppleEngine()
}
}
final class Engine {
static var current: ASREngine = pickEngine()
static var ready: Bool { current.readyForBegin }
}
// ---------- Apple 端侧 SFSpeechRecognizer ----------
final class AppleEngine: ASREngine {
static let q = DispatchQueue(label: "xtype.apple.rec")
let engine = AVAudioEngine()
var request: SFSpeechAudioBufferRecognitionRequest?
var task: SFSpeechRecognitionTask?
var recognizer = SFSpeechRecognizer(locale: Locale(identifier: "zh-CN"))
var text = ""
var finalSem = DispatchSemaphore(value: 0)
var prepared = false
var useSharedTap = false // dual 模式 = true: 由外部 tap 回调把 PCM 转发给 request.append,自己不装 tap 不启动 engine
var readyForBegin: Bool { prepared }
func prepare() {
text = ""
finalSem = DispatchSemaphore(value: 0)
// SFSpeechAudioBufferRecognitionRequest() 在 SFSpeech 内部状态不对时可能抛 NSException
// (recognizer invalidated / 系统 Speech 权限被吊销 / Tcc 阻断)——
// belt-and-suspenders 包通用桥接,失败 log + return(不置 prepared,后续 readyForBegin=false)
let reqRC = XTObjCRun { [self] in
let req = SFSpeechAudioBufferRecognitionRequest()
req.shouldReportPartialResults = true
req.addsPunctuation = true
if self.recognizer?.supportsOnDeviceRecognition == true {
req.requiresOnDeviceRecognition = true
} else {
log("apple: on-device unavailable → server fallback")
}
self.request = req
}
if reqRC != 0 {
log("apple: SFSpeechAudioBufferRecognitionRequest raised NSException")
return
}
if useSharedTap {
// dual 模式: PCM 由 XunfeiEngine 的 tap 回调转发过来,
// 自身不装 tap 不启动 engine(避免和 xunfei 抢 bus 0)
prepared = true
return
}
let input = engine.inputNode
let fmt = input.outputFormat(forBus: 0)
input.removeTap(onBus: 0)
input.installTap(onBus: 0, bufferSize: 2048, format: fmt) { [weak self] buf, _ in
self?.request?.append(buf)
}
do { try engine.start() } catch { log("apple engine start failed: \(error)") }
prepared = true
}
func beginTask() {
guard let req = request, let rec = recognizer, prepared else { return }
// recognitionTask(with:) 在 SFSpeech 内部状态不对时可能抛 NSException
// (e.g. recognizer invalidated / request 已 endAudio 后再调)——
// belt-and-suspenders 用通用桥接包一层,失败 log + 不置 task(后续会被 cancel 清掉)
let taskRC = XTObjCRun { [self] in
self.task = rec.recognitionTask(with: req) { [weak self] r, err in
guard let self = self else { return }
if let r = r {
let t = r.bestTranscription.formattedString
if !t.isEmpty { self.text = t }
if r.isFinal { self.finalSem.signal() }
}
if err != nil { log("ASR err: \(err!)"); self.finalSem.signal() }
}
}
if taskRC != 0 {
log("apple: recognitionTask(with:) raised NSException — engine may be invalidated")
return
}
log("apple: recording (on-device: \(self.recognizer?.supportsOnDeviceRecognition == true))")
}
func cancel() {
guard prepared else { return }
engine.inputNode.removeTap(onBus: 0)
engine.stop()
request = nil
task = nil
prepared = false
}
func flushNow() -> String {
// SFSpeech 不太好中途取结果;返回当前 text(可能部分)
let t = text
cancel()
return t
}
func stop() -> String {
engine.inputNode.removeTap(onBus: 0)
engine.stop()
request?.endAudio()
_ = finalSem.wait(timeout: .now() + 3)
task?.finish()
let t = text
text = ""
request = nil
task = nil
prepared = false
return t
}
}
// ---------- Apple 端侧 + LLM 异步精修 ----------
// 流程: Apple 识别 → 贴出原文 → LLM 异步精修 → 若非空且不同则删原文再贴精修版
// 失败/超时/未配 INI: 不精修,保留 Apple 原文(不翻车)
final class AppleLLMEngine: ASREngine {
let apple = AppleEngine()
let cleaner: LLMCleaner
var lastRaw = "" // 覆盖阶段用: 知道要删多少字
init() {
self.cleaner = LLMCleaner(cfg: LLMConfig.load())
}
var readyForBegin: Bool { apple.readyForBegin }
func prepare() { apple.prepare() }
func beginTask() { apple.beginTask() }
func cancel() { apple.cancel() }
func flushNow() -> String { return apple.flushNow() }
func stop() -> String {
let raw = apple.stop()
lastRaw = raw
// 异步 LLM 精修: 非空且不同 → 删原文再贴
DispatchQueue.global().async { [weak self] in
guard let self = self else { return }
self.cleaner.refine(raw) { [weak self] refined in
guard let self = self else { return }
guard !refined.isEmpty, refined != raw else {
log("apple-llm: refine skipped (empty or identical)")
return
}
log("apple-llm: llm refine overwrite (\(raw.count)→\(refined.count) chars)")
self.deleteChars(raw.count)
paste(refined)
}
}
return raw
}
private func deleteChars(_ n: Int) {
guard n > 0 else { return }
let src = CGEventSource(stateID: .combinedSessionState)
for _ in 0..<n {
let kd = CGEvent(keyboardEventSource: src, virtualKey: 51, keyDown: true)
kd?.post(tap: .cghidEventTap)
let ku = CGEvent(keyboardEventSource: src, virtualKey: 51, keyDown: false)
ku?.post(tap: .cghidEventTap)
usleep(3_000)
}
}
}
// 凭据来自环境变量(XUNFEI_APPID / XUNFEI_API_SECRET / XUNFEI_API_KEY),
// 由 xop run --secret 在启动前注入。引擎自身不读配置文件、不打 fallback —— 安静回退会让用户误以为"用了讯飞识别差"。
final class XunfeiEngine: NSObject, ASREngine {
let q = DispatchQueue(label: "xtype.xunfei")
let engine = AVAudioEngine()
var wsTask: URLSessionWebSocketTask?
// session 与 XunfeiEngine 同生命周期(进程级):lazy 首次访问时创建,永不 invalidate,
// 只在 deinit 兜底。反复 invalidate/create 会让 self delegate 同时关联 invalidated 旧 session
// + 新 session,webSocketTask(with:) 抛 NSException "INVALIDATED_SESSION"(0xD 走 SIGSEGV,
// crash 4F17E81B-DE64-491B-8D59-373FBF5CB17A 验证此路径)。
private lazy var session: URLSession = {
let delegateQ = OperationQueue()
delegateQ.maxConcurrentOperationCount = 1
return URLSession(configuration: .default, delegate: self, delegateQueue: delegateQ)
}()
var pcmBuf = Data() // 16k Int16 mono PCM 累积
var rawBuf = Data() // 硬件格式 PCM 累积(Float32 mono @ hwRate,待手动转换)
var hwFmt: AVAudioFormat? // 输入节点硬件格式(用于算下采样比)
var resultText = "" // 累积识别结果
var waitingFinal = false // status=2 已发,等任何一条后续回复即视为结束
var firstFrameSent = false
var frameTimer: DispatchSourceTimer?
var finalSem = DispatchSemaphore(value: 0)
var openedSem = DispatchSemaphore(value: 0)
var opened = false
var sharedTapForwarder: ((AVAudioPCMBuffer, AVAudioTime) -> Void)? // dual 模式: 把同一份 PCM 转发给 apple.request.append(音频实时线程上不能阻塞,SFSpeech.append 设计上允许跨线程)
var prepared = false // prepare() 成功(ws open + lastErr==nil)就 true
var lastErr: String?
var appID = ""
var readyForBegin: Bool { prepared }
private func envCreds() -> (id: String, secret: String, key: String)? {
let env = ProcessInfo.processInfo.environment
guard let id = env["XUNFEI_APPID"], !id.isEmpty,
let s = env["XUNFEI_API_SECRET"], !s.isEmpty,
let k = env["XUNFEI_API_KEY"], !k.isEmpty else { return nil }
return (id, s, k)
}
private func rfc1123() -> String {
let f = DateFormatter()
f.locale = Locale(identifier: "en_US_POSIX")
f.timeZone = TimeZone(identifier: "GMT")
f.dateFormat = "EEE, dd MMM yyyy HH:mm:ss z"
return f.string(from: Date())
}
// HMAC-SHA256 鉴权 URL(同步在 IAT v2 文档)
private func authURL(appID: String, secret: String, apiKey: String) -> URL? {
let date = rfc1123()
let signOrigin = "host: \(xunfeiHost)\ndate: \(date)\nGET /v2/iat HTTP/1.1"
let mac = HMAC<SHA256>.authenticationCode(for: Data(signOrigin.utf8), using: SymmetricKey(data: Data(secret.utf8)))
let sig = Data(mac).base64EncodedString()
let authOrigin = "api_key=\"\(apiKey)\", algorithm=\"hmac-sha256\", headers=\"host date request-line\", signature=\"\(sig)\""
let auth = Data(authOrigin.utf8).base64EncodedString()
return URL(string: "\(xunfeiURL)?authorization=\(auth)&date=\(date)&host=\(xunfeiHost)")
}
func prepare() {
// 先收尾旧的(session、wsTask),避免和上一轮并发触发 delegate 回调
cleanup()
guard let creds = envCreds() else {
lastErr = "missing env XUNFEI_APPID/API_SECRET/API_KEY — launch via `xop run --secret`"
log("XUNFEI: \(lastErr!)")
return
}
appID = creds.id
// tap 必须用输入节点硬件格式 (Float32, 通常 44.1k 或 48k) —— 不然 installTap 抛 NSException
// 转码(浮点→int16 + 下采样)放到 serial queue 上做,绝不在音频线程跑。
let input = engine.inputNode
hwFmt = input.outputFormat(forBus: 0)
guard let url = authURL(appID: creds.id, secret: creds.secret, apiKey: creds.key) else {
lastErr = "auth URL build failed"
log("XUNFEI: \(lastErr!)")
return
}
// 用串行 queue 保证 delegate 回调按序触发(session 是 lazy 复用,见字段声明)
// webSocketTask(with:) 历史抛过 INVALIDATED_SESSION NSException(crash 4F17E81B),
// 即便已切到 lazy session,belt-and-suspenders 包一层通用桥接。
let wsRC = XTObjCRun { [self] in
self.wsTask = self.session.webSocketTask(with: url)
self.opened = false
self.wsTask?.resume()
}
if wsRC != 0 {
lastErr = "webSocketTask(with:) raised NSException"
log("XUNFEI: \(lastErr!)")
wsTask = nil
return
}
// tap 回调是音频实时线程,绝对禁止在这里 log / 阻塞 / 分配大对象。
// 只做一次 unsafe 指针复制 + 把字节扔到 serial queue。
engine.inputNode.removeTap(onBus: 0)
// 用 ObjC 桥接包 installTap —— 第二次 installTap (dual 模式 AppleEngine 已经占住 bus 0)
// 会抛 NSException,Swift 接不住,异常会触发 _Unwind_RaiseException → SIGSEGV(0xD)。
// crash 0BE0B7B3-4EEF-48E8-B2FC-874281EDE95A 走的就是这条死路。
let tapRC = XTObjCTryInstallTap(engine.inputNode, 0, 2048, hwFmt!) { [weak self] buf, t in
guard let self = self,
let ch = buf.floatChannelData?[0] else { return }
let n = Int(buf.frameLength)
if n == 0 { return }
let raw = UnsafeRawBufferPointer(start: ch, count: n * 4) // Float32 = 4 bytes
let copy = Data(raw) // 拷贝出音频线程作用域
self.q.async { self.rawBuf.append(copy); self.drainAndConvert() }
// dual 模式: 把同一份 PCM 转发给 apple 的 SFSpeech,让两路共用一个 tap(不抢 bus 0)
self.sharedTapForwarder?(buf, t)
}
if tapRC != 0 {
// NSException 已被 ObjC 桥接接住,记 lastErr + 清理已 resume 的 wsTask 后返回
// (session 是 lazy 复用,不在这里 invalidate — 反复 invalidate/create 会让
// 下一次 prepare() 的 webSocketTask(with:) 抛 INVALIDATED_SESSION NSException)
lastErr = "installTap raised NSException (bus busy or format mismatch?)"
log("XUNFEI: \(lastErr!)")
wsTask?.cancel(with: .goingAway, reason: nil)
wsTask = nil
return
}
do { try engine.start() } catch {
lastErr = "engine start failed: \(error)"
log("XUNFEI: \(lastErr!)")
return
}
prepared = true
log("xunfei: ws connecting + recording (hw=\(hwFmt!.sampleRate)Hz fmt=\(hwFmt!.commonFormat.rawValue))")
}
private func cleanup() {
frameTimer?.cancel(); frameTimer = nil
wsTask?.cancel(with: .goingAway, reason: nil)
wsTask = nil
// session 是 lazy 复用(见字段声明),不在这里 invalidate —
// 反复 invalidate/create 会让 webSocketTask(with:) 抛 INVALIDATED_SESSION NSException
engine.inputNode.removeTap(onBus: 0)
if engine.isRunning { engine.stop() }
pcmBuf = Data(); rawBuf = Data(); resultText = ""
firstFrameSent = false
hwFmt = nil
waitingFinal = false
finalSem = DispatchSemaphore(value: 0)
openedSem = DispatchSemaphore(value: 0)
opened = false
}
private func sendFrames() {
q.async {
// 先把硬件格式的 rawBuf 转码成 16k int16 mono 进 pcmBuf
self.drainAndConvert()
// 1280 bytes = 640 samples × 2 bytes/sample = 40ms @ 16kHz
while self.pcmBuf.count >= 1280 {
guard let ws = self.wsTask, ws.state == .running else { return } // ws 还没握上,等下一帧再试
let chunk = self.pcmBuf.prefix(1280)
self.pcmBuf.removeFirst(1280)
let isFirst = !self.firstFrameSent
self.send(status: self.firstFrameSent ? 1 : 0, audio: Data(chunk))
self.firstFrameSent = true
if isFirst { log("xunfei: sent first chunk (\(chunk.count)B) ws.state=\(ws.state.rawValue)") }
}
}
}
// 把 rawBuf(硬件格式 Float32 mono @ hwRate)手动转成 16k int16 mono,入 pcmBuf。
// 只在 serial queue 上跑,绝不在音频线程跑。
// 解: Float32 -> Int16 (clip); 48k -> 16k 用 3:1 抽取(够 ASR 用,免 AVAudioConverter 崩)
private func drainAndConvert() {
guard hwFmt != nil else { return }
let hwFrameBytes = 4 // Float32 mono = 4 bytes/frame
let frameCount = rawBuf.count / hwFrameBytes
guard frameCount >= 3 else { return } // 至少一帧 16k 输出
let ratio = Int((hwFmt!.sampleRate / 16000).rounded())
let outFrames = frameCount / max(ratio, 1)
guard outFrames > 0 else { return }
rawBuf.withUnsafeBytes { rawPtr in
let floats = rawPtr.bindMemory(to: Float32.self).baseAddress!
var out = [Int16](repeating: 0, count: outFrames)
for i in 0..<outFrames {
let s = floats[i * ratio]
let clipped = max(-1.0, min(1.0, s))
out[i] = Int16(clipped * 32767)
}
out.withUnsafeBufferPointer { bp in
pcmBuf.append(Data(buffer: bp))
}
}
rawBuf.removeFirst(outFrames * ratio * hwFrameBytes)
}
private func send(status: Int, audio: Data) {
guard let ws = wsTask else { return }
let payload: [String: Any] = [
"common": ["app_id": appID],
"business": ["language": "zh_cn", "domain": "iat", "accent": "mandarin", "vad_eos": 10000, "ptt": 1],
"data": ["status": status, "format": "audio/L16;rate=16000", "encoding": "raw", "audio": audio.base64EncodedString()]
]
guard let data = try? JSONSerialization.data(withJSONObject: payload) else { return }
log("xunfei: send status=\(status) bytes=\(audio.count) ws=\(ws.state.rawValue)")
ws.send(.data(data)) { err in
if let err = err { log("xunfei send err: \(err)") }
}
}
private func scheduleRecv() {
wsTask?.receive { [weak self] result in
guard let self = self else { return }
switch result {
case .success(let msg):
self.handle(msg)
if self.lastErr == nil { self.scheduleRecv() }
case .failure(let err):
log("xunfei recv err: \(err)")
self.lastErr = err.localizedDescription
self.finalSem.signal()
}
}
}
private func handle(_ msg: URLSessionWebSocketTask.Message) {
let data: Data?
switch msg {
case .string(let s): data = s.data(using: .utf8)
case .data(let d): data = d
@unknown default: data = nil
}
guard let d = data, let j = try? JSONSerialization.jsonObject(with: d) as? [String: Any] else { return }
let code = j["code"] as? Int ?? -1
let payload = j["data"] as? [String: Any]
log("xunfei recv: code=\(code) payload=\(payload ?? [:])")
if code != 0 {
lastErr = "信令码 \(code) \(j["message"] ?? "")"
log("XUNFEI: \(lastErr!)")
finalSem.signal()
return
}
guard let payload = j["data"] as? [String: Any] else { return }
if let result = payload["result"] as? [String: Any],
let wsArr = result["ws"] as? [[String: Any]] {
var piece = ""
for w in wsArr {
if let cw = w["cw"] as? [[String: Any]] {
for c in cw {
if let ww = c["w"] as? String { piece += ww }
}
}
}
// pgs: "rpl" 替换全文;"apd" 追加。"apd" 时将 piece 接到尾部
let pgs = result["pgs"] as? String ?? "rpl"
if pgs == "apd" { resultText += piece } else { resultText = piece }
log("xunfei recv rg: pgs=\(pgs) piece=\"\(piece)\" total=\"\(resultText)\" ls=\(payload["ls"] ?? false)")
}
if (payload["ls"] as? Bool) == true || waitingFinal {
// status=2 之后任何一条回复 = 服务端把最终答案给我了;ls=true 也算
finalSem.signal()
}
}
func beginTask() {
guard prepared, lastErr == nil else {
log("XUNFEI: beginTask refused (not prepared: \(lastErr ?? "?"))")
return
}
// 每 40ms 切一帧推上去
let t = DispatchSource.makeTimerSource(queue: q)
t.schedule(deadline: .now() + 0.04, repeating: 0.04)
t.setEventHandler { [weak self] in self?.sendFrames() }
t.resume()
frameTimer = t
scheduleRecv()
log("xunfei: begin")
}
func cancel() {
cleanup()
lastErr = nil
prepared = false
}
func flushNow() -> String {
// 不发 status=2、不等;直接返回当前 resultText(中间识别累计)
let t = resultText
cleanup()
lastErr = nil
prepared = false
return t
}
func stop() -> String {
frameTimer?.cancel(); frameTimer = nil
if prepared, wsTask != nil {
// 最后一帧: 先把 rawBuf 转完,再把 pcmBuf 尾巴推上去 + status=2
waitingFinal = true // 发完 status=2 后,handle() 收到任何一条回复就 signal
q.sync {
self.drainAndConvert()
let tail = self.pcmBuf
self.pcmBuf = Data()
self.send(status: 2, audio: tail)
}
// 等服务端回完最终那条(经验 100~500ms),1s 兜底(waitingFinal + ws close 都会 signal)
_ = finalSem.wait(timeout: .now() + 1)
}
// 必须在 cleanup 前读 resultText —— cleanup 会把它清空
let t = resultText
cleanup()
lastErr = nil
prepared = false
return t
}
}
// ---------- LLM 清洗(可选):OpenAI Chat Completions 兼容 ----------
final class LLMCleaner {
let cfg: LLMConfig?
init(cfg: LLMConfig?) { self.cfg = cfg }
static let prompt = """
你是语音转写后处理助手。把下面这段口述语音转写文字清理成自然书面语中文:
- 去掉"嗯/啊/呃/那个/就是/然后/其实/对"等口水词
- 加标点(句号、逗号、问号等),分句
- 重组零碎短句为通顺长句
- 保留全部原意,不增删内容
只输出清理后的文字,不要任何解释、标签或前后缀。
"""
func clean(_ raw: String) -> String {
// 同步接口(给单引擎调用):1.5s 兜底超时,默认返 raw
guard let cfg = cfg, !raw.isEmpty else { return raw }
let url = URL(string: cfg.baseURL.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + "/chat/completions")!
var req = URLRequest(url: url, timeoutInterval: 1.5)
req.httpMethod = "POST"
req.setValue("application/json", forHTTPHeaderField: "Content-Type")
req.setValue("Bearer \(cfg.key)", forHTTPHeaderField: "Authorization")
var body: [String: Any] = [
"model": cfg.model,
"messages": [
["role": "system", "content": Self.prompt],
["role": "user", "content": raw],
],
"temperature": 0.1,
]
// 强制关推理: GLM-5.3-Flash 默认有 reasoning_content,7s+;关掉 <1s 且内容字段干净。
// 不支持 thinking 的代理会静默忽略。
body["thinking"] = ["type": "disabled"]
req.httpBody = try? JSONSerialization.data(withJSONObject: body)
let sem = DispatchSemaphore(value: 0)
var result = raw
URLSession.shared.dataTask(with: req) { data, _, _ in
defer { sem.signal() }
guard let data = data,
let j = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let choices = j["choices"] as? [[String: Any]],
let msg = choices.first?["message"] as? [String: Any],
let content = msg["content"] as? String else { return }
let cleaned = content.trimmingCharacters(in: .whitespacesAndNewlines)
if !cleaned.isEmpty { result = cleaned }
}.resume()
_ = sem.wait(timeout: .now() + 1.5)
return result
}
// 异步精修(给 apple-llm / dual 引擎用):长 timeout,回调返 cleaned
func refine(_ raw: String, timeout: TimeInterval = 3, _ cb: @escaping (String) -> Void) {
guard let cfg = cfg, !raw.isEmpty else { cb(raw); return }
let url = URL(string: cfg.baseURL.trimmingCharacters(in: CharacterSet(charactersIn: "/")) + "/chat/completions")!
var req = URLRequest(url: url, timeoutInterval: timeout)
req.httpMethod = "POST"
req.setValue("application/json", forHTTPHeaderField: "Content-Type")
req.setValue("Bearer \(cfg.key)", forHTTPHeaderField: "Authorization")
var body: [String: Any] = [
"model": cfg.model,
"messages": [
["role": "system", "content": Self.prompt],
["role": "user", "content": raw],
],
"temperature": 0.1,
]
body["thinking"] = ["type": "disabled"]
req.httpBody = try? JSONSerialization.data(withJSONObject: body)
let done = DispatchSemaphore(value: 0)
URLSession.shared.dataTask(with: req) { data, _, _ in
defer { done.signal() }
guard let data = data,
let j = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let choices = j["choices"] as? [[String: Any]],
let msg = choices.first?["message"] as? [String: Any],
let content = msg["content"] as? String else {
cb(raw); return
}
let cleaned = content.trimmingCharacters(in: .whitespacesAndNewlines)
cb(cleaned.isEmpty ? raw : cleaned)
}.resume()
_ = done.wait(timeout: .now() + timeout + 0.5)
}
}
// ---------- 本地清洗(无) ----------
// LLM 之前实验过本地正则去"嗯/啊/那个"等口水词,但中文没空格,
// 任何字符级启发式都会误伤"好啊""什么啊"等正常语料 → 已弃用,清洗只走 LLM
// ---------- Dual 引擎:apple + xunfei 并行,apple 先贴、xunfei 后续覆盖 ----------
final class DualEngine: ASREngine {
let apple = AppleEngine()
let xunfei = XunfeiEngine()
let cleaner: LLMCleaner
var lastAppleText = "" // 给覆盖阶段用:知道要删多少字
var overwriteScheduled = false
init() {
self.cleaner = LLMCleaner(cfg: LLMConfig.load())
}
var readyForBegin: Bool { apple.readyForBegin && xunfei.readyForBegin }
func prepare() {
// 共享一个 AVAudioEngine 池(XunfeiEngine.engine): 由 xunfei 装 tap,回调里把同一份 PCM
// 转发给 apple.request.append。两路共用一个 tap 一个 bus 0,不再撞 AVAudioNode 也不会
// 让 apple 收不到音频(原 dual 模式下 xunfei.removeTap 把 apple 的 tap 抢了)。
apple.useSharedTap = true
apple.prepare()
xunfei.sharedTapForwarder = { [weak apple] buf, _ in
apple?.request?.append(buf)
}
xunfei.prepare()
}
func beginTask() {
apple.beginTask()
xunfei.beginTask()
}
func cancel() {
apple.cancel()
xunfei.cancel()
}
func flushNow() -> String {
return apple.flushNow()
}
func stop() -> String {
// 1) 收 apple(~1s) → 立刻贴出原始文本(不本地清洗)
let appleRaw = apple.stop()
lastAppleText = appleRaw
// 2) 异步: xunfei 识别完 → 覆盖 apple;再走 LLM 异步精修 → 若更优再覆盖一次
DispatchQueue.global().async { [weak self] in
guard let self = self else { return }
let xfRaw = self.xunfei.stop()
if !xfRaw.isEmpty, xfRaw != appleRaw {
log("dual: xunfei overwrite (\(appleRaw.count)→\(xfRaw.count) chars)")
self.deleteChars(appleRaw.count)
paste(xfRaw)
}
// 3) LLM 异步精修(可关): 配了 ~/ssw.ini 就跑,返非空且不同时再覆盖一次
self.cleaner.refine(xfRaw) { [weak self] refined in
guard let self = self else { return }
guard !refined.isEmpty, refined != xfRaw else { return }
log("dual: llm refine overwrite (\(xfRaw.count)→\(refined.count) chars)")
self.deleteChars(xfRaw.count)
paste(refined)
}
}
return appleRaw
}
private func deleteChars(_ n: Int) {
guard n > 0 else { return }
let src = CGEventSource(stateID: .combinedSessionState)
// 51 = kVK_Delete(退格)
for _ in 0..<n {
let kd = CGEvent(keyboardEventSource: src, virtualKey: 51, keyDown: true)
kd?.post(tap: .cghidEventTap)
let ku = CGEvent(keyboardEventSource: src, virtualKey: 51, keyDown: false)
ku?.post(tap: .cghidEventTap)
usleep(3_000) // 3ms / 字符,13 字 ≈ 40ms
}
}
}
extension XunfeiEngine: URLSessionWebSocketDelegate {
func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask,
didOpenWithProtocol p: String?) {
opened = true
openedSem.signal()
log("xunfei: ws open")
}
func urlSession(_ session: URLSession, webSocketTask: URLSessionWebSocketTask,
didCloseWith code: URLSessionWebSocketTask.CloseCode, reason: Data?) {
log("xunfei: ws close \(code.rawValue)")
// 对端结束也算"最后一帧到了":把 finalSem 抬起来,stop() 别再傻等
finalSem.signal()
}
}
// ---------- 注入:剪贴板 + 合成 Cmd+V ----------
let vKey: CGKeyCode = 9 // kVK_ANSI_V
func paste(_ s: String) {
let pb = NSPasteboard.general
let old = pb.string(forType: .string)
pb.clearContents()
pb.setString(s, forType: .string)
let src = CGEventSource(stateID: .combinedSessionState)
let kd = CGEvent(keyboardEventSource: src, virtualKey: vKey, keyDown: true)
kd?.flags = .maskCommand
kd?.post(tap: .cghidEventTap)
usleep(60_000)
let ku = CGEvent(keyboardEventSource: src, virtualKey: vKey, keyDown: false)
ku?.flags = .maskCommand
ku?.post(tap: .cghidEventTap)
log("pasted \(s.count) chars -> cursor")
DispatchQueue.global().asyncAfter(deadline: .now() + 1.5) {
if let old = old, pb.changeCount != 0 {
pb.clearContents(); pb.setString(old, forType: .string)
log("clipboard restored")
}
}
}
func finalize() {
suppressDrag.v = false
DispatchQueue.global().async {
let t = Engine.current.stop()
log("transcript: \"\(t)\"")
if !t.isEmpty { paste(t) }
}
}
// ---------- 光标定位 app ----------
func appAtCursor(_ pt: CGPoint) -> (pid: pid_t, bundle: String?) {
guard let list = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
return (NSWorkspace.shared.frontmostApplication?.processIdentifier ?? 0,
NSWorkspace.shared.frontmostApplication?.bundleIdentifier)
}
for w in list {
guard (w[kCGWindowLayer as String] as? Int) == 0,
let b = w[kCGWindowBounds as String] as? [String: CGFloat] else { continue }
if CGRect(x: b["X"] ?? 0, y: b["Y"] ?? 0, width: b["Width"] ?? 0, height: b["Height"] ?? 0).contains(pt) {
let pid = pid_t(w[kCGWindowOwnerPID as String] as? Int ?? 0)
return (pid, NSRunningApplication(processIdentifier: pid)?.bundleIdentifier)
}
}
return (0, nil)
}
// ---------- 微信过滤 + AX 输入框判定 ----------
let textRoles: Set<String> = ["AXTextField", "AXTextArea", "AXComboBox", "AXSearchField"]
// AX 树暴露不出编辑器(如 Sublime)或只给 AXUnknown(XStudio console 插件)的 app:
// 角色查得到但不是 textRoles 时,本应 skip,这里走白名单按"AX 不透明"兜底放行
// (按 AXWebArea 走不进文本子域、Safari Web 文本输入不在同列,单独处理见下)
let editorApps: Set<String> = [
"com.sublimetext.4", // Sublime Text 4(AX 树只有 window chrome)
"com.sublimetext.3",
"com.apple.TextEdit", // macOS 内置
"com.microsoft.VSCode", // VS Code(Electron,AX 树通常残)
"com.github.atom",
"com.coteditor.CotEditor",
"com.xstudio.ui", // XStudio(AXUnknown,console 插件是 Chromium webview)
]
func frontmostIsWeChat(_ bundle: String?) -> Bool {
bundle?.hasPrefix(weChatPrefix) == true
}
func axCopy(_ el: AXUIElement, _ attr: String, _ tag: String) -> CFTypeRef? {
var val: CFTypeRef?
let err: AXError = AXUIElementCopyAttributeValue(el, attr as CFString, &val)
if err != .success { log(" ax[\(tag)] \(attr) err=\(err.rawValue)") }
return err == .success ? val : nil
}
func axSet(_ el: AXUIElement, _ attr: String, _ v: CFTypeRef) {
_ = AXUIElementSetAttributeValue(el, attr as CFString, v)
}
func nudgeAX(_ pid: pid_t) {
let el = AXUIElementCreateApplication(pid)
axSet(el, "AXManualAccessibility", kCFBooleanTrue)
axSet(el, "AXEnhancedUserInterface", kCFBooleanTrue)
}
func focusedTextRole(pid: pid_t) -> String? {
if let el = axCopy(AXUIElementCreateSystemWide(), kAXFocusedUIElementAttribute, "syswide"),
let role = axCopy(el as! AXUIElement, kAXRoleAttribute, "syswide") as? String {
return role
}
if let el = axCopy(AXUIElementCreateApplication(pid), kAXFocusedUIElementAttribute, "perapp"),
let role = axCopy(el as! AXUIElement, kAXRoleAttribute, "perapp") as? String {
return role
}
return nil
}
// ---------- 长按检测线程 ----------
Thread.detachNewThread {
while true {
let down = CGEventSource.buttonState(.combinedSessionState, button: .left)
let loc = CGEvent(source: nil)?.location ?? .zero
switch (state.v, down) {
case (.idle, true):
state.v = .press(origin: loc, start: Date())
maxMove.v = 0
DispatchQueue.global().async { Engine.current.prepare() }
case (.press(let origin, let start), true):
maxMove.v = max(maxMove.v, hypot(loc.x - origin.x, loc.y - origin.y))
if Date().timeIntervalSince(start) * 1000 >= holdMs {
if maxMove.v <= movePx {
let (pid, bundle) = appAtCursor(loc)
if frontmostIsWeChat(bundle) {
log("skip: WeChat under cursor (\(bundle ?? "?"))")
state.v = .dead
DispatchQueue.global().async { Engine.current.cancel() }
} else if pid != 0 {
var role = focusedTextRole(pid: pid)
if role == nil {
nudgeAX(pid); usleep(250_000)
// 250ms 阻塞期间用户可能已松手;再查 buttonState,
// 已松开 → 立刻 .dead,不把"已结束的点击"升级成录音
if !CGEventSource.buttonState(.combinedSessionState, button: .left) {
state.v = .dead
DispatchQueue.global().async { Engine.current.cancel() }
break // 跳出 switch 的 .press 分支,回到 polling 顶
}
role = focusedTextRole(pid: pid)
}
func startRec() {
log("activate @ \(bundle ?? "?") role=\(role ?? "opaque")")
NSSound.beep()
if !Engine.ready {
log("engine not ready: \(Engine.current is XunfeiEngine ? "xunfei creds/ws?" : "apple init?")")
state.v = .dead
DispatchQueue.global().async { Engine.current.cancel() }
return
}
suppressDrag.v = true
state.v = .recording(origin: origin, start: start)
DispatchQueue.global().async { Engine.current.beginTask() }
}
if let role, textRoles.contains(role) {
startRec()
} else if let role, let b = bundle, editorApps.contains(b) {
// AX 树不暴露编辑器小部件(Sublime)或只给 AXUnknown(XStudio console 插件),
// 按白名单当 AX 不透明走 app 级兜底激活
log("allow: editor whitelist \(b) role=\(role) (fallback)")
startRec()
} else if let role {
log("skip: role=\(role)")
state.v = .dead
DispatchQueue.global().async { Engine.current.cancel() }
} else {
log("allow: AX-opaque app \(bundle ?? "?") (fallback)")
startRec()
}
} else {
state.v = .dead
DispatchQueue.global().async { Engine.current.cancel() }
}
} else {
state.v = .dead
DispatchQueue.global().async { Engine.current.cancel() }
}
}
case (.recording(_, let start), true):
if Date().timeIntervalSince(start) >= maxRecSec {
log("max record time, force-stop (paste current)")
DispatchQueue.global().async {
let t = Engine.current.flushNow()
log("force-transcript: \"\(t)\"")
if !t.isEmpty { paste(t) }
}
state.v = .idle
}
case (.dead, true):
break
case (_, false):
switch state.v {
case .recording: finalize()
case .press: DispatchQueue.global().async { Engine.current.cancel() }
default: break
}
state.v = .idle
}
usleep(pollUs)
}
}
// ---------- 拖选抑制 tap ----------
let tapPort = Box<CFMachPort?>(nil)
func tapCallback(_ proxy: CGEventTapProxy, _ type: CGEventType, _ ev: CGEvent, _ ud: UnsafeMutableRawPointer?) -> Unmanaged<CGEvent>? {
if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
if let p = tapPort.v { CGEvent.tapEnable(tap: p, enable: true) }
return Unmanaged.passUnretained(ev)
}
return suppressDrag.v ? nil : Unmanaged.passUnretained(ev)
}
Thread.detachNewThread {
let mask = CGEventMask(1 << CGEventType.leftMouseDragged.rawValue)
guard let port = CGEvent.tapCreate(tap: .cgSessionEventTap, place: .headInsertEventTap,
options: .defaultTap, eventsOfInterest: mask,
callback: tapCallback, userInfo: nil) else {
log("drag-tap install failed (no Accessibility) — 录音期间移动鼠标会拖选文字")
return
}
let src = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, port, 0)
CFRunLoopAddSource(CFRunLoopGetCurrent(), src, .commonModes)
tapPort.v = port
CFRunLoopRun()
}
// ---------- 启动自检 + 权限请求 ----------
freopen("/tmp/xtype.log", "a", stdout)
setbuf(stdout, nil)
log("process=\(ProcessInfo.processInfo.processName) bundle=\(Bundle.main.bundleIdentifier ?? "nil")")
log("Accessibility trusted: \(AXIsProcessTrusted())")
if !AXIsProcessTrusted() {
log(">>> 系统设置 → 隐私与安全性 → 辅助功能 → 勾选 XType <<<")
}
AVCaptureDevice.requestAccess(for: .audio) { ok in log("mic granted: \(ok)") }
SFSpeechRecognizer.requestAuthorization { st in log("speech auth: \(st.rawValue) (1=authorized)") }
dispatchMain()