ShellCommand.ts 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465
  1. import type { ChildProcess } from 'child_process'
  2. import { stat } from 'fs/promises'
  3. import type { Readable } from 'stream'
  4. import treeKill from 'tree-kill'
  5. import { generateTaskId } from '../Task.js'
  6. import { formatDuration } from './format.js'
  7. import {
  8. MAX_TASK_OUTPUT_BYTES,
  9. MAX_TASK_OUTPUT_BYTES_DISPLAY,
  10. } from './task/diskOutput.js'
  11. import { TaskOutput } from './task/TaskOutput.js'
  12. export type ExecResult = {
  13. stdout: string
  14. stderr: string
  15. code: number
  16. interrupted: boolean
  17. backgroundTaskId?: string
  18. backgroundedByUser?: boolean
  19. /** Set when assistant-mode auto-backgrounded a long-running blocking command. */
  20. assistantAutoBackgrounded?: boolean
  21. /** Set when stdout was too large to fit inline — points to the output file on disk. */
  22. outputFilePath?: string
  23. /** Total size of the output file in bytes (set when outputFilePath is set). */
  24. outputFileSize?: number
  25. /** The task ID for the output file (set when outputFilePath is set). */
  26. outputTaskId?: string
  27. /** Error message when the command failed before spawning (e.g., deleted cwd). */
  28. preSpawnError?: string
  29. }
  30. export type ShellCommand = {
  31. background: (backgroundTaskId: string) => boolean
  32. result: Promise<ExecResult>
  33. kill: () => void
  34. status: 'running' | 'backgrounded' | 'completed' | 'killed'
  35. /**
  36. * Cleans up stream resources (event listeners).
  37. * Should be called after the command completes or is killed to prevent memory leaks.
  38. */
  39. cleanup: () => void
  40. onTimeout?: (
  41. callback: (backgroundFn: (taskId: string) => boolean) => void,
  42. ) => void
  43. /** The TaskOutput instance that owns all stdout/stderr data and progress. */
  44. taskOutput: TaskOutput
  45. }
  46. const SIGKILL = 137
  47. const SIGTERM = 143
  48. // Background tasks write stdout/stderr directly to a file fd (no JS involvement),
  49. // so a stuck append loop can fill the disk. Poll file size and kill when exceeded.
  50. const SIZE_WATCHDOG_INTERVAL_MS = 5_000
  51. function prependStderr(prefix: string, stderr: string): string {
  52. return stderr ? `${prefix} ${stderr}` : prefix
  53. }
  54. /**
  55. * Thin pipe from a child process stream into TaskOutput.
  56. * Used in pipe mode (hooks) for stdout and stderr.
  57. * In file mode (bash commands), both fds go to the output file —
  58. * the child process streams are null and no wrappers are created.
  59. */
  60. class StreamWrapper {
  61. #stream: Readable | null
  62. #isCleanedUp = false
  63. #taskOutput: TaskOutput | null
  64. #isStderr: boolean
  65. #onData = this.#dataHandler.bind(this)
  66. constructor(stream: Readable, taskOutput: TaskOutput, isStderr: boolean) {
  67. this.#stream = stream
  68. this.#taskOutput = taskOutput
  69. this.#isStderr = isStderr
  70. // Emit strings instead of Buffers - avoids repeated .toString() calls
  71. stream.setEncoding('utf-8')
  72. stream.on('data', this.#onData)
  73. }
  74. #dataHandler(data: Buffer | string): void {
  75. const str = typeof data === 'string' ? data : data.toString()
  76. if (this.#isStderr) {
  77. this.#taskOutput!.writeStderr(str)
  78. } else {
  79. this.#taskOutput!.writeStdout(str)
  80. }
  81. }
  82. cleanup(): void {
  83. if (this.#isCleanedUp) {
  84. return
  85. }
  86. this.#isCleanedUp = true
  87. this.#stream!.removeListener('data', this.#onData)
  88. // Release references so the stream, its StringDecoder, and
  89. // the TaskOutput can be GC'd independently of this wrapper.
  90. this.#stream = null
  91. this.#taskOutput = null
  92. this.#onData = () => {}
  93. }
  94. }
  95. /**
  96. * Implementation of ShellCommand that wraps a child process.
  97. *
  98. * For bash commands: both stdout and stderr go to a file fd via
  99. * stdio[1] and stdio[2] — no JS involvement. Progress is extracted
  100. * by polling the file tail.
  101. * For hooks: pipe mode with StreamWrappers for real-time detection.
  102. */
  103. class ShellCommandImpl implements ShellCommand {
  104. #status: 'running' | 'backgrounded' | 'completed' | 'killed' = 'running'
  105. #backgroundTaskId: string | undefined
  106. #stdoutWrapper: StreamWrapper | null
  107. #stderrWrapper: StreamWrapper | null
  108. #childProcess: ChildProcess
  109. #timeoutId: NodeJS.Timeout | null = null
  110. #sizeWatchdog: NodeJS.Timeout | null = null
  111. #killedForSize = false
  112. #maxOutputBytes: number
  113. #abortSignal: AbortSignal
  114. #onTimeoutCallback:
  115. | ((backgroundFn: (taskId: string) => boolean) => void)
  116. | undefined
  117. #timeout: number
  118. #shouldAutoBackground: boolean
  119. #resultResolver: ((result: ExecResult) => void) | null = null
  120. #exitCodeResolver: ((code: number) => void) | null = null
  121. #boundAbortHandler: (() => void) | null = null
  122. readonly taskOutput: TaskOutput
  123. static #handleTimeout(self: ShellCommandImpl): void {
  124. if (self.#shouldAutoBackground && self.#onTimeoutCallback) {
  125. self.#onTimeoutCallback(self.background.bind(self))
  126. } else {
  127. self.#doKill(SIGTERM)
  128. }
  129. }
  130. readonly result: Promise<ExecResult>
  131. readonly onTimeout?: (
  132. callback: (backgroundFn: (taskId: string) => boolean) => void,
  133. ) => void
  134. constructor(
  135. childProcess: ChildProcess,
  136. abortSignal: AbortSignal,
  137. timeout: number,
  138. taskOutput: TaskOutput,
  139. shouldAutoBackground = false,
  140. maxOutputBytes = MAX_TASK_OUTPUT_BYTES,
  141. ) {
  142. this.#childProcess = childProcess
  143. this.#abortSignal = abortSignal
  144. this.#timeout = timeout
  145. this.#shouldAutoBackground = shouldAutoBackground
  146. this.#maxOutputBytes = maxOutputBytes
  147. this.taskOutput = taskOutput
  148. // In file mode (bash commands), both stdout and stderr go to the
  149. // output file fd — childProcess.stdout/.stderr are both null.
  150. // In pipe mode (hooks), wrap streams to funnel data into TaskOutput.
  151. this.#stderrWrapper = childProcess.stderr
  152. ? new StreamWrapper(childProcess.stderr, taskOutput, true)
  153. : null
  154. this.#stdoutWrapper = childProcess.stdout
  155. ? new StreamWrapper(childProcess.stdout, taskOutput, false)
  156. : null
  157. if (shouldAutoBackground) {
  158. this.onTimeout = (callback): void => {
  159. this.#onTimeoutCallback = callback
  160. }
  161. }
  162. this.result = this.#createResultPromise()
  163. }
  164. get status(): 'running' | 'backgrounded' | 'completed' | 'killed' {
  165. return this.#status
  166. }
  167. #abortHandler(): void {
  168. // On 'interrupt' (user submitted a new message), don't kill — let the
  169. // caller background the process so the model can see partial output.
  170. if (this.#abortSignal.reason === 'interrupt') {
  171. return
  172. }
  173. this.kill()
  174. }
  175. #exitHandler(code: number | null, signal: NodeJS.Signals | null): void {
  176. const exitCode =
  177. code !== null && code !== undefined
  178. ? code
  179. : signal === 'SIGTERM'
  180. ? 144
  181. : 1
  182. this.#resolveExitCode(exitCode)
  183. }
  184. #errorHandler(): void {
  185. this.#resolveExitCode(1)
  186. }
  187. #resolveExitCode(code: number): void {
  188. if (this.#exitCodeResolver) {
  189. this.#exitCodeResolver(code)
  190. this.#exitCodeResolver = null
  191. }
  192. }
  193. // Note: exit/error listeners are NOT removed here — they're needed for
  194. // the result promise to resolve. They clean up when the child process exits.
  195. #cleanupListeners(): void {
  196. this.#clearSizeWatchdog()
  197. const timeoutId = this.#timeoutId
  198. if (timeoutId) {
  199. clearTimeout(timeoutId)
  200. this.#timeoutId = null
  201. }
  202. const boundAbortHandler = this.#boundAbortHandler
  203. if (boundAbortHandler) {
  204. this.#abortSignal.removeEventListener('abort', boundAbortHandler)
  205. this.#boundAbortHandler = null
  206. }
  207. }
  208. #clearSizeWatchdog(): void {
  209. if (this.#sizeWatchdog) {
  210. clearInterval(this.#sizeWatchdog)
  211. this.#sizeWatchdog = null
  212. }
  213. }
  214. #startSizeWatchdog(): void {
  215. this.#sizeWatchdog = setInterval(() => {
  216. void stat(this.taskOutput.path).then(
  217. s => {
  218. // Bail if the watchdog was cleared while this stat was in flight
  219. // (process exited on its own) — otherwise we'd mislabel stderr.
  220. if (
  221. s.size > this.#maxOutputBytes &&
  222. this.#status === 'backgrounded' &&
  223. this.#sizeWatchdog !== null
  224. ) {
  225. this.#killedForSize = true
  226. this.#clearSizeWatchdog()
  227. this.#doKill(SIGKILL)
  228. }
  229. },
  230. () => {
  231. // ENOENT before first write, or unlinked mid-run — skip this tick
  232. },
  233. )
  234. }, SIZE_WATCHDOG_INTERVAL_MS)
  235. this.#sizeWatchdog.unref()
  236. }
  237. #createResultPromise(): Promise<ExecResult> {
  238. this.#boundAbortHandler = this.#abortHandler.bind(this)
  239. this.#abortSignal.addEventListener('abort', this.#boundAbortHandler, {
  240. once: true,
  241. })
  242. // Use 'exit' not 'close': 'close' waits for stdio to close, which includes
  243. // grandchild processes that inherit file descriptors (e.g. `sleep 30 &`).
  244. // 'exit' fires when the shell itself exits, returning control immediately.
  245. this.#childProcess.once('exit', this.#exitHandler.bind(this))
  246. this.#childProcess.once('error', this.#errorHandler.bind(this))
  247. this.#timeoutId = setTimeout(
  248. ShellCommandImpl.#handleTimeout,
  249. this.#timeout,
  250. this,
  251. ) as NodeJS.Timeout
  252. const exitPromise = new Promise<number>(resolve => {
  253. this.#exitCodeResolver = resolve
  254. })
  255. return new Promise<ExecResult>(resolve => {
  256. this.#resultResolver = resolve
  257. void exitPromise.then(this.#handleExit.bind(this))
  258. })
  259. }
  260. async #handleExit(code: number): Promise<void> {
  261. this.#cleanupListeners()
  262. if (this.#status === 'running' || this.#status === 'backgrounded') {
  263. this.#status = 'completed'
  264. }
  265. const stdout = await this.taskOutput.getStdout()
  266. const result: ExecResult = {
  267. code,
  268. stdout,
  269. stderr: this.taskOutput.getStderr(),
  270. interrupted: code === SIGKILL,
  271. backgroundTaskId: this.#backgroundTaskId,
  272. }
  273. if (this.taskOutput.stdoutToFile && !this.#backgroundTaskId) {
  274. if (this.taskOutput.outputFileRedundant) {
  275. // Small file — full content is in result.stdout, delete the file
  276. void this.taskOutput.deleteOutputFile()
  277. } else {
  278. // Large file — tell the caller where the full output lives
  279. result.outputFilePath = this.taskOutput.path
  280. result.outputFileSize = this.taskOutput.outputFileSize
  281. result.outputTaskId = this.taskOutput.taskId
  282. }
  283. }
  284. if (this.#killedForSize) {
  285. result.stderr = prependStderr(
  286. `Background command killed: output file exceeded ${MAX_TASK_OUTPUT_BYTES_DISPLAY}`,
  287. result.stderr,
  288. )
  289. } else if (code === SIGTERM) {
  290. result.stderr = prependStderr(
  291. `Command timed out after ${formatDuration(this.#timeout)}`,
  292. result.stderr,
  293. )
  294. }
  295. const resultResolver = this.#resultResolver
  296. if (resultResolver) {
  297. this.#resultResolver = null
  298. resultResolver(result)
  299. }
  300. }
  301. #doKill(code?: number): void {
  302. this.#status = 'killed'
  303. if (this.#childProcess.pid) {
  304. treeKill(this.#childProcess.pid, 'SIGKILL')
  305. }
  306. this.#resolveExitCode(code ?? SIGKILL)
  307. }
  308. kill(): void {
  309. this.#doKill()
  310. }
  311. background(taskId: string): boolean {
  312. if (this.#status === 'running') {
  313. this.#backgroundTaskId = taskId
  314. this.#status = 'backgrounded'
  315. this.#cleanupListeners()
  316. if (this.taskOutput.stdoutToFile) {
  317. // File mode: child writes directly to the fd with no JS involvement.
  318. // The foreground timeout is gone, so watch file size to prevent
  319. // a stuck append loop from filling the disk (768GB incident).
  320. this.#startSizeWatchdog()
  321. } else {
  322. // Pipe mode: spill the in-memory buffer so readers can find it on disk.
  323. this.taskOutput.spillToDisk()
  324. }
  325. return true
  326. }
  327. return false
  328. }
  329. cleanup(): void {
  330. this.#stdoutWrapper?.cleanup()
  331. this.#stderrWrapper?.cleanup()
  332. this.taskOutput.clear()
  333. // Must run before nulling #abortSignal — #cleanupListeners() calls
  334. // removeEventListener on it. Without this, a kill()+cleanup() sequence
  335. // crashes: kill() queues #handleExit as a microtask, cleanup() nulls
  336. // #abortSignal, then #handleExit runs #cleanupListeners() on the null ref.
  337. this.#cleanupListeners()
  338. // Release references to allow GC of ChildProcess internals and AbortController chain
  339. this.#childProcess = null!
  340. this.#abortSignal = null!
  341. this.#onTimeoutCallback = undefined
  342. }
  343. }
  344. /**
  345. * Wraps a child process to enable flexible handling of shell command execution.
  346. */
  347. export function wrapSpawn(
  348. childProcess: ChildProcess,
  349. abortSignal: AbortSignal,
  350. timeout: number,
  351. taskOutput: TaskOutput,
  352. shouldAutoBackground = false,
  353. maxOutputBytes = MAX_TASK_OUTPUT_BYTES,
  354. ): ShellCommand {
  355. return new ShellCommandImpl(
  356. childProcess,
  357. abortSignal,
  358. timeout,
  359. taskOutput,
  360. shouldAutoBackground,
  361. maxOutputBytes,
  362. )
  363. }
  364. /**
  365. * Static ShellCommand implementation for commands that were aborted before execution.
  366. */
  367. class AbortedShellCommand implements ShellCommand {
  368. readonly status = 'killed' as const
  369. readonly result: Promise<ExecResult>
  370. readonly taskOutput: TaskOutput
  371. constructor(opts?: {
  372. backgroundTaskId?: string
  373. stderr?: string
  374. code?: number
  375. }) {
  376. this.taskOutput = new TaskOutput(generateTaskId('local_bash'), null)
  377. this.result = Promise.resolve({
  378. code: opts?.code ?? 145,
  379. stdout: '',
  380. stderr: opts?.stderr ?? 'Command aborted before execution',
  381. interrupted: true,
  382. backgroundTaskId: opts?.backgroundTaskId,
  383. })
  384. }
  385. background(): boolean {
  386. return false
  387. }
  388. kill(): void {}
  389. cleanup(): void {}
  390. }
  391. export function createAbortedCommand(
  392. backgroundTaskId?: string,
  393. opts?: { stderr?: string; code?: number },
  394. ): ShellCommand {
  395. return new AbortedShellCommand({
  396. backgroundTaskId,
  397. ...opts,
  398. })
  399. }
  400. export function createFailedCommand(preSpawnError: string): ShellCommand {
  401. const taskOutput = new TaskOutput(generateTaskId('local_bash'), null)
  402. return {
  403. status: 'completed' as const,
  404. result: Promise.resolve({
  405. code: 1,
  406. stdout: '',
  407. stderr: preSpawnError,
  408. interrupted: false,
  409. preSpawnError,
  410. }),
  411. taskOutput,
  412. background(): boolean {
  413. return false
  414. },
  415. kill(): void {},
  416. cleanup(): void {},
  417. }
  418. }