refactor: use vim.system (#4923)

lua/nvim-treesitter/async.lua

@@ -3,15 +3,15 @@ local co = coroutine
 local M = {}
 
 ---Executes a future with a callback when it is done
---- @param func function
+---@param func function
---- @param callback function
+---@param callback function
---- @param ... unknown
+---@param ... unknown
 local function execute(func, callback, ...)
   local thread = co.create(func)
 
   local function step(...)
     local ret = { co.resume(thread, ...) }
-    --- @type boolean, any
+    ---@type boolean, any
     local stat, nargs_or_err = unpack(ret)
 
     if not stat then
@@ -31,7 +31,7 @@ local function execute(func, callback, ...)
       return
     end
 
-    --- @type function, any[]
+    ---@type function, any[]
     local fn, args = ret[3], { unpack(ret, 4, table.maxn(ret)) }
     args[nargs_or_err] = step
     fn(unpack(args, 1, nargs_or_err))
@@ -41,13 +41,15 @@ local function execute(func, callback, ...)
 end
 
 --- Creates an async function with a callback style function.
---- @generic F: function
+---@generic F: function
---- @param func F
+---@param func F
---- @param argc integer
+---@param argc integer
---- @return F
+---@return F
 function M.wrap(func, argc)
-  --- @param ... unknown
+  vim.validate('func', func, 'function')
-  --- @return unknown
+  vim.validate('argc', argc, 'number')
+  ---@param ... unknown
+  ---@return unknown
   return function(...)
     return co.yield(argc, func, ...)
   end
@@ -56,10 +58,10 @@ end
 ---Use this to create a function which executes in an async context but
 ---called from a non-async context. Inherently this cannot return anything
 ---since it is non-blocking
---- @generic F: function
+---@generic F: function
---- @param func async F
+---@param func async F
---- @param nargs? integer
+---@param nargs? integer
---- @return F
+---@return F
 function M.sync(func, nargs)
   nargs = nargs or 0
   return function(...)
@@ -68,10 +70,10 @@ function M.sync(func, nargs)
   end
 end
 
---- @param n integer max number of concurrent jobs
+---@param n integer max number of concurrent jobs
---- @param interrupt_check? function
+---@param interrupt_check? function
---- @param thunks function[]
+---@param thunks function[]
---- @return any
+---@return any
 function M.join(n, interrupt_check, thunks)
   return co.yield(1, function(finish)
     if #thunks == 0 then
@@ -81,7 +83,7 @@ function M.join(n, interrupt_check, thunks)
     local remaining = { select(n + 1, unpack(thunks)) }
     local to_go = #thunks
 
-    local ret = {} --- @type any[]
+    local ret = {} ---@type any[]
 
     local function cb(...)
       ret[#ret + 1] = { ... }
@@ -104,7 +106,7 @@ end
 
 ---An async function that when called will yield to the Neovim scheduler to be
 ---able to call the API.
---- @type fun()
+---@type fun()
 M.main = M.wrap(vim.schedule, 1)
 
 return M

lua/nvim-treesitter/health.lua

@@ -12,12 +12,7 @@ local NVIM_TREESITTER_MINIMUM_ABI = 13
 ---@return string|nil
 local function ts_cli_version()
   if vim.fn.executable('tree-sitter') == 1 then
-    local handle = io.popen('tree-sitter -V')
+    local result = assert(vim.system({ 'tree-sitter', '-V' }):wait().stdout)
-    if not handle then
-      return
-    end
-    local result = handle:read('*a')
-    handle:close()
     return vim.split(result, '\n')[1]:match('[^tree%psitter ].*')
   end
 end
@@ -45,9 +40,7 @@ local function install_health()
   if vim.fn.executable('node') == 0 then
     health.warn('`node` executable not found (only needed for `:TSInstallFromGrammar`.')
   else
-    local handle = assert(io.popen('node --version'))
+    local result = assert(vim.system({ 'node', '--version' }):wait().stdout)
-    local result = handle:read('*a')
-    handle:close()
     local version = vim.split(result, '\n')[1]
     health.ok('`node` found ' .. version .. ' (only needed for `:TSInstallFromGrammar`)')
   end
@@ -70,7 +63,7 @@ local function install_health()
         .. ' or set `$CC` or `require"nvim-treesitter.install".compilers` explicitly.',
     })
   else
-    local version = vim.fn.systemlist(cc .. (cc == 'cl' and '' or ' --version'))[1]
+    local version = assert(vim.system({ cc, cc == 'cl' and '' or '--version' }):wait().stdout)
     health.ok(
       '`'
         .. cc

lua/nvim-treesitter/install.lua

@@ -3,7 +3,6 @@ local uv = vim.uv
 
 local a = require('nvim-treesitter.async')
 local config = require('nvim-treesitter.config')
-local job = require('nvim-treesitter.job')
 local log = require('nvim-treesitter.log')
 local parsers = require('nvim-treesitter.parsers')
 local util = require('nvim-treesitter.util')
@@ -42,6 +41,20 @@ if uv.os_getenv('CC') then
   table.insert(M.compilers, 1, uv.os_getenv('CC'))
 end
 
+local function system(cmd, opts)
+  log.trace('running job: (cwd=%s) %s', opts.cwd, table.concat(cmd, ' '))
+  local r = a.wrap(vim.system, 3)(cmd, opts) --[[@as SystemCompleted]]
+  a.main()
+  if r.stdout and r.stdout ~= '' then
+    log.trace('stdout -> %s', r.stdout)
+  end
+  if r.stderr and r.stderr ~= '' then
+    log.trace('stderr -> %s', r.stderr)
+  end
+
+  return r
+end
+
 ---
 --- PARSER INFO
 ---
@@ -156,24 +169,22 @@ local function do_generate_from_grammar(logger, repo, compile_location)
     end
 
     logger:info('Installing NPM dependencies')
-    local r = job.run({ 'npm', 'install' }, { cwd = compile_location })
+    local r = system({ 'npm', 'install' }, { cwd = compile_location })
-    a.main()
+    if r.code > 0 then
-    if r.exit_code > 0 then
       logger:error('Error during `npm install`')
     end
   end
 
   logger:info('Generating source files from grammar.js...')
 
-  local r = job.run({
+  local r = system({
     vim.fn.exepath('tree-sitter'),
     'generate',
     '--no-bindings',
     '--abi',
     tostring(vim.treesitter.language_version),
   }, { cwd = compile_location })
-  a.main()
+  if r.code > 0 then
-  if r.exit_code > 0 then
     logger:error('Error during "tree-sitter generate"')
   end
 end
@@ -201,7 +212,7 @@ local function do_download_tar(logger, repo, project_name, cache_dir, revision,
   local target = is_github and url .. '/archive/' .. revision .. '.tar.gz'
     or url .. '/-/archive/' .. revision .. '/' .. project_name .. '-' .. revision .. '.tar.gz'
 
-  local r = job.run({
+  local r = system({
     'curl',
     '--silent',
     '--show-error',
@@ -212,11 +223,8 @@ local function do_download_tar(logger, repo, project_name, cache_dir, revision,
   }, {
     cwd = cache_dir,
   })
-  a.main()
+  if r.code > 0 then
-  if r.exit_code > 0 then
+    logger:error('Error during download, please verify your internet connection: %s', r.stderr)
-    logger:error(
-      'Error during download, please verify your internet connection: ' .. vim.inspect(r.stderr)
-    )
   end
 
   logger:debug('Creating temporary directory: ' .. temp_dir)
@@ -228,7 +236,7 @@ local function do_download_tar(logger, repo, project_name, cache_dir, revision,
   end
 
   logger:info('Extracting ' .. project_name .. '...')
-  r = job.run({
+  r = system({
     'tar',
     '-xzf',
     project_name .. '.tar.gz',
@@ -238,14 +246,13 @@ local function do_download_tar(logger, repo, project_name, cache_dir, revision,
     cwd = cache_dir,
   })
 
-  a.main()
+  if r.code > 0 then
-  if r.exit_code > 0 then
+    logger:error('Error during tarball extraction: %s', r.stderr)
-    logger:error('Error during tarball extraction: ' .. vim.inspect(r.stderr))
   end
 
   err = uv_unlink(project_dir .. '.tar.gz')
   if err then
-    logger:error('Could not remove tarball: ' .. err)
+    logger:error('Could not remove tarball: %s', err)
   end
   a.main()
 
@@ -253,7 +260,7 @@ local function do_download_tar(logger, repo, project_name, cache_dir, revision,
   a.main()
 
   if err then
-    logger:error('Could not rename temp: ' .. err)
+    logger:error('Could not rename temp: %s', err)
   end
 
   util.delete(temp_dir)
@@ -268,7 +275,7 @@ end
 local function do_download_git(logger, repo, project_name, cache_dir, revision, project_dir)
   logger:info('Downloading ' .. project_name .. '...')
 
-  local r = job.run({
+  local r = system({
     'git',
     'clone',
     '--filter=blob:none',
@@ -278,16 +285,12 @@ local function do_download_git(logger, repo, project_name, cache_dir, revision,
     cwd = cache_dir,
   })
 
-  a.main()
+  if r.code > 0 then
-
+    logger:error('Error during download, please verify your internet connection: ' .. r.stderr)
-  if r.exit_code > 0 then
-    logger:error(
-      'Error during download, please verify your internet connection: ' .. vim.inspect(r.stderr)
-    )
   end
 
   logger:info('Checking out locked revision')
-  r = job.run({
+  r = system({
     'git',
     'checkout',
     revision,
@@ -295,10 +298,8 @@ local function do_download_git(logger, repo, project_name, cache_dir, revision,
     cwd = project_dir,
   })
 
-  a.main()
+  if r.code > 0 then
-
+    logger:error('Error while checking out revision: %s', r.stderr)
-  if r.exit_code > 0 then
-    logger:error('Error while checking out revision: ' .. vim.inspect(r.stderr))
   end
 end
 
@@ -388,7 +389,7 @@ end
 ---@param repo InstallInfo
 ---@param cc string
 ---@param compile_location string
----@return JobResult
+---@return SystemCompleted
 local function do_compile(repo, cc, compile_location)
   local make = M.select_executable({ 'gmake', 'make' })
 
@@ -404,9 +405,7 @@ local function do_compile(repo, cc, compile_location)
     }
   end
 
-  local r = job.run(cmd, { cwd = compile_location })
+  return system(cmd, { cwd = compile_location })
-  a.main()
-  return r
 end
 
 ---@param lang string
@@ -478,8 +477,8 @@ local function install_lang(lang, cache_dir, install_dir, force, generate_from_g
 
     logger:info('Compiling parser')
     local r = do_compile(repo, cc, compile_location)
-    if r.exit_code > 0 then
+    if r.code > 0 then
-      logger:error('Error during compilation: ' .. vim.inspect(r.stderr))
+      logger:error('Error during compilation: %s', r.stderr)
     end
 
     local parser_lib_name = fs.joinpath(install_dir, lang) .. '.so'

lua/nvim-treesitter/job.lua

@@ -1,122 +0,0 @@
--- Interface with Neovim job control and provide a simple job sequencing structure
-local uv = vim.uv
-local a = require('nvim-treesitter.async')
-local log = require('nvim-treesitter.log')
-
-local M = { JobResult = {}, Opts = {} }
-
---- @class JobResult
---- @field exit_code integer
---- @field signal integer | string
---- @field stdout string[]
---- @field stderr string[]
-
---- @class JobOpts
---- @field cwd  string
---- @field timeout  integer
---- @field env  string[]
---- @field on_stderr  fun(_: string)
---- @field on_stdout  fun(_: string)
-
---- Wrapper for vim.uv.spawn. Takes a command, options, and callback just like
---- vim.uv.spawn, but ensures that all output from the command has been
---- flushed before calling the callback.
---- @param cmd string
---- @param options uv.aliases.spawn_options
---- @param callback fun(exit_code: integer, signal: integer|string)
-local function spawn(cmd, options, callback)
-  local handle --- @type uv_process_t?
-  local timer --- @type uv_timer_t
-  log.trace('running job: (cwd=%s) %s %s', options.cwd, cmd, table.concat(options.args, ' '))
-  handle = uv.spawn(cmd, options, function(exit_code, signal)
-    ---@cast handle -nil
-
-    handle:close()
-    if timer then
-      timer:stop()
-      timer:close()
-    end
-
-    callback(exit_code, signal)
-  end)
-
-  --- @type integer?
-  --- @diagnostic disable-next-line:undefined-field
-  local timeout = options.timeout
-
-  if timeout then
-    timer = assert(uv.new_timer())
-    timer:start(timeout, 0, function()
-      timer:stop()
-      timer:close()
-      if handle and handle:is_active() then
-        log.warn('Killing %s due to timeout!', cmd)
-        handle:kill('sigint')
-        handle:close()
-        for _, pipe in
-          pairs(options.stdio --[[@as uv_pipe_t[] ]])
-        do
-          pipe:close()
-        end
-        callback(-9999, 'sigint')
-      end
-    end)
-  end
-end
-
---- Main exposed function for the jobs module. Takes a task and options and
---- returns an async function that will run the task with the given opts via
---- vim.uv.spawn
---- @param task string[]
---- @param opts JobOpts
---- @param callback fun(_: JobResult)
---- @type fun(task: string|string[], opts: JobOpts): JobResult
-M.run = a.wrap(function(task, opts, callback)
-  local stdout_data = {}
-  local stderr_data = {}
-
-  local stdout = assert(uv.new_pipe(false))
-  local stderr = assert(uv.new_pipe(false))
-
-  spawn(task[1], {
-    args = { unpack(task, 2) },
-    stdio = { nil, stdout, stderr },
-    cwd = opts.cwd,
-    timeout = opts.timeout and 1000 * opts.timeout or nil,
-    env = opts.env,
-    hide = true,
-  }, function(exit_code, signal)
-    callback({
-      exit_code = exit_code,
-      signal = signal,
-      stdout = stdout_data,
-      stderr = stderr_data,
-    })
-  end)
-
-  for kind, pipe in pairs({ stdout = stdout, stderr = stderr }) do
-    pipe:read_start(function(err, data)
-      if kind == 'stderr' and opts.on_stderr and data then
-        opts.on_stderr(data)
-      end
-      if kind == 'stdout' and opts.on_stdout and data then
-        opts.on_stdout(data)
-      end
-      if data then
-        log.trace('%s -> %s', kind, data)
-      end
-      if err then
-        log.error(err)
-      end
-      if data ~= nil then
-        local output = kind == 'stdout' and stdout_data or stderr_data
-        table.insert(output, vim.trim(data))
-      else
-        pipe:read_stop()
-        pipe:close()
-      end
-    end)
-  end
-end, 3)
-
-return M

lua/nvim-treesitter/parsers.lua

@@ -20,7 +20,7 @@ local M = {}
 
 M.tiers = { 'core', 'stable', 'community', 'unstable' }
 
----@type ParserInfo[]
+---@type table<string,ParserInfo>
 M.configs = {
   ada = {
     install_info = {

scripts/update-lockfile.lua

@@ -4,38 +4,53 @@ local util = require('nvim-treesitter.util')
 
 -- Load previous lockfile
 local filename = require('nvim-treesitter.install').get_package_path('lockfile.json')
-local lockfile = vim.json.decode(util.read_file(filename))
+-- local old_lockfile = vim.json.decode(util.read_file(filename)) --[[@as table<string,{revision:string}>]]
 
----@type string?
+---@type table<string,{revision:string}>
-local skip_lang_string = os.getenv('LOCKFILE_SKIP')
+local new_lockfile = {}
-local skip_langs = skip_lang_string and vim.split(skip_lang_string, ',') or {}
-vim.print('Skipping languages: ', skip_langs)
 
-local sorted_parsers = {}
+local parsers = require('nvim-treesitter.parsers').configs
-for k, v in pairs(require('nvim-treesitter.parsers').configs) do
+
-  table.insert(sorted_parsers, { name = k, parser = v })
+local jobs = {} --- @type table<string,SystemObj>
-end
-table.sort(sorted_parsers, function(a, b)
-  return a.name < b.name
-end)
 
 -- check for new revisions
-for _, v in ipairs(sorted_parsers) do
+for k, p in pairs(parsers) do
-  if not vim.list_contains(skip_langs, v.name) and v.parser.install_info then
+  if not p.install_info then
-    local cmd = 'git ls-remote ' .. v.parser.install_info.url
+    print('Skipping ' .. k)
-    if v.parser.install_info.branch then
-      cmd = cmd .. ' | grep refs/heads/' .. v.parser.install_info.branch
-    end
-    local sha = vim.split(vim.fn.systemlist(cmd)[1], '\t')[1]
-    lockfile[v.name] = { revision = sha }
-    print(v.name .. ': ' .. sha)
   else
-    print('Skipping ' .. v.name)
+    jobs[k] = vim.system({ 'git', 'ls-remote', p.install_info.url })
+  end
+
+  if #vim.tbl_keys(jobs) % 100 == 0 or next(parsers, k) == nil then
+    for name, job in pairs(jobs) do
+      local stdout = vim.split(job:wait().stdout, '\n')
+      jobs[name] = nil
+
+      local branch = parsers[name].install_info.branch
+
+      local line = 1
+      if branch then
+        for j, l in ipairs(stdout) do
+          if l:find(vim.pesc(branch)) then
+            line = j
+            break
+          end
+        end
+      end
+
+      local sha = vim.split(stdout[line], '\t')[1]
+      new_lockfile[name] = { revision = sha }
+      print(name .. ': ' .. sha)
+    end
   end
 end
 
-lockfile = vim.json.encode(lockfile)
+assert(#vim.tbl_keys(jobs) == 0)
+
+local lockfile_json = vim.json.encode(new_lockfile) --[[@as string]]
 if vim.fn.executable('jq') == 1 then
-  lockfile = vim.fn.system('jq --sort-keys', lockfile)
+  lockfile_json =
+    assert(vim.system({ 'jq', '--sort-keys' }, { stdin = lockfile_json }):wait().stdout)
 end
-util.write_file(filename, lockfile)
+
+util.write_file(filename, lockfile_json)