ci: remove update-lockfile shell script

.github/workflows/test-queries.yml

@@ -1,12 +1,12 @@
 name: Test queries
 
 on:
-  push:
-    branches:
-      - "master"
+  # push:
+  #   branches:
+  #     - "main"
   pull_request:
     branches:
-      - "master"
+      - "main"
 
 # Cancel any in-progress CI runs for a PR if it is updated
 concurrency:
@@ -24,29 +24,29 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-14]
         cc: [gcc, clang]
-        nvim_tag: [v0.10.4]
+        nvim_tag: [nightly]
         exclude:
           - os: ubuntu-latest
             cc: clang
-            nvim_tag: v0.10.4
+            nvim_tag: nightly
 
           - os: macos-14
             cc: gcc
-            nvim_tag: v0.10.4
-
-          - os: windows-latest
-            cc: clang
-            nvim_tag: v0.10.4
-
-        include:
-          - os: windows-latest
-            cc: cl
             nvim_tag: nightly
 
-          - os: ubuntu-latest
+          - os: windows-latest
             cc: gcc
             nvim_tag: nightly
 
+        # include:
+          # - os: windows-latest
+          #   cc: cl
+          #   nvim_tag: nightly
+
+          # - os: ubuntu-latest
+          #   cc: gcc
+          #   nvim_tag: nightly
+
     name: Parser compilation
     runs-on: ${{ matrix.os }}
     env:
@@ -69,8 +69,8 @@ jobs:
         uses: actions/cache@v4
         with:
           path: |
-            ./parser/
-            ~/AppData/Local/nvim/pack/nvim-treesitter/start/nvim-treesitter/parser/
+            ~/.local/share/nvim/site/parser/
+            ~/AppData/Local/nvim-data/site/parser/
           key: parsers-${{ join(matrix.*, '-') }}-${{ hashFiles(
             './lockfile.json',
             './lua/nvim-treesitter/install.lua',
@@ -78,11 +78,7 @@ jobs:
             './lua/nvim-treesitter/shell_command_selectors.lua') }}
 
       - name: Compile parsers
-        run: $NVIM --headless -c "lua require'nvim-treesitter.install'.prefer_git=false" -c "TSInstallSync all" -c "q"
-
-      - name: Post compile Windows
-        if: runner.os == 'Windows'
-        run: cp -r ~/AppData/Local/nvim/pack/nvim-treesitter/start/nvim-treesitter/parser/* parser
+        run: $NVIM -l ./scripts/install-parsers.lua
 
       - name: Check query files
         run: $NVIM -l ./scripts/check-queries.lua

.github/workflows/tests.yml

@@ -1,12 +1,12 @@
 name: Tests
 
 on:
-  push:
-    branches:
-      - "master"
+  # push:
+  #   branches:
+  #     - "main"
   pull_request:
     branches:
-      - "master"
+      - "main"
 
 # Cancel any in-progress CI runs for a PR if it is updated
 concurrency:
@@ -39,7 +39,8 @@ jobs:
 
       - name: Install and prepare Neovim
         env:
-          NVIM_TAG: v0.10.4
+          NVIM_TAG: nightly
+          TREE_SITTER_CLI_TAG: v0.20.8
         run: |
           bash ./scripts/ci-install.sh
 
@@ -57,8 +58,8 @@ jobs:
             './lua/nvim-treesitter/shell_selectors.lua') }}
 
       - name: Compile parsers Unix like
-        run: |
-          nvim --headless -c "TSInstallSync all" -c "q"
+        if: ${{ runner.os != 'Windows' && steps.parsers-cache.outputs.cache-hit != 'true' }}
+        run: nvim -l ./scripts/install-parsers.lua
 
       - name: Tests
         run: PATH=/usr/local/bin:$PATH ./scripts/run_tests.sh

.github/workflows/update-lockfile.yml

@@ -12,7 +12,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          ref: master
+          ref: main
 
       - uses: actions/create-github-app-token@v2
         id: app-token
@@ -34,10 +34,7 @@ jobs:
           SKIP_LOCKFILE_UPDATE_FOR_LANGS: "bp,devicetree,dhall,elm,enforce,git_config,nickel,rescript,rust,slint,sql,t32,templ,typespec,verilog,wit"
         run: |
           cp lockfile.json /tmp/old_lockfile.json
-          nvim -l scripts/write-lockfile.lua
-          # Pretty print
-          cp lockfile.json /tmp/lockfile.json
-          cat /tmp/lockfile.json | jq --sort-keys > lockfile.json
+          nvim -l ./scripts/update-lockfile.lua
           UPDATED_PARSERS=$(/tmp/jd -f merge /tmp/old_lockfile.json lockfile.json | jq -r 'keys | join(", ")')
           echo "UPDATED_PARSERS=$UPDATED_PARSERS" >> $GITHUB_ENV
 

.github/workflows/update-readme.yml

@@ -3,7 +3,7 @@ name: Update README
 on:
   push:
     branches:
-      - master
+      - main
   workflow_dispatch:
 
 jobs:

TODO.md

@@ -5,8 +5,9 @@ This document lists the planned and finished changes in this rewrite towards [Nv
 ## TODO
 
 - [ ] **`query_predicates.lua`:** upstream/remove
-- [ ] **`parsers.lua`:** modularize?
+- [ ] **`parsers.lua`:** add dependencies (and "query-only" langs like ecma, jsx, html_tags, ...)
 - [ ] **`parsers.lua`:** assign tiers
+- [ ] **`parsers.lua`:** modularize?
 - [ ] **`install.lua`:** simplify compilation:
   - hardcode one compiler + args per platform
   - provide `install.compile_command` for overriding (function that takes files, ...?)

plugin/query_predicates.lua

@@ -1,6 +1,6 @@
 local query = vim.treesitter.query
 
--- register custom predicates
+-- register custom predicates (overwrite existing; needed for CI)
 
 ---@param match (TSNode|nil)[]
 ---@param pred string[]
@@ -13,7 +13,7 @@ query.add_predicate('kind-eq?', function(match, _, _, pred)
 
   local types = { unpack(pred, 3) }
   return vim.list_contains(types, node:type())
-end)
+end, true)
 
 -- register custom directives
 
@@ -43,7 +43,7 @@ query.add_directive('set-lang-from-mimetype!', function(match, _, bufnr, pred, m
     local parts = vim.split(type_attr_value, '/', {})
     metadata['injection.language'] = parts[#parts]
   end
-end)
+end, true)
 
 local injection_aliases = {
   ex = 'elixir',
@@ -68,7 +68,7 @@ query.add_directive('set-lang-from-info-string!', function(match, _, bufnr, pred
   local injection_alias = vim.treesitter.get_node_text(node, bufnr)
   local filetype = vim.filetype.match({ filename = 'a.' .. injection_alias })
   metadata['injection.language'] = filetype or injection_aliases[injection_alias] or injection_alias
-end)
+end, true)
 
 query.add_directive('downcase!', function(match, _, bufnr, pred, metadata)
   local text, key, value ---@type string|string[], string, string|integer
@@ -95,7 +95,7 @@ query.add_directive('downcase!', function(match, _, bufnr, pred, metadata)
   else
     metadata[key] = string.lower(text)
   end
-end)
+end, true)
 
 -- Trim blank lines from end of the region
 -- Arguments are the captures to trim.
@@ -138,4 +138,4 @@ query.add_directive('trim!', function(match, _, bufnr, pred, metadata)
       metadata[id].range = { start_row, start_col, end_row, end_col }
     end
   end
-end)
+end, true)

scripts/check-queries.lua

@@ -93,17 +93,20 @@ local ok, result = pcall(do_check)
 local allowed_to_fail = vim.split(vim.env.ALLOWED_INSTALLATION_FAILURES or '', ',', true)
 
 for k, v in pairs(require('nvim-treesitter.parsers').configs) do
-  if #vim.api.nvim_get_runtime_file('parser/' .. k .. '.*', false) == 0 then
-    -- On CI all parsers that can be installed from C files should be installed
-    if
-      vim.env.CI
-      and not v.install_info.requires_generate_from_grammar
-      and not vim.list_contains(allowed_to_fail, k)
-    then
-      io_print('Error: parser for ' .. k .. ' is not installed')
-      vim.cmd('cq')
-    else
-      io_print('Warning: parser for ' .. k .. ' is not installed')
+  if v.install_info then
+    -- skip "query only" languages
+    if #vim.api.nvim_get_runtime_file('parser/' .. k .. '.*', false) == 0 then
+      -- On CI all parsers that can be installed from C files should be installed
+      if
+        vim.env.CI
+        and not v.install_info.requires_generate_from_grammar
+        and not vim.list_contains(allowed_to_fail, k)
+      then
+        io_print('Error: parser for ' .. k .. ' is not installed')
+        vim.cmd('cq')
+      else
+        io_print('Warning: parser for ' .. k .. ' is not installed')
+      end
     end
   end
 end

scripts/install-parsers.lua

@@ -0,0 +1,15 @@
+#!/usr/bin/env -S nvim -l
+
+vim.opt.runtimepath:append('.')
+
+-- needed on CI
+vim.fn.mkdir(vim.fn.stdpath('cache'), 'p')
+
+local done = false
+require('nvim-treesitter.install').install('all', {}, function()
+  done = true
+end)
+
+vim.wait(6000000, function()
+  return done
+end)

scripts/minimal_init.lua

@@ -2,11 +2,12 @@ vim.opt.runtimepath:append('.')
 vim.cmd.runtime({ 'plugin/plenary.vim', bang = true })
 vim.cmd.runtime({ 'plugin/nvim-treesitter.lua', bang = true })
 vim.cmd.runtime({ 'plugin/query_predicates.lua', bang = true })
+vim.cmd.runtime({ 'plugin/filetypes.lua', bang = true })
 
 vim.filetype.add({
   extension = {
     conf = 'hocon',
-    cmm = 't32',
+    hurl = 'hurl',
     ncl = 'nickel',
     tig = 'tiger',
     w = 'wing',
@@ -20,6 +21,6 @@ require('nvim-treesitter').setup()
 vim.api.nvim_create_autocmd('FileType', {
   callback = function(args)
     pcall(vim.treesitter.start)
-    vim.bo[args.buffer].indentexpr = 'v:lua.require"nvim-treesitter".indentexpr()'
+    vim.bo[args.buf].indentexpr = 'v:lua.require"nvim-treesitter".indentexpr()'
   end,
 })

scripts/pre-push

@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-# Can be used as a pre-push hook
-# Just symlink this file to .git/hooks/pre-push
-
-echo "Running linter..."
-luacheck .
-
-echo "Checking formatting..."
-stylua --check .

scripts/run_tests.sh

@@ -4,7 +4,7 @@ HERE="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
 cd $HERE/..
 
 run() {
-    nvim --headless --noplugin -u scripts/minimal_init.lua \
+    nvim --headless --noplugin  -u scripts/minimal_init.lua \
         -c "PlenaryBustedDirectory $1 { minimal_init = './scripts/minimal_init.lua' }"
 }
 

scripts/update-lockfile.lua

@@ -1,18 +1,18 @@
 #!/usr/bin/env -S nvim -l
 vim.opt.runtimepath:append('.')
+local util = require('nvim-treesitter.util')
 
 -- Load previous lockfile
 local filename = require('nvim-treesitter.install').get_package_path('lockfile.json')
-local lockfile = vim.json.decode(require('nvim-treesitter.util').read_file(filename))
+local lockfile = vim.json.decode(util.read_file(filename))
 
 ---@type string?
-local skip_lang_string = os.getenv('SKIP_LOCKFILE_UPDATE_FOR_LANGS')
+local skip_lang_string = os.getenv('LOCKFILE_SKIP')
 local skip_langs = skip_lang_string and vim.split(skip_lang_string, ',') or {}
 vim.print('Skipping languages: ', skip_langs)
 
 local sorted_parsers = {}
-local configs = require('nvim-treesitter.parsers').configs
-for k, v in pairs(configs) do
+for k, v in pairs(require('nvim-treesitter.parsers').configs) do
   table.insert(sorted_parsers, { name = k, parser = v })
 end
 table.sort(sorted_parsers, function(a, b)
@@ -42,7 +42,6 @@ for _, v in ipairs(sorted_parsers) do
     print('Skipping ' .. v.name)
   end
 end
-vim.print(lockfile)
 
--- write new lockfile
-require('nvim-treesitter.util').write_file(filename, vim.json.encode(lockfile))
+lockfile = vim.fn.system('jq --sort-keys', vim.json.encode(lockfile))
+util.write_file(filename, lockfile)

scripts/update-lockfile.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-make_ignored() {
-  if [ -n "$1" ]
-  then
-    while read -r lang; do
-      if [ "$lang" != "$1" ]
-      then
-        printf "%s," "$lang"
-      fi
-    done < <(jq 'keys|@sh' -c lockfile.json)
-  fi
-}
-
-TO_IGNORE=$(make_ignored $1)
-
-SKIP_LOCKFILE_UPDATE_FOR_LANGS="$TO_IGNORE" nvim -l ./scripts/write-lockfile.lua
-# Pretty print
-cp lockfile.json /tmp/lockfile.json
-cat /tmp/lockfile.json | jq --sort-keys > lockfile.json

scripts/update-readme.lua

@@ -1,21 +1,19 @@
 #!/usr/bin/env -S nvim -l
 vim.opt.runtimepath:append('.')
-
+local util = require('nvim-treesitter.util')
+local parsers = require('nvim-treesitter.parsers')
 ---@class Parser
 ---@field name string
 ---@field parser ParserInfo
 
-local parsers = require('nvim-treesitter.parsers').configs
 local sorted_parsers = {}
-for k, v in pairs(parsers) do
+for k, v in pairs(parsers.configs) do
   table.insert(sorted_parsers, { name = k, parser = v })
 end
 table.sort(sorted_parsers, function(a, b)
   return a.name < b.name
 end)
 
-local tiers = require('nvim-treesitter.parsers').tiers
-
 local generated_text = [[
 Language | Tier | Queries | CLI | NPM | Maintainer
 -------- |:----:|:-------:|:---:|:---:| ----------
@@ -38,7 +36,7 @@ for _, v in ipairs(sorted_parsers) do
   end
 
   -- tier
-  generated_text = generated_text .. (p.tier and tiers[p.tier] or '') .. ' | '
+  generated_text = generated_text .. (p.tier and parsers.tiers[p.tier] or '') .. ' | '
 
   -- queries
   generated_text = generated_text
@@ -66,7 +64,7 @@ end
 generated_text = generated_text .. footnotes
 
 local readme = 'SUPPORTED_LANGUAGES.md'
-local readme_text = require('nvim-treesitter.util').read_file(readme)
+local readme_text = util.read_file(readme)
 
 local new_readme_text = string.gsub(
   readme_text,
@@ -74,7 +72,7 @@ local new_readme_text = string.gsub(
   '<!--parserinfo-->\n' .. generated_text .. '<!--parserinfo-->'
 )
 
-require('nvim-treesitter.util').write_file(readme, new_readme_text)
+util.write_file(readme, new_readme_text)
 
 if string.find(readme_text, generated_text, 1, true) then
   print(readme .. ' is up-to-date\n')

tests/indent/common.lua

@@ -74,7 +74,7 @@ assert:register(
 local function compare_indent(before, after, xfail)
   assert:add_formatter(format_indent)
   if xfail then
-    io.stdout:write('Warning! Known failure of this test! Please help to fix it! ')
+    -- io.stdout:write('Warning! Known failure of this test! Please help to fix it! ')
     assert.is_not.same_indent(before, after)
   else
     assert.is.same_indent(before, after)