Mark failing indent tests to add them to CI

Expected failures should be monitored so that we don't have regressions
and also remove failure marks when they are resolved.

tests/indent/c_spec.lua

@@ -1,4 +1,5 @@
 local Runner = require("tests.indent.common").Runner
+local XFAIL = require("tests.indent.common").XFAIL
 
 local runner = Runner:new(it, "tests/indent/c", {
   tabstop = 4,
@@ -9,7 +10,20 @@ local runner = Runner:new(it, "tests/indent/c", {
 
 describe("indent C:", function()
   describe("whole file:", function()
-    runner:whole_file "."
+    runner:whole_file(".", {
+      expected_failures = {
+        "./ternary.c",
+        "./string.c",
+        "./preproc_func.c",
+        "./preproc_cond.c",
+        "./no_braces.c",
+        "./label.c",
+        "./func.c",
+        "./expr.c",
+        "./comment.c",
+        "./array.c",
+      },
+    })
   end)
 
   describe("new line:", function()
@@ -19,17 +33,17 @@ describe("indent C:", function()
     runner:new_line("cond.c", { on_line = 8, text = "x++;", indent = 8 })
     runner:new_line("expr.c", { on_line = 10, text = "2 *", indent = 8 })
     runner:new_line("func.c", { on_line = 17, text = "int z,", indent = 4 })
-    runner:new_line("label.c", { on_line = 3, text = "normal:", indent = 0 })
+    runner:new_line("label.c", { on_line = 3, text = "normal:", indent = 0 }, "expected failure", XFAIL)
     runner:new_line("loop.c", { on_line = 3, text = "x++;", indent = 8 })
     runner:new_line("preproc_cond.c", { on_line = 5, text = "x++;", indent = 4 })
     runner:new_line("preproc_func.c", { on_line = 3, text = "x++; \\", indent = 8 })
-    runner:new_line("string.c", { on_line = 1, text = "brave new \\", indent = 0 })
+    runner:new_line("string.c", { on_line = 1, text = "brave new \\", indent = 0 }, "expected failure", XFAIL)
     runner:new_line("string.c", { on_line = 4, text = '"brave new "', indent = 4 })
     runner:new_line("struct.c", { on_line = 4, text = "int y;", indent = 8 })
     runner:new_line("switch.c", { on_line = 3, text = "x++;", indent = 12 })
     runner:new_line("ternary.c", { on_line = 4, text = ": (x == 0) : 0", indent = 8 })
     -- the line after inserted one will be left with wrong indent but we only care about the inserted one
-    runner:new_line("no_braces.c", { on_line = 4, text = "x++;", indent = 8 })
+    runner:new_line("no_braces.c", { on_line = 4, text = "x++;", indent = 8 }, "expected failure", XFAIL)
     runner:new_line("no_braces.c", { on_line = 7, text = "x++;", indent = 8 })
     runner:new_line("no_braces.c", { on_line = 10, text = "x++;", indent = 8 })
   end)

tests/indent/common.lua

@@ -5,6 +5,8 @@ local say = require "say"
 local scan_dir = require("plenary.scandir").scan_dir
 local Path = require "plenary.path"
 
+M.XFAIL = "xfail"
+
 local function same_indent(state, arguments)
   local before = arguments[1]
   local after = arguments[2]
@@ -13,9 +15,11 @@ local function same_indent(state, arguments)
   local errors = { before = {}, after = {} }
   for line = 1, #before do
     if before[line] ~= after[line] then
-      -- store the actual indentation length for each line
+      if before[line] and after[line] then
-      errors.before[line] = #string.match(before[line], "^%s*")
+        -- store the actual indentation length for each line
-      errors.after[line] = #string.match(after[line], "^%s*")
+        errors.before[line] = #string.match(before[line], "^%s*")
+        errors.after[line] = #string.match(after[line], "^%s*")
+      end
       ok = false
     end
   end
@@ -28,6 +32,9 @@ local function same_indent(state, arguments)
 end
 
 local function format_indent(arg, fmtargs)
+  if not arg or not fmtargs then
+    return
+  end
   -- find minimal width if any line is longer
   local width = 40
   for _, line in ipairs(fmtargs.other) do
@@ -64,9 +71,14 @@ assert:register(
 )
 
 -- Custom assertion better suited for indentation diffs
-local function compare_indent(before, after)
+local function compare_indent(before, after, xfail)
   assert:add_formatter(format_indent)
-  assert.is.same_indent(before, after)
+  if xfail then
+    io.stdout:write "Warning! Known failure of this test! Please help to fix it! "
+    assert.is_not.same_indent(before, after)
+  else
+    assert.is.same_indent(before, after)
+  end
   assert:remove_formatter(format_indent)
 end
 
@@ -84,6 +96,7 @@ function M.run_indent_test(file, runner, opts)
 
   -- load reference file
   vim.cmd(string.format("edit %s", file))
+  vim.bo.indentexpr = "nvim_treesitter#indent()"
   local before = vim.api.nvim_buf_get_lines(0, 0, -1, true)
 
   assert.are.same("nvim_treesitter#indent()", vim.bo.indentexpr)
@@ -101,12 +114,12 @@ function M.run_indent_test(file, runner, opts)
   return before, after
 end
 
-function M.indent_whole_file(file, opts)
+function M.indent_whole_file(file, opts, xfail)
   local before, after = M.run_indent_test(file, function()
     vim.cmd "silent normal gg=G"
   end, opts)
 
-  compare_indent(before, after)
+  compare_indent(before, after, xfail)
 end
 
 -- Open a file, use `normal o` to insert a new line and compare results
@@ -116,7 +129,7 @@ end
 --   text: text inserted in the new line
 --   indent: expected indent before the inserted text (string or int)
 -- @param opts buffer options passed to set_buf_indent_opts
-function M.indent_new_line(file, spec, opts)
+function M.indent_new_line(file, spec, opts, xfail)
   local before, after = M.run_indent_test(file, function()
     -- move to the line and input the new one
     vim.cmd(string.format("normal! %dG", spec.on_line))
@@ -126,7 +139,7 @@ function M.indent_new_line(file, spec, opts)
   local indent = type(spec.indent) == "string" and spec.indent or string.rep(" ", spec.indent)
   table.insert(before, spec.on_line + 1, indent .. spec.text)
 
-  compare_indent(before, after)
+  compare_indent(before, after, xfail)
 end
 
 local Runner = {}
@@ -144,7 +157,12 @@ function Runner:new(it, base_dir, buf_opts)
   return setmetatable(runner, self)
 end
 
-function Runner:whole_file(dirs)
+function Runner:whole_file(dirs, opts)
+  opts = opts or {}
+  local expected_failures = opts.expected_failures or {}
+  expected_failures = vim.tbl_map(function(f)
+    return Path:new(f):make_relative(self.base_dir.filename)
+  end, expected_failures)
   dirs = type(dirs) == "table" and dirs or { dirs }
   dirs = vim.tbl_map(function(dir)
     dir = self.base_dir / Path:new(dir)
@@ -155,16 +173,16 @@ function Runner:whole_file(dirs)
   for _, file in ipairs(files) do
     local relpath = Path:new(file):make_relative(self.base_dir.filename)
     self.it(relpath, function()
-      M.indent_whole_file(file, self.buf_opts)
+      M.indent_whole_file(file, self.buf_opts, vim.tbl_contains(expected_failures, relpath))
     end)
   end
 end
 
-function Runner:new_line(file, spec, title)
+function Runner:new_line(file, spec, title, xfail)
   title = title and title or tostring(spec.on_line)
   self.it(string.format("%s[%s]", file, title), function()
     local path = self.base_dir / file
-    M.indent_new_line(path.filename, spec, self.buf_opts)
+    M.indent_new_line(path.filename, spec, self.buf_opts, xfail)
   end)
 end
 

tests/indent/cpp_spec.lua

@@ -1,4 +1,5 @@
 local Runner = require("tests.indent.common").Runner
+local XFAIL = require("tests.indent.common").XFAIL
 
 -- will use both c/ and cpp/
 local run = Runner:new(it, "tests/indent", {
@@ -11,11 +12,28 @@ local run = Runner:new(it, "tests/indent", {
 
 describe("indent C++:", function()
   describe("whole file:", function()
-    run:whole_file { "c/", "cpp/" }
+    run:whole_file({ "c/", "cpp/" }, {
+      expected_failures = {
+        -- C
+        "c/ternary.c",
+        "c/string.c",
+        "c/preproc_func.c",
+        "c/preproc_cond.c",
+        "c/no_braces.c",
+        "c/label.c",
+        "c/func.c",
+        "c/expr.c",
+        "c/comment.c",
+        "c/array.c",
+        -- C++
+        "cpp/access.cpp",
+        "cpp/stream.cpp",
+      },
+    })
   end)
 
   describe("new line:", function()
-    run:new_line("cpp/access.cpp", { on_line = 3, text = "protected:", indent = 0 })
+    run:new_line("cpp/access.cpp", { on_line = 3, text = "protected:", indent = 0 }, "expected failure", XFAIL)
     run:new_line("cpp/class.cpp", { on_line = 2, text = "using T = int;", indent = 4 })
     run:new_line("cpp/stream.cpp", { on_line = 5, text = "<< x + 3", indent = 8 })
 
@@ -25,17 +43,17 @@ describe("indent C++:", function()
     run:new_line("c/cond.c", { on_line = 8, text = "x++;", indent = 8 })
     run:new_line("c/expr.c", { on_line = 10, text = "2 *", indent = 8 })
     run:new_line("c/func.c", { on_line = 17, text = "int z,", indent = 4 })
-    run:new_line("c/label.c", { on_line = 3, text = "normal:", indent = 0 })
+    run:new_line("c/label.c", { on_line = 3, text = "normal:", indent = 0 }, "expected failure", XFAIL)
     run:new_line("c/loop.c", { on_line = 3, text = "x++;", indent = 8 })
     run:new_line("c/preproc_cond.c", { on_line = 5, text = "x++;", indent = 4 })
     run:new_line("c/preproc_func.c", { on_line = 3, text = "x++; \\", indent = 8 })
-    run:new_line("c/string.c", { on_line = 1, text = "brave new \\", indent = 0 })
+    run:new_line("c/string.c", { on_line = 1, text = "brave new \\", indent = 0 }, "expected failure", XFAIL)
     run:new_line("c/string.c", { on_line = 4, text = '"brave new "', indent = 4 })
     run:new_line("c/struct.c", { on_line = 4, text = "int y;", indent = 8 })
     run:new_line("c/switch.c", { on_line = 3, text = "x++;", indent = 12 })
     run:new_line("c/ternary.c", { on_line = 4, text = ": (x == 0) : 0", indent = 8 })
     -- the line after inserted one will be left with wrong indent but we only care about the inserted one
-    run:new_line("c/no_braces.c", { on_line = 4, text = "x++;", indent = 8 })
+    run:new_line("c/no_braces.c", { on_line = 4, text = "x++;", indent = 8 }, "expected failure", XFAIL)
     run:new_line("c/no_braces.c", { on_line = 7, text = "x++;", indent = 8 })
     run:new_line("c/no_braces.c", { on_line = 10, text = "x++;", indent = 8 })
   end)

tests/indent/lua_spec.lua

@@ -1,4 +1,5 @@
 local Runner = require("tests.indent.common").Runner
+local XFAIL = require("tests.indent.common").XFAIL
 
 local run = Runner:new(it, "tests/indent/lua", {
   tabstop = 2,
@@ -9,7 +10,10 @@ local run = Runner:new(it, "tests/indent/lua", {
 
 describe("indent Lua:", function()
   describe("whole file:", function()
-    run:whole_file "."
+    run:whole_file(".", { expected_failures = {
+      "./string.lua",
+      "./comment.lua",
+    } })
   end)
 
   describe("new line:", function()
@@ -18,10 +22,10 @@ describe("indent Lua:", function()
     run:new_line("func.lua", { on_line = 1, text = "x = x + 1", indent = 2 })
     run:new_line("func.lua", { on_line = 2, text = "y = y + 1", indent = 4 })
     run:new_line("func.lua", { on_line = 5, text = "3,", indent = 4 })
-    run:new_line("string.lua", { on_line = 1, text = "x", indent = 0 })
+    run:new_line("string.lua", { on_line = 1, text = "x", indent = 0 }, "expected failure", XFAIL)
-    run:new_line("string.lua", { on_line = 2, text = "x", indent = 0 })
+    run:new_line("string.lua", { on_line = 2, text = "x", indent = 0 }, "expected failure", XFAIL)
     run:new_line("string.lua", { on_line = 3, text = "x", indent = 2 })
-    run:new_line("string.lua", { on_line = 4, text = "x", indent = 4 })
+    run:new_line("string.lua", { on_line = 4, text = "x", indent = 4 }, "expected failure", XFAIL)
     run:new_line("table.lua", { on_line = 1, text = "b = 0,", indent = 2 })
     run:new_line("table.lua", { on_line = 5, text = "4,", indent = 4 })
     run:new_line("table.lua", { on_line = 7, text = "4,", indent = 4 })

tests/indent/python_spec.lua

@@ -1,4 +1,5 @@
 local Runner = require("tests.indent.common").Runner
+local XFAIL = require("tests.indent.common").XFAIL
 
 local run = Runner:new(it, "tests/indent/python", {
   tabstop = 4,
@@ -9,11 +10,23 @@ local run = Runner:new(it, "tests/indent/python", {
 
 describe("indent Python:", function()
   describe("whole file:", function()
-    run:whole_file "."
+    run:whole_file(".", {
+      expected_failures = {
+        "./aligned_indent.py",
+        "./basic_blocks.py",
+        "./branches.py",
+        "./control_flow.py",
+        "./hanging_indent.py",
+        "./join_lines.py",
+        "./nested_collections.py",
+        "./strings.py",
+        "./control_flow.py",
+      },
+    })
   end)
 
   describe("new line:", function()
-    run:new_line("aligned_indent.py", { on_line = 1, text = "arg3,", indent = 19 })
+    run:new_line("aligned_indent.py", { on_line = 1, text = "arg3,", indent = 19 }, "xfail", XFAIL)
     run:new_line("basic_blocks.py", { on_line = 1, text = "wait,", indent = 4 })
     run:new_line("basic_blocks.py", { on_line = 6, text = "x += 1", indent = 4 })
     run:new_line("basic_blocks.py", { on_line = 10, text = "x += 1", indent = 8 })
@@ -21,8 +34,8 @@ describe("indent Python:", function()
     run:new_line("basic_blocks.py", { on_line = 11, text = "x += 1", indent = 8 }, "11, after last line of a block")
     run:new_line("basic_collections.py", { on_line = 3, text = "4,", indent = 4 })
     run:new_line("comprehensions.py", { on_line = 8, text = "if x != 2", indent = 4 })
-    run:new_line("control_flow.py", { on_line = 23, text = "x = 4", indent = 4 })
+    run:new_line("control_flow.py", { on_line = 23, text = "x = 4", indent = 4 }, "expected failure", XFAIL)
-    run:new_line("hanging_indent.py", { on_line = 1, text = "arg0,", indent = 8 })
+    run:new_line("hanging_indent.py", { on_line = 1, text = "arg0,", indent = 8 }, "expected failure", XFAIL)
     run:new_line("hanging_indent.py", { on_line = 5, text = "0,", indent = 4 })
     run:new_line("join_lines.py", { on_line = 1, text = "+ 1 \\", indent = 4 })
     run:new_line("join_lines.py", { on_line = 4, text = "+ 1 \\", indent = 4 })
@@ -32,7 +45,7 @@ describe("indent Python:", function()
     run:new_line("nested_collections.py", { on_line = 29, text = "[1, 2],", indent = 12 })
     run:new_line("nested_collections.py", { on_line = 39, text = "0,", indent = 5 })
     run:new_line("strings.py", { on_line = 14, text = "x", indent = 4 })
-    run:new_line("strings.py", { on_line = 15, text = "x", indent = 0 })
+    run:new_line("strings.py", { on_line = 15, text = "x", indent = 0 }, nil, XFAIL)
-    run:new_line("strings.py", { on_line = 16, text = "x", indent = 8 })
+    run:new_line("strings.py", { on_line = 16, text = "x", indent = 8 }, nil, XFAIL)
   end)
 end)

tests/indent/rust_spec.lua

@@ -1,4 +1,5 @@
 local Runner = require("tests.indent.common").Runner
+local XFAIL = require("tests.indent.common").XFAIL
 
 local run = Runner:new(it, "tests/indent/rust", {
   tabstop = 4,
@@ -9,7 +10,17 @@ local run = Runner:new(it, "tests/indent/rust", {
 
 describe("indent Rust:", function()
   describe("whole file:", function()
-    run:whole_file "."
+    run:whole_file(".", {
+      expected_failures = {
+        "./enum.rs",
+        "./func.rs",
+        "./array.rs",
+        "./where.rs",
+        "./trait.rs",
+        "./string.rs",
+        "./macro.rs",
+      },
+    })
   end)
 
   describe("new line:", function()
@@ -18,8 +29,8 @@ describe("indent Rust:", function()
     run:new_line("comment.rs", { on_line = 3, text = "a", indent = "/// " })
     run:new_line("cond.rs", { on_line = 11, text = "x += 1;", indent = 12 })
     run:new_line("cond.rs", { on_line = 2, text = "x += 1;", indent = 8 })
-    run:new_line("cond.rs", { on_line = 4, text = "x += 1;", indent = 8 })
+    run:new_line("cond.rs", { on_line = 4, text = "x += 1;", indent = 8 }, "expected_failures", XFAIL)
-    run:new_line("cond.rs", { on_line = 6, text = "x += 1;", indent = 8 })
+    run:new_line("cond.rs", { on_line = 6, text = "x += 1;", indent = 8 }, "expected_failures", XFAIL)
     run:new_line("enum.rs", { on_line = 2, text = "Q,", indent = 4 })
     run:new_line("enum.rs", { on_line = 4, text = "i32,", indent = 8 })
     run:new_line("enum.rs", { on_line = 8, text = "z: u32,", indent = 8 })
@@ -38,7 +49,7 @@ describe("indent Rust:", function()
     run:new_line("mod.rs", { on_line = 1, text = "const Z: i32 = 1;", indent = 4 })
     run:new_line("mod.rs", { on_line = 2, text = "const Z: i32 = 1;", indent = 4 })
     run:new_line("mod.rs", { on_line = 6, text = "const Z: i32 = 1;", indent = 8 })
-    run:new_line("string.rs", { on_line = 2, text = "brave new", indent = 0 })
+    run:new_line("string.rs", { on_line = 2, text = "brave new", indent = 0 }, "expected_failures", XFAIL)
     run:new_line("string.rs", { on_line = 5, text = "brave new \\", indent = 8 })
     run:new_line("string.rs", { on_line = 9, text = "brave new \\", indent = 8 })
     run:new_line("struct.rs", { on_line = 1, text = "z: i32,", indent = 4 })