initial commit

tools/highlight-test-generator/.gitignore

@@ -0,0 +1,2 @@
+bin/
+obj/

tools/highlight-test-generator/.vscode/launch.json

@@ -0,0 +1,26 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            // Use IntelliSense to find out which attributes exist for C# debugging
+            // Use hover for the description of the existing attributes
+            // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
+            "name": ".NET Core Launch (console)",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceFolder}/bin/Debug/net6.0/Generator.dll",
+            "args": [],
+            "cwd": "${workspaceFolder}",
+            // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
+            "console": "internalConsole",
+            "stopAtEntry": false
+        },
+        {
+            "name": ".NET Core Attach",
+            "type": "coreclr",
+            "request": "attach"
+        }
+    ]
+}

tools/highlight-test-generator/.vscode/tasks.json

@@ -0,0 +1,41 @@
+{
+    "version": "2.0.0",
+    "tasks": [
+        {
+            "label": "build",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "build",
+                "${workspaceFolder}/Generator.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "publish",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "publish",
+                "${workspaceFolder}/Generator.csproj",
+                "/property:GenerateFullPaths=true",
+                "/consoleloggerparameters:NoSummary"
+            ],
+            "problemMatcher": "$msCompile"
+        },
+        {
+            "label": "watch",
+            "command": "dotnet",
+            "type": "process",
+            "args": [
+                "watch",
+                "run",
+                "--project",
+                "${workspaceFolder}/Generator.csproj"
+            ],
+            "problemMatcher": "$msCompile"
+        }
+    ]
+}

tools/highlight-test-generator/Generator.cs

@@ -0,0 +1,177 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text.RegularExpressions;
+
+if (args.Length != 1)
+{
+    Console.WriteLine("Add the path to the file as an argument. The path needs to be fully qualified and point to an existing file in [REPO ROOT]/test/highlight.");
+    return;
+}
+
+var filePath = args[0];
+
+// Some basic tests on the path, so that we have a chance:
+if (!filePath.Contains("/test/highlight/") ||
+    !filePath.EndsWith(".cs") ||
+    !File.Exists(filePath) ||
+    !Path.IsPathFullyQualified(filePath))
+{
+    Console.WriteLine("The file needs to exist in [REPO ROOT]/test/highlight, and the path needs to be fully qualified.");
+    return;
+}
+
+// Random variable name prefix, so that we don't accidentally replace something in the file:
+var idPrefix = "a" + new Random(filePath.GetHashCode()).NextInt64(10000000) + "_";
+
+var originalLines = File.ReadAllLines(filePath);
+
+
+/// <summary>
+/// Adds tree-sitter highlighting comments to the input file.
+/// Comments start with either `// <-` or `// ^`, depending on the position of the hiughlighted token.
+/// For highlight category, a unique random identifier is used.
+/// </summary>
+void AddCommentsToFile()
+{
+    var newLines = new List<string>();
+    var index = 0;
+
+    foreach (var line in originalLines)
+    {
+        newLines.Add(line);
+
+        var leadingWhitespaces = line[..^line.TrimStart().Length];
+        var first = true;
+
+        var position = leadingWhitespaces.Length;
+        while (position < line.Length)
+        {
+            var ch = line[position];
+
+            bool HandleToken(Func<char, bool> isOfType)
+            {
+                if (!isOfType(ch))
+                {
+                    return false;
+                }
+
+                var variable = $"{idPrefix}{index++}";
+                if (first)
+                {
+                    newLines.Add($"{leadingWhitespaces}// <- {variable}");
+                    first = false;
+                }
+                else
+                {
+                    var spacesLength = position - leadingWhitespaces.Length - 2;
+                    if (spacesLength < 0)
+                    {
+                        // Handle case when the first two characters need different highlight categories:
+                        // Shift // by one space to the right.
+                        newLines.Add($"{leadingWhitespaces} // <- {variable}");
+                    }
+                    else
+                    {
+                        var spaces = new string(' ', position - leadingWhitespaces.Length - 2);
+                        newLines.Add($"{leadingWhitespaces}//{spaces}^ {variable}");
+                    }
+                }
+
+                while (position < line.Length && isOfType(line[position]))
+                {
+                    position++;
+                }
+
+                return true;
+            }
+
+            // The below char methods are not exactly what we need for token parsing, but good enough.
+            // For example
+            // - `_abc` is an identifier, but has both letter and punctuation characters.
+            // - string literals are parsed pretty badly, considering they can have all sorts of characters, even spaces, on which we split.
+            if (!HandleToken(char.IsLetterOrDigit) &&
+                !HandleToken(c => char.IsPunctuation(c) || char.IsSymbol(c)))
+            {
+                position++;
+            }
+        }
+    }
+
+    File.WriteAllLines(filePath, newLines.ToArray());
+}
+
+string GetHighlighterOutput()
+{
+    var process = new Process
+    {
+        StartInfo = new ProcessStartInfo
+        {
+            FileName = "tree-sitter",
+            Arguments = $"test --filter skip-all-corpus-tests",
+            UseShellExecute = false,
+            RedirectStandardOutput = true,
+            WorkingDirectory = Path.GetFullPath(Path.Combine(filePath, "..", "..", "..")),
+
+        }
+    };
+    process.Start();
+    var output = process.StandardOutput.ReadToEnd();
+    process.WaitForExit();
+
+    return output;
+}
+
+var regexWithHighlight = new Regex($@"Failure - row: \d+, column: \d+, expected highlight '{idPrefix}(\d+)', actual highlights: '(.*)'", RegexOptions.Compiled);
+var regexWithNone = new Regex($@"Failure - row: \d+, column: \d+, expected highlight '{idPrefix}(\d+)', actual highlights: none.", RegexOptions.Compiled);
+
+/// <summary>
+/// Runs the tree-sitter test command, and tries to find a single highlighting failure.
+/// If a failure is found, the category is extracted from the output, and the corresponding variable is replaced with the category.
+/// </summary>
+bool FindAndFixHighlightFailure()
+{
+    Console.Write(".");
+    var output = GetHighlighterOutput();
+
+    if (output.IndexOf("✗") != output.LastIndexOf("✗"))
+    {
+        Console.WriteLine("\nThe tree-sitter test execution identified multiple files with failed highlighting. Aborting.");
+        File.WriteAllLines(filePath, originalLines);
+        Environment.Exit(1);
+    }
+
+    var match = regexWithHighlight.Match(output);
+    if (match.Success && match.Groups.Count == 3)
+    {
+        // Highlight found for position, so replace with expected category.
+        var variableCat = $"{idPrefix}{match.Groups[1].Captures[0].Value}";
+        var category = match.Groups[2].Captures[0].Value;
+        File.WriteAllText(filePath, File.ReadAllText(filePath).Replace(variableCat + "\n", category + "\n"));
+        return true;
+    }
+
+    match = regexWithNone.Match(output);
+    if (!match.Success || match.Groups.Count != 2)
+    {
+        // Couldn't match any of the expected patterns.
+        return false;
+    }
+
+    // No highlight found for position, so remove entire line.
+    var variableNone = $"{idPrefix}{match.Groups[1].Captures[0].Value}";
+    var lines = File.ReadAllLines(filePath).Where(line => !line.EndsWith(variableNone)).ToArray();
+    File.WriteAllLines(filePath, lines);
+    return true;
+}
+
+AddCommentsToFile();
+
+Console.WriteLine("Calling tree-sitter highlighter several times. This might take a while.");
+while (FindAndFixHighlightFailure())
+{ }
+Console.WriteLine("");
+Console.WriteLine("Done modifying the input file. It may require some manual cleanup.");
+

tools/highlight-test-generator/Generator.csproj

@@ -0,0 +1,6 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+</Project>

tools/highlight-test-generator/run-generator

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+ROOT="$(git rev-parse --show-toplevel)"
+echo $ROOT/tools/highlight-test-generator/Generator.csproj
+dotnet run --project $ROOT/tools/highlight-test-generator/Generator.csproj $1