using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text.RegularExpressions; if (args.Length != 1) { Console.WriteLine("Add the path to the file as an argument. The path needs to be fully qualified and point to an existing file in [REPO ROOT]/test/highlight."); return; } var filePath = args[0]; // Some basic tests on the path, so that we have a chance: if (!filePath.Contains("/test/highlight/") || !filePath.EndsWith(".cs") || !File.Exists(filePath) || !Path.IsPathFullyQualified(filePath)) { Console.WriteLine("The file needs to exist in [REPO ROOT]/test/highlight, and the path needs to be fully qualified."); return; } // Random variable name prefix, so that we don't accidentally replace something in the file: var idPrefix = "a" + new Random(filePath.GetHashCode()).NextInt64(10000000) + "_"; var originalLines = File.ReadAllLines(filePath); /// /// Adds tree-sitter highlighting comments to the input file. /// Comments start with either `// <-` or `// ^`, depending on the position of the hiughlighted token. /// For highlight category, a unique random identifier is used. /// void AddCommentsToFile() { var newLines = new List(); var index = 0; foreach (var line in originalLines) { newLines.Add(line); var leadingWhitespaces = line[..^line.TrimStart().Length]; var first = true; var position = leadingWhitespaces.Length; while (position < line.Length) { var ch = line[position]; bool HandleToken(Func isOfType) { if (!isOfType(ch)) { return false; } var variable = $"{idPrefix}{index++}"; if (first) { newLines.Add($"{leadingWhitespaces}// <- {variable}"); first = false; } else { var spacesLength = position - leadingWhitespaces.Length - 2; if (spacesLength < 0) { // Handle case when the first two characters need different highlight categories: // Shift // by one space to the right. newLines.Add($"{leadingWhitespaces} // <- {variable}"); } else { var spaces = new string(' ', position - leadingWhitespaces.Length - 2); newLines.Add($"{leadingWhitespaces}//{spaces}^ {variable}"); } } while (position < line.Length && isOfType(line[position])) { position++; } return true; } // The below char methods are not exactly what we need for token parsing, but good enough. // For example // - `_abc` is an identifier, but has both letter and punctuation characters. // - string literals are parsed pretty badly, considering they can have all sorts of characters, even spaces, on which we split. if (!HandleToken(char.IsLetterOrDigit) && !HandleToken(c => char.IsPunctuation(c) || char.IsSymbol(c))) { position++; } } } File.WriteAllLines(filePath, newLines.ToArray()); } string GetHighlighterOutput() { var process = new Process { StartInfo = new ProcessStartInfo { FileName = "tree-sitter", Arguments = $"test --filter skip-all-corpus-tests", UseShellExecute = false, RedirectStandardOutput = true, WorkingDirectory = Path.GetFullPath(Path.Combine(filePath, "..", "..", "..")), } }; process.Start(); var output = process.StandardOutput.ReadToEnd(); process.WaitForExit(); return output; } var regexWithHighlight = new Regex($@"Failure - row: \d+, column: \d+, expected highlight '{idPrefix}(\d+)', actual highlights: '(.*)'", RegexOptions.Compiled); var regexWithNone = new Regex($@"Failure - row: \d+, column: \d+, expected highlight '{idPrefix}(\d+)', actual highlights: none.", RegexOptions.Compiled); /// /// Runs the tree-sitter test command, and tries to find a single highlighting failure. /// If a failure is found, the category is extracted from the output, and the corresponding variable is replaced with the category. /// bool FindAndFixHighlightFailure() { Console.Write("."); var output = GetHighlighterOutput(); if (output.IndexOf("✗") != output.LastIndexOf("✗")) { Console.WriteLine("\nThe tree-sitter test execution identified multiple files with failed highlighting. Aborting."); File.WriteAllLines(filePath, originalLines); Environment.Exit(1); } var match = regexWithHighlight.Match(output); if (match.Success && match.Groups.Count == 3) { // Highlight found for position, so replace with expected category. var variableCat = $"{idPrefix}{match.Groups[1].Captures[0].Value}"; var category = match.Groups[2].Captures[0].Value; File.WriteAllText(filePath, File.ReadAllText(filePath).Replace(variableCat + "\n", category + "\n")); return true; } match = regexWithNone.Match(output); if (!match.Success || match.Groups.Count != 2) { // Couldn't match any of the expected patterns. return false; } // No highlight found for position, so remove entire line. var variableNone = $"{idPrefix}{match.Groups[1].Captures[0].Value}"; var lines = File.ReadAllLines(filePath).Where(line => !line.EndsWith(variableNone)).ToArray(); File.WriteAllLines(filePath, lines); return true; } AddCommentsToFile(); Console.WriteLine("Calling tree-sitter highlighter several times. This might take a while."); while (FindAndFixHighlightFailure()) { } Console.WriteLine(""); Console.WriteLine("Done modifying the input file. It may require some manual cleanup.");