diff --git a/.gitignore b/.gitignore index b3a52364a..8b256e0c4 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ build/ build_*/ install/ install_*/ + +# Mac OS specific files +.DS_Store + +# Local dev docker compose +local-dev/ diff --git a/plugins/csharp/parser/src/csharpparser.cpp b/plugins/csharp/parser/src/csharpparser.cpp index 9666fafea..790212299 100644 --- a/plugins/csharp/parser/src/csharpparser.cpp +++ b/plugins/csharp/parser/src/csharpparser.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include @@ -18,9 +20,6 @@ #include -#include -#include - namespace cc { namespace parser @@ -31,9 +30,9 @@ CsharpParser::CsharpParser(ParserContext& ctx_): AbstractParser(ctx_) _threadNum = _ctx.options["jobs"].as(); } -bool CsharpParser::acceptProjectBuildPath(const std::string& buildPath_) +bool CsharpParser::acceptProjectBuildPath(const std::string& path_) { - return fs::is_directory(buildPath_); + return fs::exists(path_); } bool CsharpParser::parse() @@ -42,36 +41,33 @@ bool CsharpParser::parse() std::vector paths = _ctx.options["input"].as>(); - if (!paths.empty()) - { - LOG(debug) << "C# parser parse path: " << paths.size(); - success = success && parseProjectBuildPath(paths); - } - else - { - LOG(error) << "No input directories provided for C# parser!"; - success = false; - } + if (!paths.empty()) + { + LOG(debug) << "C# parser input paths count: " << paths.size(); + success = success && parseProjectBuildPath(paths); + } + else + { + LOG(error) << "No input paths provided for C# parser!"; + success = false; + } return success; } -bool CsharpParser::parseProjectBuildPath( - const std::vector& paths_ //, - ) +bool CsharpParser::parseProjectBuildPath(const std::vector& paths_) { namespace ch = std::chrono; std::future log; fs::path csharp_path = util::findCurrentExecutableDir() + "/../lib/csharp/"; /* - * Concatenate the command parameters to pass to the C# parser. + * Concatenate the command parameters to pass to the C# parser: * 1) C# parser binary * 2) Database connection string - * 3) Project build directory path - * 4) CC lib directory path - * 5) Thread number - * 6+) Source directories + * 3) CC lib directory path + * 4) Thread number + * 5+) Input paths (directories, .sln, .slnx, or .csproj files) */ std::string command("./CSharpParser "); command.append("'"); @@ -81,14 +77,18 @@ bool CsharpParser::parseProjectBuildPath( command.append("' "); command.append(std::to_string(_ctx.options["jobs"].as())); - for (auto p : paths_) + for (const auto& p : paths_) { - if (fs::is_directory(p)) + if (fs::exists(p)) { command.append(" '"); command.append(p); command.append("' "); } + else + { + LOG(warning) << "Input path does not exist: " << p; + } } LOG(debug) << "CSharpParser command: " << command; @@ -105,7 +105,7 @@ bool CsharpParser::parseProjectBuildPath( std::stringstream log_str(log.get()); int countFull = 0, countPart = 0; -while(std::getline(log_str, line, '\n')) + while (std::getline(log_str, line, '\n')) { line.erase(std::remove(line.begin(), line.end(), '\r'), line.end()); // remove \r if (line.empty()) continue; @@ -137,7 +137,7 @@ while(std::getline(log_str, line, '\n')) } catch (const boost::property_tree::ptree_error& e) { - LOG(warning) << "Missing expected JSON field from C# parser: " << e.what() << " | Line: " << jsonString; + LOG(warning) << "Missing expected JSON field from C# parser: " << e.what() << " | Line: " << jsonString; } } else @@ -147,12 +147,11 @@ while(std::getline(log_str, line, '\n')) } ch::steady_clock::time_point after = ch::steady_clock::now(); - elapsed_time = - ch::duration_cast(after - current).count(); + elapsed_time = ch::duration_cast(after - current).count(); LOG(debug) << "C# source manage time: " << elapsed_time << " ms"; LOG(info) << "Number of files fully parsed: " << countFull << - ", partially parsed: " << countPart << ", total: " << countFull+countPart; + ", partially parsed: " << countPart << ", total: " << countFull + countPart; return result == 0; } @@ -173,16 +172,28 @@ void CsharpParser::addSource(const std::string& filepath_, const std::string& ta buildSource.file->type = "CS"; buildSource.action = buildAction; + model::BuildTarget buildTarget; + if (!targetDll_.empty()) + { + buildTarget.file = _ctx.srcMgr.getFile(targetDll_); + buildTarget.file->type = "Binary"; + buildTarget.action = buildAction; + _ctx.srcMgr.updateFile(*buildTarget.file); + } + _ctx.srcMgr.updateFile(*buildSource.file); _ctx.srcMgr.persistFiles(); transaction([&, this] { _ctx.db->persist(buildAction); _ctx.db->persist(buildSource); + if (!targetDll_.empty()) + { + _ctx.db->persist(buildTarget); + } }); } - CsharpParser::~CsharpParser() { } @@ -194,7 +205,6 @@ extern "C" boost::program_options::options_description getOptions() { boost::program_options::options_description description("C# Plugin"); - return description; } @@ -206,4 +216,4 @@ extern "C" #pragma clang diagnostic pop } // parser -} // cc +} // cc \ No newline at end of file diff --git a/plugins/csharp/parser/src_csharp/CSharpParser.csproj b/plugins/csharp/parser/src_csharp/CSharpParser.csproj index ee513d5b9..173050c6b 100644 --- a/plugins/csharp/parser/src_csharp/CSharpParser.csproj +++ b/plugins/csharp/parser/src_csharp/CSharpParser.csproj @@ -15,6 +15,7 @@ + all diff --git a/plugins/csharp/parser/src_csharp/Program.cs b/plugins/csharp/parser/src_csharp/Program.cs index ee66bba7f..f7eeb43e4 100644 --- a/plugins/csharp/parser/src_csharp/Program.cs +++ b/plugins/csharp/parser/src_csharp/Program.cs @@ -1,5 +1,4 @@ -using static System.Console; -using System.Linq; +using System.Linq; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.EntityFrameworkCore; @@ -7,8 +6,11 @@ using System.IO; using System.Collections; using System.Collections.Generic; +using System.Text.RegularExpressions; using System.Threading.Tasks; using CSharpParser.model; +using Microsoft.Build.Locator; +using Microsoft.CodeAnalysis.MSBuild; using System.Xml.Linq; using System.Text.Json; @@ -20,15 +22,25 @@ class Program private static string _buildDirBase = ""; private static string _connectionString = ""; - static int Main(string[] args) + static async Task Main(string[] args) { + try + { + MSBuildLocator.RegisterDefaults(); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Failed to register MSBuild: {ex.Message}"); + return 1; + } + _rootDir = new List(); int threadNum = 4; try { _connectionString = args[0].Replace("'", ""); - _buildDirBase = args[1].Replace("'", ""); //indexes + _buildDirBase = args[1].Replace("'", ""); // indexes threadNum = int.Parse(args[2]); for (int i = 3; i < args.Length; ++i) @@ -36,46 +48,135 @@ static int Main(string[] args) _rootDir.Add(args[i].Replace("'", "")); } } - catch (Exception e) + catch (Exception) + { + Console.WriteLine("Error in parsing command!"); + return 1; + } + + if (_rootDir.Count == 0) { - WriteLine("Error in parsing command!"); + Console.WriteLine("Missing input path argument in CSharpParser!"); return 1; } - //Converting the connectionstring into entiy framwork style connectionstring + string primaryInput = _rootDir[0]; + + // Converting the connectionstring into entity framework style connectionstring string csharpConnectionString = transformConnectionString(); var options = new DbContextOptionsBuilder() .UseNpgsql(csharpConnectionString) .Options; - CsharpDbContext _context = new CsharpDbContext(options); + await using var _context = new CsharpDbContext(options); _context.Database.Migrate(); + if (IsSolutionInput(primaryInput)) + { + await ParseSolutionPathAsync(primaryInput, csharpConnectionString, threadNum); + return 0; + } + + if (IsProjectInput(primaryInput)) + { + var projectInputs = _rootDir + .Where(IsProjectInput) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + await ParseProjectsByPathAsync(projectInputs, csharpConnectionString, threadNum); + return 0; + } + + if (Directory.Exists(primaryInput)) + { + var solutionFiles = Directory.GetFiles(primaryInput, "*.sln", SearchOption.AllDirectories) + .Concat(Directory.GetFiles(primaryInput, "*.slnx", SearchOption.AllDirectories)) + .ToList(); + + if (solutionFiles.Count > 0) + { + Console.WriteLine($"Found solution file in directory: {solutionFiles[0]}"); + await ParseSolutionPathAsync(solutionFiles[0], csharpConnectionString, threadNum); + return 0; + } + + var projectFiles = Directory.GetFiles(primaryInput, "*.csproj", SearchOption.AllDirectories) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (projectFiles.Count > 0) + { + Console.WriteLine($"Found {projectFiles.Count} .csproj file(s) in directory."); + await ParseProjectsByPathAsync(projectFiles, csharpConnectionString, threadNum); + return 0; + } + await ParseLegacyMode(csharpConnectionString, threadNum); + return 0; + } + + if (File.Exists(primaryInput)) + { + Console.Error.WriteLine($"Unsupported file type: {primaryInput}"); + Console.Error.WriteLine("Supported: .sln, .slnx, .csproj"); + return 1; + } + + Console.WriteLine("Unsupported input path in CSharpParser!"); + return 1; + } + + private static bool IsSolutionInput(string inputPath) + { + return File.Exists(inputPath) + && (inputPath.EndsWith(".sln", StringComparison.OrdinalIgnoreCase) + || inputPath.EndsWith(".slnx", StringComparison.OrdinalIgnoreCase)); + } + + private static bool IsProjectInput(string inputPath) + { + return File.Exists(inputPath) + && inputPath.EndsWith(".csproj", StringComparison.OrdinalIgnoreCase); + } + + private static async Task ParseLegacyMode(string csharpConnectionString, int threadNum) + { List allFiles = new List(); // This dictionary will remember which file belongs to which DLL Dictionary fileToTargetDll = new Dictionary(); foreach (var p in _rootDir) { + if (!Directory.Exists(p)) + { + Console.WriteLine("Skipping non-directory input in legacy mode: " + p); + continue; + } + + Console.WriteLine(p); allFiles.AddRange(GetSourceFilesFromDir(p, ".cs")); } allFiles = allFiles.Distinct().ToList(); foreach (var f in allFiles) { - WriteLine(f); + Console.WriteLine(f); } + IEnumerable assemblies_base = Directory.Exists(_buildDirBase) + ? GetSourceFilesFromDir(_buildDirBase, ".dll") + : Enumerable.Empty(); - IEnumerable assemblies_base = GetSourceFilesFromDir(_buildDirBase, ".dll"); //loading basic dlls - List assemblies = new List(); foreach (var p in _rootDir) { - // We search for all .dll files in all input directories - assemblies.AddRange(GetSourceFilesFromDir(p, ".dll")); + if (Directory.Exists(p)) + { + // We search for all .dll files in all input directories + assemblies.AddRange(GetSourceFilesFromDir(p, ".dll")); + } } // Let's keep only one of each DLL based on the file name! assemblies = assemblies.GroupBy(x => System.IO.Path.GetFileName(x)) @@ -89,12 +190,12 @@ static int Main(string[] args) SyntaxTree tree = CSharpSyntaxTree.ParseText(programText, null, file); trees.Add(tree); } - Write(trees.Count); + Console.Write(trees.Count); CSharpCompilation compilation = CSharpCompilation.Create("CSharpCompilation") .AddReferences(MetadataReference.CreateFromFile(typeof(object).Assembly.Location)) .AddSyntaxTrees(trees); - + foreach (string file in assemblies_base) { compilation = compilation.AddReferences(MetadataReference.CreateFromFile(file)); @@ -104,10 +205,7 @@ static int Main(string[] args) compilation = compilation.AddReferences(MetadataReference.CreateFromFile(file)); } - var runtask = ParalellRun(csharpConnectionString, threadNum, trees, compilation, fileToTargetDll); - int ret = runtask.Result; - - return 0; + await ParalellRun(csharpConnectionString, threadNum, trees, compilation, fileToTargetDll); } private static async Task ParalellRun(string csharpConnectionString, int threadNum, @@ -128,10 +226,10 @@ private static async Task ParalellRun(string csharpConnectionString, int th var ParsingTasks = new List>(); int maxThread = threadNum < trees.Count() ? threadNum : trees.Count(); - WriteLine(threadNum); + Console.WriteLine(threadNum); for (int i = 0; i < maxThread; i++) - { - ParsingTasks.Add(ParseTree(contextList[i],trees[i],compilation,i,fileToTargetDll)); + { + ParsingTasks.Add(ParseTree(contextList[i], trees[i], compilation, i, fileToTargetDll)); } int nextTreeIndex = maxThread; @@ -144,7 +242,7 @@ private static async Task ParalellRun(string csharpConnectionString, int th if (nextTreeIndex < trees.Count) { ParsingTasks.Add(ParseTree(contextList[nextContextIndex], - trees[nextTreeIndex],compilation,nextContextIndex, fileToTargetDll)); + trees[nextTreeIndex], compilation, nextContextIndex, fileToTargetDll)); ++nextTreeIndex; } } @@ -157,24 +255,23 @@ private static async Task ParalellRun(string csharpConnectionString, int th return 0; } - private static async Task ParseTree(CsharpDbContext context, - SyntaxTree tree, CSharpCompilation compilation, int index, + private static async Task ParseTree(CsharpDbContext context, + SyntaxTree tree, CSharpCompilation compilation, int index, Dictionary fileToTargetDll) { var ParsingTask = Task.Run(() => { - WriteLine("ParallelRun " + tree.FilePath); + Console.WriteLine("ParallelRun " + tree.FilePath); SemanticModel model = compilation.GetSemanticModel(tree); var visitor = new AstVisitor(context, model, tree); - visitor.Visit(tree.GetCompilationUnitRoot()); + visitor.Visit(tree.GetCompilationUnitRoot()); context.SaveChanges(); - context.ChangeTracker.Clear(); + context.ChangeTracker.Clear(); - // Find the DLL name and append a | to the filename. string target = fileToTargetDll.ContainsKey(tree.FilePath) ? fileToTargetDll[tree.FilePath] : ""; - - var resultData = new + + var resultData = new { fullyParsed = visitor.FullyParsed, filePath = tree.FilePath, @@ -190,8 +287,7 @@ private static async Task ParseTree(CsharpDbContext context, public static IEnumerable GetSourceFilesFromDir(string root, string extension) { - IEnumerable allFiles = new string[]{}; - // Data structure to hold names of subfolders. + IEnumerable allFiles = new string[] { }; ArrayList dirs = new ArrayList(); if (!System.IO.Directory.Exists(root)) @@ -211,16 +307,15 @@ public static IEnumerable GetSourceFilesFromDir(string root, string exte } catch (UnauthorizedAccessException e) { - WriteLine(e.Message); + Console.WriteLine(e.Message); continue; } catch (System.IO.DirectoryNotFoundException e) { - WriteLine(e.Message); + Console.WriteLine(e.Message); continue; } - // Add the subdirectories for traversal. dirs.AddRange(subDirs); string[] files = null; @@ -244,13 +339,13 @@ public static IEnumerable GetSourceFilesFromDir(string root, string exte try { System.IO.FileInfo fi = new System.IO.FileInfo(file); - if (fi.Extension == extension) { + if (fi.Extension == extension) + { allFiles = allFiles.Append(file); } } catch (System.IO.FileNotFoundException e) { - // If file was deleted by a separate application Console.WriteLine(e.Message); } } @@ -261,15 +356,15 @@ public static IEnumerable GetSourceFilesFromDir(string root, string exte private static string transformConnectionString() { - _connectionString = _connectionString.Substring(_connectionString.IndexOf(':')+1); + _connectionString = _connectionString.Substring(_connectionString.IndexOf(':') + 1); _connectionString = _connectionString.Replace("user", "username"); - string [] properties = _connectionString.Split(';'); + string[] properties = _connectionString.Split(';'); string csharpConnectionString = ""; for (int i = 0; i < properties.Length; ++i) { - csharpConnectionString += properties[i].Substring(0,1).ToUpper() + csharpConnectionString += properties[i].Substring(0, 1).ToUpper() + properties[i].Substring(1); - if (i < properties.Length-1) + if (i < properties.Length - 1) { csharpConnectionString += ";"; } @@ -277,5 +372,285 @@ private static string transformConnectionString() return csharpConnectionString; } + + private static async Task ParseSolutionPathAsync( + string solutionPath, + string csharpConnectionString, + int threadCount) + { + try + { + var solution = await LoadSolutionAsync(solutionPath); + int documentCount = solution.Projects + .SelectMany(p => p.Documents) + .Count(d => !string.IsNullOrEmpty(d.FilePath)); + + if (solution.Projects.Any() && documentCount > 0) + { + await ParseSolutionAsync(solution, csharpConnectionString, threadCount); + return; + } + + Console.WriteLine( + "OpenSolutionAsync returned no usable documents, trying manual project discovery."); + } + catch (Exception ex) + { + Console.Error.WriteLine($"OpenSolutionAsync failed: {ex.Message}"); + Console.WriteLine("Trying manual project discovery from solution file."); + } + + var projectPaths = ExtractProjectPathsFromSolution(solutionPath); + if (projectPaths.Count == 0) + { + Console.Error.WriteLine($"No C# projects found in solution: {solutionPath}"); + throw new InvalidOperationException( + $"No C# projects found in solution: {solutionPath}"); + } + + await ParseProjectsByPathAsync(projectPaths, csharpConnectionString, threadCount); + } + + private static List ExtractProjectPathsFromSolution(string solutionPath) + { + var projectPaths = new List(); + var solutionDir = Path.GetDirectoryName(solutionPath) ?? string.Empty; + + var csprojRegex = new Regex( + "\"([^\"]+\\.csproj)\"", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + IEnumerable solutionLines; + try + { + solutionLines = File.ReadLines(solutionPath); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Failed to read solution file '{solutionPath}': {ex.Message}"); + return projectPaths; + } + + foreach (var line in solutionLines) + { + var matches = csprojRegex.Matches(line); + foreach (Match match in matches) + { + var relativeProjectPath = match.Groups[1].Value.Trim(); + if (string.IsNullOrEmpty(relativeProjectPath)) + { + continue; + } + + var normalizedProjectPath = relativeProjectPath + .Replace('\\', Path.DirectorySeparatorChar) + .Replace('/', Path.DirectorySeparatorChar); + + var fullProjectPath = Path.GetFullPath( + Path.Combine(solutionDir, normalizedProjectPath)); + + if (!File.Exists(fullProjectPath)) + { + Console.Error.WriteLine( + $"Project path from solution not found: {fullProjectPath}"); + continue; + } + + projectPaths.Add(fullProjectPath); + } + } + + return projectPaths + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static async Task ParseProjectsByPathAsync( + List projectPaths, + string csharpConnectionString, + int threadCount) + { + Console.WriteLine($"Fallback project count: {projectPaths.Count}"); + foreach (var projectPath in projectPaths) + { + try + { + Console.WriteLine($"Fallback loading project: {projectPath}"); + using var workspace = CreateMsBuildWorkspace(); + var project = await workspace.OpenProjectAsync(projectPath); + await ParseProjectAsync( + project, + csharpConnectionString, + threadCount); + } + catch (Exception ex) + { + Console.Error.WriteLine( + $"Fallback load failed for '{projectPath}': {ex.Message}"); + } + } + } + + private static MSBuildWorkspace CreateMsBuildWorkspace() + { + var workspace = MSBuildWorkspace.Create(); + workspace.LoadMetadataForReferencedProjects = true; + workspace.WorkspaceFailed += (sender, args) => + { + Console.Error.WriteLine($"Workspace warning: {args.Diagnostic.Message}"); + }; + + return workspace; + } + + private static async Task LoadSolutionAsync(string solutionPath) + { + Console.WriteLine($"Loading solution: {solutionPath}"); + using var workspace = CreateMsBuildWorkspace(); + + var solution = await workspace.OpenSolutionAsync(solutionPath); + Console.WriteLine($"Solution loaded: {solution.FilePath}"); + Console.WriteLine($"Projects found: {solution.Projects.Count()}"); + return solution; + } + + private static async Task ParseSolutionAsync( + Solution solution, + string csharpConnectionString, + int threadCount) + { + Console.WriteLine($"Solution loaded: {solution.FilePath}"); + Console.WriteLine($"Projects found: {solution.Projects.Count()}"); + + foreach (var project in solution.Projects) + { + await ParseProjectAsync(project, csharpConnectionString, threadCount); + } + } + + private static async Task ParseProjectAsync( + Project project, + string csharpConnectionString, + int threadCount) + { + Console.WriteLine($"Processing project: {project.Name}"); + + var compilation = await project.GetCompilationAsync(); + + if (compilation == null) + { + Console.WriteLine($"Compilation failed for {project.Name}"); + return; + } + + var documents = project.Documents + .Where(d => !string.IsNullOrEmpty(d.FilePath)) + .ToList(); + Console.WriteLine($"Documents in {project.Name}: {documents.Count}"); + + string targetDll = project.OutputFilePath ?? $"{project.Name}.dll"; + + await ProcessDocumentsInParallel( + documents, + compilation, + csharpConnectionString, + threadCount, + project.Name, + targetDll + ); + } + + private static async Task ProcessDocumentsInParallel( + List documents, + Compilation compilation, + string csharpConnectionString, + int threadCount, + string projectName, + string targetDll) + { + if (documents.Count == 0) + { + return; + } + + var tasks = new List(); + int effectiveThreadCount = Math.Max(1, Math.Min(threadCount, documents.Count)); + var documentGroups = documents.Chunk(documents.Count / effectiveThreadCount + 1); + + var options = new DbContextOptionsBuilder() + .UseNpgsql(csharpConnectionString) + .Options; + + foreach (var group in documentGroups) + { + tasks.Add(Task.Run(async () => + { + await using var localDbContext = new CsharpDbContext(options); + + foreach (var document in group) + { + await ProcessSingleDocument( + document, + compilation, + localDbContext, + projectName, + targetDll + ); + } + + await localDbContext.SaveChangesAsync(); + })); + } + await Task.WhenAll(tasks); + } + + private static async Task ProcessSingleDocument( + Document document, + Compilation compilation, + CsharpDbContext dbContext, + string projectName, + string targetDll) + { + try + { + var syntaxTree = await document.GetSyntaxTreeAsync(); + if (syntaxTree == null) return; + + var semanticModel = compilation.GetSemanticModel(syntaxTree); + + var visitor = new AstVisitor( + dbContext, + semanticModel, + syntaxTree); + + var root = await syntaxTree.GetRootAsync(); + visitor.Visit(root); + + Console.WriteLine($"[{projectName}] {document.FilePath}"); + + var resultData = new + { + fullyParsed = visitor.FullyParsed, + filePath = document.FilePath, + targetDll = targetDll + }; + + Console.WriteLine("JSON_READY:" + JsonSerializer.Serialize(resultData)); + } + catch (Exception ex) + { + Console.Error.WriteLine($"Error parsing {document.FilePath}: {ex.Message}"); + Console.WriteLine($"[{projectName}] {document.FilePath}"); + + var resultData = new + { + fullyParsed = false, + filePath = document.FilePath ?? "", + targetDll = targetDll + }; + + Console.WriteLine("JSON_READY:" + JsonSerializer.Serialize(resultData)); + } + } } -} +} \ No newline at end of file