From b126a856ceba5e96b14d777efe1f23077d3f4f73 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 14 Sep 2025 05:46:45 +0300 Subject: [PATCH 1/3] Initial commit with task details for issue #131 Adding CLAUDE.md with task information for AI processing. This file will be removed when the task is complete. Issue: https://github.com/linksplatform/Data.Doublets/issues/131 --- CLAUDE.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..093276334 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +Issue to solve: https://github.com/linksplatform/Data.Doublets/issues/131 +Your prepared branch: issue-131-173bd186 +Your prepared working directory: /tmp/gh-issue-solver-1757817948559 + +Proceed. \ No newline at end of file From 1f045b67c7ccc7d33b9b716effbf0a9ae10c4436 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 14 Sep 2025 06:04:24 +0300 Subject: [PATCH 2/3] Implement configurable memory pre-allocation for reduced reallocations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This addresses issue #131 by adding configurable memory pre-allocation strategies that can allocate up to 75% of available RAM or drive space at startup to reduce the frequency of memory reallocations during link operations. ## Key Features - **SystemMemoryInfo**: Cross-platform utility for detecting available RAM and disk space - **MemoryAllocationConfiguration**: Flexible configuration system supporting multiple strategies: - Incremental (existing behavior) - PreAllocateBySystemRAM (75% of available RAM) - PreAllocateByDiskSpace (75% of available disk space) - PreAllocateCustom (user-specified amount) - **EnhancedSplitMemoryLinks**: Enhanced version supporting pre-allocation for split memory model - **EnhancedUnitedMemoryLinks**: Enhanced version supporting pre-allocation for united memory model ## Usage Examples ```csharp // Use 75% of available RAM var config = MemoryAllocationConfiguration.Default75PercentRAM; using var links = new EnhancedUnitedMemoryLinks( new FileMappedResizableDirectMemory("data.links"), config); // Custom pre-allocation var customConfig = new MemoryAllocationConfiguration { Strategy = MemoryAllocationStrategy.PreAllocateCustom, CustomPreAllocationSize = 100 * 1024 * 1024 // 100 MB }; ``` ## Benefits - Reduces memory reallocation frequency during intensive link operations - Configurable strategies for different deployment scenarios - Cross-platform system resource detection - Comprehensive logging and statistics reporting - Maintains backward compatibility with existing code ## Testing Added comprehensive test suites covering: - System memory detection across platforms - Configuration validation and calculation - Enhanced memory links functionality - Usage examples and performance comparisons 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- .../Memory/EnhancedMemoryLinksTests.cs | 345 ++++++++++++++++++ .../MemoryAllocationConfigurationTests.cs | 198 ++++++++++ .../Memory/PreAllocationUsageExample.cs | 258 +++++++++++++ .../Memory/SystemMemoryInfoTests.cs | 99 +++++ .../Memory/MemoryAllocationStrategy.cs | 225 ++++++++++++ .../Split/Generic/EnhancedSplitMemoryLinks.cs | 245 +++++++++++++ .../Memory/SystemMemoryInfo.cs | 251 +++++++++++++ .../Generic/EnhancedUnitedMemoryLinks.cs | 229 ++++++++++++ 8 files changed, 1850 insertions(+) create mode 100644 csharp/Platform.Data.Doublets.Tests/Memory/EnhancedMemoryLinksTests.cs create mode 100644 csharp/Platform.Data.Doublets.Tests/Memory/MemoryAllocationConfigurationTests.cs create mode 100644 csharp/Platform.Data.Doublets.Tests/Memory/PreAllocationUsageExample.cs create mode 100644 csharp/Platform.Data.Doublets.Tests/Memory/SystemMemoryInfoTests.cs create mode 100644 csharp/Platform.Data.Doublets/Memory/MemoryAllocationStrategy.cs create mode 100644 csharp/Platform.Data.Doublets/Memory/Split/Generic/EnhancedSplitMemoryLinks.cs create mode 100644 csharp/Platform.Data.Doublets/Memory/SystemMemoryInfo.cs create mode 100644 csharp/Platform.Data.Doublets/Memory/United/Generic/EnhancedUnitedMemoryLinks.cs diff --git a/csharp/Platform.Data.Doublets.Tests/Memory/EnhancedMemoryLinksTests.cs b/csharp/Platform.Data.Doublets.Tests/Memory/EnhancedMemoryLinksTests.cs new file mode 100644 index 000000000..853e1dc7c --- /dev/null +++ b/csharp/Platform.Data.Doublets.Tests/Memory/EnhancedMemoryLinksTests.cs @@ -0,0 +1,345 @@ +using System; +using System.IO; +using System.Numerics; +using Platform.Data.Doublets.Decorators; +using Platform.Data.Doublets.Memory; +using Platform.Data.Doublets.Memory.Split.Generic; +using Platform.Data.Doublets.Memory.United.Generic; +using Platform.Memory; +using Xunit; + +namespace Platform.Data.Doublets.Tests.Memory +{ + public static class EnhancedMemoryLinksTests + { + [Fact] + public static void EnhancedSplitMemoryLinksBasicTest() + { + var tempDataFile = Path.GetTempFileName(); + var tempIndexFile = Path.GetTempFileName(); + + try + { + var config = MemoryAllocationConfiguration.DefaultIncremental; + + using var links = new EnhancedSplitMemoryLinks( + new FileMappedResizableDirectMemory(tempDataFile), + new FileMappedResizableDirectMemory(tempIndexFile), + config); + + // Basic CRUD operations should work + var link1 = links.Create(); + Assert.True(link1 > 0); + + var retrievedLink = links.GetLinkStruct(link1); + Assert.Equal(link1, retrievedLink[0]); + + var statistics = links.GetAllocationStatistics(); + Assert.Contains("Strategy: Incremental", statistics); + Assert.Contains("Memory Statistics:", statistics); + } + finally + { + File.Delete(tempDataFile); + File.Delete(tempIndexFile); + } + } + + [Fact] + public static void EnhancedUnitedMemoryLinksBasicTest() + { + var tempFile = Path.GetTempFileName(); + + try + { + var config = MemoryAllocationConfiguration.DefaultIncremental; + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config); + + // Basic CRUD operations should work + var link1 = links.Create(); + Assert.True(link1 > 0); + + var retrievedLink = links.GetLinkStruct(link1); + Assert.Equal(link1, retrievedLink[0]); + + var statistics = links.GetAllocationStatistics(); + Assert.Contains("Strategy: Incremental", statistics); + Assert.Contains("Memory Statistics:", statistics); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void EnhancedSplitMemoryLinksCustomPreAllocationTest() + { + var tempDataFile = Path.GetTempFileName(); + var tempIndexFile = Path.GetTempFileName(); + + try + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 10 * 1024 * 1024, // 10 MB + EnableAllocationLogging = true + }; + + var logMessages = new System.Collections.Generic.List(); + + using var links = new EnhancedSplitMemoryLinks( + new FileMappedResizableDirectMemory(tempDataFile), + new FileMappedResizableDirectMemory(tempIndexFile), + config, + message => logMessages.Add(message)); + + // Should have pre-allocated memory + var statistics = links.GetAllocationStatistics(); + Assert.Contains("Strategy: PreAllocateCustom", statistics); + + // Should have logged the allocation + Assert.True(logMessages.Count > 0); + Assert.True(logMessages.Exists(msg => msg.Contains("Pre-allocating"))); + + // Memory should be pre-allocated + Assert.True(links.DataMemory.ReservedCapacity > 0); + Assert.True(links.IndexMemory.ReservedCapacity > 0); + } + finally + { + File.Delete(tempDataFile); + File.Delete(tempIndexFile); + } + } + + [Fact] + public static void EnhancedUnitedMemoryLinksCustomPreAllocationTest() + { + var tempFile = Path.GetTempFileName(); + + try + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 5 * 1024 * 1024, // 5 MB + EnableAllocationLogging = true + }; + + var logMessages = new System.Collections.Generic.List(); + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config, + message => logMessages.Add(message)); + + // Should have pre-allocated memory + var statistics = links.GetAllocationStatistics(); + Assert.Contains("Strategy: PreAllocateCustom", statistics); + + // Should have logged the allocation + Assert.True(logMessages.Count > 0); + Assert.True(logMessages.Exists(msg => msg.Contains("Pre-allocating"))); + + // Memory should be pre-allocated + Assert.True(links.Memory.ReservedCapacity > 0); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void EnhancedLinksRAMPreAllocationTest() + { + var tempFile = Path.GetTempFileName(); + + try + { + // Use a conservative percentage to avoid issues on low-memory systems + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateBySystemRAM, + PreAllocationPercentage = 0.01, // 1% of RAM + MaximumPreAllocationSize = 50 * 1024 * 1024, // Cap at 50 MB for testing + EnableAllocationLogging = true + }; + + var logMessages = new System.Collections.Generic.List(); + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config, + message => logMessages.Add(message)); + + var statistics = links.GetAllocationStatistics(); + Assert.Contains("Strategy: PreAllocateBySystemRAM", statistics); + + // Should have some pre-allocation + Assert.True(links.Memory.ReservedCapacity > 0); + + // Verify the allocation is reasonable + var expectedSize = config.CalculatePreAllocationSize(); + Assert.True(expectedSize > 0); + Assert.True(expectedSize <= config.MaximumPreAllocationSize); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void EnhancedLinksDiskPreAllocationTest() + { + var tempFile = Path.GetTempFileName(); + + try + { + // Use a very small percentage to avoid filling up the disk + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateByDiskSpace, + PreAllocationPercentage = 0.001, // 0.1% of disk space + MaximumPreAllocationSize = 100 * 1024 * 1024, // Cap at 100 MB for testing + EnableAllocationLogging = true + }; + + var logMessages = new System.Collections.Generic.List(); + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config, + message => logMessages.Add(message)); + + var statistics = links.GetAllocationStatistics(); + Assert.Contains("Strategy: PreAllocateByDiskSpace", statistics); + + // Should have some pre-allocation + Assert.True(links.Memory.ReservedCapacity > 0); + + // Verify the allocation is reasonable + var expectedSize = config.CalculatePreAllocationSize(Path.GetDirectoryName(tempFile)); + Assert.True(expectedSize > 0); + Assert.True(expectedSize <= config.MaximumPreAllocationSize); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void EnhancedLinksWithLoggingTest() + { + var tempFile = Path.GetTempFileName(); + + try + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 1024 * 1024, // 1 MB + EnableAllocationLogging = true + }; + + var logMessages = new System.Collections.Generic.List(); + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config, + message => logMessages.Add(message)); + + // Should have logged messages + Assert.True(logMessages.Count > 0); + + // Check that log messages have timestamps + Assert.True(logMessages.Exists(msg => msg.Contains("EnhancedUnitedMemoryLinks:"))); + Assert.True(logMessages.Exists(msg => msg.Contains("Pre-allocating") || msg.Contains("Pre-allocation completed"))); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void EnhancedLinksStatisticsTest() + { + var tempFile = Path.GetTempFileName(); + + try + { + var config = MemoryAllocationConfiguration.DefaultIncremental; + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config); + + // Create some links + for (int i = 0; i < 10; i++) + { + links.Create(); + } + + var statistics = links.GetAllocationStatistics(); + + // Should contain expected information + Assert.Contains("Memory Statistics:", statistics); + Assert.Contains("Strategy: Incremental", statistics); + Assert.Contains("bytes", statistics); + Assert.Contains("Links:", statistics); + Assert.Contains("Link Size:", statistics); + + // Should show some utilization + Assert.Contains("utilized", statistics); + } + finally + { + File.Delete(tempFile); + } + } + + [Theory] + [InlineData(typeof(byte))] + [InlineData(typeof(ushort))] + [InlineData(typeof(uint))] + [InlineData(typeof(ulong))] + public static void EnhancedLinksGenericTypesTest(Type linkAddressType) + { + var tempFile = Path.GetTempFileName(); + + try + { + var config = MemoryAllocationConfiguration.DefaultIncremental; + + // Use reflection to create the appropriate generic type + var enhancedLinksType = typeof(EnhancedUnitedMemoryLinks<>).MakeGenericType(linkAddressType); + var memoryType = typeof(FileMappedResizableDirectMemory); + var memory = Activator.CreateInstance(memoryType, tempFile); + + // Use the constructor that takes (memory, config) + using var links = (IDisposable)Activator.CreateInstance(enhancedLinksType, memory, config, null); + + Assert.NotNull(links); + + // Test that we can call GetAllocationStatistics + var statisticsMethod = enhancedLinksType.GetMethod("GetAllocationStatistics"); + var statistics = (string)statisticsMethod!.Invoke(links, null); + + Assert.Contains("Memory Statistics:", statistics); + Assert.Contains("Strategy: Incremental", statistics); + } + finally + { + File.Delete(tempFile); + } + } + } +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets.Tests/Memory/MemoryAllocationConfigurationTests.cs b/csharp/Platform.Data.Doublets.Tests/Memory/MemoryAllocationConfigurationTests.cs new file mode 100644 index 000000000..2752e5461 --- /dev/null +++ b/csharp/Platform.Data.Doublets.Tests/Memory/MemoryAllocationConfigurationTests.cs @@ -0,0 +1,198 @@ +using System; +using Platform.Data.Doublets.Memory; +using Xunit; + +namespace Platform.Data.Doublets.Tests.Memory +{ + public static class MemoryAllocationConfigurationTests + { + [Fact] + public static void DefaultConfigurationsTest() + { + var ramConfig = MemoryAllocationConfiguration.Default75PercentRAM; + Assert.Equal(MemoryAllocationStrategy.PreAllocateBySystemRAM, ramConfig.Strategy); + Assert.Equal(0.75, ramConfig.PreAllocationPercentage); + + var diskConfig = MemoryAllocationConfiguration.Default75PercentDisk; + Assert.Equal(MemoryAllocationStrategy.PreAllocateByDiskSpace, diskConfig.Strategy); + Assert.Equal(0.75, diskConfig.PreAllocationPercentage); + + var incrementalConfig = MemoryAllocationConfiguration.DefaultIncremental; + Assert.Equal(MemoryAllocationStrategy.Incremental, incrementalConfig.Strategy); + } + + [Fact] + public static void ValidateValidConfigurationTest() + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateBySystemRAM, + PreAllocationPercentage = 0.5, + MinimumPreAllocationSize = 1024, + MaximumPreAllocationSize = 1024 * 1024 * 1024 + }; + + // Should not throw + config.Validate(); + } + + [Theory] + [InlineData(-0.1)] + [InlineData(1.1)] + public static void ValidateInvalidPercentageTest(double percentage) + { + var config = new MemoryAllocationConfiguration + { + PreAllocationPercentage = percentage + }; + + Assert.Throws(() => config.Validate()); + } + + [Fact] + public static void ValidateNegativeMinimumTest() + { + var config = new MemoryAllocationConfiguration + { + MinimumPreAllocationSize = -1 + }; + + Assert.Throws(() => config.Validate()); + } + + [Fact] + public static void ValidateMaximumLessThanMinimumTest() + { + var config = new MemoryAllocationConfiguration + { + MinimumPreAllocationSize = 1000, + MaximumPreAllocationSize = 500 + }; + + Assert.Throws(() => config.Validate()); + } + + [Fact] + public static void ValidateCustomStrategyWithInvalidSizeTest() + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 0 + }; + + Assert.Throws(() => config.Validate()); + } + + [Fact] + public static void CalculatePreAllocationSizeIncrementalTest() + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.Incremental, + MinimumPreAllocationSize = 1000 + }; + + var size = config.CalculatePreAllocationSize(); + Assert.Equal(1000, size); + } + + [Fact] + public static void CalculatePreAllocationSizeCustomTest() + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 5000, + MinimumPreAllocationSize = 1000, + MaximumPreAllocationSize = 10000 + }; + + var size = config.CalculatePreAllocationSize(); + Assert.Equal(5000, size); + } + + [Fact] + public static void CalculatePreAllocationSizeWithBoundsTest() + { + // Test lower bound + var configLow = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 500, + MinimumPreAllocationSize = 1000, + MaximumPreAllocationSize = 10000 + }; + + var sizeLow = configLow.CalculatePreAllocationSize(); + Assert.Equal(1000, sizeLow); // Should be clamped to minimum + + // Test upper bound + var configHigh = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 15000, + MinimumPreAllocationSize = 1000, + MaximumPreAllocationSize = 10000 + }; + + var sizeHigh = configHigh.CalculatePreAllocationSize(); + Assert.Equal(10000, sizeHigh); // Should be clamped to maximum + } + + [Fact] + public static void CalculatePreAllocationSizeRAMTest() + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateBySystemRAM, + PreAllocationPercentage = 0.5, + MinimumPreAllocationSize = 1000, + MaximumPreAllocationSize = long.MaxValue + }; + + var size = config.CalculatePreAllocationSize(); + + // Should be positive and at least the minimum + Assert.True(size >= 1000); + + // Should be reasonable (not more than available RAM) + var availableRAM = SystemMemoryInfo.GetAvailablePhysicalMemory(); + Assert.True(size <= availableRAM); + } + + [Fact] + public static void CalculatePreAllocationSizeDiskTest() + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateByDiskSpace, + PreAllocationPercentage = 0.01, // Use small percentage for testing + MinimumPreAllocationSize = 1000, + MaximumPreAllocationSize = long.MaxValue + }; + + var size = config.CalculatePreAllocationSize(); + + // Should be positive and at least the minimum + Assert.True(size >= 1000); + + // Should be reasonable (not more than available disk space) + var availableDisk = SystemMemoryInfo.GetAvailableDiskSpace(Environment.CurrentDirectory); + Assert.True(size <= availableDisk); + } + + [Fact] + public static void ConfigurationImmutabilityTest() + { + var original = MemoryAllocationConfiguration.Default75PercentRAM; + var modified = MemoryAllocationConfiguration.Default75PercentRAM; + + modified.PreAllocationPercentage = 0.5; + + // Original should not be affected + Assert.Equal(0.75, original.PreAllocationPercentage); + Assert.Equal(0.5, modified.PreAllocationPercentage); + } + } +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets.Tests/Memory/PreAllocationUsageExample.cs b/csharp/Platform.Data.Doublets.Tests/Memory/PreAllocationUsageExample.cs new file mode 100644 index 000000000..0c34ce719 --- /dev/null +++ b/csharp/Platform.Data.Doublets.Tests/Memory/PreAllocationUsageExample.cs @@ -0,0 +1,258 @@ +using System; +using System.IO; +using Platform.Data.Doublets.Memory; +using Platform.Data.Doublets.Memory.Split.Generic; +using Platform.Data.Doublets.Memory.United.Generic; +using Platform.Memory; +using Xunit; + +namespace Platform.Data.Doublets.Tests.Memory +{ + /// + /// + /// Usage examples for the enhanced memory allocation strategies. + /// These examples demonstrate how to use the new pre-allocation features + /// to reduce memory reallocations and improve performance. + /// + /// + /// + public static class PreAllocationUsageExample + { + [Fact] + public static void BasicRAMPreAllocationExample() + { + // Example: Pre-allocate 75% of available RAM for optimal performance + var tempFile = Path.GetTempFileName(); + + try + { + // Configuration for 75% RAM pre-allocation + var config = MemoryAllocationConfiguration.Default75PercentRAM; + config.EnableAllocationLogging = true; + + var logMessages = new System.Collections.Generic.List(); + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(tempFile), + config, + message => Console.WriteLine(message) // In practice, log to your preferred system + ); + + Console.WriteLine("=== Memory Allocation Example ==="); + Console.WriteLine(links.GetAllocationStatistics()); + Console.WriteLine(); + + // Perform operations - these should have minimal reallocations + Console.WriteLine("Creating 1000 links..."); + for (int i = 0; i < 1000; i++) + { + var link = links.Create(); + if (i == 0) + { + Console.WriteLine($"First link created: {link}"); + } + } + + Console.WriteLine("Final statistics:"); + Console.WriteLine(links.GetAllocationStatistics()); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void CustomPreAllocationExample() + { + // Example: Pre-allocate a specific amount of memory + var tempFile = Path.GetTempFileName(); + + try + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 50 * 1024 * 1024, // 50 MB + EnableAllocationLogging = true + }; + + using var links = new EnhancedUnitedMemoryLinks(tempFile, config, + message => Console.WriteLine($"[LOG] {message}")); + + Console.WriteLine("=== Custom Pre-allocation Example ==="); + Console.WriteLine(links.GetAllocationStatistics()); + + // Test the pre-allocated capacity + var maxLinks = (links.Memory.ReservedCapacity - EnhancedUnitedMemoryLinks.LinkHeaderSizeInBytes) + / EnhancedUnitedMemoryLinks.LinkSizeInBytes; + + Console.WriteLine($"Theoretical max links with current allocation: {maxLinks}"); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void SplitMemoryPreAllocationExample() + { + // Example: Using split memory with disk-based pre-allocation + var tempDataFile = Path.GetTempFileName(); + var tempIndexFile = Path.GetTempFileName(); + + try + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateByDiskSpace, + PreAllocationPercentage = 0.001, // 0.1% of disk space (conservative for testing) + MaximumPreAllocationSize = 10 * 1024 * 1024, // Cap at 10MB for example + EnableAllocationLogging = true + }; + + using var links = new EnhancedSplitMemoryLinks( + new FileMappedResizableDirectMemory(tempDataFile), + new FileMappedResizableDirectMemory(tempIndexFile), + config, + message => Console.WriteLine($"[SPLIT LOG] {message}") + ); + + Console.WriteLine("=== Split Memory Pre-allocation Example ==="); + Console.WriteLine(links.GetAllocationStatistics()); + + // Demonstrate that data and index are handled separately + Console.WriteLine($"Data memory reserved: {links.DataMemory.ReservedCapacity:N0} bytes"); + Console.WriteLine($"Index memory reserved: {links.IndexMemory.ReservedCapacity:N0} bytes"); + } + finally + { + File.Delete(tempDataFile); + File.Delete(tempIndexFile); + } + } + + [Fact] + public static void ConservativePreAllocationExample() + { + // Example: Conservative approach with bounds and fallback + var tempFile = Path.GetTempFileName(); + + try + { + var config = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateBySystemRAM, + PreAllocationPercentage = 0.25, // 25% of RAM + MinimumPreAllocationSize = 1 * 1024 * 1024, // At least 1MB + MaximumPreAllocationSize = 100 * 1024 * 1024, // At most 100MB + EnableAllocationLogging = true + }; + + using var links = new EnhancedUnitedMemoryLinks(tempFile, config, + message => Console.WriteLine($"[CONSERVATIVE] {message}")); + + Console.WriteLine("=== Conservative Pre-allocation Example ==="); + Console.WriteLine(links.GetAllocationStatistics()); + + // Show how bounds are applied + var calculatedSize = config.CalculatePreAllocationSize(); + Console.WriteLine($"Calculated pre-allocation size: {calculatedSize:N0} bytes"); + Console.WriteLine($"Actual reserved capacity: {links.Memory.ReservedCapacity:N0} bytes"); + } + finally + { + File.Delete(tempFile); + } + } + + [Fact] + public static void PerformanceComparisonExample() + { + // Example: Compare performance between incremental and pre-allocation + var incrementalFile = Path.GetTempFileName(); + var preAllocatedFile = Path.GetTempFileName(); + + try + { + Console.WriteLine("=== Performance Comparison Example ==="); + + // Test incremental allocation + var incrementalConfig = MemoryAllocationConfiguration.DefaultIncremental; + var incrementalTime = MeasureCreationTime(incrementalFile, incrementalConfig, 5000); + Console.WriteLine($"Incremental allocation time: {incrementalTime.TotalMilliseconds:F2} ms"); + + // Test pre-allocation + var preAllocationConfig = new MemoryAllocationConfiguration + { + Strategy = MemoryAllocationStrategy.PreAllocateCustom, + CustomPreAllocationSize = 10 * 1024 * 1024 // 10 MB + }; + var preAllocationTime = MeasureCreationTime(preAllocatedFile, preAllocationConfig, 5000); + Console.WriteLine($"Pre-allocation time: {preAllocationTime.TotalMilliseconds:F2} ms"); + + if (preAllocationTime < incrementalTime) + { + var improvement = ((incrementalTime.TotalMilliseconds - preAllocationTime.TotalMilliseconds) + / incrementalTime.TotalMilliseconds) * 100; + Console.WriteLine($"Pre-allocation was {improvement:F1}% faster"); + } + } + finally + { + File.Delete(incrementalFile); + File.Delete(preAllocatedFile); + } + } + + private static TimeSpan MeasureCreationTime(string filename, MemoryAllocationConfiguration config, int linkCount) + { + var stopwatch = System.Diagnostics.Stopwatch.StartNew(); + + using var links = new EnhancedUnitedMemoryLinks( + new FileMappedResizableDirectMemory(filename), config); + + for (int i = 0; i < linkCount; i++) + { + links.Create(); + } + + stopwatch.Stop(); + return stopwatch.Elapsed; + } + + [Fact] + public static void SystemInfoExample() + { + // Example: Display system information used for pre-allocation decisions + Console.WriteLine("=== System Information Example ==="); + + try + { + var totalRAM = SystemMemoryInfo.GetTotalPhysicalMemory(); + var availableRAM = SystemMemoryInfo.GetAvailablePhysicalMemory(); + var totalDisk = SystemMemoryInfo.GetTotalDiskSpace(Environment.CurrentDirectory); + var availableDisk = SystemMemoryInfo.GetAvailableDiskSpace(Environment.CurrentDirectory); + + Console.WriteLine($"Total RAM: {totalRAM / (1024.0 * 1024 * 1024):F2} GB"); + Console.WriteLine($"Available RAM: {availableRAM / (1024.0 * 1024 * 1024):F2} GB"); + Console.WriteLine($"Total Disk: {totalDisk / (1024.0 * 1024 * 1024):F2} GB"); + Console.WriteLine($"Available Disk: {availableDisk / (1024.0 * 1024 * 1024):F2} GB"); + + // Show what 75% allocation would mean + var ram75Config = MemoryAllocationConfiguration.Default75PercentRAM; + var ram75Size = ram75Config.CalculatePreAllocationSize(); + Console.WriteLine($"75% RAM pre-allocation would be: {ram75Size / (1024.0 * 1024):F2} MB"); + + var disk75Config = MemoryAllocationConfiguration.Default75PercentDisk; + var disk75Size = disk75Config.CalculatePreAllocationSize(); + Console.WriteLine($"75% Disk pre-allocation would be: {disk75Size / (1024.0 * 1024 * 1024):F2} GB"); + } + catch (Exception ex) + { + Console.WriteLine($"Error getting system info: {ex.Message}"); + } + } + } +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets.Tests/Memory/SystemMemoryInfoTests.cs b/csharp/Platform.Data.Doublets.Tests/Memory/SystemMemoryInfoTests.cs new file mode 100644 index 000000000..e9c0d7636 --- /dev/null +++ b/csharp/Platform.Data.Doublets.Tests/Memory/SystemMemoryInfoTests.cs @@ -0,0 +1,99 @@ +using System; +using Platform.Data.Doublets.Memory; +using Xunit; + +namespace Platform.Data.Doublets.Tests.Memory +{ + public static class SystemMemoryInfoTests + { + [Fact] + public static void GetTotalPhysicalMemoryTest() + { + var totalMemory = SystemMemoryInfo.GetTotalPhysicalMemory(); + + // Should return a positive value + Assert.True(totalMemory > 0); + + // Should be at least 1 MB (very conservative lower bound) + Assert.True(totalMemory >= 1024 * 1024); + + // Should be less than 1 TB (conservative upper bound for testing) + Assert.True(totalMemory <= 1024L * 1024 * 1024 * 1024); + } + + [Fact] + public static void GetAvailablePhysicalMemoryTest() + { + var availableMemory = SystemMemoryInfo.GetAvailablePhysicalMemory(); + var totalMemory = SystemMemoryInfo.GetTotalPhysicalMemory(); + + // Should return a positive value + Assert.True(availableMemory > 0); + + // Available memory should not exceed total memory + Assert.True(availableMemory <= totalMemory); + + // Should be at least some memory available (conservative check) + Assert.True(availableMemory >= 1024 * 1024); // At least 1 MB + } + + [Fact] + public static void GetTotalDiskSpaceTest() + { + var currentDirectory = Environment.CurrentDirectory; + var totalDiskSpace = SystemMemoryInfo.GetTotalDiskSpace(currentDirectory); + + // Should return a positive value + Assert.True(totalDiskSpace > 0); + + // Should be at least 1 GB (modern systems) + Assert.True(totalDiskSpace >= 1024L * 1024 * 1024); + } + + [Fact] + public static void GetAvailableDiskSpaceTest() + { + var currentDirectory = Environment.CurrentDirectory; + var availableDiskSpace = SystemMemoryInfo.GetAvailableDiskSpace(currentDirectory); + var totalDiskSpace = SystemMemoryInfo.GetTotalDiskSpace(currentDirectory); + + // Should return a positive value + Assert.True(availableDiskSpace > 0); + + // Available disk space should not exceed total disk space + Assert.True(availableDiskSpace <= totalDiskSpace); + } + + [Fact] + public static void DiskSpaceConsistencyTest() + { + var path1 = Environment.CurrentDirectory; + var path2 = System.IO.Path.Combine(Environment.CurrentDirectory, "subdir"); + + var totalSpace1 = SystemMemoryInfo.GetTotalDiskSpace(path1); + var totalSpace2 = SystemMemoryInfo.GetTotalDiskSpace(path2); + + // Both paths should report the same total disk space (same drive) + Assert.Equal(totalSpace1, totalSpace2); + + var availableSpace1 = SystemMemoryInfo.GetAvailableDiskSpace(path1); + var availableSpace2 = SystemMemoryInfo.GetAvailableDiskSpace(path2); + + // Available space should be the same (or very close) for paths on same drive + Assert.Equal(availableSpace1, availableSpace2); + } + + [Fact] + public static void MemoryValuesReasonablenessTest() + { + var totalMemory = SystemMemoryInfo.GetTotalPhysicalMemory(); + var availableMemory = SystemMemoryInfo.GetAvailablePhysicalMemory(); + + // Available should be at least 10% of total (systems usually have some free RAM) + Assert.True(availableMemory >= totalMemory * 0.1); + + // Available should be at most 95% of total (some is always used by OS) + Assert.True(availableMemory <= totalMemory * 0.95); + } + } +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets/Memory/MemoryAllocationStrategy.cs b/csharp/Platform.Data.Doublets/Memory/MemoryAllocationStrategy.cs new file mode 100644 index 000000000..1123eb8d8 --- /dev/null +++ b/csharp/Platform.Data.Doublets/Memory/MemoryAllocationStrategy.cs @@ -0,0 +1,225 @@ +using System; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Platform.Data.Doublets.Memory; + +/// +/// +/// Defines memory allocation strategy options. +/// +/// +/// +public enum MemoryAllocationStrategy +{ + /// + /// + /// Default incremental allocation strategy. + /// + /// + /// + Incremental, + + /// + /// + /// Pre-allocate based on system RAM. + /// + /// + /// + PreAllocateBySystemRAM, + + /// + /// + /// Pre-allocate based on available disk space. + /// + /// + /// + PreAllocateByDiskSpace, + + /// + /// + /// Pre-allocate a custom amount. + /// + /// + /// + PreAllocateCustom +} + +/// +/// +/// Configuration for memory allocation strategies. +/// +/// +/// +public class MemoryAllocationConfiguration +{ + /// + /// + /// Gets or sets the allocation strategy. + /// + /// + /// + public MemoryAllocationStrategy Strategy { get; set; } = MemoryAllocationStrategy.Incremental; + + /// + /// + /// Gets or sets the percentage of system resource to pre-allocate (0.0 to 1.0). + /// Default is 0.75 (75%). + /// + /// + /// + public double PreAllocationPercentage { get; set; } = 0.75; + + /// + /// + /// Gets or sets the custom pre-allocation size in bytes. + /// Only used when Strategy is PreAllocateCustom. + /// + /// + /// + public long CustomPreAllocationSize { get; set; } + + /// + /// + /// Gets or sets the minimum pre-allocation size in bytes. + /// Used as a safety lower bound. + /// + /// + /// + public long MinimumPreAllocationSize { get; set; } = 1024 * 1024; // 1 MB + + /// + /// + /// Gets or sets the maximum pre-allocation size in bytes. + /// Used as a safety upper bound. + /// + /// + /// + public long MaximumPreAllocationSize { get; set; } = 16L * 1024 * 1024 * 1024; // 16 GB + + /// + /// + /// Gets or sets whether to enable logging for allocation decisions. + /// + /// + /// + public bool EnableAllocationLogging { get; set; } = false; + + /// + /// + /// Gets or sets the path for disk space calculation. + /// Only used when Strategy is PreAllocateByDiskSpace. + /// If null, uses current directory. + /// + /// + /// + public string? DiskPath { get; set; } + + /// + /// + /// Gets a default configuration that enables 75% RAM pre-allocation. + /// + /// + /// + public static MemoryAllocationConfiguration Default75PercentRAM => new() + { + Strategy = MemoryAllocationStrategy.PreAllocateBySystemRAM, + PreAllocationPercentage = 0.75 + }; + + /// + /// + /// Gets a default configuration that enables 75% disk space pre-allocation. + /// + /// + /// + public static MemoryAllocationConfiguration Default75PercentDisk => new() + { + Strategy = MemoryAllocationStrategy.PreAllocateByDiskSpace, + PreAllocationPercentage = 0.75 + }; + + /// + /// + /// Gets a default configuration that uses incremental allocation. + /// + /// + /// + public static MemoryAllocationConfiguration DefaultIncremental => new() + { + Strategy = MemoryAllocationStrategy.Incremental + }; + + /// + /// + /// Validates the configuration and throws if invalid. + /// + /// + /// + /// + /// Thrown when configuration values are out of valid range + /// + /// + public void Validate() + { + if (PreAllocationPercentage < 0.0 || PreAllocationPercentage > 1.0) + { + throw new ArgumentOutOfRangeException(nameof(PreAllocationPercentage), + "PreAllocationPercentage must be between 0.0 and 1.0"); + } + + if (MinimumPreAllocationSize < 0) + { + throw new ArgumentOutOfRangeException(nameof(MinimumPreAllocationSize), + "MinimumPreAllocationSize must be non-negative"); + } + + if (MaximumPreAllocationSize < MinimumPreAllocationSize) + { + throw new ArgumentOutOfRangeException(nameof(MaximumPreAllocationSize), + "MaximumPreAllocationSize must be greater than or equal to MinimumPreAllocationSize"); + } + + if (Strategy == MemoryAllocationStrategy.PreAllocateCustom && CustomPreAllocationSize <= 0) + { + throw new ArgumentOutOfRangeException(nameof(CustomPreAllocationSize), + "CustomPreAllocationSize must be positive when using PreAllocateCustom strategy"); + } + } + + /// + /// + /// Calculates the optimal pre-allocation size based on the current configuration. + /// + /// + /// + /// + /// Path for disk space calculations (optional) + /// + /// + /// + /// The calculated pre-allocation size in bytes + /// + /// + public long CalculatePreAllocationSize(string? dataPath = null) + { + Validate(); + + long calculatedSize = Strategy switch + { + MemoryAllocationStrategy.PreAllocateBySystemRAM => + (long)(SystemMemoryInfo.GetAvailablePhysicalMemory() * PreAllocationPercentage), + + MemoryAllocationStrategy.PreAllocateByDiskSpace => + (long)(SystemMemoryInfo.GetAvailableDiskSpace(DiskPath ?? dataPath ?? Environment.CurrentDirectory) * PreAllocationPercentage), + + MemoryAllocationStrategy.PreAllocateCustom => + CustomPreAllocationSize, + + _ => MinimumPreAllocationSize + }; + + // Apply bounds + return Math.Max(MinimumPreAllocationSize, Math.Min(MaximumPreAllocationSize, calculatedSize)); + } +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets/Memory/Split/Generic/EnhancedSplitMemoryLinks.cs b/csharp/Platform.Data.Doublets/Memory/Split/Generic/EnhancedSplitMemoryLinks.cs new file mode 100644 index 000000000..122478157 --- /dev/null +++ b/csharp/Platform.Data.Doublets/Memory/Split/Generic/EnhancedSplitMemoryLinks.cs @@ -0,0 +1,245 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Platform.Memory; +using Platform.Singletons; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Platform.Data.Doublets.Memory.Split.Generic; + +/// +/// +/// Represents enhanced split memory links with configurable pre-allocation strategies. +/// +/// +/// +/// +public unsafe class EnhancedSplitMemoryLinks : SplitMemoryLinks where TLinkAddress : IUnsignedNumber, IComparisonOperators +{ + private readonly MemoryAllocationConfiguration _allocationConfiguration; + private readonly Action? _logger; + + /// + /// + /// Initializes a new instance with 75% RAM pre-allocation. + /// + /// + /// + /// + /// A data memory. + /// + /// + /// + /// A index memory. + /// + /// + [MethodImpl(methodImplOptions: MethodImplOptions.AggressiveInlining)] + public EnhancedSplitMemoryLinks(string dataMemory, string indexMemory) + : this(dataMemory: new FileMappedResizableDirectMemory(path: dataMemory), + indexMemory: new FileMappedResizableDirectMemory(path: indexMemory), + allocationConfiguration: MemoryAllocationConfiguration.Default75PercentRAM) + { } + + /// + /// + /// Initializes a new instance with custom allocation configuration. + /// + /// + /// + /// + /// A data memory. + /// + /// + /// + /// A index memory. + /// + /// + /// + /// Memory allocation configuration. + /// + /// + /// + /// Optional logger for allocation decisions. + /// + /// + [MethodImpl(methodImplOptions: MethodImplOptions.AggressiveInlining)] + public EnhancedSplitMemoryLinks(IResizableDirectMemory dataMemory, IResizableDirectMemory indexMemory, + MemoryAllocationConfiguration allocationConfiguration, Action? logger = null) + : this(dataMemory: dataMemory, indexMemory: indexMemory, allocationConfiguration: allocationConfiguration, + constants: Default>.Instance, indexTreeType: IndexTreeType.Default, + useLinkedList: true, logger: logger) + { } + + /// + /// + /// Initializes a new instance with full configuration options. + /// + /// + /// + /// + /// A data memory. + /// + /// + /// + /// A index memory. + /// + /// + /// + /// Memory allocation configuration. + /// + /// + /// + /// A constants. + /// + /// + /// + /// A index tree type. + /// + /// + /// + /// A use linked list. + /// + /// + /// + /// Optional logger for allocation decisions. + /// + /// + [MethodImpl(methodImplOptions: MethodImplOptions.AggressiveInlining)] + public EnhancedSplitMemoryLinks(IResizableDirectMemory dataMemory, IResizableDirectMemory indexMemory, + MemoryAllocationConfiguration allocationConfiguration, LinksConstants constants, + IndexTreeType indexTreeType, bool useLinkedList, Action? logger = null) + : base(dataMemory: dataMemory, indexMemory: indexMemory, + memoryReservationStep: CalculateOptimalReservationStep(allocationConfiguration, dataMemory, indexMemory), + constants: constants, indexTreeType: indexTreeType, useLinkedList: useLinkedList) + { + _allocationConfiguration = allocationConfiguration ?? throw new ArgumentNullException(nameof(allocationConfiguration)); + _logger = logger; + + // Apply pre-allocation strategy + ApplyPreAllocationStrategy(dataMemory, indexMemory); + } + + /// + /// + /// Gets the current memory allocation configuration. + /// + /// + /// + public MemoryAllocationConfiguration AllocationConfiguration => _allocationConfiguration; + + /// + /// + /// Gets the data memory instance for statistics and monitoring. + /// + /// + /// + public IResizableDirectMemory DataMemory => _dataMemory; + + /// + /// + /// Gets the index memory instance for statistics and monitoring. + /// + /// + /// + public IResizableDirectMemory IndexMemory => _indexMemory; + + private static long CalculateOptimalReservationStep(MemoryAllocationConfiguration config, IResizableDirectMemory dataMemory, IResizableDirectMemory indexMemory) + { + if (config.Strategy == MemoryAllocationStrategy.Incremental) + { + return DefaultLinksSizeStep; + } + + // For pre-allocation strategies, calculate a step based on the pre-allocated size + var dataPath = dataMemory is FileMappedResizableDirectMemory ? + Environment.CurrentDirectory : null; + + var preAllocationSize = config.CalculatePreAllocationSize(dataPath); + var linksCount = preAllocationSize / (LinkDataPartSizeInBytes + LinkIndexPartSizeInBytes); + + // Use a step that's about 10% of the pre-allocated size, but at least 1MB + return Math.Max(DefaultLinksSizeStep, linksCount / 10); + } + + private void ApplyPreAllocationStrategy(IResizableDirectMemory dataMemory, IResizableDirectMemory indexMemory) + { + if (_allocationConfiguration.Strategy == MemoryAllocationStrategy.Incremental) + { + LogAllocation("Using incremental allocation strategy"); + return; + } + + try + { + var dataPath = dataMemory is FileMappedResizableDirectMemory ? + Environment.CurrentDirectory : null; + + var preAllocationSize = _allocationConfiguration.CalculatePreAllocationSize(dataPath); + + // Calculate how much capacity to allocate for data and index + // Split the allocation between data and index based on their size ratios + var totalSizePerLink = LinkDataPartSizeInBytes + LinkIndexPartSizeInBytes; + var maxLinks = preAllocationSize / totalSizePerLink; + + var targetDataCapacity = maxLinks * LinkDataPartSizeInBytes; + var targetIndexCapacity = maxLinks * LinkIndexPartSizeInBytes; + + LogAllocation($"Pre-allocating memory: {preAllocationSize:N0} bytes total (strategy: {_allocationConfiguration.Strategy})"); + + // Only pre-allocate if it's more than current capacity + if (targetDataCapacity > dataMemory.ReservedCapacity) + { + LogAllocation($"Pre-allocating data memory: {targetDataCapacity:N0} bytes"); + dataMemory.ReservedCapacity = targetDataCapacity; + } + + if (targetIndexCapacity > indexMemory.ReservedCapacity) + { + LogAllocation($"Pre-allocating index memory: {targetIndexCapacity:N0} bytes"); + indexMemory.ReservedCapacity = targetIndexCapacity; + } + + LogAllocation($"Pre-allocation completed. Max links capacity: {maxLinks:N0}"); + } + catch (Exception ex) + { + LogAllocation($"Pre-allocation failed, falling back to incremental: {ex.Message}"); + // Don't throw - fall back to incremental allocation + } + } + + private void LogAllocation(string message) + { + if (_allocationConfiguration.EnableAllocationLogging) + { + _logger?.Invoke($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss}] EnhancedSplitMemoryLinks: {message}"); + } + } + + /// + /// + /// Gets memory allocation statistics. + /// + /// + /// + /// + /// Memory allocation statistics as a formatted string + /// + /// + public string GetAllocationStatistics() + { + var dataMemory = _dataMemory; + var indexMemory = _indexMemory; + var totalReserved = dataMemory.ReservedCapacity + indexMemory.ReservedCapacity; + var totalUsed = dataMemory.UsedCapacity + indexMemory.UsedCapacity; + var utilizationPercent = totalReserved > 0 ? (totalUsed * 100.0 / totalReserved) : 0; + + return $"Memory Statistics:\n" + + $" Strategy: {_allocationConfiguration.Strategy}\n" + + $" Data Memory: {dataMemory.UsedCapacity:N0} / {dataMemory.ReservedCapacity:N0} bytes\n" + + $" Index Memory: {indexMemory.UsedCapacity:N0} / {indexMemory.ReservedCapacity:N0} bytes\n" + + $" Total: {totalUsed:N0} / {totalReserved:N0} bytes ({utilizationPercent:F1}% utilized)\n" + + $" Links: {Total} allocated"; + } +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets/Memory/SystemMemoryInfo.cs b/csharp/Platform.Data.Doublets/Memory/SystemMemoryInfo.cs new file mode 100644 index 000000000..e30edd7a5 --- /dev/null +++ b/csharp/Platform.Data.Doublets/Memory/SystemMemoryInfo.cs @@ -0,0 +1,251 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Platform.Data.Doublets.Memory; + +/// +/// +/// Provides system memory and storage information. +/// +/// +/// +public static class SystemMemoryInfo +{ + /// + /// + /// Gets the total system RAM in bytes. + /// + /// + /// + /// + /// Total system RAM in bytes + /// + /// + public static long GetTotalPhysicalMemory() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return GetTotalPhysicalMemoryWindows(); + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return GetTotalPhysicalMemoryLinux(); + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return GetTotalPhysicalMemoryMacOS(); + } + + // Fallback: use GC memory limit as approximation + return GC.GetTotalMemory(false) * 10; // Rough approximation + } + + /// + /// + /// Gets the available system RAM in bytes. + /// + /// + /// + /// + /// Available system RAM in bytes + /// + /// + public static long GetAvailablePhysicalMemory() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return GetAvailablePhysicalMemoryWindows(); + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + return GetAvailablePhysicalMemoryLinux(); + } + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + return GetAvailablePhysicalMemoryMacOS(); + } + + // Fallback: assume 50% of total memory is available + return GetTotalPhysicalMemory() / 2; + } + + /// + /// + /// Gets the total disk space for the specified path in bytes. + /// + /// + /// + /// + /// The file or directory path to check + /// + /// + /// + /// Total disk space in bytes + /// + /// + public static long GetTotalDiskSpace(string path) + { + var driveInfo = new DriveInfo(Path.GetPathRoot(Path.GetFullPath(path)) ?? "C:\\"); + return driveInfo.TotalSize; + } + + /// + /// + /// Gets the available disk space for the specified path in bytes. + /// + /// + /// + /// + /// The file or directory path to check + /// + /// + /// + /// Available disk space in bytes + /// + /// + public static long GetAvailableDiskSpace(string path) + { + var driveInfo = new DriveInfo(Path.GetPathRoot(Path.GetFullPath(path)) ?? "C:\\"); + return driveInfo.AvailableFreeSpace; + } + + private static long GetTotalPhysicalMemoryWindows() + { + try + { + var memStatus = new MEMORYSTATUSEX(); + memStatus.dwLength = (uint)Marshal.SizeOf(); + if (GlobalMemoryStatusEx(ref memStatus)) + { + return (long)memStatus.ullTotalPhys; + } + } + catch + { + // Fall through to alternative method + } + + // Alternative method using performance counter + return Environment.WorkingSet; // This is not ideal but serves as fallback + } + + private static long GetAvailablePhysicalMemoryWindows() + { + try + { + var memStatus = new MEMORYSTATUSEX(); + memStatus.dwLength = (uint)Marshal.SizeOf(); + if (GlobalMemoryStatusEx(ref memStatus)) + { + return (long)memStatus.ullAvailPhys; + } + } + catch + { + // Fall through to fallback + } + + return GetTotalPhysicalMemoryWindows() / 2; // Assume 50% available as fallback + } + + private static long GetTotalPhysicalMemoryLinux() + { + try + { + var memInfo = File.ReadAllText("/proc/meminfo"); + foreach (var line in memInfo.Split('\n')) + { + if (line.StartsWith("MemTotal:", StringComparison.OrdinalIgnoreCase)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && long.TryParse(parts[1], out var memKb)) + { + return memKb * 1024; // Convert KB to bytes + } + } + } + } + catch + { + // Fall through to fallback + } + + return Environment.WorkingSet; // Fallback + } + + private static long GetAvailablePhysicalMemoryLinux() + { + try + { + var memInfo = File.ReadAllText("/proc/meminfo"); + long available = 0; + foreach (var line in memInfo.Split('\n')) + { + if (line.StartsWith("MemAvailable:", StringComparison.OrdinalIgnoreCase)) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && long.TryParse(parts[1], out var memKb)) + { + return memKb * 1024; // Convert KB to bytes + } + } + } + + // If MemAvailable is not present, calculate from MemFree + Buffers + Cached + long memFree = 0, buffers = 0, cached = 0; + foreach (var line in memInfo.Split('\n')) + { + var parts = line.Split(' ', StringSplitOptions.RemoveEmptyEntries); + if (parts.Length >= 2 && long.TryParse(parts[1], out var value)) + { + if (line.StartsWith("MemFree:", StringComparison.OrdinalIgnoreCase)) + memFree = value; + else if (line.StartsWith("Buffers:", StringComparison.OrdinalIgnoreCase)) + buffers = value; + else if (line.StartsWith("Cached:", StringComparison.OrdinalIgnoreCase)) + cached = value; + } + } + return (memFree + buffers + cached) * 1024; // Convert KB to bytes + } + catch + { + // Fall through to fallback + } + + return GetTotalPhysicalMemoryLinux() / 2; // Assume 50% available as fallback + } + + private static long GetTotalPhysicalMemoryMacOS() + { + // For macOS, we would need to use sysctlbyname or similar + // For now, use a simple fallback + return Environment.WorkingSet * 4; // Rough approximation + } + + private static long GetAvailablePhysicalMemoryMacOS() + { + return GetTotalPhysicalMemoryMacOS() / 2; // Assume 50% available as fallback + } + + [StructLayout(LayoutKind.Sequential)] + private struct MEMORYSTATUSEX + { + public uint dwLength; + public uint dwMemoryLoad; + public ulong ullTotalPhys; + public ulong ullAvailPhys; + public ulong ullTotalPageFile; + public ulong ullAvailPageFile; + public ulong ullTotalVirtual; + public ulong ullAvailVirtual; + public ulong ullAvailExtendedVirtual; + } + + [DllImport("kernel32.dll", SetLastError = true)] + [return: MarshalAs(UnmanagedType.Bool)] + private static extern bool GlobalMemoryStatusEx(ref MEMORYSTATUSEX lpBuffer); +} \ No newline at end of file diff --git a/csharp/Platform.Data.Doublets/Memory/United/Generic/EnhancedUnitedMemoryLinks.cs b/csharp/Platform.Data.Doublets/Memory/United/Generic/EnhancedUnitedMemoryLinks.cs new file mode 100644 index 000000000..4f67364e4 --- /dev/null +++ b/csharp/Platform.Data.Doublets/Memory/United/Generic/EnhancedUnitedMemoryLinks.cs @@ -0,0 +1,229 @@ +using System; +using System.Numerics; +using System.Runtime.CompilerServices; +using Platform.Memory; +using Platform.Singletons; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace Platform.Data.Doublets.Memory.United.Generic; + +/// +/// +/// Represents enhanced united memory links with configurable pre-allocation strategies. +/// +/// +/// +/// +public unsafe class EnhancedUnitedMemoryLinks : UnitedMemoryLinks where TLinkAddress : IUnsignedNumber, IShiftOperators, IBitwiseOperators, IMinMaxValue, IComparisonOperators +{ + private readonly MemoryAllocationConfiguration _allocationConfiguration; + private readonly Action? _logger; + + /// + /// + /// Initializes a new instance with 75% RAM pre-allocation. + /// + /// + /// + /// + /// File address. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public EnhancedUnitedMemoryLinks(string address) + : this(address: address, allocationConfiguration: MemoryAllocationConfiguration.Default75PercentRAM) + { } + + /// + /// + /// Initializes a new instance with custom allocation configuration. + /// + /// + /// + /// + /// File address. + /// + /// + /// + /// Memory allocation configuration. + /// + /// + /// + /// Optional logger for allocation decisions. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public EnhancedUnitedMemoryLinks(string address, MemoryAllocationConfiguration allocationConfiguration, Action? logger = null) + : this(memory: new FileMappedResizableDirectMemory(path: address), + allocationConfiguration: allocationConfiguration, logger: logger) + { } + + /// + /// + /// Initializes a new instance with memory and allocation configuration. + /// + /// + /// + /// + /// A memory. + /// + /// + /// + /// Memory allocation configuration. + /// + /// + /// + /// Optional logger for allocation decisions. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public EnhancedUnitedMemoryLinks(IResizableDirectMemory memory, MemoryAllocationConfiguration allocationConfiguration, Action? logger = null) + : this(memory: memory, allocationConfiguration: allocationConfiguration, + constants: Default>.Instance, + indexTreeType: IndexTreeType.Default, logger: logger) + { } + + /// + /// + /// Initializes a new instance with full configuration options. + /// + /// + /// + /// + /// A memory. + /// + /// + /// + /// Memory allocation configuration. + /// + /// + /// + /// A constants. + /// + /// + /// + /// A index tree type. + /// + /// + /// + /// Optional logger for allocation decisions. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public EnhancedUnitedMemoryLinks(IResizableDirectMemory memory, MemoryAllocationConfiguration allocationConfiguration, + LinksConstants constants, IndexTreeType indexTreeType, Action? logger = null) + : base(memory: memory, + memoryReservationStep: CalculateOptimalReservationStep(allocationConfiguration, memory), + constants: constants, indexTreeType: indexTreeType) + { + _allocationConfiguration = allocationConfiguration ?? throw new ArgumentNullException(nameof(allocationConfiguration)); + _logger = logger; + + // Apply pre-allocation strategy + ApplyPreAllocationStrategy(memory); + } + + /// + /// + /// Gets the current memory allocation configuration. + /// + /// + /// + public MemoryAllocationConfiguration AllocationConfiguration => _allocationConfiguration; + + /// + /// + /// Gets the memory instance for statistics and monitoring. + /// + /// + /// + public IResizableDirectMemory Memory => _memory; + + private static long CalculateOptimalReservationStep(MemoryAllocationConfiguration config, IResizableDirectMemory memory) + { + if (config.Strategy == MemoryAllocationStrategy.Incremental) + { + return DefaultLinksSizeStep; + } + + // For pre-allocation strategies, calculate a step based on the pre-allocated size + var dataPath = memory is FileMappedResizableDirectMemory ? + Environment.CurrentDirectory : null; + + var preAllocationSize = config.CalculatePreAllocationSize(dataPath); + var linksCount = preAllocationSize / LinkSizeInBytes; + + // Use a step that's about 10% of the pre-allocated size, but at least the default + return Math.Max(DefaultLinksSizeStep, (linksCount * LinkSizeInBytes) / 10); + } + + private void ApplyPreAllocationStrategy(IResizableDirectMemory memory) + { + if (_allocationConfiguration.Strategy == MemoryAllocationStrategy.Incremental) + { + LogAllocation("Using incremental allocation strategy"); + return; + } + + try + { + var dataPath = memory is FileMappedResizableDirectMemory ? + Environment.CurrentDirectory : null; + + var preAllocationSize = _allocationConfiguration.CalculatePreAllocationSize(dataPath); + + // Calculate target capacity ensuring we have space for the header + var targetCapacity = Math.Max(preAllocationSize, LinkHeaderSizeInBytes + LinkSizeInBytes); + + LogAllocation($"Pre-allocating memory: {preAllocationSize:N0} bytes requested (strategy: {_allocationConfiguration.Strategy})"); + + // Only pre-allocate if it's more than current capacity + if (targetCapacity > memory.ReservedCapacity) + { + LogAllocation($"Pre-allocating memory: {targetCapacity:N0} bytes"); + memory.ReservedCapacity = targetCapacity; + } + + var maxLinks = (targetCapacity - LinkHeaderSizeInBytes) / LinkSizeInBytes; + LogAllocation($"Pre-allocation completed. Max links capacity: {maxLinks:N0}"); + } + catch (Exception ex) + { + LogAllocation($"Pre-allocation failed, falling back to incremental: {ex.Message}"); + // Don't throw - fall back to incremental allocation + } + } + + private void LogAllocation(string message) + { + if (_allocationConfiguration.EnableAllocationLogging) + { + _logger?.Invoke($"[{DateTime.Now:yyyy-MM-dd HH:mm:ss}] EnhancedUnitedMemoryLinks: {message}"); + } + } + + /// + /// + /// Gets memory allocation statistics. + /// + /// + /// + /// + /// Memory allocation statistics as a formatted string + /// + /// + public string GetAllocationStatistics() + { + var memory = _memory; + var utilizationPercent = memory.ReservedCapacity > 0 ? (memory.UsedCapacity * 100.0 / memory.ReservedCapacity) : 0; + var maxLinks = memory.ReservedCapacity > LinkHeaderSizeInBytes ? + (memory.ReservedCapacity - LinkHeaderSizeInBytes) / LinkSizeInBytes : 0; + + return $"Memory Statistics:\n" + + $" Strategy: {_allocationConfiguration.Strategy}\n" + + $" Memory: {memory.UsedCapacity:N0} / {memory.ReservedCapacity:N0} bytes ({utilizationPercent:F1}% utilized)\n" + + $" Links: {Total} / {maxLinks:N0} (max capacity)\n" + + $" Link Size: {LinkSizeInBytes} bytes"; + } +} \ No newline at end of file From 48a60639153c505be2063694242524b65f98ec28 Mon Sep 17 00:00:00 2001 From: konard Date: Sun, 14 Sep 2025 06:05:25 +0300 Subject: [PATCH 3/3] Remove CLAUDE.md - Claude command completed --- CLAUDE.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 093276334..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,5 +0,0 @@ -Issue to solve: https://github.com/linksplatform/Data.Doublets/issues/131 -Your prepared branch: issue-131-173bd186 -Your prepared working directory: /tmp/gh-issue-solver-1757817948559 - -Proceed. \ No newline at end of file