Advanced Tools & Utilities

Overview

The Beep Data Management Engine includes a comprehensive set of advanced tools and utilities that support various aspects of data management, from dynamic code generation to sophisticated configuration management and type handling.

ClassCreator

The ClassCreator component provides dynamic class generation capabilities for creating entity classes from database schemas.

public interface IClassCreator
{
    IDMEEditor DMEEditor { get; set; }
    string outputFileName { get; set; }
    string outputpath { get; set; }

    string CreateClass(string classname, EntityStructure entity, string poutputpath,
        string NameSpacestring = "TheTechIdea.ProjectClasses", bool GenerateCSharpCodeFiles = true);

    string CreatePOCOClass(string classname, EntityStructure entity, string usingheader, string implementations,
        string extracode, string outputpath, string nameSpacestring = "TheTechIdea.ProjectClasses",
        bool GenerateCSharpCodeFiles = true);

    string CreateDLL(string dllname, List<EntityStructure> entities, string outputpath,
        IProgress<PassedArgs> progress, CancellationToken token,
        string NameSpacestring = "TheTechIdea.ProjectClasses", bool GenerateCSharpCodeFiles = true);

    Type CreateEntityTypeAtRuntime(EntityStructure entity,
        string namespaceString = "TheTechIdea.ProjectClasses");

    // Plus additional batch, EF Core conversion, and code-generation methods.
}

// Access the class creator
var classCreator = dmeEditor.classCreator;

// Get entity structure from data source
var dataSource = dmeEditor.GetDataSource("CustomerDB");
var customerEntity = dataSource.GetEntityStructure("Customers", true);

// Generate POCO class code
string classCode = classCreator.CreatePOCOClass(customerEntity, "MyCompany.DataModels");

Console.WriteLine("Generated Class:");
Console.WriteLine(classCode);

// Output:
// namespace MyCompany.DataModels
// {
//     public class Customer
//     {
//         public int CustomerID { get; set; }
//         public string Name { get; set; }
//         public string Email { get; set; }
//         public DateTime CreatedDate { get; set; }
//     }
// }

// Create runtime type
Type customerType = classCreator.CreatePOCOType(customerEntity, "MyCompany.DataModels");

// Create instance dynamically
var customerInstance = Activator.CreateInstance(customerType);

// Set properties using reflection
customerType.GetProperty("Name")?.SetValue(customerInstance, "John Doe");
customerType.GetProperty("Email")?.SetValue(customerInstance, "john@example.com");

// Generate classes for multiple entities
var entities = new List<EntityStructure>();

// Get all entities from a data source
var dataSource = dmeEditor.GetDataSource("NorthwindDB");
var entityNames = dataSource.GetEntitesList();

foreach (var entityName in entityNames)
{
    var entity = dataSource.GetEntityStructure(entityName, true);
    entities.Add(entity);
}

// Compile to assembly
Assembly generatedAssembly = classCreator.CompileTOAssembly(entities, "NorthwindModels");


// Or create DLL file
string dllPath = classCreator.CreateDLL(entities, @"C:\Output", "NorthwindModels");

Console.WriteLine($"Generated assembly with {entities.Count} entity classes");
Console.WriteLine($"DLL created at: {dllPath}");

ClassCreator — Modular Architecture & Helpers

ClassCreator is implemented as a set of C# partial classes, each delegating to a focused generator helper. All helpers accept an IDMEEditor and are initialized in the main constructor, so every partial has access to the full DME context.

public ClassCreator(IDMEEditor dmeEditor)
{
    DMEEditor = dmeEditor ?? throw new ArgumentNullException(nameof(dmeEditor));

    // Core helpers (always initialized)
    _generationHelper   = new ClassGenerationHelper(dmeEditor);    // naming, type mapping, validation
    _pocoHelper         = new PocoClassGeneratorHelper(dmeEditor);  // POCO / domain classes
    _modernHelper       = new ModernClassGeneratorHelper(dmeEditor); // record / init-only classes

    // Specialized helpers
    _validationTestingHelper = new ValidationAndTestingGeneratorHelper(dmeEditor);
    _documentationHelper     = new DocumentationGeneratorHelper(dmeEditor);
    _uiComponentHelper       = new UiComponentGeneratorHelper(dmeEditor);
    _serverlessHelper        = new ServerlessGeneratorHelper(dmeEditor);
    _databaseHelper          = new DatabaseClassGeneratorHelper(dmeEditor);
    _webApiHelper            = new WebApiGeneratorHelper(dmeEditor);
    _pocoToEntityHelper      = new PocoToEntityGeneratorHelper(dmeEditor);
    _efCoreToEntityHelper    = new EfCoreToEntityGeneratorHelper(dmeEditor);
}

ClassCreator — Database & EF Core Generation

The ClassCreator.Database.cs partial (backed by DatabaseClassGeneratorHelper) generates data-access artifacts directly from EntityStructure metadata.

// Generate a repository-style DAL class for a single entity
var entity = dataSource.GetEntityStructure("Orders", true);
string dalPath = classCreator.GenerateDataAccessLayer(entity, @"C:\Output\DAL");
// Writes Orders.DAL.cs with CRUD methods wired to the Beep IDataSource API

// Generate an EF Core DbContext for all entities in a data source
var entities = entityNames
    .Select(n => dataSource.GetEntityStructure(n, true))
    .ToList();

string ctxPath = classCreator.GenerateDbContext(
    entities,
    namespaceString: "MyApp.Data",
    outputPath: @"C:\Output\EFCore"
);
// Writes MyAppDbContext.cs with DbSet<T> properties and OnModelCreating overrides

// Generate IEntityTypeConfiguration<T> from an EntityStructure
string cfgPath = classCreator.GenerateEntityConfiguration(
    entity,
    namespaceString: "MyApp.Data.Configurations",
    outputPath: @"C:\Output\EFCore\Config"
);
// Writes OrdersConfiguration.cs with column mappings, key, and index declarations

// Generate interface + implementation
string repoPath = classCreator.GenerateRepositoryImplementation(
    entity,
    outputPath: @"C:\Output\Repositories",
    namespaceName: "MyApp.Repositories",
    interfaceOnly: false   // set true to emit only IOrdersRepository.cs
);
// Writes IOrdersRepository.cs and OrdersRepository.cs with async CRUD + filtering

ClassCreator — Web API Generation

The ClassCreator.WebApi.cs partial wraps WebApiGeneratorHelper to produce ASP.NET Core controller classes and .NET 8+ Minimal API endpoint files.

// Scaffold one ApiController per entity
List<string> paths = classCreator.GenerateWebApiControllers(
    dataSourceName: "NorthwindDB",
    entities: entities,
    outputPath: @"C:\Output\Controllers",
    namespaceName: "MyApp.Controllers"
);

foreach (var path in paths)
    Console.WriteLine($"Created: {path}");

// Each file contains:
// [ApiController] [Route("api/[controller]")]
// GET / GET {id} / POST / PUT {id} / DELETE {id}
// wired to IBeepService.DMEEditor.GetDataSource(...)

// One generic controller that accepts dsName + entityName at runtime
string path = classCreator.GenerateWebApiControllerForEntityWithParams(
    className: "BeepDynamicController",
    outputPath: @"C:\Output\Controllers",
    namespaceName: "MyApp.Controllers"
);
// Useful for multi-tenant or data-explorer scenarios

// Generate a Program.cs-style Minimal API with MapGet / MapPost / MapPut / MapDelete
string apiPath = classCreator.GenerateMinimalWebApi(
    outputPath: @"C:\Output\MinimalApi",
    namespaceName: "MyApp.MinimalApi"
);
// Produces a single file with builder.Services + app.MapXxx registrations

ClassCreator — Testing & Validation Generation

The ClassCreator.Testing.cs partial (backed by ValidationAndTestingGeneratorHelper) generates xUnit/NUnit test class templates and FluentValidation validator classes from entity metadata.

// Generate an xUnit test scaffold for CRUD operations against an entity
string testPath = classCreator.GenerateUnitTestClass(entity, @"C:\Output\Tests");

// Generated class includes:
// - Arrange/Act/Assert patterns for Create, Read, Update, Delete
// - Mocked IDataSource via Moq
// - Constructor and field assertions for all EntityField entries

// Generate a FluentValidation AbstractValidator<T> from EntityStructure field metadata
string validatorPath = classCreator.GenerateFluentValidators(
    entity,
    outputPath: @"C:\Output\Validators",
    namespaceName: "MyApp.Validators"
);

// Fields with IsRequired=true emit .NotEmpty()
// Fields with Size1 > 0 emit .MaximumLength(n)
// Fields typed as email convention emit .EmailAddress()
// Nullable numeric fields emit .InclusiveBetween(min, max) if range metadata present

ClassCreator — UI Components & GraphQL

The ClassCreator.Advanced.cs partial exposes UI artifact generation (Blazor components, GraphQL schemas) and documentation utilities.

// Generate a Blazor edit/display component for an entity
string componentPath = classCreator.GenerateBlazorComponent(
    entity,
    outputPath: @"C:\Output\Components",
    namespaceName: "MyApp.Components"
);
// Writes Orders.razor + Orders.razor.cs:
// - @inject IBeepService BeepService
// - edit form bound to entity fields
// - Save / Cancel buttons wired to UnitofWork

// Generate SDL type definitions for multiple entities
string schemaPath = classCreator.GenerateGraphQLSchema(
    entities,
    outputPath: @"C:\Output\GraphQL",
    namespaceName: "TheTechIdea.ProjectGraphQL"
);
// Emits type Order { id: ID!, customerId: String, total: Float }
// plus Query { orders: [Order] order(id: ID!): Order }
// and Mutation { createOrder / updateOrder / deleteOrder }

// Generate XML documentation for an entity
string docPath = classCreator.GenerateEntityDocumentation(entity, @"C:\Output\Docs");

// Compare two schema versions and produce a human-readable diff
string diffReport = classCreator.GenerateEntityDiffReport(originalEntity, updatedEntity);
// Reports: added fields, removed fields, type changes, nullable changes

ClassCreator — POCO-to-Entity & Type Conversion

Two partial files handle reverse mapping — going from existing CLR types or data structures back to EntityStructure metadata that the rest of Beep can consume: ClassCreator.PocoToEntity.cs (namespace scanning) and ClassCreator.EntityExtensions.cs (EntitiesExtensions integration).

// Discover all POCO types in a namespace
List<Type> pocoTypes = classCreator.ScanNamespaceForPocos(
    "MyApp.Domain",
    assembly: Assembly.GetExecutingAssembly()
);

// Convert a single POCO type (with relationship detection)
EntityStructure entity = classCreator.ConvertPocoToEntity(
    typeof(Order),
    detectRelationships: true  // scans navigation properties for FK relationships
);

// Generate entity class code from the POCO (optionally write to file)
string code = classCreator.GenerateEntityClassFromPoco(
    typeof(Order),
    outputPath: @"C:\Output",
    namespaceString: "MyApp.Entities"
);

// Bulk conversion: every POCO in the namespace
List<string> codes = classCreator.GenerateEntityClassesFromNamespace(
    "MyApp.Domain",
    outputPath: @"C:\Output",
    targetNamespace: "MyApp.Entities"
);

// From a CLR type
EntityStructure fromType = classCreator.CreateEntityStructureFromType(
    typeof(Customer), entityName: "Customer");

// Generic overload
EntityStructure fromGeneric = classCreator.CreateEntityStructureFromType<Customer>();

// From an ADO.NET DataTable
EntityStructure fromTable = classCreator.CreateEntityStructureFromDataTable(myDataTable);

// From a typed list (infers fields from item type)
EntityStructure fromList = classCreator.CreateEntityStructureFromList(customers, "Customer");

// Batch: multiple types at once
List<EntityStructure> all = classCreator.CreateEntityStructuresFromTypes(
    typeof(Customer), typeof(Order), typeof(Product));

ConfigEditor

The refactored ConfigEditor uses specialized managers for different configuration responsibilities.

public class ConfigEditor : IConfigEditor
{
    // Specialized managers
    private readonly ConfigPathManager _pathManager;
    private readonly DataConnectionManager _connectionManager;
    private readonly QueryManager _queryManager;
    private readonly EntityMappingManager _entityManager;
    private readonly ComponentConfigManager _componentManager;
    private readonly MigrationHistoryManager _migrationHistoryManager;
    
    // Configuration properties
    public BeepConfigType ConfigType { get; set; } = BeepConfigType.Application;
    public string ConfigPath { get; set; }
    public List<ConnectionProperties> DataConnections
    {
        get => _connectionManager.DataConnections;
        set => _connectionManager.DataConnections = value;
    }
    public List<QuerySqlRepo> QueryList
    {
        get => _queryManager.QueryList;
        set => _queryManager.QueryList = value;
    }
    public List<ConnectionDriversConfig> DataDriversClasses
    {
        get => _componentManager.DataDriversClasses;
        set => _componentManager.DataDriversClasses = value;
    }
    public List<AssemblyClassDefinition> DataSourcesClasses { get; set; } = new();
}

// Initialize ConfigEditor
var configEditor = dmeEditor.ConfigEditor;

// Data Connection Management
var connection = new ConnectionProperties
{
    ConnectionName = "ProductionDB",
    DatabaseType = DataSourceType.SqlServer,
    ConnectionString = "Server=prod;Database=MyDB;Integrated Security=true;",
    Category = DatasourceCategory.RDBMS
};

configEditor.AddDataConnection(connection);
configEditor.SaveDataconnectionsValues();

// Component Configuration
var componentConfig = new AssemblyClassDefinition
{
    className = "SqlServerDataSource",
    PackageName = "TheTechIdea.Beep.SqlServer",
    AddinType = "DataSource"
};

configEditor.DataSourcesClasses.Add(componentConfig);

// Entity Mapping Configuration
var entityMap = new EntityDataMap
{
    EntityName = "CustomerMapping",
    SourceEntityName = "Customers",
    DestinationEntityName = "Customer_Archive"
};

configEditor.EntityMaps.Add(entityMap);
configEditor.SaveEntityMaps();

// Query Management
var savedQuery = new QuerySqlRepo
{
    ID = Guid.NewGuid().ToString(),
    QueryName = "ActiveCustomers",
    SqlQuery = "SELECT * FROM Customers WHERE Status = 'Active'",
    DataSourceName = "ProductionDB"
};

configEditor.AddQuery(savedQuery);
configEditor.SaveQueryFile();

DataTypesHelper

The DataTypesHelper provides comprehensive type handling and conversion utilities.

// Access helper from IDMEEditor
var typesHelper = dmeEditor.typesHelper;

// DataTypesHelper public surface in current code
string[] netTypes = typesHelper.GetNetDataTypes();
string[] netTypes2 = typesHelper.GetNetDataTypes2();
List<string> dataClasses = typesHelper.GetDataClasses();

// Resolve DB type from field metadata
var fld = new EntityField { FieldName = "Amount", fieldtype = "System.Decimal", Size1 = 18, Size2 = 2 };
string dbType = typesHelper.GetFieldTypeWoConversion("SQLSERVER", fld);
string clrType = typesHelper.GetDataType("SQLSERVER", fld);

// Mapping cache/query helpers
List<DatatypeMapping> mappings = typesHelper.GetDataTypeMappings("SQLSERVER");
bool valid = typesHelper.IsValidDataTypeMapping("SQLSERVER", fld);

public class DataTypesHelper : IDataTypesHelper
{
    private readonly ConcurrentDictionary<string, List<DatatypeMapping>> _mappingCache;
    private readonly ConcurrentDictionary<string, string[]> _netTypesCache;
    private volatile bool _isInitialized = false;

    public IDMEEditor DMEEditor { get; set; }
    public List<DatatypeMapping> mapping { get; set; }
    public int DefaultStringSize { get; set; } = 250;
}

// Map .NET types to different database systems
var fieldMappings = new Dictionary<string, string>();

fieldMappings["SqlServer"] = typesHelper.GetDataType("SqlServer", typeof(decimal), 18, 2);
// Returns: "DECIMAL(18,2)"

fieldMappings["MySQL"] = typesHelper.GetDataType("MySQL", typeof(decimal), 18, 2);
// Returns: "DECIMAL(18,2)"

fieldMappings["PostgreSQL"] = typesHelper.GetDataType("PostgreSQL", typeof(decimal), 18, 2);
// Returns: "NUMERIC(18,2)"

// Validate type mappings
foreach (var mapping in fieldMappings)
{
    bool isValid = typesHelper.IsValidDataType(mapping.Key, mapping.Value);
    Console.WriteLine($"{mapping.Key}: {mapping.Value} - Valid: {isValid}");
}

Runtime Components (Services, Rules, DataView, Caching, Proxy)

These signatures are synchronized with current implementations under Services, Rules, DataView, Caching, and Proxy.

Dedicated pages: DataViewDataSource, ICacheProvider + CacheManager, IProxyDataSource.

public interface IBeepServiceBuilder
{
    IBeepServiceBuilder WithDirectory(string directoryPath);
    IBeepServiceBuilder WithAppRepo(string appRepoName);
    IBeepServiceBuilder WithConfigType(BeepConfigType configType);
    IBeepServiceBuilder WithMapping(bool enable = true);
    IBeepServiceBuilder WithAssemblyLoading(bool enable = true);
    IBeepServiceBuilder AsSingleton();
    IBeepServiceBuilder AsScoped();
    IBeepServiceBuilder AsTransient();
    IBeepService Build();
}

public partial class DataViewDataSource : IDataSource, IDataViewOperations, IDisposable
{
    public event EventHandler<PassedArgs> PassEvent;
    public DataSourceType DatasourceType { get; set; }
    public IDMEEditor DMEEditor { get; set; }
    public IDMDataView DataView { get; set; } = new DMDataView();
    public FederationExecutionMode ExecutionMode { get; set; }
    public List<FederatedJoinDefinition> JoinDefinitions { get; set; }
}

public interface ICacheProvider : IDisposable
{
    string Name { get; }
    bool IsAvailable { get; }
    CacheStatistics Statistics { get; }
    Task<T> GetAsync<T>(string key, CancellationToken cancellationToken = default);
    Task<bool> SetAsync<T>(string key, T value, TimeSpan? expiry = null, CancellationToken cancellationToken = default);
    Task<bool> ExistsAsync(string key, CancellationToken cancellationToken = default);
    Task<long> ClearAsync(string pattern = null, CancellationToken cancellationToken = default);
}

public static partial class CacheManager
{
    public static void Initialize(CacheConfiguration configuration = null,
        CacheProviderType primaryProviderType = CacheProviderType.InMemory,
        CacheProviderType? fallbackProviderType = null);

    public static Task<T> GetOrCreateAsync<T>(string key, Func<Task<T>> factory,
        TimeSpan? expiry = null, CancellationToken cancellationToken = default);
}

public interface IProxyDataSource : IDataSource, IDisposable
{
    event EventHandler<FailoverEventArgs> OnFailover;
    event EventHandler<RecoveryEventArgs> OnRecovery;
    void ApplyPolicy(ProxyPolicy policy);
    Task<T> ExecuteWithLoadBalancing<T>(Func<IDataSource, Task<T>> operation,
        bool isWrite = false, CancellationToken cancellationToken = default);
    IDictionary<string, DataSourceMetrics> GetMetrics();
    ProxySloSnapshot GetSloSnapshot(string dsName);
    IProxyAuditSink AuditSink { get; set; }
}

AssemblyHandler

The AssemblyHandler runtime surface includes assembly loading, scanning, NuGet package acquisition, source management, and driver-package provenance tracking.

public interface IAssemblyHandler : IDisposable
{
    Assembly LoadAssembly(string path);
    string LoadAssembly(string path, FolderFileTypes fileTypes);
    IErrorsInfo LoadAllAssembly(IProgress<PassedArgs> progress, CancellationToken token);

    Task<List<NuGetSearchResult>> SearchNuGetPackagesAsync(string searchTerm, int skip = 0, int take = 20,
        bool includePrerelease = false, CancellationToken token = default);
    Task<List<string>> GetNuGetPackageVersionsAsync(string packageId, bool includePrerelease = false,
        CancellationToken token = default);
    Task<List<Assembly>> LoadNuggetFromNuGetAsync(string packageName, string version = null,
        IEnumerable<string> sources = null, bool useSingleSharedContext = true, string appInstallPath = null,
        bool useProcessHost = false);

    void TrackDriverPackage(string packageId, string version, string driverClassName, DataSourceType dsType);
    bool IsDriverFromNuGet(string driverClassName);
}

File Utilities

Advanced file processing utilities for various file formats and operations.

// File operation utilities
var fileHelper = new FileHelper(dmeEditor);

// Detect file encoding
Encoding encoding = fileHelper.DetectFileEncoding(@"C:\Data\customers.csv");
Console.WriteLine($"File encoding: {encoding.EncodingName}");

// Get file metadata
var fileInfo = fileHelper.GetFileMetadata(@"C:\Data\customers.csv");
Console.WriteLine($"Size: {fileInfo.Size} bytes");
Console.WriteLine($"Created: {fileInfo.CreatedDate}");
Console.WriteLine($"Modified: {fileInfo.ModifiedDate}");

// Validate file format
bool isValidCsv = fileHelper.ValidateFileFormat(@"C:\Data\customers.csv", "CSV");
bool isValidExcel = fileHelper.ValidateFileFormat(@"C:\Data\customers.xlsx", "Excel");

// Copy file with progress
var progress = new Progress<FileProgressArgs>(args =>
{
    Console.WriteLine($"Copying: {args.BytesCopied}/{args.TotalBytes} ({args.PercentComplete:F1}%)");
});

await fileHelper.CopyFileAsync(@"C:\Source\largefile.dat", @"C:\Destination\largefile.dat", progress);

// Split large file into chunks
var chunks = await fileHelper.SplitFileAsync(@"C:\Data\hugefile.csv", 1024 * 1024 * 100); // 100MB chunks
Console.WriteLine($"File split into {chunks.Count} chunks");

// Merge chunks back
await fileHelper.MergeChunksAsync(chunks, @"C:\Data\merged_hugefile.csv");

// File connection utilities
FileConnectionHelper.Initialize(dmeEditor);

// Create file-based data source connection
var csvConnection = FileConnectionHelper.CreateFileConnection(
    @"C:\Data\customers.csv", 
    DataSourceType.CSV,
    true // Has headers
);

dmeEditor.ConfigEditor.AddDataConnection(csvConnection);

// Auto-detect file type and create appropriate connection
var autoConnection = FileConnectionHelper.AutoDetectAndCreateConnection(@"C:\Data\unknown_file.dat");

if (autoConnection != null)
{
    Console.WriteLine($"Auto-detected as: {autoConnection.DatabaseType}");
}

// Batch create connections for directory
var directory = @"C:\Data\Files";
var connections = FileConnectionHelper.CreateConnectionsForDirectory(directory, "*.csv");

Console.WriteLine($"Created {connections.Count} connections for CSV files");

foreach (var conn in connections)
{
    dmeEditor.ConfigEditor.AddDataConnection(conn);
}

// Validate file accessibility
bool canAccess = FileConnectionHelper.ValidateFileAccess(@"C:\Data\protected_file.csv");
if (!canAccess)
{
    Console.WriteLine("File is not accessible or does not exist");
}

CircuitBreaker

The CircuitBreaker pattern provides fault tolerance for external service calls.

// Create circuit breaker with configuration
var circuitBreaker = new CircuitBreaker(
    name: "DatabaseCircuitBreaker",
    failureThreshold: 5,           // Fail after 5 consecutive failures
    timeout: TimeSpan.FromMinutes(2), // Stay open for 2 minutes
    retryTimeout: TimeSpan.FromSeconds(30) // Try again after 30 seconds
);

// Execute operation with circuit breaker protection
async Task<List<Customer>> GetCustomersWithProtection()
{
    return await circuitBreaker.ExecuteAsync(async () =>
    {
        var dataSource = dmeEditor.GetDataSource("CustomerDB");
        
        if (dataSource.ConnectionStatus != ConnectionState.Open)
        {
            dataSource.Openconnection();
        }
        
        var customers = await dataSource.GetEntityAsync("Customers", null);
        return customers.Cast<Customer>().ToList();
    });
}

// Handle circuit breaker events
circuitBreaker.OnCircuitOpened += (sender, args) =>
{
    Console.WriteLine($"Circuit breaker opened: {args.Reason}");
    // Send alert, switch to cache, etc.
};

circuitBreaker.OnCircuitClosed += (sender, args) =>
{
    Console.WriteLine("Circuit breaker closed - service recovered");
};

circuitBreaker.OnCircuitHalfOpened += (sender, args) =>
{
    Console.WriteLine("Circuit breaker half-open - testing service");
};

// Use the protected method
try
{
    var customers = await GetCustomersWithProtection();
    Console.WriteLine($"Retrieved {customers.Count} customers");
}
catch (CircuitBreakerOpenException)
{
    Console.WriteLine("Service is currently unavailable - circuit breaker is open");
    // Fallback to cached data or alternative service
}
catch (Exception ex)
{
    Console.WriteLine($"Operation failed: {ex.Message}");
}

? BatchExtensions

The BatchExtensions provide high-performance batch processing capabilities.

// Batch insert with progress reporting
var customers = new List<Customer>
{
    new Customer { Name = "John Doe", Email = "john@example.com" },
    new Customer { Name = "Jane Smith", Email = "jane@example.com" },
    // ... thousands more records
};

var dataSource = dmeEditor.GetDataSource("CustomerDB");

// Configure batch settings
var batchConfig = new BatchConfiguration
{
    BatchSize = 1000,
    MaxConcurrency = 4,
    ContinueOnError = true,
    EnableProgressReporting = true
}

// Progress reporting
var progress = new Progress<BatchProgressArgs>(args =>
{
    Console.WriteLine($"Batch Progress: {args.ProcessedCount}/{args.TotalCount} " +
                     $"({args.ProgressPercentage:F1}%) - Errors: {args.ErrorCount}");
});

// Execute batch insert
var result = await dataSource.BatchInsertAsync("Customers", customers, batchConfig, progress);

Console.WriteLine($"Batch insert completed:");
Console.WriteLine($"  Processed: {result.ProcessedCount}");
Console.WriteLine($"  Successful: {result.SuccessCount}");
Console.WriteLine($"  Errors: {result.ErrorCount}");
Console.WriteLine($"  Duration: {result.Duration}");

// Review errors if any
if (result.Errors.Any())
{
    Console.WriteLine("Errors encountered:");
    foreach (var error in result.Errors.Take(10)) // Show first 10 errors
    {
        Console.WriteLine($"  Record {error.RecordIndex}: {error.ErrorMessage}");
    }
}

// Process multiple entities in parallel
var entityBatches = new Dictionary<string, IEnumerable<object>>
{
    ["Customers"] = customers,
    ["Orders"] = orders,
    ["Products"] = products
};

var parallelResults = await dataSource.BatchProcessParallelAsync(
    entityBatches,
    new ParallelBatchConfiguration
    {
        MaxDegreeOfParallelism = Environment.ProcessorCount,
        BatchSize = 500,
        ContinueOnError = true
    },
    progress
);

foreach (var result in parallelResults)
{
    Console.WriteLine($"{result.Key}: {result.Value.SuccessCount} successful, {result.Value.ErrorCount} errors");
}

// Bulk update with conditions
var updateCriteria = new BatchUpdateCriteria
{
    WhereClause = "Status = 'Pending'",
    UpdateFields = new Dictionary<string, object>
    {
        ["Status"] = "Processed",
        ["ProcessedDate"] = DateTime.Now
    }
};

var updateResult = await dataSource.BatchUpdateAsync("Orders", updateCriteria, batchConfig);
Console.WriteLine($"Updated {updateResult.AffectedRows} records");

CSV Analyzer & Tools

Advanced CSV analysis and processing tools for data import and validation.

// Initialize CSV analyzer
var csvAnalyzer = new CSVAnalyser(dmeEditor);

// Analyze CSV file structure
var analysisResult = await csvAnalyzer.AnalyzeFileAsync(@"C:\Data\customers.csv");

Console.WriteLine($"CSV Analysis Results:");
Console.WriteLine($"  Rows: {analysisResult.TotalRows}");
Console.WriteLine($"  Columns: {analysisResult.ColumnCount}");
Console.WriteLine($"  Encoding: {analysisResult.DetectedEncoding}");
Console.WriteLine($"  Delimiter: '{analysisResult.DetectedDelimiter}'");
Console.WriteLine($"  Has Headers: {analysisResult.HasHeaders}");

// Column analysis
foreach (var column in analysisResult.Columns)
{
    Console.WriteLine($"Column '{column.Name}':");
    Console.WriteLine($"  Type: {column.DetectedType}");
    Console.WriteLine($"  Null Count: {column.NullCount}");
    Console.WriteLine($"  Unique Values: {column.UniqueValueCount}");
    Console.WriteLine($"  Max Length: {column.MaxLength}");
}

// Validate data quality
var qualityReport = csvAnalyzer.ValidateDataQuality(analysisResult);

Console.WriteLine($"Data Quality Report:");
Console.WriteLine($"  Overall Score: {qualityReport.OverallScore:F2}/10");
Console.WriteLine($"  Issues Found: {qualityReport.Issues.Count}");

foreach (var issue in qualityReport.Issues.Take(5))
{
    Console.WriteLine($"  - {issue.IssueType}: {issue.Description} (Severity: {issue.Severity})");
}

// Initialize type mapper
var typeMapper = new CSVTypeMapper(dmeEditor);

// Auto-detect column types
var columnMappings = typeMapper.DetectColumnTypes(@"C:\Data\customers.csv");

foreach (var mapping in columnMappings)
{
    Console.WriteLine($"Column: {mapping.ColumnName}");
    Console.WriteLine($"  Detected Type: {mapping.DetectedType}");
    Console.WriteLine($"  Confidence: {mapping.Confidence:F2}");
    Console.WriteLine($"  Suggested .NET Type: {mapping.SuggestedNetType}");
    Console.WriteLine($"  Suggested SQL Type: {mapping.SuggestedSqlType}");
}

// Create entity structure from CSV
var entityStructure = typeMapper.CreateEntityStructureFromCSV(
    @"C:\Data\customers.csv",
    "Customer",
    "CustomerDB"
);

Console.WriteLine($"Generated Entity Structure:");
Console.WriteLine($"  Entity Name: {entityStructure.EntityName}");
Console.WriteLine($"  Fields: {entityStructure.Fields.Count}");

foreach (var field in entityStructure.Fields)
{
    Console.WriteLine($"  - {field.FieldName}: {field.Fieldtype} (Size: {field.Size1})");
}

// Generate CREATE TABLE script
string createTableScript = typeMapper.GenerateCreateTableScript(entityStructure, "SqlServer");
Console.WriteLine($"Generated SQL Script:");
Console.WriteLine(createTableScript);

Utils — Internal Engine Helpers

The DataManagementEngineStandard/Utils/ folder contains a set of internal static helpers and one public utility class (Util) that are reused across the engine to avoid duplication in data source drivers, editors, and feature modules.

Util / IUtil

Util is the only public class in the Utils namespace. It implements IUtil and is registered in the DI container so engine consumers can inject it via IDMEEditor.Utilfunction.

public class Util : IUtil
{
    public IDMLogger Logger { get; set; }
    public IErrorsInfo ErrorObject { get; set; }
    public IConfigEditor ConfigEditor { get; set; }
    public IDMEEditor DME { get; set; }

    // DataTable → typed ObservableBindingList (runtime type via MakeGenericType)
    public IBindingList ConvertDataTableToObservableList(DataTable dataTable, Type type);

    // DataTable → generic ObservableBindingList<T> (requires T : Entity, new())
    public ObservableBindingList<T> ConvertDataTableToObservableBindingList<T>(DataTable dt) where T : Entity, new();

    // DataTable → List<T>
    public List<T> ConvertDataTable<T>(DataTable dt);

    // Activator through compiled Expression tree
    public T CreateInstance<T>(params object[] paramArray);
    public static ObjectActivator<T> GetActivator<T>(ConstructorInfo ctor);

    // Type helpers
    public TypeCode GetTypeCode(Type dest);
    public bool IsObjectNumeric(object o);

    // Data helpers
    public DataTable JsonToDataTable(string jsonString);

    // Driver resolution: matches a ConnectionProperties to the best-fit ConnectionDriversConfig
    public ConnectionDriversConfig LinkConnection2Drivers(IConnectionProperties cn);
}

// Convert a DataTable returned by a data source to a typed list
var dt = (DataTable)dataSource.GetEntity("Orders", null);
var orders = dmeEditor.Utilfunction.ConvertDataTable<Order>(dt);

// Resolve the correct driver config for a known connection
var conn = dmeEditor.ConfigEditor.DataConnections.First(c => c.ConnectionName == "NorthwindDB");
ConnectionDriversConfig drv = dmeEditor.Utilfunction.LinkConnection2Drivers(conn);
Console.WriteLine($"Driver package: {drv.PackageName} v{drv.version}");

// Parse a JSON string to DataTable
DataTable tbl = dmeEditor.Utilfunction.JsonToDataTable(jsonPayload);

DataConversionHelper

DataConversionHelper is an internal static class used internally by the engine whenever a DataTable needs to be materialized into a typed binding or list structure. It handles the standard edge-cases: DBNull, single-character strings, enums from strings, and safe numeric conversions.

// Runtime-typed ObservableBindingList (type known only at runtime)
IBindingList ConvertDataTableToObservableList(DataTable dataTable, Type type);

// Strongly-typed ObservableBindingList<T> (compile-time type)
ObservableBindingList<T> ConvertDataTableToObservableBindingList<T>(DataTable dt)
    where T : Entity, new();

// Simple List<T>
List<T> ConvertDataTable<T>(DataTable dt);

// Row-level conversion (supports DBNull, char, enum, numeric ChangeType)
T GetItem<T>(DataRow dr);

// Guard helper
bool IsNumericType(Type type); // covers byte → decimal

EntityStructureHelper

Builds EntityStructure and EntityField metadata from either an ADO.NET DataTable or a CLR Type. This is the low-level implementation behind ClassCreator.CreateEntityStructureFromDataTable and ClassCreator.CreateEntityStructureFromType.

// DataTable → EntityStructure (uses column metadata)
EntityStructure FromDataTable(DataTable tb);

// CLR Type → EntityStructure (uses public properties; detects [Key] attribute)
EntityStructure FromType(Type type);

// Column → EntityField (sets FieldCategory, AllowDBNull, IsAutoIncrement, IsUnique, Size1)
EntityField BuildFieldFromColumn(DataColumn col);

// PropertyInfo → EntityField (sets FieldCategory, IsKey from [Key] attribute)
EntityField BuildFieldFromProperty(PropertyInfo prop);

FilterHelper

Translates a List<AppFilter> into a DataView row filter (fast, index-aware) or a LINQ predicate chain (flexible, streaming). Both paths support the same set of comparison operators.

// Returns DataView with RowFilter built from filters (empty filters = full view)
DataView ApplyFilters(DataTable records, List<AppFilter> filters);

// Returns IEnumerable<DataRow> filtered via LINQ predicates
IEnumerable<DataRow> ApplyFiltersEnumerable(DataTable records, List<AppFilter> filters);

var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "Status",   Operator = "equals",  FilterValue = "Active" },
    new AppFilter { FieldName = "Total",    Operator = ">=",     FilterValue = 100 },
    new AppFilter { FieldName = "OrderDate",Operator = "between", FilterValue = new DateTime(2025,1,1),
                                                                   FilterValue1 = new DateTime(2025,12,31) }
};

// DataView path (preferred for DataGridView binding)
DataView view = FilterHelper.ApplyFilters(ordersTable, filters);

// LINQ path (preferred for server-side pagination)
var rows = FilterHelper.ApplyFiltersEnumerable(ordersTable, filters)
    .Skip(page * pageSize).Take(pageSize);

FileDataSourceHelper

Converts file-system paths into ConnectionProperties objects by auto-resolving the correct ConnectionDriversConfig for each file extension. Supports all DataSourceType values registered in ConfigEditor.DataDriversClasses.

// Create ConnectionProperties for multiple files (skips missing files and already-registered connections)
List<ConnectionProperties> CreateFileConnections(IDMEEditor dme, string[] filenames);

// Create a single ConnectionProperties from one file path
ConnectionProperties CreateFileDataConnection(IDMEEditor dme, string file);

// Extension-to-type switch (full list in FileDataSourceHelper.MapExtToType)
"csv"      → DataSourceType.CSV
"json"     → DataSourceType.Json
"xml"      → DataSourceType.XML
"xls/xlsx" → DataSourceType.Xls
"parquet"  → DataSourceType.Parquet
"yaml/yml" → DataSourceType.YAML
"avro"     → DataSourceType.Avro
"md"       → DataSourceType.Markdown
// ... plus tsv, orc, onnx, ini, log, pdf, doc/docx, ppt/pptx,
//         feather, tfrecord, recordio, libsvm, graphml, dicom, las

// Register all CSV files in a folder as data connections
var files = Directory.GetFiles(@"C:\Data", "*.csv");
var connections = FileDataSourceHelper.CreateFileConnections(dmeEditor, files);
foreach (var cn in connections)
    dmeEditor.ConfigEditor.AddDataConnection(cn);

// Register a single file
var conn = FileDataSourceHelper.CreateFileDataConnection(dmeEditor, @"C:\Data\sales.parquet");
if (conn != null)
    dmeEditor.ConfigEditor.AddDataConnection(conn);

ReflectionHelper

Provides high-performance compiled-Expression activators, runtime numeric checks, generic ExpandoObject projection, and field-mapping via reflection. Used internally by ETL, data conversion, and object-mapping pipelines.

// Create instance via Activator (simple path)
T CreateInstance<T>(params object[] args);

// Build a compiled Expression-tree activator for a constructor (cached by callers for hot paths)
Util.ObjectActivator<T> GetActivator<T>(ConstructorInfo ctor);

// Check whether an object's runtime type is a numeric primitive
bool IsObjectNumeric(object o);

// Get List<T> element type from a runtime list reference
Type GetListElementType(object list);

// Project all public properties of obj to an ExpandoObject / IDictionary<string,object>
ExpandoObject ToExpando(object obj);

// Map source object fields to a dest object using an EntityDataMap_DTL field-mapping descriptor
object MapObject(IDMEEditor dme, string destEntityName, EntityDataMap_DTL map, object sourceObj);

// One-time: compile the activator for Order's parameterless constructor
var ctor = typeof(Order).GetConstructor(Type.EmptyTypes);
var activator = ReflectionHelper.GetActivator<Order>(ctor);

// Hot-path reuse (avoids Activator.CreateInstance overhead in tight loops)
for (int i = 0; i < 100_000; i++)
{
    var order = activator(); // compiled IL — ~3x faster than Activator.CreateInstance
    // populate from DataRow ...
}

// Project a typed object to a loosely-typed dictionary (e.g. for dynamic JSON serialization)
var order = new Order { Id = 1, Total = 99.50m };
ExpandoObject exp = ReflectionHelper.ToExpando(order);
var dict = (IDictionary<string, object>)exp;
Console.WriteLine(dict["Total"]); // 99.50

// Map between two entity types using a stored field-mapping descriptor
var mappedObj = ReflectionHelper.MapObject(
    dmeEditor,
    destEntityName: "OrderArchive",
    map: entityDataMapDetail,
    sourceObj: sourceOrder
);

Tools Module — Full API Reference

IClassCreator — Complete Interface

public interface IClassCreator
{
    IDMEEditor DMEEditor { get; set; }
    string outputFileName { get; set; }
    string outputpath { get; set; }

    // Core generation
    string CreateClass(string classname, EntityStructure entity, string poutputpath,
        string NameSpacestring = "TheTechIdea.ProjectClasses",
        bool GenerateCSharpCodeFiles = true);
    string CreatePOCOClass(string classname, EntityStructure entity, string usingheader,
        string implementations, string extracode, string outputpath,
        string nameSpacestring = "TheTechIdea.ProjectClasses",
        bool GenerateCSharpCodeFiles = true);
    void CompileClassFromText(string sourceString, string output);
    void GenerateCSharpCode(string fileName);

    // DLL creation
    string CreateDLL(string dllname, List<EntityStructure> entities,
        string outputpath, IProgress<PassedArgs> progress, CancellationToken token,
        string NameSpacestring = "TheTechIdea.ProjectClasses",
        bool GenerateCSharpCodeFiles = true);
    string CreateDLLFromPoco(string dllname, Type pocoType, string outputpath,
        IProgress<PassedArgs> progress, CancellationToken token,
        System.Text.Encoding encoding = null);
    string CreateBatchDLL(string dllname, List<EntityStructure> entities,
        string outputpath, string NameSpacestring = "TheTechIdea.ProjectClasses");

    // Dynamic runtime types
    Type CreateEntityTypeAtRuntime(EntityStructure entity,
        string namespaceString = "TheTechIdea.ProjectClasses");
    Type CreateMultipleEntityTypesAtRuntime(List<EntityStructure> entities,
        string namespaceString = "TheTechIdea.ProjectClasses");

    // Database / EF Core
    string GenerateDataAccessLayer(EntityStructure entity, string outputPath,
        string namespaceName = "TheTechIdea.DAL");
    string GenerateDbContext(List<EntityStructure> entities, string outputPath,
        string namespaceString = "MyApp.Data");
    string GenerateEntityConfiguration(EntityStructure entity, string outputPath,
        string namespaceString = "MyApp.Data.Configurations");
    string GenerateRepositoryImplementation(EntityStructure entity,
        string outputPath, string namespaceName = "MyApp.Repositories",
        bool interfaceOnly = false);

    // Web API
    List<string> GenerateWebApiControllers(string dataSourceName,
        List<EntityStructure> entities, string outputPath,
        string namespaceName = "MyApp.Controllers");
    string GenerateWebApiControllerForEntityWithParams(string className,
        string outputPath, string namespaceName = "MyApp.Controllers");
    string GenerateMinimalWebApi(string outputPath,
        string namespaceName = "MyApp.MinimalApi");

    // Testing & Validation
    string GenerateUnitTestClass(EntityStructure entity, string outputPath,
        string namespaceName = "MyApp.Tests", string testFramework = "xUnit");
    string GenerateFluentValidators(EntityStructure entity, string outputPath,
        string namespaceName = "MyApp.Validators");

    // UI & Docs
    string GenerateBlazorComponent(EntityStructure entity, string outputPath,
        string namespaceName = "MyApp.Components");
    string GenerateGraphQLSchema(List<EntityStructure> entities,
        string outputPath, string namespaceName = "TheTechIdea.ProjectGraphQL");
    string GenerateEntityDocumentation(EntityStructure entity, string outputPath);
    string GenerateEntityDiffReport(EntityStructure original,
        EntityStructure updated);

    // POCO → Entity
    List<Type> ScanNamespaceForPocos(string namespaceName, Assembly assembly);
    EntityStructure ConvertPocoToEntity(Type pocoType,
        bool detectRelationships = true);
    string GenerateEntityClassFromPoco(Type pocoType, string outputPath,
        string namespaceString = "MyApp.Entities");
    List<string> GenerateEntityClassesFromNamespace(string namespaceName,
        string outputPath, string targetNamespace = "MyApp.Entities");

    // Type conversions
    EntityStructure CreateEntityStructureFromType(Type type,
        string entityName = null);
    EntityStructure CreateEntityStructureFromType<T>();
    EntityStructure CreateEntityStructureFromDataTable(DataTable table);
    EntityStructure CreateEntityStructureFromList(IEnumerable items, string name);
    List<EntityStructure> CreateEntityStructuresFromTypes(params Type[] types);
}

ClassCreator — 10 Generator Helpers

WebApiGenerator — Standalone Utility (not ClassCreator)

public class WebApiGenerator
{
    // Direct API endpoint generation from data sources
    public static List<string> GenerateControllers(
        string dataSourceName, List<EntityStructure> entities,
        string outputPath, string namespaceName = "MyApp.Controllers");

    // Emit with OpenAPI annotations
    public static string GenerateWithSwaggerAnnotations(
        EntityStructure entity, string outputPath,
        string namespaceName = "MyApp.Controllers");

    // Emit with versioning
    public static string GenerateVersionedController(
        EntityStructure entity, string apiVersion, string outputPath,
        string namespaceName = "MyApp.Controllers");

    // Emit with custom authorization policy
    public static string GenerateWithAuthorization(
        EntityStructure entity, string policyName, string outputPath,
        string namespaceName = "MyApp.Controllers");
}

Tools Source File Map (21 files)