ConfigEditor

Overview

The ConfigEditor class is the centralized configuration management system for the Beep Data Management Engine. It provides a unified interface for managing data connections, component configurations, entity mappings, queries, and application settings. The class has been refactored to use specialized managers for different configuration responsibilities, improving maintainability and separation of concerns.

Specialized Manager Architecture

ConfigEditor uses a modular architecture with specialized managers that handle specific aspects of configuration:

public class ConfigEditor : IConfigEditor
{
    // Specialized managers
    private readonly ConfigPathManager _pathManager;
    private readonly DataConnectionManager _connectionManager;
    private readonly QueryManager _queryManager;
    private readonly EntityMappingManager _entityManager;
    private readonly ComponentConfigManager _componentManager;
    private readonly MigrationHistoryManager _migrationHistoryManager;

    // Core properties
    public BeepConfigType ConfigType { get; set; }
    public string ConfigPath { get; set; }
    public ConfigandSettings Config { get; set; }
    
    // Delegated properties
    public List DataConnections { get; set; }
    public List DataDriversClasses { get; set; }
    public List QueryList { get; set; }
    public List WorkFlows { get; set; }
}

Initialization & Setup

// Full application configuration
var configEditor = new ConfigEditor(
    logger,                          // IDMLogger instance
    errorObject,                     // IErrorsInfo for error handling
    jsonLoader,                      // IJsonLoader for serialization
    folderPath: @"C:\MyApp\Data",   // Base path (optional)
    containerFolder: "MyContainer", // Container name (optional)
    configType: BeepConfigType.Application
);

// Data connector only (minimal setup)
var dataConfigEditor = new ConfigEditor(
    logger, 
    errorObject, 
    jsonLoader,
    configType: BeepConfigType.DataConnector
);

// The ConfigEditor will automatically:
// 1. Create necessary directory structure
// 2. Initialize all specialized managers
// 3. Load existing configurations
// 4. Set up default values

Data Connection Management

// Create a new SQL Server connection
var sqlConnection = new ConnectionProperties
{
    ConnectionName = "ProductionDB",
    DatabaseType = DataSourceType.SqlServer,
    ConnectionString = "Server=prod-server;Database=ProductionDB;Integrated Security=true;",
    Category = DatasourceCategory.RDBMS,
    GuidID = Guid.NewGuid().ToString(),
    Host = "prod-server",
    Database = "ProductionDB",
    Port = 1433
};

// Add the connection
if (configEditor.AddDataConnection(sqlConnection))
{
    Console.WriteLine("? Connection added successfully");
    
    // Save to persistent storage
    configEditor.SaveDataconnectionsValues();
}

// Check if a connection exists
if (configEditor.DataConnectionExist("ProductionDB"))
{
    Console.WriteLine("Connection already exists");
}

// Load all existing connections
var allConnections = configEditor.LoadDataConnectionsValues();
foreach (var conn in allConnections)
{
    Console.WriteLine($"Found connection: {conn.ConnectionName} ({conn.DatabaseType})");
}

// Update a connection
var updatedConnection = new ConnectionProperties
{
    ConnectionName = "ProductionDB",
    DatabaseType = DataSourceType.SqlServer,
    ConnectionString = "Server=new-server;Database=ProductionDB;Integrated Security=true;",
    Category = DatasourceCategory.RDBMS,
    Host = "new-server",
    Database = "ProductionDB",
    Port = 1433
};

if (configEditor.UpdateDataConnection(updatedConnection, sqlConnection.GuidID))
{
    Console.WriteLine("? Connection updated successfully");
    configEditor.SaveDataconnectionsValues();
}

// Remove a connection
if (configEditor.RemoveDataConnection("ProductionDB"))
{
    Console.WriteLine("? Connection removed successfully");
    configEditor.SaveDataconnectionsValues();
}

Component Configuration

// Add a custom driver configuration
var customDriver = new ConnectionDriversConfig
{
    GuidID = Guid.NewGuid().ToString(),
    PackageName = "CustomPostgreSQLDriver",
    DriverClass = "CustomPostgreSQLDataSource",
    version = "1.0.0.0",
    dllname = "MyCustomDrivers.dll",
    classHandler = "MyCustomDrivers.CustomPostgreSQLDataSource",
    DatasourceType = DataSourceType.Postgre,
    DatasourceCategory = DatasourceCategory.RDBMS,
    ConnectionString = "Host={Host};Port={Port};Database={Database};Username={UserID};Password={Password}",
    ADOType = true,
    CreateLocal = false,
    InMemory = false
};

// Add the driver
int driverIndex = configEditor.AddDriver(customDriver);
if (driverIndex >= 0)
{
    Console.WriteLine($"? Driver added at index {driverIndex}");
    
    // Save driver configurations
    configEditor.SaveConnectionDriversConfigValues();
}

// Register a data source class
var dataSourceClass = new AssemblyClassDefinition
{
    className = "CustomPostgreSQLDataSource",
    PackageName = "MyCustomDrivers",
    type = "TheTechIdea.Beep.DataBase.IDataSource",
    AddinType = "DataSource",
    DllPath = "MyCustomDrivers.dll",
    Order = 1,
    version = "1.0.0.0"
};

configEditor.DataSourcesClasses.Add(dataSourceClass);

// Register workflow actions
var workflowAction = new AssemblyClassDefinition
{
    className = "CustomDataValidationAction",
    PackageName = "MyWorkflowActions",
    type = "TheTechIdea.Beep.Workflow.IWorkFlowAction",
    AddinType = "WorkFlowAction",
    DllPath = "MyWorkflowActions.dll"
};

configEditor.WorkFlowActions.Add(workflowAction);

Entity & Mapping Management

// Create an entity structure
var customerEntity = new EntityStructure
{
    EntityName = "Customers",
    DatasourceEntityName = "Customers",
    DataSourceID = "ProductionDB",
    SchemaOrOwnerOrDatabase = "dbo",
    Fields = new List
    {
        new EntityField
        {
           FieldName = "CustomerID",
            Fieldtype = "System.Int32",
            Size1 = 4,
            AllowDBNull = false,
            IsKey = true,
            IsIdentity = true
        },
        new EntityField
        {
           FieldName = "CustomerName",
            Fieldtype = "System.String",
            Size1 = 100,
            AllowDBNull = false
        },
        new EntityField
        {
           FieldName = "Email",
            Fieldtype = "System.String",
            Size1 = 255,
            AllowDBNull = true
        },
        new EntityField
        {
           FieldName = "CreatedDate",
            Fieldtype = "System.DateTime",
            AllowDBNull = false,
            DefaultValue = "GETDATE()"
        }
    }
};

// Save the entity structure
string entityPath = Path.Combine(configEditor.Config.EntitiesPath, "Customers.json");
configEditor.SaveEntityStructure(entityPath, customerEntity);

// Check if entity exists
if (configEditor.EntityStructureExist(entityPath, "Customers", "ProductionDB"))
{
    // Load the entity structure
    var loadedEntity = configEditor.LoadEntityStructure(entityPath, "Customers", "ProductionDB");
    Console.WriteLine($"? Loaded entity with {loadedEntity.Fields.Count} fields");
}

// Create a field mapping for ETL operations
var entityMapping = new EntityDataMap
{
    EntityName = "CustomerMapping",
    SourceEntityName = "LegacyCustomers",
    DestinationEntityName = "Customers",
    FieldMapping = new List
    {
        new EntityDataMap_DTL
        {
            SourceFieldName = "CUST_ID",
            DestinationFieldName = "CustomerID",
            DataType = "System.Int32",
            MappingType = "Direct"
        },
        new EntityDataMap_DTL
        {
            SourceFieldName = "CUST_NAME",
            DestinationFieldName = "CustomerName",
            DataType = "System.String",
            MappingType = "Direct"
        },
        new EntityDataMap_DTL
        {
            SourceFieldName = "EMAIL_ADDR",
            DestinationFieldName = "Email",
            DataType = "System.String",
            MappingType = "Transform",
            TransformRule = "LOWER(TRIM({value}))"
        }
    }
};

// Save the mapping
configEditor.SaveMappingValues("Customers", "ProductionDB", entityMapping);

// Load the mapping
var loadedMapping = configEditor.LoadMappingValues("Customers", "ProductionDB");
Console.WriteLine($"? Loaded mapping with {loadedMapping.FieldMapping.Count} field mappings");

Query Management

// Add custom queries to the repository
var customQueries = new List
{
    new QuerySqlRepo
    {
        ID = Guid.NewGuid().ToString(),
        QueryName = "ActiveCustomers",
        SqlQuery = "SELECT * FROM Customers WHERE IsActive = 1",
        DatabaseType = DataSourceType.SqlServer,
        CommandType = Sqlcommandtype.SELECT
    },
    new QuerySqlRepo
    {
        ID = Guid.NewGuid().ToString(),
        QueryName = "CustomersByRegion",
        SqlQuery = "SELECT * FROM Customers WHERE Region = @Region",
        DatabaseType = DataSourceType.SqlServer,
        CommandType = Sqlcommandtype.SELECT,
        Parameters = "@Region"
    },
    new QuerySqlRepo
    {
        ID = Guid.NewGuid().ToString(),
        QueryName = "InsertCustomer",
        SqlQuery = "INSERT INTO Customers (CustomerName, Email, Region) VALUES (@Name, @Email, @Region)",
        DatabaseType = DataSourceType.SqlServer,
        CommandType = Sqlcommandtype.INSERT,
        Parameters = "@Name,@Email,@Region"
    }
};

// Add queries to the configuration
configEditor.QueryList.AddRange(customQueries);

// Save the query repository
configEditor.SaveQueryFile();

// Generate SQL using the repository
string selectSql = configEditor.GetSql(
    Sqlcommandtype.SELECT, 
    "Customers", 
    "dbo", 
    "WHERE IsActive = 1", 
    DataSourceType.SqlServer
);

Console.WriteLine($"Generated SQL: {selectSql}");

// Get multiple SQL variations
var sqlVariations = configEditor.GetSqlList(
    Sqlcommandtype.SELECT,
    "Orders",
    "dbo",
    "WHERE OrderDate >= @StartDate",
    DataSourceType.SqlServer
);

foreach (string sql in sqlVariations)
{
    Console.WriteLine($"SQL Variation: {sql}");
}

// Load existing queries
var loadedQueries = configEditor.LoadQueryFile();
console.WriteLine($"? Loaded {loadedQueries.Count} saved queries");

// Use custom query with parameters
string customSql = configEditor.GetSqlFromCustomQuery(
    Sqlcommandtype.SELECT,
    "Customers",
    "SELECT * FROM Customers WHERE Region = '{Region}' AND Status = 'Active'",
    DataSourceType.SqlServer
);

Console.WriteLine($"Custom SQL: {customSql}");

Configuration Persistence

// Create a comprehensive configuration backup
public static class ConfigurationManager
{
    public static void BackupConfiguration(IConfigEditor configEditor, string backupPath)
    {
        try
        {
            // Create backup directory
            if (!Directory.Exists(backupPath))
            {
                Directory.CreateDirectory(backupPath);
            }

            // Backup main configuration
            configEditor.SaveConfigValues();
            File.Copy(
                Path.Combine(configEditor.ConfigPath, "Config.json"),
                Path.Combine(backupPath, $"Config_{DateTime.Now:yyyyMMdd_HHmmss}.json")
            );

            // Backup data connections
            configEditor.SaveDataconnectionsValues();
            File.Copy(
                Path.Combine(configEditor.ConfigPath, "DataConnections.json"),
                Path.Combine(backupPath, $"DataConnections_{DateTime.Now:yyyyMMdd_HHmmss}.json")
            );

            // Backup driver configurations
            configEditor.SaveConnectionDriversConfigValues();
            File.Copy(
                Path.Combine(configEditor.ConfigPath, "ConnectionConfig.json"),
                Path.Combine(backupPath, $"Drivers_{DateTime.Now:yyyyMMdd_HHmmss}.json")
            );

            // Backup query repository
            configEditor.SaveQueryFile();
            File.Copy(
                Path.Combine(configEditor.ConfigPath, "QueryList.json"),
                Path.Combine(backupPath, $"Queries_{DateTime.Now:yyyyMMdd_HHmmss}.json")
            );

            Console.WriteLine($"? Configuration backup completed: {backupPath}");
        }
        catch (Exception ex)
        {
            Console.WriteLine($"? Backup failed: {ex.Message}");
        }
    }

    public static void RestoreConfiguration(IConfigEditor configEditor, string backupPath)
    {
        try
        {
            // Restore configurations from backup
            var configFiles = Directory.GetFiles(backupPath, "*.json");
            
            foreach (var file in configFiles)
            {
                var fileName = Path.GetFileName(file);
                string targetPath = null;
                if (fileName.StartsWith("Config_", StringComparison.OrdinalIgnoreCase))
                    targetPath = Path.Combine(configEditor.ConfigPath, "Config.json");
                else if (fileName.StartsWith("DataConnections_", StringComparison.OrdinalIgnoreCase))
                    targetPath = Path.Combine(configEditor.ConfigPath, "DataConnections.json");
                else if (fileName.StartsWith("Drivers_", StringComparison.OrdinalIgnoreCase))
                    targetPath = Path.Combine(configEditor.ConfigPath, "ConnectionConfig.json");
                else if (fileName.StartsWith("Queries_", StringComparison.OrdinalIgnoreCase))
                    targetPath = Path.Combine(configEditor.ConfigPath, "QueryList.json");

                if (targetPath == null)
                    continue;

                File.Copy(file, targetPath, overwrite: true);
                Console.WriteLine($"? Restored: {Path.GetFileName(targetPath)}");
            }

            // Reload configurations
            configEditor.LoadConfigValues();
            configEditor.LoadDataConnectionsValues();
            configEditor.LoadConnectionDriversConfigValues();
            configEditor.LoadQueryFile();

            Console.WriteLine("? Configuration restore completed");
        }
        catch (Exception ex)
        {
            Console.WriteLine($"? Restore failed: {ex.Message}");
        }
    }
}

DMTypeBuilder — Dynamic Type Generation

DMTypeBuilder creates runtime dynamic types from EntityField definitions using both System.Reflection.Emit and RoslynCompiler:

public static class DMTypeBuilder
{
    public static IDMEEditor DMEEditor { get; set; }
    public static Type MyType { get; set; }
    public static object MyObject { get; set; }
    public static Dictionary<string, string> DataSourceNameSpace { get; set; }

    // Type cache for performance
    public static readonly Dictionary<string, Type> typeCache;

    // Primary: creates dynamic object from EntityField list
    public static object CreateNewObject(IDMEEditor editor, string classNamespace,
        string dataSourceName, string typeName, List<EntityField> fields);

    // Overload without dataSourceName  
    public static object CreateNewObject(IDMEEditor editor, string classNamespace,
        string typeName, List<EntityField> fields);

    // Create dynamic type from an existing object (reflection-based)
    public static Type CreateDynamicTypeFromObject(object sourceObject,
        string typeName = "DynamicType");
}

How It Works

1. Converts EntityField list → EntityStructure
2. Calls ConvertPOCOClassToEntity → generates C# code via ClassCreator
3. Compiles via RoslynCompiler.CompileClassTypeandAssembly()
4. Caches result in typeCache for subsequent calls
5. Returns Activator.CreateInstance(type)

Usage

var fields = new List<EntityField>
{
    new EntityField { FieldName = "Id", FieldType = "System.Int32", IsKey = true },
    new EntityField { FieldName = "Name", FieldType = "System.String", MaxLength = 100 },
    new EntityField { FieldName = "Email", FieldType = "System.String" }
};

// Create dynamic object
var obj = DMTypeBuilder.CreateNewObject(editor, "MyApp.Models",
    "MyDB", "DynamicCustomer", fields);

// Set properties via reflection
var type = obj.GetType();
type.GetProperty("Name").SetValue(obj, "John");
Console.WriteLine(type.GetProperty("Name").GetValue(obj));

CreateDynamicTypeFromObject

Reflection-based type generation from an existing instance:

var source = new { Id = 1, Name = "Test", Date = DateTime.Now };
Type dynamicType = DMTypeBuilder.CreateDynamicTypeFromObject(source, "CustomType");
var instance = Activator.CreateInstance(dynamicType);
// Instance has matching Id, Name, Date properties

MigrationHistory — Schema Version Tracking

public class MigrationHistory
{
    public string DataSourceName { get; set; }
    public DataSourceType DataSourceType { get; set; }
    public List<MigrationRecord> Migrations { get; set; }
}

public class MigrationRecord
{
    public string MigrationId { get; set; }
    public string Name { get; set; }
    public DateTime AppliedOnUtc { get; set; }
    public bool Success { get; set; }
    public string Notes { get; set; }
    public List<MigrationStep> Steps { get; set; }
}

public class MigrationStep
{
    public string Operation { get; set; }   // Create, Alter, Drop, Rename
    public string EntityName { get; set; }
    public string ColumnName { get; set; }
    public string Sql { get; set; }
    public bool Success { get; set; }
    public string Message { get; set; }
}

MigrationHistoryManager (in Managers/MigrationHistoryManager.cs) persists this data per datasource and provides query/export operations. Stored in ConfigEditor._migrationHistory and loaded via ConfigEditor.LoadMigrationHistory().

Usage Examples

// Initialize ConfigEditor for a data-intensive application
public static class DataApplicationConfigManager
{
    private static IConfigEditor _configEditor;

    public static void InitializeConfiguration()
    {
        // Create necessary services
        var logger = new DMLogger();
        var errorObject = new ErrorsInfo();
        var jsonLoader = new JsonLoader();

        // Initialize ConfigEditor for full application
        _configEditor = new ConfigEditor(
            logger, 
            errorObject, 
            jsonLoader,
            folderPath: @"C:\MyDataApp\Config",
            containerFolder: "Production",
            configType: BeepConfigType.Application
        );

        Console.WriteLine("? ConfigEditor initialized");
        
        // Setup data connections
        SetupDataConnections();
        
        // Configure drivers
        SetupDrivers();
        
        // Setup entity mappings
        SetupEntityMappings();
        
        // Create query repository
        SetupQueryRepository();
        
        // Save all configurations
        PersistAllConfigurations();
    }

    private static void SetupDataConnections()
    {
        var connections = new List
        {
            new ConnectionProperties
            {
                ConnectionName = "MainDatabase",
                DatabaseType = DataSourceType.SqlServer,
                ConnectionString = "Server=prod-sql;Database=MainDB;Integrated Security=true;",
                Category = DatasourceCategory.RDBMS,
                GuidID = Guid.NewGuid().ToString()
            },
            new ConnectionProperties
            {
                ConnectionName = "AnalyticsDB",
                DatabaseType = DataSourceType.Postgre,
                ConnectionString = "Host=analytics-pg;Database=analytics;Username=admin;Password=secret;",
                Category = DatasourceCategory.RDBMS,
                GuidID = Guid.NewGuid().ToString()
            },
            new ConnectionProperties
            {
                ConnectionName = "DocumentStore",
                DatabaseType = DataSourceType.MongoDB,
                ConnectionString = "mongodb://localhost:27017/documents",
                Category = DatasourceCategory.NOSQL,
                GuidID = Guid.NewGuid().ToString()
            }
        };

        foreach (var connection in connections)
        {
            if (_configEditor.AddDataConnection(connection))
            {
                Console.WriteLine($"? Added connection: {connection.ConnectionName}");
            }
        }
    }

    private static void SetupDrivers()
    {
        // Add custom drivers if needed
        var customDriver = new ConnectionDriversConfig
        {
            GuidID = Guid.NewGuid().ToString(),
            PackageName = "CustomAnalyticsDriver",
            DriverClass = "CustomAnalyticsDataSource",
            version = "1.0.0.0",
            DatasourceType = DataSourceType.Postgre,
            DatasourceCategory = DatasourceCategory.RDBMS,
            ConnectionString = "Host={Host};Database={Database};Username={UserID};Password={Password}"
        };

        _configEditor.AddDriver(customDriver);
        Console.WriteLine("? Custom driver added");
    }

    private static void SetupEntityMappings()
    {
        // Create entity structures for key entities
        var customerEntity = new EntityStructure
        {
            EntityName = "Customers",
            DataSourceID = "MainDatabase",
            Fields = new List
            {
                new EntityField {FieldName = "CustomerID", Fieldtype = "System.Int32", IsKey = true },
                new EntityField {FieldName = "CustomerName", Fieldtype = "System.String", Size1 = 100 },
                new EntityField {FieldName = "Email", Fieldtype = "System.String", Size1 = 255 }
            }
        };

        string entityPath = Path.Combine(_configEditor.Config.EntitiesPath, "Customers.json");
        _configEditor.SaveEntityStructure(entityPath, customerEntity);
        Console.WriteLine("? Customer entity structure saved");
    }

    private static void SetupQueryRepository()
    {
        var commonQueries = new List
        {
            new QuerySqlRepo
            {
                ID = Guid.NewGuid().ToString(),
                QueryName = "GetActiveCustomers",
                SqlQuery = "SELECT * FROM Customers WHERE IsActive = 1",
                DatabaseType = DataSourceType.SqlServer,
                CommandType = Sqlcommandtype.SELECT
            },
            new QuerySqlRepo
            {
                ID = Guid.NewGuid().ToString(),
                QueryName = "GetOrdersByDateRange",
                SqlQuery = "SELECT * FROM Orders WHERE OrderDate BETWEEN @StartDate AND @EndDate",
                DatabaseType = DataSourceType.SqlServer,
                CommandType = Sqlcommandtype.SELECT,
                Parameters = "@StartDate,@EndDate"
            }
        };

        _configEditor.QueryList.AddRange(commonQueries);
        Console.WriteLine("? Query repository populated");
    }

    private static void PersistAllConfigurations()
    {
        _configEditor.SaveConfigValues();
        _configEditor.SaveDataconnectionsValues();
        _configEditor.SaveConnectionDriversConfigValues();
        _configEditor.SaveQueryFile();
        
        Console.WriteLine("? All configurations persisted");
    }

    // Usage in application
    public static void UseConfiguredConnections()
    {
        // Check available connections
        foreach (var connection in _configEditor.DataConnections)
        {
            Console.WriteLine($"Available: {connection.ConnectionName} ({connection.DatabaseType})");
        }

        // Get SQL for a specific operation
        string sql = _configEditor.GetSql(
            Sqlcommandtype.SELECT,
            "Customers",
            "dbo",
            "WHERE Region = 'North'",
            DataSourceType.SqlServer
        );

        Console.WriteLine($"Generated SQL: {sql}");
    }
}

Best Practices