JSON DataSource

Overview

The Json folder (DataManagementEngineStandard/Json/) delivers file-document-style JSON data handling under the standard BeepDM IDataSource abstraction. A JSON file becomes a fully queryable, writable, and schema-aware data source with no external database engine.

Folder Structure

Json/
├── JsonDataSource.cs            ← basic datasource
├── JsonDataSourceAdvanced.cs    ← advanced datasource (helper pipeline)
├── JsonExtensions.cs            ← shared JToken/JObject extension methods
└── Helpers/
    ├── JsonDataHelper.cs        ← synchronous entity query
    ├── JsonAsyncDataHelper.cs   ← async streaming, pagination
    ├── JsonCrudHelper.cs        ← Insert / Update / Delete on JArray
    ├── JsonSchemaHelper.cs      ← schema inference, entity tree construction
    ├── JsonSchemaSyncHelper.cs  ← live field sync, PK integrity
    ├── JsonSchemaPersistenceHelper.cs  ← save/load derived schemas
    ├── JsonFilterHelper.cs      ← AppFilter → JObject predicate compilation
    ├── JsonCacheManager.cs      ← predicate, property-map, child-entity caches
    ├── JsonGraphHelper.cs       ← graph node hydration with depth control
    ├── JsonRelationHelper.cs    ← FK parent/child traversal
    ├── JsonPathNavigator.cs     ← JSONPath array resolution from EntityStructure
    ├── JsonDeepEntityResolver.cs← deep multi-level entity resolution
    └── GraphHydrationOptions.cs ← depth / ancestor-chain / circular-ref options

JsonDataSource

JsonDataSource is the lightweight implementation. It loads the entire JSON file into memory on construction and performs all CRUD in-memory, flushing back to disk on write operations.

[AddinAttribute(
    Category       = DatasourceCategory.FILE,
    DatasourceType = DataSourceType.Json)]
public class JsonDataSource : IDataSource, IDisposable { }

// Constructor resolves the connection from ConfigEditor,
// then reads the file immediately.
public JsonDataSource(
    string         datasourcename,
    IDMLogger      logger,
    IDMEEditor     pDMEEditor,
    DataSourceType databasetype,
    IErrorsInfo    per)

// Internal after ctor:
//   Dataconnection.ConnectionProp = ConfigEditor.DataConnections[datasourcename]
//   FileName = Path.Combine(FilePath, FileName)
//   if (File.Exists(FileName)) ReadJson(FileName);

// Core state
DataSourceType DatasourceType { get; set; }   // Json
DatasourceCategory Category   { get; set; }   // FILE
ConnectionState ConnectionStatus { get; set; }

// Entity management
List<string>          EntitiesNames { get; set; }
List<EntityStructure> Entities      { get; set; }

// Standard CRUD (delegates to JToken in-memory tree)
IErrorsInfo  InsertEntity   (string entityName, object entity);
IErrorsInfo  UpdateEntities (string entityName, IErrorsInfo errObject, object entity);
IErrorsInfo  DeleteEntity   (string entityName, object entity);
object       GetEntity      (string entityName, List<AppFilter> filters);
Task<object> GetEntityAsync (string entityName, List<AppFilter> filters);

// Schema
EntityStructure GetEntityStructure(string entityname, bool refresh = false);
List<string>    GetEntitiesNames  (string schemaName);

dmeEditor.ConfigEditor.AddDataConnection(new ConnectionProperties
{
    ConnectionName = "products",
    FilePath       = @"C:\data",
    FileName       = "products.json",
    DatabaseType   = DataSourceType.Json,
    DriverName     = "JsonDataSource"
});

var ds = dmeEditor.GetDataSource("products") as JsonDataSource;
ds.Openconnection();

// Get all entities inferred from the file
var entityNames = ds.GetEntitiesNames(null);
Console.WriteLine(string.Join(", ", entityNames));

JsonDataSourceAdvanced

JsonDataSourceAdvanced composes the full helper pipeline. Each concern is delegated to a dedicated helper class injected at connection-open time.

// Fields wired up inside Openconnection():
private JsonDataHelper             _dataHelper;      // sync entity query
private JsonAsyncDataHelper        _asyncHelper;     // streaming / pagination
private JsonCrudHelper             _crudHelper;      // insert / update / delete
private JsonGraphHelper            _graphHelper;     // graph hydration
private JsonSchemaPersistenceHelper _schemaPersistence; // schema save/load

// Root JSON state
private JToken _rootJson;     // original root (object or array)
private JArray _rootArray;    // normalized array root
private bool   _rootWasObject; // true when root was {}, not []

// Thread safety
private readonly object _lock = new();
private bool _dataDirty;     // true after any write, triggers SaveJson on close

// Async streaming — yields objects one at a time without loading all rows
IAsyncEnumerable<object> StreamEntityAsync(
    string entityName, List<AppFilter> filters,
    CancellationToken cancellationToken = default);

// Paged fetch
Task<List<object>> GetPageAsync(
    string entityName, List<AppFilter> filters,
    int pageNumber, int pageSize,
    CancellationToken cancellationToken = default);

// Count without materializing all rows
Task<int> CountAsync(
    string entityName, List<AppFilter> filters,
    CancellationToken cancellationToken = default);

// Graph hydration with depth and options
IEnumerable<object> MaterializeGraph(
    string rootEntityName, List<AppFilter> rootFilters,
    GraphHydrationOptions options,
    Func<EntityStructure, bool> includeChildPredicate = null);

// Schema sync — scan data rows and add previously-unseen fields
bool SyncSchema(string entityName);

Schema Inference (JsonSchemaHelper)

JsonSchemaHelper traverses the JSON tree and produces a list of EntityStructure objects — one per array-of-objects node — plus RelationShipKeys linking parent arrays to their nested children.

// Static entry point — called by both datasource classes during Openconnection()
(List<EntityStructure> entities, List<RelationShipKeys> relations) =
    JsonSchemaHelper.BuildEntityStructures(
        root:           _rootJson,
        rootEntityName: "products",   // derived from file name
        dataSourceId:   DatasourceName,
        dsType:         DataSourceType.Json);

// JArray of JObject  → new EntityStructure, recurse into first element's nested arrays
// JObject (root)     → single-row entity, recurse into nested arrays/objects
// JArray of primitives → ignored (not entity-modelled)
// Nested array       → child entity named "parent_propertyName", parent FK added automatically

// PK auto-detection order (PreferredKeyNames):
//   "_id", "id", "Id", "ID"
// If none found → synthetic field "__id" added with type Guid

// Add fields seen in data but absent from current EntityStructure
// Returns true if any fields were added
bool changed = JsonSchemaSyncHelper.SyncFieldsFromData(dataArray, entityStructure);

// Ensure the synthetic PK field is still present in es.Fields
bool fixed   = JsonSchemaSyncHelper.EnsurePrimaryKeyIntegrity(entityStructure);

// CLR type inference for new fields:
//   JTokenType.Integer → long
//   JTokenType.Float   → double
//   JTokenType.Boolean → bool
//   JTokenType.Date    → DateTime
//   everything else    → string

Filter Compilation (JsonFilterHelper)

JsonFilterHelper.CompileFilters converts a List<AppFilter> into a list of Func<JObject, bool> predicates that are applied to each row during query execution.

// Equality
"="  | "equals"           → case-insensitive string equality
"!=" | "<>" | "notequals"  → negated equality

// Containment
"contains"                → IndexOf ≥ 0 (case-insensitive)
"notcontains" | "!contains"
"startswith" | "starts with"
"endswith"   | "ends with"

// Set membership
"in"     → FilterValue is comma-separated list
"notin"  | "not in"

// Pattern
"regex" | "matches"       → Regex.IsMatch(value, FilterValue)

// Range
"between"  → FilterValue + FilterValue1 (or single comma-separated FilterValue)
">" | ">=" | "<" | "<="

// Null checks
"is null"     | "isnull"
"is not null" | "isnotnull"

var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "Category", Operator = "=",        FilterValue  = "Electronics" },
    new AppFilter { FieldName = "Price",    Operator = "between",  FilterValue  = "100",
                                                                    FilterValue1 = "500" },
    new AppFilter { FieldName = "Name",     Operator = "contains", FilterValue  = "Pro" }
};

var predicates = JsonFilterHelper.CompileFilters(filters, entityStructure);

// Returns objects that match ALL predicates (AND semantics)
var results = ds.GetEntity("products", filters);

// MongoDB-style $oid fields are unwrapped automatically before comparison
// DateTime fields are serialised to ISO-8601 for string comparisons

CRUD Operations (JsonCrudHelper)

JsonCrudHelper operates on an in-memory JToken tree. All mutations are accumulated until Closeconnection() (or an explicit SaveJson call) flushes the file.

// Resolves the target JArray via JsonPathNavigator, then:
// 1. Converts the POCO to JObject via JObject.FromObject(data)
// 2. Ensures a synthetic PK value exists (adds GUID if missing)
// 3. Adds parent-FK reference if entity is a child
// 4. Appends to the resolved array
bool ok = _crudHelper.Insert("products", new Product { Name = "Laptop", Price = 999.99m });

// Finds row where PrimaryKey field matches, then merges all properties
// Returns false if PK field is missing from entity definition or data object
bool ok = _crudHelper.Update("products", updatedProduct);

// Deletes by AppFilter key match (typically PK equality filter)
var keyFilter = new AppFilter
{
    FieldName   = "id",
    Operator    = "=",
    FilterValue = "abc-123"
};
bool ok = _crudHelper.Delete("products", keyFilter);

// All three operations go through _crudHelper internally
IErrorsInfo result;

result = ds.InsertEntity("products", new Product { Name = "Tablet", Price = 499m });
result = ds.UpdateEntities("products", result, updatedTablet);
result = ds.DeleteEntity("products", new AppFilter { FieldName = "id", Operator = "=", FilterValue = "xyz" });

// File is written on Closeconnection() if _dataDirty == true
ds.Closeconnection();

Async Streaming (JsonAsyncDataHelper)

JsonAsyncDataHelper provides IAsyncEnumerable-based streaming and paged access. It inserts a Task.Yield() between each row to keep the calling thread responsive when processing large files.

// Yields materialized objects one at a time, applying filter predicates per row.
// Supports typed POCO materialization when the entity type is registered.
await foreach (var product in ds.StreamEntityAsync(
    "products",
    filters,
    cancellationToken))
{
    Console.WriteLine(((Product)product).Name);
}

// Page 2, 25 rows per page
var page = await ds.GetPageAsync(
    entityName:  "products",
    filters:     filters,
    pageNumber:  2,
    pageSize:    25,
    cancellationToken: ct);

Console.WriteLine($"Got {page.Count} rows");

// Counts without materializing objects — useful for pagination metadata
int total = await ds.CountAsync("products", filters, ct);
int totalPages = (int)Math.Ceiling((double)total / pageSize);

Graph Hydration & Relations

JsonGraphHelper — MaterializeGraph

Recursively builds a graph of nested dictionaries from the in-memory JSON tree, controlled by GraphHydrationOptions.

public class GraphHydrationOptions
{
    public int    Depth               { get; set; } = 1;         // 0 = root only
    public bool   IncludeAncestorChain { get; set; } = false;
    public string AncestorsKey        { get; set; } = "__ancestors";
    public bool   BreakCircularRefs   { get; set; } = true;
    // Func to include/exclude child entity types dynamically
}

var options = new GraphHydrationOptions
{
    Depth              = 2,
    IncludeAncestorChain = true
};

// Returns IEnumerable<object> where each object is a
// Dictionary<string, object> with child collections inlined.
var graph = ds.MaterializeGraph(
    rootEntityName:       "orders",
    rootFilters:          new List<AppFilter> { /* ... */ },
    options:              options,
    includeChildPredicate: es => es.EntityName != "orders_audit");

foreach (Dictionary<string, object> order in graph)
{
    var lines = (List<Dictionary<string, object>>)order["orders_lines"];
    Console.WriteLine($"Order {order["id"]} has {lines.Count} lines");
}

JsonRelationHelper — FK Traversal

Provides direct parent/child traversal using EntityStructure.Relations metadata without materializing the full graph.

// Resolve child entities for a given parent row key value
IEnumerable<JObject> lines = JsonRelationHelper.GetChildren(
    root:           _rootJson,
    parent:         ordersEntity,
    child:          orderLinesEntity,
    parentKeyValue: "order-abc-123");

// Resolve parent rows from a child FK value
IEnumerable<JObject> parents = JsonRelationHelper.GetParents(
    root:                   _rootJson,
    child:                  orderLinesEntity,
    parent:                 ordersEntity,
    childRowParentKeyValue: "order-abc-123");

// Relation metadata used:
//   child.Relations[x].RelatedEntityID  == parent.EntityName
//   child.Relations[x].EntityColumnID   == FK field name in child

Caching (JsonCacheManager)

JsonCacheManager is a static, thread-safe cache using ConcurrentDictionary. It amortises the cost of predicate compilation, property-map reflection, and child-entity list construction across repeated calls.

// 1. Predicate cache — key: "entityName|field:op:val;field:op:val;..."
List<Func<JObject, bool>> preds =
    JsonCacheManager.GetOrAddPredicates(entityName, filters, entity);

// 2. Property-map cache — key: type.FullName
//    Maps FieldName → PropertyInfo for POCO materialization
Dictionary<string, PropertyInfo> propMap =
    JsonCacheManager.GetOrAddPropertyMap(typeof(Product), entity);

// 3. Child-entity list cache — key: parent.EntityName
List<EntityStructure> children =
    JsonCacheManager.GetOrAddChildren(parentEntity, allEntities);

// Partial invalidation (e.g. after schema sync)
JsonCacheManager.InvalidateEntity("products");

// Full reset (e.g. after file reload)
JsonCacheManager.InvalidateAll();

End-to-End Examples

// 1. Register connection
dmeEditor.ConfigEditor.AddDataConnection(new ConnectionProperties
{
    ConnectionName = "catalog",
    FilePath       = @"C:\data",
    FileName       = "catalog.json",
    DatabaseType   = DataSourceType.Json,
    DriverName     = "JsonDataSource"
});

// 2. Open
var ds = dmeEditor.GetDataSource("catalog") as JsonDataSource;
ds.Openconnection();

// 3. List entities inferred from schema
foreach (var name in ds.GetEntitiesNames(null))
    Console.WriteLine(name);

// 4. Query with filters
var filters = new List<AppFilter>
{
    new AppFilter { FieldName = "inStock", Operator = "=", FilterValue = "true" }
};
var rows = (IEnumerable<object>)ds.GetEntity("products", filters);
ds.Closeconnection();

var ads = dmeEditor.GetDataSource("catalog") as JsonDataSourceAdvanced;
ads.Openconnection();

int total     = await ads.CountAsync("products", null, CancellationToken.None);
int pageSize  = 50;
int pageCount = (int)Math.Ceiling((double)total / pageSize);

for (int p = 1; p <= pageCount; p++)
{
    var page = await ads.GetPageAsync("products", null, p, pageSize);
    Console.WriteLine($"Page {p}: {page.Count} rows");
}

ads.Closeconnection();

var ads = dmeEditor.GetDataSource("catalog") as JsonDataSourceAdvanced;
ads.Openconnection();

// Insert
ads.InsertEntity("products", new { id = Guid.NewGuid(), name = "Widget", price = 9.99 });

// Update
ads.UpdateEntities("products", ads.ErrorObject,
    new { id = existingId, name = "Widget Pro", price = 19.99 });

// Delete
ads.DeleteEntity("products",
    new AppFilter { FieldName = "id", Operator = "=", FilterValue = existingId.ToString() });

// File is written here
ads.Closeconnection();

// Given catalog.json:
// { "orders": [ { "id": 1, "lines": [ { "sku": "A", "qty": 2 } ] } ] }

var ads = dmeEditor.GetDataSource("catalog") as JsonDataSourceAdvanced;
ads.Openconnection();

var options = new GraphHydrationOptions { Depth = 2 };

var graph = ads.MaterializeGraph(
    rootEntityName: "orders",
    rootFilters:    null,
    options:        options);

foreach (Dictionary<string, object> order in graph)
{
    var lines = order["orders_lines"] as List<Dictionary<string, object>>;
    Console.WriteLine($"Order {order["id"]}: {lines?.Count ?? 0} lines");
}

ads.Closeconnection();

// A new field "tags" appears in some product objects — sync the schema
bool changed = ads.SyncSchema("products");
if (changed)
{
    // Invalidate predicate and property-map caches so next query uses new fields
    JsonCacheManager.InvalidateEntity("products");
    Console.WriteLine("Schema updated: new fields discovered");
}

Full API — All Helpers & Source Map

JsonExtensions

public static class JsonExtensions
{
    // Map JSON property to EntityField with DbFieldCategory detection
    public static EntityField MapEntityField(JProperty prop, string entityName, int ordinal, bool deep = false);

    // Digest JSON root: single entity object, flat or nested array, or heterogeneous root
    public static EntityStructure DigestJsonToEntityStructure(string json, string entityName, bool useRootAsEntity = false);

    // Deduce the base entity name from the JSON root token
    public static string GetBaseEntityFromToken(JToken rootToken);
}

JsonSchemaSyncHelper

Schema drift detection and reconciliation for JSON-based data sources.

public class JsonSchemaSyncHelper
{
    // Compare in-memory structure to persisted schema. Returns true if changes were applied.
    public bool SyncSchema(string entityName);

    // Detect drift: added columns, removed columns, type changes
    public SchemaDriftReport DetectDrift(string entityName);

    // Rebuild EntityStructure from raw JSON data, overwriting current schema
    public void RebuildSchema(string entityName, JArray data);
}

JsonSchemaPersistenceHelper

Save/load EntityStructure metadata alongside the JSON data file.

public class JsonSchemaPersistenceHelper
{
    public void SaveSchema(string dataFilePath, EntityStructure schema);
    public EntityStructure LoadSchema(string dataFilePath);
    public bool SchemaFileExists(string dataFilePath);
    public void DeleteSchema(string dataFilePath);

    // Schema file convention: data.json → data.json.schema
    private string GetSchemaFilePath(string dataFilePath);
}

JsonDataHelper

Core data transformation: CLR object ↔ JSON.

public class JsonDataHelper
{
    // Convert a POCO to a JObject with optional field filtering
    public JObject ConvertToJObject(object entity,
        EntityStructure structure = null, List<string> includeFields = null);

    // Convert a JObject or JToken to a typed CLR object
    public object ConvertToTypedObject(JToken token, Type targetType,
        EntityStructure structure = null);

    // Serialize/deserialize with Newtonsoft settings
    public string Serialize(object obj, Formatting formatting = Formatting.Indented);
    public T Deserialize<T>(string json);
    public object Deserialize(string json, Type targetType);
}

GraphHydrationOptions

public class GraphHydrationOptions
{
    public int MaxDepth { get; set; } = 3;
    public bool FollowBackReferences { get; set; } = false;
    public bool IncludeDeleted { get; set; } = false;
    public List<string> ExcludedRelations { get; set; } = new();
    public List<string> IncludedRelations { get; set; } = new();   // null = all
}

JsonDeepEntityResolver

public class JsonDeepEntityResolver
{
    // Resolve a nested entity by dot-path: "order.customer.address.city"
    public JToken ResolveDeep(JToken root, string dottedPath);

    // Flatten nested JSON into a flat dictionary of dotted keys
    public Dictionary<string, JToken> Flatten(JToken root, int maxDepth = 10);

    // Unflatten a flat dotted-key dictionary back into nested JObject
    public JObject Unflatten(Dictionary<string, JToken> flatMap);
}

JsonPathNavigator

public class JsonPathNavigator
{
    // Navigate to a token using JSONPath expression: "$.orders[0].items[*].price"
    public JToken Navigate(string jsonPath, JToken root = null);

    // Navigate and project selected fields
    public List<Dictionary<string, object>> NavigateAndProject(
        string jsonPath, string[] projectedFields);

    // Navigate and filter
    public List<JToken> NavigateAndFilter(
        string jsonPath, Func<JToken, bool> predicate);
}

Source File Map (16 files)