@page "/admin/sync-structure" @page "/admin/sync-structure/{DatabaseStr}" @page "/admin/sync-structure/{DatabaseStr}/{DatabaseInstanceStr}" @attribute [Authorize] @using Rms.Risk.Mango.Components.Commands @using Rms.Risk.Mango.Controllers @using Rms.Risk.Mango.Pivot.Core.MongoDb @using Rms.Service.Bootstrap.Security @inject NavigationManager NavigationManager @inject IUserSession UserSession @inject IJSRuntime JsRuntime @inject ITempFileStorage TempFileStorage @inject IPasswordManager PasswordManager @inject ISingleUseTokenService SingleUseTokenService @* * dbMango * * Copyright 2025 Deutsche Bank AG * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. *@

Sync structure

@if ( _loading ) { } else {
@if ( item.Data == null ) // index label { if ( item.Label != "Collections") {
@item.Label
} else // root node {
@item.Label
} } else if ( item.Data.Index != null ) // index body {
} else if (item.Data.Collection != null ) // collection node { if (item.Data.Collection.Name.EndsWith("-Meta")) {
@item.Label
} else {
@item.Label
} if ( item.Data?.IsSharded ?? false ) {
Sharded
} }
}
@if ( item.Data == null ) // index label { if ( !item.Label.Contains( "Differences", StringComparison.OrdinalIgnoreCase )) { @item.Label } else // root node { @item.Label } } else if ( item.Data.Index != null ) // index body {
} else if (item.Data.Collection != null ) // collection node { if (item.Data.Collection.Name.EndsWith("-Meta")) { @item.Label } else { @item.Label } if ( item.Data.IsSharded ) { Sharded } } else { @item.Label if ( item.Data.IsSharded ) { Sharded } }
@foreach( var (success, message) in _progress ) {
@message
}
@if (Difference.ToBeSharded.Count > 0 ) {
There are collections that needs to be sharded. This is needs to be done manually because you have to specify sharding index. Usually it's {_id: "hashed"}, but this is not always true.
} @if (Difference.ToBeUnSharded.Count > 0 ) {
There are collections that needs to be unsharded. This is needs to be done manually because it's impossible to automatically select shard to host all the collection's data.
}
@code { [CascadingParameter] public IModalService Modal { get; set; } = null!; [Parameter] public string? DatabaseStr { get; set; } [Parameter] public string? DatabaseInstanceStr { get; set; } private const string ToAddLabel = "To add/change"; private const string ToRemoveLabel = "To remove"; private const string ToBeShardedLabel = "To be sharded"; private const string ToBeUnshardedLabel = "To be unsharded"; public class IndexTreeItem { public DatabaseStructureLoader.CollectionStructure? Collection { get; set; } public DatabaseStructureLoader.IndexStructure? Index { get; set; } public bool IsSelected { get; set; } public bool IsSharded { get; init; } } private DatabaseStructureLoader.StructureDifference Difference { get; set; } = new(); private List<(bool Success, string Message)> _progress = new(); private List _currentCollections = []; private List _targetCollections = []; private bool _loading; private readonly DatabaseStructureLoader.SyncStructureOptions _syncOptions = new() { RemoveCollections = false, RemoveIndexes = false }; private readonly List> _currentNodes = [ new () { Label = "Current", Children = [] } ]; private readonly List> _targetNodes = [ new () { Label = "Upload to see differences...", Children = [] } ]; private string Database { get => UserSession.Database; set { if (UserSession.Database == value) return; UserSession.Database = value; SyncUrl(); } } private string DatabaseInstance { get => UserSession.DatabaseInstance; set { if (UserSession.DatabaseInstance == value) return; UserSession.DatabaseInstance = value; SyncUrl(); } } protected override async Task OnAfterRenderAsync(bool firstRender) { if (!firstRender) return; if (string.IsNullOrWhiteSpace(DatabaseStr)) DatabaseStr = Database; else Database = DatabaseStr; if (string.IsNullOrWhiteSpace(DatabaseInstanceStr)) DatabaseInstanceStr = DatabaseInstance; else DatabaseInstance = DatabaseInstanceStr; SyncUrl(); try { _loading = true; StateHasChanged(); var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(2)); var (collections, root) = await LoadStructure(UserSession.MongoDbAdmin, UserSession.MongoDbAdminForAdminDatabase, cts2.Token); _currentCollections = collections; _currentNodes.Clear(); _currentNodes.Add(root); // #if DEBUG // var json = await File.ReadAllTextAsync("C:\\shabale\\Downloads\\Forge UAT-structure.json"); // _targetCollections = DatabaseStructureLoader.ParseCollections(json); // CompareStructure(); // #endif } catch (Exception ex) { _loading = false; await ModalDialogUtils.ShowExceptionDialog(Modal, $"Error loading structure for {Database}", ex); } finally { _loading = false; StateHasChanged(); } } private void SyncUrl() { var url = NavigationManager.BaseUri + $"admin/sync-structure/{Database}"; if (!string.IsNullOrWhiteSpace(DatabaseInstance)) url += $"/{DatabaseInstance}"; JsRuntime.InvokeAsync("DashboardUtils.ChangeUrl", url); } public static async Task<(List, TreeNode)> LoadStructure(IMongoDbDatabaseAdminService db, IMongoDbDatabaseAdminService admin, CancellationToken token) { var root = new TreeNode { Label = "Collections" }; var collections = (await DatabaseStructureLoader.LoadCollections(db, admin, token)) .Select(x => new { Group = x.Name.EndsWith("-Meta", StringComparison.OrdinalIgnoreCase) ? " Meta" : x.Name.EndsWith("-Cache", StringComparison.OrdinalIgnoreCase) ? " Cache" : " Data", Value = x } ) .ToList(); var allCollections = collections .OrderBy(x => x.Value.Name) .Select( x => x.Value ) .ToList(); // Add collections to the root node var collectionNodes = collections .GroupBy(x => x.Group) .OrderBy(x => x.Key) .Select(gr => new TreeNode { Label = gr.Key, IsExpanded = true, Children = gr .OrderBy(x => x.Value.Name) .Select(coll => new TreeNode { Label = coll.Value.Name, IsExpanded = false, Data = new() { Collection = coll.Value, IsSharded = coll.Value.IsSharded }, Children = coll.Value.Indexes .OrderBy(idx => idx.Name) .Select(idx => new TreeNode { Label = idx.Name, IsExpanded = false, Children = [ new TreeNode { Label = idx.Name, LabelFragment = _ => null!, Data = new() { Collection = coll.Value, Index = idx }, } ] }) .ToList() }) .ToList() }) .ToList(); root.Children = collectionNodes; root.IsExpanded = true; Set(root, true); return (allCollections, root); } private string ConvertCurrentCollectionsToJson() => Newtonsoft.Json.JsonConvert.SerializeObject(_currentCollections.Where(IsSelected).ToList(), Newtonsoft.Json.Formatting.Indented); private bool IsSelected(DatabaseStructureLoader.CollectionStructure coll) => IsSelected(_currentNodes, coll); private static bool IsSelected(List> nodes, DatabaseStructureLoader.CollectionStructure coll) { foreach (var node in nodes) { if (node.Data?.Collection?.Name.Equals(coll.Name, StringComparison.OrdinalIgnoreCase) ?? false) { return node.Data.IsSelected; } if (IsSelected(node.Children, coll)) { return true; } } return false; } private async Task OnDownload() { var url = await DownloadController.GetDownloadLink( TempFileStorage, PasswordManager, SingleUseTokenService, async fileName => await File.WriteAllTextAsync(fileName, ConvertCurrentCollectionsToJson()), $"{FileUtils.Shield(Database)}-structure.json" ); await JsRuntime.InvokeVoidAsync("open", $"{NavigationManager.BaseUri}{url}", "_blank"); } private async Task OnSync() { var toAdd = ExtractSelected(ToAddLabel); var toRemove = ExtractSelected(ToRemoveLabel); if (toAdd.Count == 0 && toRemove.Count == 0) { await ModalDialogUtils.ShowInfoDialog(Modal, "Sync structure", "No changes detected."); return; } if ( !_syncOptions.DryRun ) { var res = await ModalDialogUtils.ShowConfirmationDialog(Modal, "Sync structure", "Are you sure you want to synchronize the database structure? This will add or remove collections and indexes as needed."); if (res.Cancelled) return; } var admin = UserSession.MongoDbAdmin; try { var newDiff = new DatabaseStructureLoader.StructureDifference(); newDiff.ToAdd.AddRange(Difference.ToAdd.Where(x => toAdd.Contains(x.Name))); newDiff.ToRemove.AddRange(Difference.ToRemove.Where(x => toRemove.Contains(x.Name))); _progress = await DatabaseStructureLoader.SyncStructure(admin, newDiff, _syncOptions); await InvokeAsync(StateHasChanged); await ModalDialogUtils.ShowInfoDialog(Modal, "Sync structure", "Synchronization completed successfully."); } catch (Exception ex3) { await InvokeAsync(StateHasChanged); await ModalDialogUtils.ShowExceptionDialog(Modal, "Error during synchronization", ex3); } return; HashSet ExtractSelected(string label) { var idx = _targetNodes[0].Children.FindIndex(x => x.Label.Equals(label, StringComparison.OrdinalIgnoreCase)); var res = idx >= 0 ? _targetNodes[0].Children[idx].Children .Where(x => x.Data is { IsSelected: true, Collection: not null } ) .Select(x => x.Data!.Collection!.Name) .ToHashSet(StringComparer.OrdinalIgnoreCase) : [] ; return res; } } private async Task OnFileUploaded(InputFileChangeEventArgs args) { try { if (args.FileCount == 0) return; var file = args.GetMultipleFiles(1).FirstOrDefault(); if (file == null) return; await using var stream = file.OpenReadStream(10 * 1024 * 1024); // 10 MB limit var json = await new StreamReader(stream).ReadToEndAsync(); _targetCollections = DatabaseStructureLoader.ParseCollections(json); if (_targetCollections.Count == 0) { await ModalDialogUtils.ShowInfoDialog(Modal,"Sync structure", "No collections found in the uploaded file."); return; } CompareStructure(); await InvokeAsync(StateHasChanged); } catch (Exception ex) { await ModalDialogUtils.ShowExceptionDialog(Modal, "Error uploading structure", ex); } } private bool SelectedInCurrent(DatabaseStructureLoader.CollectionStructure node) { var found = _currentNodes[0].Children.FirstOrDefault(x => x.Data?.Collection?.Name.Equals(node.Name, StringComparison.OrdinalIgnoreCase) ?? false); if ( found == null ) return true; return found.Data?.IsSelected ?? true; } private void CompareStructure() { var targetCollections = _targetCollections.Where(SelectedInCurrent).ToList(); if (targetCollections.Count == 0) { if (Difference.ToAdd.Count == 0 && Difference.ToRemove.Count == 0 && _targetNodes.Count == 1) return; Difference = new(); _targetNodes.Clear(); _targetNodes.Add( new() { Label = "Upload to see differences...", Children = [] } ); return; } Difference = DatabaseStructureLoader.GetStructureDifference(_currentCollections.Where(SelectedInCurrent).ToList(), targetCollections); _targetNodes[0].IsExpanded = true; _targetNodes[0].Label = $"Differences (Add: {Difference.ToAdd.Count + Difference.ToAdd.Sum(x => x.Indexes.Count)} Remove: {Difference.ToRemove.Count + Difference.ToRemove.Sum(x => x.Indexes.Count)})"; _targetNodes[0].Children.Clear(); if (Difference.ToAdd.Count > 0) { _targetNodes[0].Children.Add( new() { Label = ToAddLabel, IsExpanded = true, Children = Difference.ToAdd.OrderBy(x => x.Name).Select(coll => new TreeNode { Label = $"{coll.Name} ({coll.Type})", Data = new() { Collection = coll }, Children = coll.Indexes.OfType().Select(idx => new TreeNode { Label = $"{idx.Name} ({idx.Type})", Data = new() { Collection = coll, Index = idx } }).ToList() }).ToList() } ); } if (Difference.ToRemove.Count > 0) { _targetNodes[0].Children.Add( new() { Label = ToRemoveLabel, IsExpanded = true, Children = Difference.ToRemove.OrderBy(x => x.Name).Select(coll => new TreeNode { Label = $"{coll.Name} ({coll.Type})", Data = new() { Collection = coll }, Children = coll.Indexes.OfType().Select(idx => new TreeNode { Label = $"{idx.Name} ({idx.Type})", Data = new() { Collection = coll, Index = idx } }).ToList() }).ToList() } ); } if ( Difference.ToBeSharded.Count > 0 ) { _targetNodes[0].Children.Add( new() { Label = ToBeShardedLabel, IsExpanded = true, Children = Difference.ToBeSharded.OrderBy(x => x).Select(coll => new TreeNode { Label = $"{coll}", Data = new() { IsSharded = false }, }).ToList() } ); } if ( Difference.ToBeUnSharded.Count > 0 ) { _targetNodes[0].Children.Add( new() { Label = ToBeUnshardedLabel, IsExpanded = true, Children = Difference.ToBeUnSharded.OrderBy(x => x).Select(coll => new TreeNode { Label = $"{coll}", Data = new() { IsSharded = false }, }).ToList() } ); } OnSelectAllTarget(); } private static void Set(TreeNode item, bool value) { if (item.Data != null) { item.Data.IsSelected = value; } foreach (var child in item.Children) { Set(child, value); } } private Task OnSelectAllCurrent() { foreach( var node in _currentNodes) Set(node, true); CompareStructure(); return InvokeAsync(StateHasChanged); } private Task OnSelectNoneCurrent() { foreach( var node in _currentNodes) Set(node, false); CompareStructure(); return InvokeAsync(StateHasChanged); } private Task OnSelectAllTarget() { foreach( var node in _targetNodes) Set(node, true); return InvokeAsync(StateHasChanged); } private Task OnSelectNoneTarget() { foreach( var node in _targetNodes) Set(node, false); return InvokeAsync(StateHasChanged); } private Task SetSourceItem(TreeNode item, bool b) { Set(item, b); CompareStructure(); return InvokeAsync(StateHasChanged); } }