@page "/admin/sync-structure"
@page "/admin/sync-structure/{DatabaseStr}"
@page "/admin/sync-structure/{DatabaseStr}/{DatabaseInstanceStr}"
@attribute [Authorize]
@using Rms.Risk.Mango.Components.Commands
@using Rms.Risk.Mango.Controllers
@using Rms.Risk.Mango.Pivot.Core.MongoDb
@using Rms.Service.Bootstrap.Security
@inject NavigationManager NavigationManager
@inject IUserSession UserSession
@inject IJSRuntime JsRuntime
@inject ITempFileStorage TempFileStorage
@inject IPasswordManager PasswordManager
@inject ISingleUseTokenService SingleUseTokenService
@*
* dbMango
*
* Copyright 2025 Deutsche Bank AG
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*@
Sync structure
@if ( _loading )
{
}
else
{
@if ( item.Data == null ) // index label
{
if ( item.Label != "Collections")
{
@item.Label
}
else // root node
{
@item.Label
}
}
else if ( item.Data.Index != null ) // index body
{
}
else if (item.Data.Collection != null ) // collection node
{
if (item.Data.Collection.Name.EndsWith("-Meta"))
{
@item.Label
}
else
{
@item.Label
}
if ( item.Data?.IsSharded ?? false )
{
Sharded
}
}
}
@if ( item.Data == null ) // index label
{
if ( !item.Label.Contains( "Differences", StringComparison.OrdinalIgnoreCase ))
{
@item.Label
}
else // root node
{
@item.Label
}
}
else if ( item.Data.Index != null ) // index body
{
}
else if (item.Data.Collection != null ) // collection node
{
if (item.Data.Collection.Name.EndsWith("-Meta"))
{
@item.Label
}
else
{
@item.Label
}
if ( item.Data.IsSharded )
{
Sharded
}
}
else
{
@item.Label
if ( item.Data.IsSharded )
{
Sharded
}
}
@foreach( var (success, message) in _progress )
{
@message
}
@if (Difference.ToBeSharded.Count > 0 )
{
There are collections that needs to be sharded. This is needs to be done manually because you have to specify sharding index. Usually it's {_id: "hashed"}, but this is not always true.
}
@if (Difference.ToBeUnSharded.Count > 0 )
{
There are collections that needs to be unsharded. This is needs to be done manually because it's impossible to automatically select shard to host all the collection's data.
}
@code {
[CascadingParameter] public IModalService Modal { get; set; } = null!;
[Parameter] public string? DatabaseStr { get; set; }
[Parameter] public string? DatabaseInstanceStr { get; set; }
private const string ToAddLabel = "To add/change";
private const string ToRemoveLabel = "To remove";
private const string ToBeShardedLabel = "To be sharded";
private const string ToBeUnshardedLabel = "To be unsharded";
public class IndexTreeItem
{
public DatabaseStructureLoader.CollectionStructure? Collection { get; set; }
public DatabaseStructureLoader.IndexStructure? Index { get; set; }
public bool IsSelected { get; set; }
public bool IsSharded { get; init; }
}
private DatabaseStructureLoader.StructureDifference Difference { get; set; } = new();
private List<(bool Success, string Message)> _progress = new();
private List _currentCollections = [];
private List _targetCollections = [];
private bool _loading;
private readonly DatabaseStructureLoader.SyncStructureOptions _syncOptions = new()
{
RemoveCollections = false,
RemoveIndexes = false
};
private readonly List> _currentNodes =
[
new ()
{
Label = "Current",
Children = []
}
];
private readonly List> _targetNodes =
[
new ()
{
Label = "Upload to see differences...",
Children = []
}
];
private string Database
{
get => UserSession.Database;
set
{
if (UserSession.Database == value)
return;
UserSession.Database = value;
SyncUrl();
}
}
private string DatabaseInstance
{
get => UserSession.DatabaseInstance;
set
{
if (UserSession.DatabaseInstance == value)
return;
UserSession.DatabaseInstance = value;
SyncUrl();
}
}
protected override async Task OnAfterRenderAsync(bool firstRender)
{
if (!firstRender)
return;
if (string.IsNullOrWhiteSpace(DatabaseStr))
DatabaseStr = Database;
else
Database = DatabaseStr;
if (string.IsNullOrWhiteSpace(DatabaseInstanceStr))
DatabaseInstanceStr = DatabaseInstance;
else
DatabaseInstance = DatabaseInstanceStr;
SyncUrl();
try
{
_loading = true;
StateHasChanged();
var cts2 = new CancellationTokenSource(TimeSpan.FromMinutes(2));
var (collections, root) = await LoadStructure(UserSession.MongoDbAdmin, UserSession.MongoDbAdminForAdminDatabase, cts2.Token);
_currentCollections = collections;
_currentNodes.Clear();
_currentNodes.Add(root);
// #if DEBUG
// var json = await File.ReadAllTextAsync("C:\\shabale\\Downloads\\Forge UAT-structure.json");
// _targetCollections = DatabaseStructureLoader.ParseCollections(json);
// CompareStructure();
// #endif
}
catch (Exception ex)
{
_loading = false;
await ModalDialogUtils.ShowExceptionDialog(Modal, $"Error loading structure for {Database}", ex);
}
finally
{
_loading = false;
StateHasChanged();
}
}
private void SyncUrl()
{
var url = NavigationManager.BaseUri + $"admin/sync-structure/{Database}";
if (!string.IsNullOrWhiteSpace(DatabaseInstance))
url += $"/{DatabaseInstance}";
JsRuntime.InvokeAsync("DashboardUtils.ChangeUrl", url);
}
public static async Task<(List, TreeNode)> LoadStructure(IMongoDbDatabaseAdminService db, IMongoDbDatabaseAdminService admin, CancellationToken token)
{
var root = new TreeNode
{
Label = "Collections"
};
var collections = (await DatabaseStructureLoader.LoadCollections(db, admin, token))
.Select(x =>
new
{
Group =
x.Name.EndsWith("-Meta", StringComparison.OrdinalIgnoreCase)
? " Meta"
: x.Name.EndsWith("-Cache", StringComparison.OrdinalIgnoreCase)
? " Cache"
: " Data",
Value = x
}
)
.ToList();
var allCollections = collections
.OrderBy(x => x.Value.Name)
.Select( x => x.Value )
.ToList();
// Add collections to the root node
var collectionNodes = collections
.GroupBy(x => x.Group)
.OrderBy(x => x.Key)
.Select(gr => new TreeNode
{
Label = gr.Key,
IsExpanded = true,
Children = gr
.OrderBy(x => x.Value.Name)
.Select(coll => new TreeNode
{
Label = coll.Value.Name,
IsExpanded = false,
Data = new()
{
Collection = coll.Value,
IsSharded = coll.Value.IsSharded
},
Children = coll.Value.Indexes
.OrderBy(idx => idx.Name)
.Select(idx => new TreeNode
{
Label = idx.Name,
IsExpanded = false,
Children =
[
new TreeNode
{
Label = idx.Name,
LabelFragment = _ => null!,
Data = new() { Collection = coll.Value, Index = idx },
}
]
})
.ToList()
})
.ToList()
})
.ToList();
root.Children = collectionNodes;
root.IsExpanded = true;
Set(root, true);
return (allCollections, root);
}
private string ConvertCurrentCollectionsToJson() => Newtonsoft.Json.JsonConvert.SerializeObject(_currentCollections.Where(IsSelected).ToList(), Newtonsoft.Json.Formatting.Indented);
private bool IsSelected(DatabaseStructureLoader.CollectionStructure coll)
=> IsSelected(_currentNodes, coll);
private static bool IsSelected(List> nodes, DatabaseStructureLoader.CollectionStructure coll)
{
foreach (var node in nodes)
{
if (node.Data?.Collection?.Name.Equals(coll.Name, StringComparison.OrdinalIgnoreCase) ?? false)
{
return node.Data.IsSelected;
}
if (IsSelected(node.Children, coll))
{
return true;
}
}
return false;
}
private async Task OnDownload()
{
var url = await DownloadController.GetDownloadLink(
TempFileStorage,
PasswordManager,
SingleUseTokenService,
async fileName => await File.WriteAllTextAsync(fileName, ConvertCurrentCollectionsToJson()),
$"{FileUtils.Shield(Database)}-structure.json"
);
await JsRuntime.InvokeVoidAsync("open", $"{NavigationManager.BaseUri}{url}", "_blank");
}
private async Task OnSync()
{
var toAdd = ExtractSelected(ToAddLabel);
var toRemove = ExtractSelected(ToRemoveLabel);
if (toAdd.Count == 0 && toRemove.Count == 0)
{
await ModalDialogUtils.ShowInfoDialog(Modal, "Sync structure", "No changes detected.");
return;
}
if ( !_syncOptions.DryRun )
{
var res = await ModalDialogUtils.ShowConfirmationDialog(Modal, "Sync structure", "Are you sure you want to synchronize the database structure? This will add or remove collections and indexes as needed.");
if (res.Cancelled)
return;
}
var admin = UserSession.MongoDbAdmin;
try
{
var newDiff = new DatabaseStructureLoader.StructureDifference();
newDiff.ToAdd.AddRange(Difference.ToAdd.Where(x => toAdd.Contains(x.Name)));
newDiff.ToRemove.AddRange(Difference.ToRemove.Where(x => toRemove.Contains(x.Name)));
_progress = await DatabaseStructureLoader.SyncStructure(admin, newDiff, _syncOptions);
await InvokeAsync(StateHasChanged);
await ModalDialogUtils.ShowInfoDialog(Modal, "Sync structure", "Synchronization completed successfully.");
}
catch (Exception ex3)
{
await InvokeAsync(StateHasChanged);
await ModalDialogUtils.ShowExceptionDialog(Modal, "Error during synchronization", ex3);
}
return;
HashSet ExtractSelected(string label)
{
var idx = _targetNodes[0].Children.FindIndex(x => x.Label.Equals(label, StringComparison.OrdinalIgnoreCase));
var res = idx >= 0
? _targetNodes[0].Children[idx].Children
.Where(x => x.Data is { IsSelected: true, Collection: not null } )
.Select(x => x.Data!.Collection!.Name)
.ToHashSet(StringComparer.OrdinalIgnoreCase)
: []
;
return res;
}
}
private async Task OnFileUploaded(InputFileChangeEventArgs args)
{
try
{
if (args.FileCount == 0)
return;
var file = args.GetMultipleFiles(1).FirstOrDefault();
if (file == null)
return;
await using var stream = file.OpenReadStream(10 * 1024 * 1024); // 10 MB limit
var json = await new StreamReader(stream).ReadToEndAsync();
_targetCollections = DatabaseStructureLoader.ParseCollections(json);
if (_targetCollections.Count == 0)
{
await ModalDialogUtils.ShowInfoDialog(Modal,"Sync structure", "No collections found in the uploaded file.");
return;
}
CompareStructure();
await InvokeAsync(StateHasChanged);
}
catch (Exception ex)
{
await ModalDialogUtils.ShowExceptionDialog(Modal, "Error uploading structure", ex);
}
}
private bool SelectedInCurrent(DatabaseStructureLoader.CollectionStructure node)
{
var found = _currentNodes[0].Children.FirstOrDefault(x => x.Data?.Collection?.Name.Equals(node.Name, StringComparison.OrdinalIgnoreCase) ?? false);
if ( found == null )
return true;
return found.Data?.IsSelected ?? true;
}
private void CompareStructure()
{
var targetCollections = _targetCollections.Where(SelectedInCurrent).ToList();
if (targetCollections.Count == 0)
{
if (Difference.ToAdd.Count == 0 && Difference.ToRemove.Count == 0 && _targetNodes.Count == 1)
return;
Difference = new();
_targetNodes.Clear();
_targetNodes.Add(
new()
{
Label = "Upload to see differences...",
Children = []
}
);
return;
}
Difference = DatabaseStructureLoader.GetStructureDifference(_currentCollections.Where(SelectedInCurrent).ToList(), targetCollections);
_targetNodes[0].IsExpanded = true;
_targetNodes[0].Label = $"Differences (Add: {Difference.ToAdd.Count + Difference.ToAdd.Sum(x => x.Indexes.Count)} Remove: {Difference.ToRemove.Count + Difference.ToRemove.Sum(x => x.Indexes.Count)})";
_targetNodes[0].Children.Clear();
if (Difference.ToAdd.Count > 0)
{
_targetNodes[0].Children.Add(
new()
{
Label = ToAddLabel,
IsExpanded = true,
Children = Difference.ToAdd.OrderBy(x => x.Name).Select(coll => new TreeNode
{
Label = $"{coll.Name} ({coll.Type})",
Data = new() { Collection = coll },
Children = coll.Indexes.OfType().Select(idx => new TreeNode
{
Label = $"{idx.Name} ({idx.Type})",
Data = new() { Collection = coll, Index = idx }
}).ToList()
}).ToList()
}
);
}
if (Difference.ToRemove.Count > 0)
{
_targetNodes[0].Children.Add(
new()
{
Label = ToRemoveLabel,
IsExpanded = true,
Children = Difference.ToRemove.OrderBy(x => x.Name).Select(coll => new TreeNode
{
Label = $"{coll.Name} ({coll.Type})",
Data = new() { Collection = coll },
Children = coll.Indexes.OfType().Select(idx => new TreeNode
{
Label = $"{idx.Name} ({idx.Type})",
Data = new() { Collection = coll, Index = idx }
}).ToList()
}).ToList()
}
);
}
if ( Difference.ToBeSharded.Count > 0 )
{
_targetNodes[0].Children.Add(
new()
{
Label = ToBeShardedLabel,
IsExpanded = true,
Children = Difference.ToBeSharded.OrderBy(x => x).Select(coll => new TreeNode
{
Label = $"{coll}",
Data = new() { IsSharded = false },
}).ToList()
}
);
}
if ( Difference.ToBeUnSharded.Count > 0 )
{
_targetNodes[0].Children.Add(
new()
{
Label = ToBeUnshardedLabel,
IsExpanded = true,
Children = Difference.ToBeUnSharded.OrderBy(x => x).Select(coll => new TreeNode
{
Label = $"{coll}",
Data = new() { IsSharded = false },
}).ToList()
}
);
}
OnSelectAllTarget();
}
private static void Set(TreeNode item, bool value)
{
if (item.Data != null)
{
item.Data.IsSelected = value;
}
foreach (var child in item.Children)
{
Set(child, value);
}
}
private Task OnSelectAllCurrent()
{
foreach( var node in _currentNodes)
Set(node, true);
CompareStructure();
return InvokeAsync(StateHasChanged);
}
private Task OnSelectNoneCurrent()
{
foreach( var node in _currentNodes)
Set(node, false);
CompareStructure();
return InvokeAsync(StateHasChanged);
}
private Task OnSelectAllTarget()
{
foreach( var node in _targetNodes)
Set(node, true);
return InvokeAsync(StateHasChanged);
}
private Task OnSelectNoneTarget()
{
foreach( var node in _targetNodes)
Set(node, false);
return InvokeAsync(StateHasChanged);
}
private Task SetSourceItem(TreeNode item, bool b)
{
Set(item, b);
CompareStructure();
return InvokeAsync(StateHasChanged);
}
}