Support reconfiguration of BlobDepot (#29641)

Author: alexvru

ydb/apps/dstool/lib/commands.py

@@ -27,6 +27,7 @@
 import ydb.apps.dstool.lib.dstool_cmd_group_take_snapshot as group_take_snapshot
 import ydb.apps.dstool.lib.dstool_cmd_group_virtual_create as group_virtual_create
 import ydb.apps.dstool.lib.dstool_cmd_group_virtual_cancel as group_virtual_cancel
+import ydb.apps.dstool.lib.dstool_cmd_group_virtual_reconfigure as group_virtual_reconfigure
 
 import ydb.apps.dstool.lib.dstool_cmd_pool_create_virtual as pool_create_virtual
 import ydb.apps.dstool.lib.dstool_cmd_pool_list as pool_list
@@ -52,7 +53,7 @@
     box_list,
     pool_list, pool_create_virtual,
     group_check, group_decommit, group_show_blob_info, group_show_storage_efficiency, group_show_usage_by_tablets,
-    group_state, group_take_snapshot, group_add, group_list, group_virtual_create, group_virtual_cancel,
+    group_state, group_take_snapshot, group_add, group_list, group_virtual_create, group_virtual_cancel, group_virtual_reconfigure,
     pdisk_add_by_serial, pdisk_remove_by_serial, pdisk_set, pdisk_list, pdisk_stop, pdisk_restart, pdisk_readonly, pdisk_move,
     vdisk_evict, vdisk_list, vdisk_set_read_only, vdisk_remove_donor, vdisk_wipe, vdisk_compact, device_list,
 ]
@@ -61,7 +62,7 @@
     ('device', ['list']),
     ('pdisk', ['add-by-serial', 'remove-by-serial', 'set', 'list', 'stop', 'restart', 'readonly', 'move']),
     ('vdisk', ['evict', 'list', 'set-read-only', 'remove-donor', 'wipe', 'compact']),
-    ('group', ['add', 'check', 'decommit', ('show', ['blob-info', 'storage-efficiency', 'usage-by-tablets']), 'state', 'take-snapshot', 'list', ('virtual', ['create', 'cancel'])]),
+    ('group', ['add', 'check', 'decommit', ('show', ['blob-info', 'storage-efficiency', 'usage-by-tablets']), 'state', 'take-snapshot', 'list', ('virtual', ['create', 'cancel', 'reconfigure'])]),
     ('pool', ['list', ('create', ['virtual'])]),
     ('box', ['list']),
     ('node', ['list']),

ydb/apps/dstool/lib/dstool_cmd_group_virtual_reconfigure.py

@@ -0,0 +1,28 @@
+import ydb.apps.dstool.lib.common as common
+from argparse import FileType
+import google.protobuf.json_format as pb_json
+
+description = 'Create virtual group backed by BlobDepot'
+
+
+def add_options(p):
+    p.add_argument('--name', type=str, required=True, help='name of virtual group that requires reconfiguring')
+    p.add_argument('--s3-settings', type=FileType('r', encoding='utf-8'), metavar='JSON_FILE', help='path to JSON file containing S3 settings')
+
+
+def do(args):
+    request = common.create_bsc_request(args)
+
+    if args.s3_settings:
+        s3_settings = args.s3_settings.read()
+    else:
+        s3_settings = None
+
+    cmd = request.Command.add().ReconfigureVirtualGroup
+    cmd.Name = args.name
+
+    if s3_settings is not None:
+        pb_json.Parse(s3_settings, cmd.S3BackendSettings)
+
+    response = common.invoke_bsc_request(request)
+    common.print_request_result(args, request, response)

ydb/apps/dstool/lib/ya.make

@@ -38,6 +38,7 @@ PY_SRCS(
     dstool_cmd_group_take_snapshot.py
     dstool_cmd_group_virtual_create.py
     dstool_cmd_group_virtual_cancel.py
+    dstool_cmd_group_virtual_reconfigure.py
 
     dstool_cmd_pool_create_virtual.py
     dstool_cmd_pool_list.py

ydb/core/blob_depot/data.cpp

@@ -88,6 +88,10 @@ namespace NKikimr::NBlobDepot {
             return false; // no such key existed and will not be created as it hits the barrier
         }
 
+        Y_DEFER {
+            Self->JsonHandler.Invalidate();
+        };
+
         const auto [it, inserted] = Data.try_emplace(std::move(key), std::forward<TArgs>(args)...);
         {
             auto& [key, value] = *it;

ydb/core/blob_depot/op_apply_config.cpp

@@ -50,6 +50,9 @@ namespace NKikimr::NBlobDepot {
 
                 if (!std::exchange(Self->Configured, true)) {
                     Self->StartOperation();
+                } else {
+                    // TODO(alexvru): handle it in a better way, without tablet restart
+                    Self->Send(Self->Tablet(), new TEvents::TEvPoison);
                 }
 
                 TActivationContext::Send(Response.release());

ydb/core/mind/bscontroller/bsc.cpp

@@ -1159,6 +1159,7 @@ ui32 TBlobStorageController::GetEventPriority(IEventHandle *ev) {
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kGetInterfaceVersion:
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kMovePDisk:
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kUpdateBridgeGroupInfo:
+                    case NKikimrBlobStorage::TConfigRequest::TCommand::kReconfigureVirtualGroup:
                         return 2; // read-write commands go with higher priority as they are needed to keep cluster intact
 
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kReadHostConfig:

ydb/core/mind/bscontroller/config.h

@@ -366,6 +366,7 @@ namespace NKikimr {
             void ExecuteStep(const NKikimrBlobStorage::TGetInterfaceVersion& cmd, TStatus& status);
             void ExecuteStep(const NKikimrBlobStorage::TMovePDisk& cmd, TStatus& status);
             void ExecuteStep(const NKikimrBlobStorage::TUpdateBridgeGroupInfo& cmd, TStatus& status);
+            void ExecuteStep(const NKikimrBlobStorage::TReconfigureVirtualGroup& cmd, TStatus& status);
         };
 
     } // NBsController

ydb/core/mind/bscontroller/config_cmd.cpp

@@ -254,6 +254,7 @@ namespace NKikimr::NBsController {
                             MAP_TIMING(SanitizeGroup, SANITIZE_GROUP)
                             MAP_TIMING(CancelVirtualGroup, CANCEL_VIRTUAL_GROUP)
                             MAP_TIMING(ChangeGroupSizeInUnits, CHANGE_GROUP_SIZE_IN_UNITS)
+                            MAP_TIMING(ReconfigureVirtualGroup, RECONFIGURE_VIRTUAL_GROUP)
 
                             default:
                                 break;
@@ -370,6 +371,7 @@ namespace NKikimr::NBsController {
                     HANDLE_COMMAND(GetInterfaceVersion)
                     HANDLE_COMMAND(MovePDisk)
                     HANDLE_COMMAND(UpdateBridgeGroupInfo)
+                    HANDLE_COMMAND(ReconfigureVirtualGroup)
 
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kAddMigrationPlan:
                     case NKikimrBlobStorage::TConfigRequest::TCommand::kDeleteMigrationPlan:

ydb/core/mind/bscontroller/virtual_group.cpp

@@ -209,6 +209,52 @@ namespace NKikimr::NBsController {
         }
     }
 
+    void TBlobStorageController::TConfigState::ExecuteStep(const NKikimrBlobStorage::TReconfigureVirtualGroup& cmd,
+            TStatus& /*status*/) {
+        if (!cmd.GetName()) {
+            throw TExError() << "TReconfigureVirtualGroup.Name must be set and be nonempty";
+        }
+
+        std::optional<TGroupId> groupId;
+        Groups.ForEach([&](TGroupId key, const TGroupInfo& value) {
+            if (value.VirtualGroupName == cmd.GetName()) {
+                if (groupId) {
+                    throw TExError() << "Duplicate virtual group name";
+                }
+                groupId.emplace(key);
+            }
+        });
+        if (!groupId) {
+            throw TExError() << "Virtual group not found by designated name";
+        }
+
+        TGroupInfo *group = Groups.FindForUpdate(*groupId);
+        if (!group->BlobDepotConfig) {
+            Y_DEBUG_ABORT(); // this should never happen
+            throw TExError() << "Virtual group does not contain BlobDepotConfig";
+        }
+
+        // parse current config
+        NKikimrBlobDepot::TBlobDepotConfig config;
+        bool success = config.ParseFromString(*group->BlobDepotConfig);
+        if (!success) {
+            Y_DEBUG_ABORT();
+            throw TExError() << "Failed to parse BlobDepotConfig";
+        }
+
+        // update fields in current config
+        if (cmd.HasS3BackendSettings()) {
+            config.MutableS3BackendSettings()->CopyFrom(cmd.GetS3BackendSettings());
+        }
+
+        // overwrite the config
+        success = config.SerializeToString(&group->BlobDepotConfig.ConstructInPlace());
+        Y_ABORT_UNLESS(success);
+
+        // start altering machinery
+        group->NeedAlter = true;
+    }
+
     ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
     class TBlobStorageController::TVirtualGroupSetupMachine : public TActorBootstrapped<TVirtualGroupSetupMachine> {

ydb/core/protos/blobstorage_config.proto

@@ -591,6 +591,11 @@ message TUpdateBridgeGroupInfo {
     NKikimrBlobStorage.TGroupInfo BridgeGroupInfo = 3; // new value for bridge group info
 }
 
+message TReconfigureVirtualGroup {
+    string Name = 1;
+    optional NKikimrBlobDepot.TS3BackendSettings S3BackendSettings = 2;
+}
+
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 // INTERFACE PART
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -641,6 +646,7 @@ message TConfigRequest {
             TChangeGroupSizeInUnits ChangeGroupSizeInUnits = 52;
             TMovePDisk MovePDisk = 53;
             TUpdateBridgeGroupInfo UpdateBridgeGroupInfo = 54;
+            TReconfigureVirtualGroup ReconfigureVirtualGroup = 55;
 
             // commands intended for internal use
             TReassignGroupDisk ReassignGroupDisk = 19;