#!/usr/bin/env python
# Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

# Interactive shell based on Django:
#
# Copyright (c) 2005, the Lawrence Journal-World
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#     1. Redistributions of source code must retain the above copyright notice,
#        this list of conditions and the following disclaimer.
#
#     2. Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in the
#        documentation and/or other materials provided with the distribution.
#
#     3. Neither the name of Django nor the names of its contributors may be
#        used to endorse or promote products derived from this software without
#        specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


"""CLI interface for cinder management."""

from __future__ import annotations

import collections
import collections.abc as collections_abc
import errno
import glob
import itertools
import logging as python_logging
import os
import re
import sys
import time
import typing
from typing import Any, Callable, Optional, Tuple, Union

from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_log import log as logging
from oslo_utils import timeutils
import tabulate

# Need to register global_opts
from cinder.backup import rpcapi as backup_rpcapi
from cinder.common import config  # noqa
from cinder import context
from cinder import db
from cinder.db import migration as db_migration
from cinder.db.sqlalchemy import api as db_api
from cinder.db.sqlalchemy import models
from cinder import exception
from cinder.i18n import _
from cinder import objects
from cinder.objects import base as ovo_base
from cinder import quota
from cinder import rpc
from cinder.scheduler import rpcapi as scheduler_rpcapi
from cinder import version
from cinder.volume import rpcapi as volume_rpcapi
from cinder.volume import volume_utils


CONF = cfg.CONF

LOG = logging.getLogger(__name__)

RPC_VERSIONS = {
    'cinder-scheduler': scheduler_rpcapi.SchedulerAPI.RPC_API_VERSION,
    'cinder-volume': volume_rpcapi.VolumeAPI.RPC_API_VERSION,
    'cinder-backup': backup_rpcapi.BackupAPI.RPC_API_VERSION,
}

OVO_VERSION = ovo_base.OBJ_VERSIONS.get_current()


# Decorators for actions
@typing.no_type_check
def args(*args, **kwargs):
    args = list(args)
    if not args[0].startswith('-') and '-' in args[0]:
        kwargs.setdefault('metavar', args[0])
        args[0] = args[0].replace('-', '_')

    def _decorator(func):
        func.__dict__.setdefault('args', []).insert(0, (args, kwargs))
        return func
    return _decorator


class HostCommands(object):
    """List hosts."""

    @args('zone', nargs='?', default=None,
          help='Availability Zone (default: %(default)s)')
    def list(self, zone: Optional[str] = None) -> None:
        """Show a list of all physical hosts.

        Can be filtered by zone.
        args: [zone]
        """
        print(_("%(host)-25s\t%(zone)-15s") % {'host': 'host', 'zone': 'zone'})
        ctxt = context.get_admin_context()
        services = objects.ServiceList.get_all(ctxt)
        if zone:
            services = [s for s in services if s.availability_zone == zone]
        hosts: list[dict[str, Any]] = []
        for srv in services:
            if not [h for h in hosts if h['host'] == srv['host']]:
                hosts.append(srv)

        for h in hosts:
            print(_("%(host)-25s\t%(availability_zone)-15s")
                  % {'host': h['host'],
                     'availability_zone': h['availability_zone']})


class DbCommands(object):
    """Class for managing the database."""

    # NOTE: Online migrations cannot depend on having Cinder services running.
    # Migrations can be called during Fast-Forward Upgrades without having any
    # Cinder services up.
    # NOTE: Online migrations must be removed at the beginning of the next
    # release to the one they've been introduced.  A comment with the release
    # a migration is introduced and the one where it must be removed must
    # preceed any element of the "online_migrations" tuple, like this:
    #    # Added in Queens remove in Rocky
    #    db.service_uuids_online_data_migration,
    online_migrations: Tuple[Callable[[context.RequestContext, int],
                                      Tuple[int, int]], ...] = (
        # TODO: (D Release) Remove next line and this comment
        db.remove_temporary_admin_metadata_data_migration,
    )

    def __init__(self):
        pass

    @args('version', nargs='?', default=None, type=int,
          help='Database version')
    @args('--bump-versions', dest='bump_versions', default=False,
          action='store_true',
          help='Update RPC and Objects versions when doing offline upgrades, '
               'with this we no longer need to restart the services twice '
               'after the upgrade to prevent ServiceTooOld exceptions.')
    def sync(self,
             version: Optional[int] = None,
             bump_versions: bool = False) -> None:
        """Sync the database up to the most recent version."""
        if version is not None and version > db.MAX_INT:
            print(_('Version should be less than or equal to '
                    '%(max_version)d.') % {'max_version': db.MAX_INT})
            sys.exit(1)
        try:
            db_migration.db_sync(version)
        except db_exc.DBMigrationError as ex:
            print("Error during database migration: %s" % ex)
            sys.exit(1)

        try:
            if bump_versions:
                ctxt = context.get_admin_context()
                services = objects.ServiceList.get_all(ctxt)
                for service in services:
                    rpc_version = RPC_VERSIONS[service.binary]
                    if (service.rpc_current_version != rpc_version or
                            service.object_current_version != OVO_VERSION):
                        service.rpc_current_version = rpc_version
                        service.object_current_version = OVO_VERSION
                        service.save()
        except Exception as ex:
            print(_('Error during service version bump: %s') % ex)
            sys.exit(2)

    def version(self) -> None:
        """Print the current database version."""
        print(db_migration.db_version())

    @args('age_in_days', type=int,
          help='Purge deleted rows older than age in days')
    def purge(self, age_in_days: int) -> None:
        """Purge deleted rows older than a given age from cinder tables."""
        age_in_days = int(age_in_days)
        if age_in_days < 0:
            print(_("Must supply a positive value for age"))
            sys.exit(1)
        if age_in_days >= (int(time.time()) / 86400):
            print(_("Maximum age is count of days since epoch."))
            sys.exit(1)
        ctxt = context.get_admin_context()

        try:
            db.purge_deleted_rows(ctxt, age_in_days)
        except db_exc.DBReferenceError:
            print(_("Purge command failed, check cinder-manage "
                    "logs for more details."))
            sys.exit(1)

    def _run_migration(self,
                       ctxt: context.RequestContext,
                       max_count: int) -> Tuple[dict, bool]:
        ran = 0
        exceptions = False
        migrations = {}
        for migration_meth in self.online_migrations:
            count = max_count - ran
            try:
                found, done = migration_meth(ctxt, count)
            except Exception:
                msg = (_("Error attempting to run %(method)s") %
                       {'method': migration_meth.__name__})
                print(msg)
                LOG.exception(msg)
                exceptions = True
                found = done = 0

            name = migration_meth.__name__
            if found:
                print(_('%(found)i rows matched query %(meth)s, %(done)i '
                        'migrated') % {'found': found,
                                       'meth': name,
                                       'done': done})
            migrations[name] = found, done
            if max_count is not None:
                ran += done
                if ran >= max_count:
                    break
        return migrations, exceptions

    @args('--max_count', metavar='<number>', dest='max_count', type=int,
          help='Maximum number of objects to consider.')
    def online_data_migrations(self, max_count: Optional[int] = None) -> None:
        """Perform online data migrations for the release in batches."""
        ctxt = context.get_admin_context()
        if max_count is not None:
            unlimited = False
            if max_count < 1:
                print(_('Must supply a positive value for max_count.'))
                sys.exit(127)
        else:
            unlimited = True
            max_count = 50
            print(_('Running batches of %i until complete.') % max_count)

        ran = None
        exceptions = False
        migration_info: dict[str, Any] = {}
        while ran is None or ran != 0:
            migrations, exceptions = self._run_migration(ctxt, max_count)
            ran = 0
            for name in migrations:
                migration_info.setdefault(name, (0, 0))
                migration_info[name] = (
                    max(migration_info[name][0], migrations[name][0]),
                    migration_info[name][1] + migrations[name][1],
                )
                ran += migrations[name][1]
            if not unlimited:
                break
        headers = ["{}".format(_('Migration')),
                   "{}".format(_('Total Needed')),
                   "{}".format(_('Completed')), ]
        rows = []
        for name in sorted(migration_info.keys()):
            info = migration_info[name]
            rows.append([name, info[0], info[1]])
        print(tabulate.tabulate(rows, headers=headers, tablefmt='psql'))

        # NOTE(imacdonn): In the "unlimited" case, the loop above will only
        # terminate when all possible migrations have been effected. If we're
        # still getting exceptions, there's a problem that requires
        # intervention. In the max-count case, exceptions are only considered
        # fatal if no work was done by any other migrations ("not ran"),
        # because otherwise work may still remain to be done, and that work
        # may resolve dependencies for the failing migrations.
        if exceptions and (unlimited or not ran):
            print(_("Some migrations failed unexpectedly. Check log for "
                    "details."))
            sys.exit(2)

        sys.exit(1 if ran else 0)

    @args('--enable-replication', action='store_true', default=False,
          help='Set replication status to enabled (default: %(default)s).')
    @args('--active-backend-id', default=None,
          help='Change the active backend ID (default: %(default)s).')
    @args('--backend-host', required=True,
          help='The backend host name.')
    def reset_active_backend(self,
                             enable_replication: bool,
                             active_backend_id: Optional[str],
                             backend_host: str) -> None:
        """Reset the active backend for a host."""

        ctxt = context.get_admin_context()

        try:
            db.reset_active_backend(ctxt, enable_replication,
                                    active_backend_id, backend_host)
        except db_exc.DBReferenceError:
            print(_("Failed to reset active backend for host %s, "
                    "check cinder-manage logs for more details.") %
                  backend_host)
            sys.exit(1)


class QuotaCommands(object):
    """Class for managing quota issues."""

    def __init__(self):
        pass

    @args('--project-id', default=None,
          help=('The ID of the project where we want to check the quotas '
                '(defaults to all projects).'))
    def check(self, project_id: Optional[str]) -> None:
        """Check if quotas and reservations are correct

        This action checks quotas and reservations, for a specific project or
        for all projects, to see if they are out of sync.

        The check will also look for duplicated entries.

        One way to use this check in combination with the sync action is to
        run the check for all projects, take note of those that are out of
        sync, and then sync them one by one at intervals to reduce stress on
        the DB.
        """
        result = self._check_sync(project_id, do_fix=False)
        if result:
            sys.exit(1)

    @args('--project-id', default=None,
          help=('The ID of the project where we want to sync the quotas '
                '(defaults to all projects).'))
    def sync(self, project_id: Optional[str]) -> None:
        """Fix quotas and reservations

        This action refreshes existing quota usage and reservation count for a
        specific project or for all projects.

        The refresh will also remove duplicated entries.

        This operation is best executed when Cinder is not running, but it can
        be run with cinder services running as well.

        A different transaction is used for each project's quota sync, so an
        action failure will only rollback the current project's changes.
        """
        self._check_sync(project_id, do_fix=True)

    @db_api.main_context_manager.reader
    def _get_quota_projects(self,
                            ctxt: context.RequestContext,
                            project_id: Optional[str]) -> list[str]:
        """Get project ids that have quota_usage entries."""
        if project_id:
            model = models.QuotaUsage
            # If the project does not exist
            if not ctxt.session.query(
                db_api.sql.exists()
                .where(
                    db_api.and_(
                        model.project_id == project_id,
                        ~model.deleted,
                    ),
                )
            ).scalar():
                print(
                    'Project id %s has no quota usage. Nothing to do.' %
                    project_id,
                )
                return []
            return [project_id]

        projects = db_api.model_query(
            ctxt,
            models.QuotaUsage,
            read_deleted="no"
        ).with_entities('project_id').distinct().all()
        project_ids = [row.project_id for row in projects]
        return project_ids

    def _get_usages(self,
                    ctxt: context.RequestContext,
                    resources,
                    project_id: str) -> list:
        """Get data necessary to check out of sync quota usage.

        Returns a list of QuotaUsage instances for the specific project
        """
        usages = db_api.model_query(
            ctxt,
            db_api.models.QuotaUsage,
            read_deleted="no",
        ).filter_by(project_id=project_id).with_for_update().all()
        return usages

    def _get_reservations(self,
                          ctxt: context.RequestContext,
                          project_id: str,
                          usage_id: str) -> list:
        """Get reservations for a given project and usage id."""
        reservations = (
            db_api.model_query(
                ctxt,
                models.Reservation,
                read_deleted="no",
            )
            .filter_by(project_id=project_id, usage_id=usage_id)
            .with_for_update()
            .all()
        )
        return reservations

    def _check_duplicates(self,
                          ctxt: context.RequestContext,
                          usages,
                          do_fix: bool) -> tuple[list, bool]:
        """Look for duplicated quota used entries (bug#1484343)

        If we have duplicates and we are fixing them, then we reassign the
        reservations of the usage we are removing.
        """
        resources = collections.defaultdict(list)
        for usage in usages:
            resources[usage.resource].append(usage)

        duplicates_found = False
        result = []
        for resource_usages in resources.values():
            keep_usage = resource_usages[0]
            if len(resource_usages) > 1:
                duplicates_found = True
                print('\t%s: %s duplicated usage entries - ' %
                      (keep_usage.resource, len(resource_usages) - 1),
                      end='')

                if do_fix:
                    # Each of the duplicates can have reservations
                    reassigned = 0
                    for usage in resource_usages[1:]:
                        reservations = self._get_reservations(
                            ctxt,
                            usage.project_id,
                            usage.id,
                        )
                        reassigned += len(reservations)
                        for reservation in reservations:
                            reservation.usage_id = keep_usage.id
                        keep_usage.in_use += usage.in_use
                        keep_usage.reserved += usage.reserved
                        usage.delete(ctxt.session)
                    print('duplicates removed & %s reservations reassigned' %
                          reassigned)
                else:
                    print('ignored')
            result.append(keep_usage)
        return result, duplicates_found

    def _check_sync(self, project_id: Optional[str], do_fix: bool) -> bool:
        """Check the quotas and reservations optionally fixing them."""

        ctxt = context.get_admin_context()
        # Get the quota usage types and their sync methods
        resources = quota.QUOTAS.resources

        # Get all project ids that have quota usage. Method doesn't lock
        # projects, since newly added projects should not be out of sync and
        # projects removed will just turn nothing on the quota usage.
        projects = self._get_quota_projects(ctxt, project_id)

        discrepancy = False

        for project in projects:
            discrepancy &= self._check_project_sync(
                ctxt,
                project,
                do_fix,
                resources,
            )

        print('Action successfully completed')
        return discrepancy

    @db_api.main_context_manager.reader
    def _check_project_sync(self,
                            ctxt: context.RequestContext,
                            project: str,
                            do_fix: bool,
                            resources) -> bool:
        print('Processing quota usage for project %s' % project)

        discrepancy = False
        action_msg = ' - fixed' if do_fix else ''

        # NOTE: It's important to always get the quota first and then the
        # reservations to prevent deadlocks with quota commit and rollback from
        # running Cinder services.

        # We only want to sync existing quota usage rows
        usages = self._get_usages(ctxt, resources, project)

        # Check for duplicated entries (bug#1484343)
        usages, duplicates_found = self._check_duplicates(
            ctxt, usages, do_fix,
        )
        if duplicates_found:
            discrepancy = True

        # Check quota and reservations
        for usage in usages:
            resource_name = usage.resource
            # Get the correct value for this quota usage resource
            updates = db_api._get_sync_updates(
                ctxt,
                project,
                resources,
                resource_name,
            )
            in_use = updates[resource_name]
            if in_use != usage.in_use:
                print(
                    '\t%s: invalid usage saved=%s actual=%s%s' %
                    (resource_name, usage.in_use, in_use, action_msg)
                )
                discrepancy = True
                if do_fix:
                    usage.in_use = in_use

            reservations = self._get_reservations(
                ctxt,
                project,
                usage.id,
            )
            num_reservations = sum(
                r.delta for r in reservations if r.delta > 0
            )
            if num_reservations != usage.reserved:
                print(
                    '\t%s: invalid reserved saved=%s actual=%s%s' %
                    (
                        resource_name,
                        usage.reserved,
                        num_reservations,
                        action_msg,
                    )
                )
                discrepancy = True
                if do_fix:
                    usage.reserved = num_reservations

        return discrepancy


class VersionCommands(object):
    """Class for exposing the codebase version."""

    def __init__(self):
        pass

    def list(self):
        print(version.version_string())

    def __call__(self):
        self.list()


class VolumeCommands(object):
    """Methods for dealing with a cloud in an odd state."""

    @args('volume_id',
          help='Volume ID to be deleted')
    def delete(self, volume_id: str) -> None:
        """Delete a volume, bypassing the check that it must be available."""
        ctxt = context.get_admin_context()
        volume = objects.Volume.get_by_id(ctxt, volume_id)
        host = volume_utils.extract_host(volume.host) if volume.host else None

        if not host:
            print(_("Volume not yet assigned to host."))
            print(_("Deleting volume from database and skipping rpc."))
            volume.destroy()
            return

        if volume.status == 'in-use':
            print(_("Volume is in-use."))
            print(_("Detach volume from instance and then try again."))
            return

        rpc.init(CONF)
        rpcapi = volume_rpcapi.VolumeAPI()
        rpcapi.delete_volume(ctxt, volume)

    @args('--currenthost', required=True, help='Existing volume host name in '
                                               'the format host@backend#pool')
    @args('--newhost', required=True, help='New volume host name in the '
                                           'format host@backend#pool')
    def update_host(self, currenthost: str, newhost: str) -> None:
        """Modify the host name associated with a volume.

        Particularly to recover from cases where one has moved
        their Cinder Volume node, or modified their backend_name in a
        multi-backend config.
        """
        ctxt = context.get_admin_context()
        volumes = db.volume_get_all_by_host(ctxt,
                                            currenthost)
        for v in volumes:
            db.volume_update(ctxt, v['id'],
                             {'host': newhost})


class ConfigCommands(object):
    """Class for exposing the flags defined by flag_file(s)."""

    def __init__(self):
        pass

    @args('param', nargs='?', default=None,
          help='Configuration parameter to display (default: %(default)s)')
    def list(self, param: Optional[str] = None) -> None:
        """List parameters configured for cinder.

        Lists all parameters configured for cinder unless an optional argument
        is specified.  If the parameter is specified we only print the
        requested parameter.  If the parameter is not found an appropriate
        error is produced by .get*().
        """
        param = param and param.strip()
        if param:
            print('%s = %s' % (param, CONF.get(param)))
        else:
            for key, value in CONF.items():
                print('%s = %s' % (key, value))


class BackupCommands(object):
    """Methods for managing backups."""

    def list(self) -> None:
        """List all backups.

        List all backups (including ones in progress) and the host
        on which the backup operation is running.
        """
        ctxt = context.get_admin_context()
        backups = objects.BackupList.get_all(ctxt)

        hdr = "%-32s\t%-32s\t%-32s\t%-24s\t%-24s\t%-12s\t%-12s\t%-12s\t%-12s"
        print(hdr % (_('ID'),
                     _('User ID'),
                     _('Project ID'),
                     _('Host'),
                     _('Name'),
                     _('Container'),
                     _('Status'),
                     _('Size'),
                     _('Object Count')))

        res = "%-32s\t%-32s\t%-32s\t%-24s\t%-24s\t%-12s\t%-12s\t%-12d\t%-12d"
        for backup in backups:
            object_count = 0
            if backup['object_count'] is not None:
                object_count = backup['object_count']
            print(res % (backup['id'],
                         backup['user_id'],
                         backup['project_id'],
                         backup['host'],
                         backup['display_name'],
                         backup['container'],
                         backup['status'],
                         backup['size'],
                         object_count))

    @args('--currenthost', required=True, help='Existing backup host name')
    @args('--newhost', required=True, help='New backup host name')
    def update_backup_host(self, currenthost: str, newhost: str) -> None:
        """Modify the host name associated with a backup.

        Particularly to recover from cases where one has moved
        their Cinder Backup node, and not set backup_use_same_backend.
        """
        ctxt = context.get_admin_context()
        backups = objects.BackupList.get_all_by_host(ctxt, currenthost)
        for bk in backups:
            bk.host = newhost
            bk.save()


class BaseCommand(object):
    @staticmethod
    def _normalize_time(time_field):
        return time_field and timeutils.normalize_time(time_field)

    @staticmethod
    def _state_repr(is_up):
        return ':-)' if is_up else 'XXX'


class ServiceCommands(BaseCommand):
    """Methods for managing services."""
    def list(self):
        """Show a list of all cinder services."""
        ctxt = context.get_admin_context()
        services = objects.ServiceList.get_all(ctxt)
        print_format = "%-16s %-36s %-16s %-10s %-5s %-20s %-12s %-15s %-36s"
        print(print_format % (_('Binary'),
                              _('Host'),
                              _('Zone'),
                              _('Status'),
                              _('State'),
                              _('Updated At'),
                              _('RPC Version'),
                              _('Object Version'),
                              _('Cluster')))
        for svc in services:
            art = self._state_repr(svc.is_up)
            status = 'disabled' if svc.disabled else 'enabled'
            updated_at = self._normalize_time(svc.updated_at)
            rpc_version = svc.rpc_current_version
            object_version = svc.object_current_version
            cluster = svc.cluster_name or ''
            print(print_format % (svc.binary, svc.host,
                                  svc.availability_zone, status, art,
                                  updated_at, rpc_version, object_version,
                                  cluster))

    @args('binary', type=str,
          help='Service to delete from the host.')
    @args('host_name', type=str,
          help='Host from which to remove the service.')
    def remove(self, binary: str, host_name: str) -> Optional[int]:
        """Completely removes a service."""
        ctxt = context.get_admin_context()
        try:
            svc = objects.Service.get_by_args(ctxt, host_name, binary)
            svc.destroy()
        except exception.ServiceNotFound as e:
            print(_("Host not found. Failed to remove %(service)s"
                    " on %(host)s.") %
                  {'service': binary, 'host': host_name})
            print(u"%s" % e.args)
            return 2
        print(_("Service %(service)s on host %(host)s removed.") %
              {'service': binary, 'host': host_name})

        return None


class ClusterCommands(BaseCommand):
    """Methods for managing clusters."""
    def list(self) -> None:
        """Show a list of all cinder services."""
        ctxt = context.get_admin_context()
        clusters = objects.ClusterList.get_all(ctxt, services_summary=True)
        print_format = "%-36s %-16s %-10s %-5s %-20s %-7s %-12s %-20s"
        print(print_format % (_('Name'),
                              _('Binary'),
                              _('Status'),
                              _('State'),
                              _('Heartbeat'),
                              _('Hosts'),
                              _('Down Hosts'),
                              _('Updated At')))
        for cluster in clusters:
            art = self._state_repr(cluster.is_up)
            status = 'disabled' if cluster.disabled else 'enabled'
            heartbeat = self._normalize_time(cluster.last_heartbeat)
            updated_at = self._normalize_time(cluster.updated_at)
            print(print_format % (cluster.name, cluster.binary, status, art,
                                  heartbeat, cluster.num_hosts,
                                  cluster.num_down_hosts, updated_at))

    @args('--recursive', action='store_true', default=False,
          help='Delete associated hosts.')
    @args('binary', type=str,
          help='Service to delete from the cluster.')
    @args('cluster-name', type=str, help='Cluster to delete.')
    def remove(self,
               recursive: bool,
               binary: str,
               cluster_name: str) -> Optional[int]:
        """Completely removes a cluster."""
        ctxt = context.get_admin_context()
        try:
            cluster = objects.Cluster.get_by_id(ctxt, None, name=cluster_name,
                                                binary=binary,
                                                get_services=recursive)
        except exception.ClusterNotFound:
            print(_("Couldn't remove cluster %s because it doesn't exist.") %
                  cluster_name)
            return 2

        if recursive:
            for service in cluster.services:
                service.destroy()

        try:
            cluster.destroy()
        except exception.ClusterHasHosts:
            print(_("Couldn't remove cluster %s because it still has hosts.") %
                  cluster_name)
            return 2

        msg = _('Cluster %s successfully removed.') % cluster_name
        if recursive:
            msg = (_('%(msg)s And %(num)s services from the cluster were also '
                     'removed.') % {'msg': msg, 'num': len(cluster.services)})
        print(msg)

        return None

    @args('--full-rename', dest='partial',
          action='store_false', default=True,
          help='Do full cluster rename instead of just replacing provided '
               'current cluster name and preserving backend and/or pool info.')
    @args('current', help='Current cluster name.')
    @args('new', help='New cluster name.')
    def rename(self,
               partial: bool,
               current: Optional[str],
               new: Optional[str]) -> Optional[int]:
        """Rename cluster name for Volumes and Consistency Groups.

        Useful when you want to rename a cluster, particularly when the
        backend_name has been modified in a multi-backend config or we have
        moved from a single backend to multi-backend.
        """
        ctxt = context.get_admin_context()

        # Convert empty strings to None
        current = current or None
        new = new or None

        # Update Volumes
        num_vols = objects.VolumeList.include_in_cluster(
            ctxt, new, partial_rename=partial, cluster_name=current)

        # Update Consistency Groups
        num_cgs = objects.ConsistencyGroupList.include_in_cluster(
            ctxt, new, partial_rename=partial, cluster_name=current)

        if num_vols or num_cgs:
            msg = _('Successfully renamed %(num_vols)s volumes and '
                    '%(num_cgs)s consistency groups from cluster %(current)s '
                    'to %(new)s')
            print(msg % {'num_vols': num_vols, 'num_cgs': num_cgs, 'new': new,
                         'current': current})
        else:
            msg = _('No volumes or consistency groups exist in cluster '
                    '%(current)s.')
            print(msg % {'current': current})
            return 2

        return None


class ConsistencyGroupCommands(object):
    """Methods for managing consistency groups."""

    @args('--currenthost', required=True, help='Existing CG host name')
    @args('--newhost', required=True, help='New CG host name')
    def update_cg_host(self, currenthost: str, newhost: str) -> None:
        """Modify the host name associated with a Consistency Group.

        Particularly to recover from cases where one has moved
        a host from single backend to multi-backend, or changed the host
        configuration option, or modified the backend_name in a multi-backend
        config.
        """

        ctxt = context.get_admin_context()
        groups = objects.ConsistencyGroupList.get_all(
            ctxt, {'host': currenthost})
        for gr in groups:
            gr.host = newhost
            gr.save()


class UtilCommands(object):
    """Generic utils."""

    @staticmethod
    def _get_resources_locks() -> Tuple[collections.defaultdict,
                                        collections.defaultdict,
                                        collections.defaultdict]:
        """Get all vol/snap/backup file lock paths."""
        backup_locks_prefix = 'cinder-cleanup_incomplete_backups_'
        oslo_dir = os.path.abspath(cfg.CONF.oslo_concurrency.lock_path)
        filenames = glob.glob(os.path.join(oslo_dir, 'cinder-*'))

        backend_url = cfg.CONF.coordination.backend_url
        if backend_url.startswith('file://'):
            tooz_dir = os.path.abspath(backend_url[7:])
            if tooz_dir != oslo_dir:
                filenames += glob.glob(os.path.join(tooz_dir, 'cinder-*'))

        volumes: collections.defaultdict = collections.defaultdict(list)
        snapshots: collections.defaultdict = collections.defaultdict(list)
        backups = collections.defaultdict(list)
        matcher = re.compile('.*?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-'
                             '[0-9a-f]{4}-[0-9a-f]{12}).*?', re.IGNORECASE)
        for filename in filenames:
            basename = os.path.basename(filename)
            match = matcher.match(basename)
            if match:
                dest = snapshots if 'snapshot' in basename else volumes
                res_id = match.group(1)
                dest[res_id].append(filename)
            elif basename.startswith(backup_locks_prefix):
                pgrp = basename[34:]
                backups[pgrp].append(filename)

        return volumes, snapshots, backups

    def _exclude_running_backups(self, backups: dict) -> None:
        """Remove backup entries from the dict for running backup services."""
        for backup_pgrp in list(backups.keys()):
            # The PGRP is the same as the PID of the parent process, so we know
            # the lock could be in use if the process is running and it's the
            # cinder-backup command (the PID could have been reused).
            cmdline_file = os.path.join('/proc', backup_pgrp, 'cmdline')
            try:
                with open(cmdline_file, 'r') as f:
                    if 'cinder-backup' in f.read():
                        del backups[backup_pgrp]
            except FileNotFoundError:
                continue
            except Exception:
                # Unexpected error, leaving the lock file just in case
                del backups[backup_pgrp]

    @args('--services-offline', dest='online',
          action='store_false', default=True,
          help='All locks can be deleted as Cinder services are not running.')
    def clean_locks(self, online: bool) -> None:
        """Clean file locks for vols, snaps, and backups on the current host.

        Should be run on any host where we are running a Cinder service (API,
        Scheduler, Volume, Backup) and can be run with the Cinder services
        running or stopped.

        If the services are running it will check existing resources in the
        Cinder database in order to know which resources are still available
        (it's not safe to remove their file locks) and will only remove the
        file locks for the resources that are no longer present.  Deleting
        locks while the services are offline is faster as there's no need to
        check the database.

        For backups, the way to know if we can remove the startup lock is by
        checking if the PGRP in the file name is currently running
        cinder-backup.

        Default assumes that services are online, must pass
        ``--services-offline`` to specify that they are offline.

        Doesn't clean DLM locks (except when using file locks), as those don't
        leave lock leftovers.
        """
        self.ctxt = context.get_admin_context()
        # Find volume and snapshots ids, and backups PGRP based on the existing
        # file locks
        volumes: Union[collections.defaultdict, dict]
        snapshots: Union[collections.defaultdict, dict]
        volumes, snapshots, backups = self._get_resources_locks()

        # If services are online we cannot delete locks for existing resources
        if online:
            # We don't want to delete file locks for existing resources
            volumes = {vol_id: files for vol_id, files in volumes.items()
                       if not objects.Volume.exists(self.ctxt, vol_id)}
            snapshots = {snap_id: files for snap_id, files in snapshots.items()
                         if not objects.Snapshot.exists(self.ctxt, snap_id)}
            self._exclude_running_backups(backups)

        def _err(filename: str, exc: Exception) -> None:
            print('Failed to cleanup lock %(name)s: %(exc)s',
                  {'name': filename, 'exc': exc})

        # Now clean
        for filenames in itertools.chain(volumes.values(),
                                         snapshots.values(),
                                         backups.values()):
            for filename in filenames:
                try:
                    os.remove(filename)
                except OSError as exc:
                    if (exc.errno != errno.ENOENT):
                        _err(filename, exc)
                except Exception as exc:
                    _err(filename, exc)


CATEGORIES = {
    'backup': BackupCommands,
    'config': ConfigCommands,
    'cluster': ClusterCommands,
    'cg': ConsistencyGroupCommands,
    'db': DbCommands,
    'host': HostCommands,
    'quota': QuotaCommands,
    'service': ServiceCommands,
    'version': VersionCommands,
    'volume': VolumeCommands,
    'util': UtilCommands,
}


def methods_of(obj) -> list:
    """Return non-private methods from an object.

    Get all callable methods of an object that don't start with underscore
    :return: a list of tuples of the form (method_name, method)
    """
    result = []
    for i in dir(obj):
        if (isinstance(getattr(obj, i),
                       collections_abc.Callable) and  # type: ignore
                not i.startswith('_')):
            result.append((i, getattr(obj, i)))
    return result


def missing_action(help_func: Callable) -> Callable:
    def wrapped():
        help_func()
        exit(2)
    return wrapped


def add_command_parsers(subparsers):
    for category in sorted(CATEGORIES):
        command_object = CATEGORIES[category]()

        parser = subparsers.add_parser(category)
        parser.set_defaults(command_object=command_object)
        parser.set_defaults(action_fn=missing_action(parser.print_help))

        category_subparsers = parser.add_subparsers(dest='action')

        for (action, action_fn) in methods_of(command_object):
            parser = category_subparsers.add_parser(action)

            action_kwargs: list = []
            for args, kwargs in getattr(action_fn, 'args', []):
                parser.add_argument(*args, **kwargs)

            parser.set_defaults(action_fn=action_fn)
            parser.set_defaults(action_kwargs=action_kwargs)


category_opt = cfg.SubCommandOpt('category',
                                 title='Command categories',
                                 handler=add_command_parsers)


def get_arg_string(args):
    if args[0] == '-':
        # (Note)zhiteng: args starts with FLAGS.oparser.prefix_chars
        # is optional args. Notice that cfg module takes care of
        # actual ArgParser so prefix_chars is always '-'.
        if args[1] == '-':
            # This is long optional arg
            args = args[2:]
        else:
            args = args[1:]  # pylint: disable=E1136

    # We convert dashes to underscores so we can have cleaner optional arg
    # names
    if args:
        args = args.replace('-', '_')

    return args


def fetch_func_args(func):
    fn_kwargs = {}
    for args, kwargs in getattr(func, 'args', []):
        # Argparser `dest` configuration option takes precedence for the name
        arg = kwargs.get('dest') or get_arg_string(args[0])
        fn_kwargs[arg] = getattr(CONF.category, arg)

    return fn_kwargs


def main():
    objects.register_all()
    """Parse options and call the appropriate class/method."""
    CONF.register_cli_opt(category_opt)
    script_name = sys.argv[0]
    if len(sys.argv) < 2:
        print(_("\nOpenStack Cinder version: %(version)s\n") %
              {'version': version.version_string()})
        print(script_name + " category action [<args>]")
        print(_("Available categories:"))
        for category in CATEGORIES:
            print(_("\t%s") % category)
        sys.exit(2)

    try:
        CONF(sys.argv[1:], project='cinder',
             version=version.version_string())
        logging.setup(CONF, "cinder")
        python_logging.captureWarnings(True)
    except cfg.ConfigDirNotFoundError as details:
        print(_("Invalid directory: %s") % details)
        sys.exit(2)
    except cfg.ConfigFilesNotFoundError as e:
        cfg_files = e.config_files
        print(_("Failed to read configuration file(s): %s") % cfg_files)
        sys.exit(2)

    fn = CONF.category.action_fn
    fn_kwargs = fetch_func_args(fn)
    fn(**fn_kwargs)
