/*
 * SPDX-FileCopyrightText: Copyright (c) 2020-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */
#include "containers/map.h"
#include "containers/multimap.h"
#include "nvctassert.h"
#include "nvmisc.h"
#include "nvport/sync.h"
#include "nvrm_registry.h"
#include "os/os.h"
#include "rmapi/control.h"
#include "rmapi/rmapi.h"
#include "rmapi/rmapi_utils.h"
#include "ctrl/ctrl0000/ctrl0000system.h"

typedef struct
{
    void* params;
} RmapiControlCacheEntry;

//
// Stores the mapping of client object to the corresponding GPU instance
// The key is generated by _handlesToGpuInstKey with client and object handle
// The value is the instance of the GPU (pGpu->gpuInstance) the object is linked to
//
MAKE_MAP(ObjectToGpuInstMap, NvU64);

//
// Stores the cached control value.
// Each submap in the multimap stores the cached control value for one GPU.
// The key to find a submap is GPU Instance stored in ObjectToGpuInstMap
// 
// The key inside the submap is the control command
// The value inside the submap is the cached control value for the command
//
MAKE_MULTIMAP(GpusControlCache, RmapiControlCacheEntry);

ct_assert(sizeof(NvHandle) <= 4);

#define CLIENT_KEY_SHIFT (sizeof(NvHandle) * 8)

static NvBool _isCmdSystemWide(NvU32 cmd)
{
    return DRF_VAL(XXXX, _CTRL_CMD, _CLASS, cmd) == NV01_ROOT;
}

static NvHandle _gpuInstKeyToClient(NvU64 key)
{
    return (key >> CLIENT_KEY_SHIFT);
}

static NvU64 _handlesToGpuInstKey(NvHandle hClient, NvHandle hObject)
{
    return ((NvU64)hClient << CLIENT_KEY_SHIFT) | hObject;
}

static struct {
    /* NOTE: Size unbounded for now */
    GpusControlCache gpusControlCache;
    ObjectToGpuInstMap objectToGpuInstMap;
    NvU32 mode;
    PORT_RWLOCK *pLock;
} RmapiControlCache;

NvBool rmapiControlIsCacheable(NvU32 flags, NvU32 accessRight, NvBool bAllowInternal)
{
    if (RmapiControlCache.mode == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_DISABLE)
        return NV_FALSE;

    if (!(flags & RMCTRL_FLAGS_CACHEABLE))
        return NV_FALSE;

    //
    // RMCTRL with access right requires access right check.
    // We need resource ref and client object to check the access right, both not
    // available here. Do not cache RMCTRLs with access right.
    //
    if (accessRight != 0)
        return NV_FALSE;

    // Allow internal RMCTRL of all privilege with bAllowInternal flag
    if (flags & RMCTRL_FLAGS_INTERNAL)
        return bAllowInternal;

    //
    // For non-internal calls, allow cache only for non-privileged RMCTRL since
    // we don't have the client object to validate client.
    //
    if (!(flags & RMCTRL_FLAGS_NON_PRIVILEGED))
    {
        NV_PRINTF(LEVEL_WARNING, "Do not allow cache for priviliged ctrls\n");
        return NV_FALSE;
    }

    return NV_TRUE;
}

NvBool rmapiCmdIsCacheable(NvU32 cmd, NvBool bAllowInternal)
{
    NvU32 flags;
    NvU32 accessRight;

    if (rmapiutilGetControlInfo(cmd, &flags, &accessRight) != NV_OK)
        return NV_FALSE;

    return rmapiControlIsCacheable(flags, accessRight, bAllowInternal);
}

NV_STATUS rmapiControlCacheInit(void)
{
#if defined(DEBUG)
    RmapiControlCache.mode = NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY;
#else
    RmapiControlCache.mode = NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_ENABLE;
#endif

    NvU32 mode;

    if (osReadRegistryDword(NULL, NV_REG_STR_RM_CACHEABLE_CONTROLS, &mode) == NV_OK)
    {
        RmapiControlCache.mode = mode;
    }
    NV_PRINTF(LEVEL_INFO, "using cache mode %d\n", RmapiControlCache.mode);

    multimapInit(&RmapiControlCache.gpusControlCache, portMemAllocatorGetGlobalNonPaged());
    mapInit(&RmapiControlCache.objectToGpuInstMap, portMemAllocatorGetGlobalNonPaged());
    RmapiControlCache.pLock = portSyncRwLockCreate(portMemAllocatorGetGlobalNonPaged());
    if (RmapiControlCache.pLock == NULL)
    {
        NV_PRINTF(LEVEL_ERROR, "failed to create rw lock\n");
        multimapDestroy(&RmapiControlCache.gpusControlCache);
        mapDestroy(&RmapiControlCache.objectToGpuInstMap);
        return NV_ERR_NO_MEMORY;
    }
    return NV_OK;
}

NV_STATUS rmapiControlCacheSetGpuInstForObject
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 gpuInst
)
{
    NV_STATUS status = NV_OK;
    NvU64 *entry;

    if (gpuInst >= NV_MAX_DEVICES)
        return NV_ERR_INVALID_PARAMETER;

    NV_PRINTF(LEVEL_INFO, "gpu inst set for 0x%x 0x%x: 0x%x\n", hClient, hObject, gpuInst);

    portSyncRwLockAcquireWrite(RmapiControlCache.pLock);
    entry = mapFind(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, hObject));

    if (entry != NULL)
    {
        // set happens in object allocation, should not exist in cache already
        NV_PRINTF(LEVEL_WARNING,
                  "set existing gpu inst 0x%x 0x%x was 0x%llx is 0x%x\n",
                  hClient, hObject, *entry, gpuInst);
        status = NV_ERR_INVALID_PARAMETER;
        goto done;
    }

    entry = mapInsertNew(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, hObject));

    if (entry == NULL)
    {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    *entry = gpuInst;

done:
    portSyncRwLockReleaseWrite(RmapiControlCache.pLock);
    return status;
}

// Need to hold rmapi control cache read/write lock
static NV_STATUS _rmapiControlCacheGetGpuInstForObject
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 *pGpuInst
)
{
    NV_STATUS status = NV_ERR_OBJECT_NOT_FOUND;
    NvU64* entry = mapFind(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, hObject));

    NV_PRINTF(LEVEL_INFO, "cached gpu inst lookup for 0x%x 0x%x\n", hClient, hObject);

    if (entry != NULL)
    {
        NV_PRINTF(LEVEL_INFO, "cached gpu inst for 0x%x 0x%x: 0x%llx\n", hClient, hObject, *entry);
        *pGpuInst = *entry;
        status = NV_OK;
    }

    return status;
}

// Need to hold rmapi control cache write lock
static void _rmapiControlCacheFreeGpuInstForObject
(
    NvHandle hClient,
    NvHandle hObject
)
{
    const NvU64 key = _handlesToGpuInstKey(hClient, hObject);
    NvU64* entry = mapFind(&RmapiControlCache.objectToGpuInstMap, key);

    if (entry != NULL)
    {
        mapRemove(&RmapiControlCache.objectToGpuInstMap, entry);
        NV_PRINTF(LEVEL_INFO, "Gpu Inst entry with key 0x%llx freed\n", key);
    }
}

// Need to hold rmapi control cache write lock
static void _rmapiControlCacheFreeGpuInstForClient(NvHandle hClient)
{
    while (NV_TRUE)
    {
        NvU64* entry = mapFindGEQ(&RmapiControlCache.objectToGpuInstMap, _handlesToGpuInstKey(hClient, 0));
        NvU64 key;

        if (entry == NULL)
            break;

        key = mapKey(&RmapiControlCache.objectToGpuInstMap, entry);

        if (_gpuInstKeyToClient(key) != hClient)
            break;

        mapRemove(&RmapiControlCache.objectToGpuInstMap, entry);
        NV_PRINTF(LEVEL_INFO, "Gpu Inst entry with key 0x%llx freed\n", key);
    }
}

NV_STATUS rmapiControlCacheGet
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    void* params,
    NvU32 paramsSize
)
{
    RmapiControlCacheEntry *entry;
    NvU32 gpuInst;
    NV_STATUS status = NV_OK;

    if (RmapiControlCache.mode == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY)
        return NV_ERR_OBJECT_NOT_FOUND;

    NV_PRINTF(LEVEL_INFO, "control cache lookup for 0x%x 0x%x 0x%x\n", hClient, hObject, cmd);
    portSyncRwLockAcquireRead(RmapiControlCache.pLock);

    if (_isCmdSystemWide(cmd))
    {
       gpuInst = NV_MAX_DEVICES;
    }
    else
    {
        status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
        if (status != NV_OK)
            goto done;
    }

    entry = multimapFindItem(&RmapiControlCache.gpusControlCache, gpuInst, cmd);
    if (entry == NULL || entry->params == NULL)
    {
        status = NV_ERR_OBJECT_NOT_FOUND;
        goto done;
    }

    portMemCopy(params, paramsSize, entry->params, paramsSize);
    NV_PRINTF(LEVEL_INFO, "control cache for 0x%x 0x%x 0x%x: entry %p\n", hClient, hObject, cmd, entry);
done:
    portSyncRwLockReleaseRead(RmapiControlCache.pLock);
    return status;
}

NV_STATUS rmapiControlCacheSet
(
    NvHandle hClient,
    NvHandle hObject,
    NvU32 cmd,
    const void* params,
    NvU32 paramsSize
)
{
    NV_STATUS status = NV_OK;
    RmapiControlCacheEntry* entry = NULL;
    GpusControlCacheSubmap* insertedSubmap = NULL;
    NvU32 gpuInst;

    NV_PRINTF(LEVEL_INFO, "control cache set for 0x%x 0x%x 0x%x\n", hClient, hObject, cmd);

    portSyncRwLockAcquireWrite(RmapiControlCache.pLock);

    if (_isCmdSystemWide(cmd))
    {
        gpuInst = NV_MAX_DEVICES;
    }
    else
    {
        status = _rmapiControlCacheGetGpuInstForObject(hClient, hObject, &gpuInst);
        if (status != NV_OK)
            goto done;
    }

    entry = multimapFindItem(&RmapiControlCache.gpusControlCache, gpuInst, cmd);

    if (entry == NULL)
    {
        if (multimapFindSubmap(&RmapiControlCache.gpusControlCache, gpuInst) == NULL)
        {
            insertedSubmap = multimapInsertSubmap(&RmapiControlCache.gpusControlCache, gpuInst);
            if (insertedSubmap == NULL)
            {
                status = NV_ERR_NO_MEMORY;
                goto done;
            }
        }

        entry = multimapInsertItemNew(&RmapiControlCache.gpusControlCache, gpuInst, cmd);
    }

    if (entry == NULL)
    {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    //
    // Skip duplicated cache insertion. Duplicated cache set happens when
    // 1. Parallel controls call into RM before first cache set.
    //    All threads will attempt cache set after the control calls.
    // 2. Cache already set by RPC to GSP path
    // 3. Cache in verify only mode
    //
    if (entry->params != NULL)
    {
        if (RmapiControlCache.mode == NV0000_CTRL_SYSTEM_RMCTRL_CACHE_MODE_CTRL_MODE_VERIFY_ONLY)
        {
            NV_ASSERT(portMemCmp(entry->params, params, paramsSize) == 0);
        }
        status = NV_OK;
        goto done;
    }

    entry->params = portMemAllocNonPaged(paramsSize);
    if (entry->params == NULL)
    {
        status = NV_ERR_NO_MEMORY;
        goto done;
    }

    portMemCopy(entry->params, paramsSize, params, paramsSize);
    NV_PRINTF(LEVEL_INFO,
              "control cache set for 0x%x 0x%x 0x%x succeed, entry: %p\n",
              hClient, hObject, cmd, entry);

done:
    if (status != NV_OK)
    {
        /* To avoid leaking memory, remove the newly inserted empty submap and entry */
        if (entry != NULL)
        {
            portMemFree(entry->params);
            multimapRemoveItem(&RmapiControlCache.gpusControlCache, entry);
        }

        if (insertedSubmap != NULL)
            multimapRemoveSubmap(&RmapiControlCache.gpusControlCache, insertedSubmap);
    }

    portSyncRwLockReleaseWrite(RmapiControlCache.pLock);

    return status;
}

// Need to hold rmapi control cache write lock
static void _freeSubmap(GpusControlCacheSubmap* submap)
{
    /* (Sub)map modification invalidates the iterator, so we have to restart */
    while (NV_TRUE)
    {
        GpusControlCacheIter it = multimapSubmapIterItems(&RmapiControlCache.gpusControlCache, submap);

        if (multimapItemIterNext(&it))
        {
            RmapiControlCacheEntry* entry = it.pValue;
            portMemFree(entry->params);
            multimapRemoveItem(&RmapiControlCache.gpusControlCache, entry);
        }
        else
        {
            break;
        }
    }
    multimapRemoveSubmap(&RmapiControlCache.gpusControlCache, submap);
}

void rmapiControlCacheFreeAllCacheForGpu
(
    NvU32 gpuInst
)
{
    GpusControlCacheSubmap* submap;

    portSyncRwLockAcquireWrite(RmapiControlCache.pLock);
    submap = multimapFindSubmap(&RmapiControlCache.gpusControlCache, gpuInst);

    if (submap != NULL)
        _freeSubmap(submap);

    portSyncRwLockReleaseWrite(RmapiControlCache.pLock);
}

void rmapiControlCacheFreeClientEntry(NvHandle hClient)
{
    portSyncRwLockAcquireWrite(RmapiControlCache.pLock);
    _rmapiControlCacheFreeGpuInstForClient(hClient);
    portSyncRwLockReleaseWrite(RmapiControlCache.pLock);
}

void rmapiControlCacheFreeObjectEntry(NvHandle hClient, NvHandle hObject)
{
    if (hClient == hObject)
    {
        rmapiControlCacheFreeClientEntry(hClient);
        return;
    }

    portSyncRwLockAcquireWrite(RmapiControlCache.pLock);
    _rmapiControlCacheFreeGpuInstForObject(hClient, hObject);
    portSyncRwLockReleaseWrite(RmapiControlCache.pLock);
}

void rmapiControlCacheFree(void)
{
    GpusControlCacheIter it;

    it = multimapItemIterAll(&RmapiControlCache.gpusControlCache);
    while (multimapItemIterNext(&it))
    {
        RmapiControlCacheEntry* entry = it.pValue;
        portMemFree(entry->params);
    }

    multimapDestroy(&RmapiControlCache.gpusControlCache);
    mapDestroy(&RmapiControlCache.objectToGpuInstMap);
    portSyncRwLockDestroy(RmapiControlCache.pLock);
}

//
// Changing cache mode cause race conditions on cacheability check and cache get/set.
// When the race condition happens, we may
//   1. Skip cache access when cache enabled
//   2. Access cache when cache disabled
// In the current design, these do not cause any error to the control data.
//
void rmapiControlCacheSetMode(NvU32 mode)
{
    NV_PRINTF(LEVEL_INFO, "Set rmapi control cache mode to 0x%x\n", mode);
    RmapiControlCache.mode = mode;
}

NvU32 rmapiControlCacheGetMode(void)
{
    return RmapiControlCache.mode;
}
