fix: use internal engine for checking peer access

Signed-off-by: Maciej Plewka <[email protected]>

Related-To: NEO-9721

Author: MaciejPlewka

level_zero/core/source/cmdlist/cmdlist.h

@@ -186,7 +186,8 @@ struct CommandList : _ze_command_list_handle_t {
     virtual ze_result_t hostSynchronize(uint64_t timeout) = 0;
 
     static CommandList *create(uint32_t productFamily, Device *device, NEO::EngineGroupType engineGroupType,
-                               ze_command_list_flags_t flags, ze_result_t &resultValue);
+                               ze_command_list_flags_t flags, ze_result_t &resultValue,
+                               bool internalUsage);
     static CommandList *createImmediate(uint32_t productFamily, Device *device,
                                         const ze_command_queue_desc_t *desc,
                                         bool internalUsage, NEO::EngineGroupType engineGroupType,

level_zero/core/source/cmdlist/cmdlist_imp.cpp

@@ -102,7 +102,8 @@ ze_result_t CommandListImp::appendMetricQueryEnd(zet_metric_query_handle_t hMetr
 }
 
 CommandList *CommandList::create(uint32_t productFamily, Device *device, NEO::EngineGroupType engineGroupType,
-                                 ze_command_list_flags_t flags, ze_result_t &returnValue) {
+                                 ze_command_list_flags_t flags, ze_result_t &returnValue,
+                                 bool internalUsage) {
     CommandListAllocatorFn allocator = nullptr;
     if (productFamily < IGFX_MAX_PRODUCT) {
         allocator = commandListFactory[productFamily];
@@ -113,6 +114,7 @@ CommandList *CommandList::create(uint32_t productFamily, Device *device, NEO::En
 
     if (allocator) {
         commandList = static_cast<CommandListImp *>((*allocator)(CommandList::defaultNumIddsPerBlock));
+        commandList->internalUsage = internalUsage;
         returnValue = commandList->initialize(device, engineGroupType, flags);
         if (returnValue != ZE_RESULT_SUCCESS) {
             commandList->destroy();
@@ -147,9 +149,7 @@ CommandList *CommandList::createImmediate(uint32_t productFamily, Device *device
             } else {
                 auto internalEngine = deviceImp->getActiveDevice()->getInternalEngine();
                 csr = internalEngine.commandStreamReceiver;
-                auto internalEngineType = internalEngine.getEngineType();
-                auto internalEngineUsage = internalEngine.getEngineUsage();
-                engineGroupType = gfxCoreHelper.getEngineGroupType(internalEngineType, internalEngineUsage, hwInfo);
+                engineGroupType = deviceImp->getInternalEngineGroupType();
             }
         } else {
             returnValue = device->getCsrForOrdinalAndIndex(&csr, desc->ordinal, desc->index);

level_zero/core/source/cmdqueue/cmdqueue.h

@@ -7,6 +7,7 @@
 
 #pragma once
 #include "shared/source/command_stream/task_count_helper.h"
+#include "shared/source/helpers/definitions/engine_group_types.h"
 #include "shared/source/helpers/heap_base_address_model.h"
 
 #include <level_zero/ze_api.h>

level_zero/core/source/device/device_imp.cpp

@@ -23,6 +23,7 @@
 #include "shared/source/helpers/constants.h"
 #include "shared/source/helpers/engine_node_helper.h"
 #include "shared/source/helpers/gfx_core_helper.h"
+#include "shared/source/helpers/hw_info.h"
 #include "shared/source/helpers/ray_tracing_helper.h"
 #include "shared/source/helpers/string.h"
 #include "shared/source/helpers/topology_map.h"
@@ -96,6 +97,7 @@ ze_result_t DeviceImp::getStatus() {
 ze_result_t DeviceImp::submitCopyForP2P(ze_device_handle_t hPeerDevice, ze_bool_t *value) {
     DeviceImp *pPeerDevice = static_cast<DeviceImp *>(Device::fromHandle(hPeerDevice));
     uint32_t peerRootDeviceIndex = pPeerDevice->getNEODevice()->getRootDeviceIndex();
+    *value = false;
 
     ze_command_list_handle_t commandList = nullptr;
     ze_command_list_desc_t listDescriptor = {};
@@ -114,8 +116,10 @@ ze_result_t DeviceImp::submitCopyForP2P(ze_device_handle_t hPeerDevice, ze_bool_
     queueDescriptor.ordinal = 0;
     queueDescriptor.index = 0;
 
-    this->createCommandList(&listDescriptor, &commandList);
-    this->createCommandQueue(&queueDescriptor, &commandQueue);
+    auto ret = this->createInternalCommandList(&listDescriptor, &commandList);
+    UNRECOVERABLE_IF(ret != ZE_RESULT_SUCCESS);
+    ret = this->createInternalCommandQueue(&queueDescriptor, &commandQueue);
+    UNRECOVERABLE_IF(ret != ZE_RESULT_SUCCESS);
 
     auto driverHandle = this->getDriverHandle();
     DriverHandleImp *driverHandleImp = static_cast<DriverHandleImp *>(driverHandle);
@@ -144,7 +148,7 @@ ze_result_t DeviceImp::submitCopyForP2P(ze_device_handle_t hPeerDevice, ze_bool_
     contextImp->allocDeviceMem(this->toHandle(), &deviceDesc, 8, 1, &memory);
     contextImp->allocDeviceMem(hPeerDevice, &peerDeviceDesc, 8, 1, &peerMemory);
 
-    auto ret = L0::CommandList::fromHandle(commandList)->appendMemoryCopy(peerMemory, memory, 8, nullptr, 0, nullptr, false, false);
+    ret = L0::CommandList::fromHandle(commandList)->appendMemoryCopy(peerMemory, memory, 8, nullptr, 0, nullptr, false, false);
     L0::CommandList::fromHandle(commandList)->close();
 
     if (ret == ZE_RESULT_SUCCESS) {
@@ -163,9 +167,9 @@ ze_result_t DeviceImp::submitCopyForP2P(ze_device_handle_t hPeerDevice, ze_bool_
     contextImp->freeMem(peerMemory);
     contextImp->freeMem(memory);
 
-    L0::Context::fromHandle(context)->destroy();
-    L0::CommandQueue::fromHandle(commandQueue)->destroy();
     L0::CommandList::fromHandle(commandList)->destroy();
+    L0::CommandQueue::fromHandle(commandQueue)->destroy();
+    L0::Context::fromHandle(context)->destroy();
 
     if (ret == ZE_RESULT_ERROR_DEVICE_LOST) {
         return ZE_RESULT_ERROR_DEVICE_LOST;
@@ -221,7 +225,19 @@ ze_result_t DeviceImp::createCommandList(const ze_command_list_desc_t *desc,
     auto productFamily = neoDevice->getHardwareInfo().platform.eProductFamily;
     ze_result_t returnValue = ZE_RESULT_SUCCESS;
     auto createCommandList = getCmdListCreateFunc(desc);
-    *commandList = createCommandList(productFamily, this, engineGroupType, desc->flags, returnValue);
+    *commandList = createCommandList(productFamily, this, engineGroupType, desc->flags, returnValue, false);
+
+    return returnValue;
+}
+
+ze_result_t DeviceImp::createInternalCommandList(const ze_command_list_desc_t *desc,
+                                                 ze_command_list_handle_t *commandList) {
+    NEO::EngineGroupType engineGroupType = getInternalEngineGroupType();
+
+    auto productFamily = neoDevice->getHardwareInfo().platform.eProductFamily;
+    ze_result_t returnValue = ZE_RESULT_SUCCESS;
+    auto createCommandList = getCmdListCreateFunc(desc);
+    *commandList = createCommandList(productFamily, this, engineGroupType, desc->flags, returnValue, true);
 
     return returnValue;
 }
@@ -300,7 +316,22 @@ ze_result_t DeviceImp::createCommandQueue(const ze_command_queue_desc_t *desc,
 
     ze_result_t returnValue = ZE_RESULT_SUCCESS;
     *commandQueue = CommandQueue::create(platform.eProductFamily, this, csr, &commandQueueDesc, isCopyOnly, false, false, returnValue);
+    return returnValue;
+}
+
+ze_result_t DeviceImp::createInternalCommandQueue(const ze_command_queue_desc_t *desc,
+                                                  ze_command_queue_handle_t *commandQueue) {
+    auto &platform = neoDevice->getHardwareInfo().platform;
+
+    auto internalEngine = this->getActiveDevice()->getInternalEngine();
+    auto csr = internalEngine.commandStreamReceiver;
+    auto engineGroupType = getInternalEngineGroupType();
+    auto isCopyOnly = NEO::EngineHelper::isCopyOnlyEngineType(engineGroupType);
 
+    UNRECOVERABLE_IF(csr == nullptr);
+
+    ze_result_t returnValue = ZE_RESULT_SUCCESS;
+    *commandQueue = CommandQueue::create(platform.eProductFamily, this, csr, desc, isCopyOnly, true, false, returnValue);
     return returnValue;
 }
 
@@ -479,6 +510,14 @@ ze_result_t DeviceImp::getComputeProperties(ze_device_compute_properties_t *pCom
     return ZE_RESULT_SUCCESS;
 }
 
+NEO::EngineGroupType DeviceImp::getInternalEngineGroupType() {
+    auto &gfxCoreHelper = neoDevice->getGfxCoreHelper();
+    auto internalEngine = this->getActiveDevice()->getInternalEngine();
+    auto internalEngineType = internalEngine.getEngineType();
+    auto internalEngineUsage = internalEngine.getEngineUsage();
+    return gfxCoreHelper.getEngineGroupType(internalEngineType, internalEngineUsage, getHwInfo());
+}
+
 void DeviceImp::getP2PPropertiesDirectFabricConnection(DeviceImp *peerDeviceImp,
                                                        ze_device_p2p_bandwidth_exp_properties_t *bandwidthPropertiesDesc) {
 

level_zero/core/source/device/device_imp.h

@@ -38,10 +38,14 @@ struct DeviceImp : public Device, NEO::NonCopyableOrMovableClass {
     ze_result_t canAccessPeer(ze_device_handle_t hPeerDevice, ze_bool_t *value) override;
     ze_result_t createCommandList(const ze_command_list_desc_t *desc,
                                   ze_command_list_handle_t *commandList) override;
+    MOCKABLE_VIRTUAL ze_result_t createInternalCommandList(const ze_command_list_desc_t *desc,
+                                                           ze_command_list_handle_t *commandList);
     ze_result_t createCommandListImmediate(const ze_command_queue_desc_t *desc,
                                            ze_command_list_handle_t *phCommandList) override;
     ze_result_t createCommandQueue(const ze_command_queue_desc_t *desc,
                                    ze_command_queue_handle_t *commandQueue) override;
+    MOCKABLE_VIRTUAL ze_result_t createInternalCommandQueue(const ze_command_queue_desc_t *desc,
+                                                            ze_command_queue_handle_t *commandQueue);
     ze_result_t createImage(const ze_image_desc_t *desc, ze_image_handle_t *phImage) override;
     ze_result_t createModule(const ze_module_desc_t *desc, ze_module_handle_t *module,
                              ze_module_build_log_handle_t *buildLog, ModuleType type) override;
@@ -153,7 +157,7 @@ struct DeviceImp : public Device, NEO::NonCopyableOrMovableClass {
     bool isQueueGroupOrdinalValid(uint32_t ordinal);
     void setFabricVertex(FabricVertex *inFabricVertex) { fabricVertex = inFabricVertex; }
 
-    using CmdListCreateFunPtrT = L0::CommandList *(*)(uint32_t, Device *, NEO::EngineGroupType, ze_command_list_flags_t, ze_result_t &);
+    using CmdListCreateFunPtrT = L0::CommandList *(*)(uint32_t, Device *, NEO::EngineGroupType, ze_command_list_flags_t, ze_result_t &, bool);
     CmdListCreateFunPtrT getCmdListCreateFunc(const ze_command_list_desc_t *desc);
     ze_result_t getFabricVertex(ze_fabric_vertex_handle_t *phVertex) override;
 
@@ -162,6 +166,7 @@ struct DeviceImp : public Device, NEO::NonCopyableOrMovableClass {
     uint32_t getEventMaxPacketCount() const override;
     uint32_t getEventMaxKernelCount() const override;
     uint32_t queryDeviceNodeMask();
+    NEO::EngineGroupType getInternalEngineGroupType();
 
   protected:
     void adjustCommandQueueDesc(uint32_t &ordinal, uint32_t &index);

level_zero/core/test/aub_tests/fixtures/aub_fixture.cpp

@@ -77,7 +77,7 @@ void AUBFixtureL0::setUp(const NEO::HardwareInfo *hardwareInfo, bool debuggingEn
     context = static_cast<ContextImp *>(Context::fromHandle(hContext));
 
     ze_result_t returnValue;
-    commandList.reset(ult::CommandList::whiteboxCast(CommandList::create(hwInfo.platform.eProductFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue)));
+    commandList.reset(ult::CommandList::whiteboxCast(CommandList::create(hwInfo.platform.eProductFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)));
 
     returnValue = ZE_RESULT_ERROR_UNINITIALIZED;
     ze_command_queue_desc_t queueDesc = {};

level_zero/core/test/unit_tests/fixtures/cmdlist_fixture.cpp

@@ -34,7 +34,7 @@ CommandListFixture ::~CommandListFixture() = default;
 void CommandListFixture::setUp() {
     DeviceFixture::setUp();
     ze_result_t returnValue;
-    commandList.reset(CommandList::whiteboxCast(CommandList::create(device->getHwInfo().platform.eProductFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue)));
+    commandList.reset(CommandList::whiteboxCast(CommandList::create(device->getHwInfo().platform.eProductFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false)));
 
     ze_event_pool_desc_t eventPoolDesc = {ZE_STRUCTURE_TYPE_EVENT_POOL_DESC};
     eventPoolDesc.flags = ZE_EVENT_POOL_FLAG_HOST_VISIBLE;
@@ -72,7 +72,7 @@ void MultiTileCommandListFixtureInit::setUpParams(bool createImmediate, bool cre
     NEO::EngineGroupType cmdListEngineType = createCopy ? NEO::EngineGroupType::copy : NEO::EngineGroupType::renderCompute;
 
     if (!createImmediate) {
-        commandList.reset(CommandList::whiteboxCast(CommandList::create(device->getHwInfo().platform.eProductFamily, device, cmdListEngineType, 0u, returnValue)));
+        commandList.reset(CommandList::whiteboxCast(CommandList::create(device->getHwInfo().platform.eProductFamily, device, cmdListEngineType, 0u, returnValue, false)));
     } else {
         const ze_command_queue_desc_t desc = {};
         commandList.reset(CommandList::whiteboxCast(CommandList::createImmediate(device->getHwInfo().platform.eProductFamily, device, &desc, createInternal, cmdListEngineType, returnValue)));
@@ -112,7 +112,7 @@ void ModuleMutableCommandListFixture::setUpImpl() {
     auto &gfxCoreHelper = device->getGfxCoreHelper();
     engineGroupType = gfxCoreHelper.getEngineGroupType(neoDevice->getDefaultEngine().getEngineType(), neoDevice->getDefaultEngine().getEngineUsage(), device->getHwInfo());
 
-    commandList.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue)));
+    commandList.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue, false)));
     commandListImmediate.reset(CommandList::whiteboxCast(CommandList::createImmediate(productFamily, device, &queueDesc, false, engineGroupType, returnValue)));
     commandListImmediate->isFlushTaskSubmissionEnabled = true;
 
@@ -161,7 +161,7 @@ void CmdListPipelineSelectStateFixture::setUp() {
     ModuleMutableCommandListFixture::setUp();
 
     auto result = ZE_RESULT_SUCCESS;
-    commandList2.reset(CommandList::whiteboxCast(CommandList::create(productFamily, this->device, this->engineGroupType, 0u, result)));
+    commandList2.reset(CommandList::whiteboxCast(CommandList::create(productFamily, this->device, this->engineGroupType, 0u, result, false)));
     ASSERT_EQ(ZE_RESULT_SUCCESS, result);
 }
 
@@ -225,7 +225,7 @@ void CommandListGlobalHeapsFixtureInit::setUpParams(int32_t globalHeapMode) {
     debugManager.flags.SelectCmdListHeapAddressModel.set(static_cast<int32_t>(NEO::HeapAddressModel::privateHeaps));
 
     ze_result_t returnValue;
-    commandListPrivateHeap.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue)));
+    commandListPrivateHeap.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue, false)));
 
     debugManager.flags.SelectCmdListHeapAddressModel.set(globalHeapMode);
 }
@@ -449,7 +449,7 @@ void CommandListAppendLaunchRayTracingKernelFixture::setUp() {
     ASSERT_NE(nullptr, buffer2);
 
     ze_result_t returnValue;
-    commandList = CommandList::whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    commandList = CommandList::whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
     ASSERT_NE(commandList->getCmdContainer().getCommandStream(), nullptr);
 
     dispatchKernelArguments.groupCountX = 1u;
@@ -482,8 +482,8 @@ void PrimaryBatchBufferPreamblelessCmdListFixture::setUp() {
     PrimaryBatchBufferCmdListFixture::setUp();
 
     ze_result_t returnValue;
-    commandList2.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue)));
-    commandList3.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue)));
+    commandList2.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue, false)));
+    commandList3.reset(CommandList::whiteboxCast(CommandList::create(productFamily, device, engineGroupType, 0u, returnValue, false)));
 }
 
 void PrimaryBatchBufferPreamblelessCmdListFixture::tearDown() {
@@ -567,7 +567,7 @@ void CommandQueueThreadArbitrationPolicyFixture::setUp() {
                                                      returnValue));
     ASSERT_NE(nullptr, commandQueue);
 
-    commandList = CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue);
+    commandList = CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false);
     ASSERT_NE(nullptr, commandList);
 
     commandList->close();

level_zero/core/test/unit_tests/fixtures/multi_tile_fixture.cpp

@@ -39,7 +39,7 @@ void MultiTileCommandListAppendLaunchKernelFixture::setUp() {
     contextImp = static_cast<ContextImp *>(Context::fromHandle(hContext));
 
     ze_result_t returnValue;
-    commandList = CommandList::whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    commandList = CommandList::whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
     EXPECT_EQ(ZE_RESULT_SUCCESS, returnValue);
 }
 

level_zero/core/test/unit_tests/gen9/test_cmdlist_append_launch_kernel_gen9.cpp

@@ -22,7 +22,7 @@ HWTEST2_F(CommandListAppendLaunchKernel, givenKernelWithSLMThenL3IsProgrammedWit
     using MI_LOAD_REGISTER_IMM = typename FamilyType::MI_LOAD_REGISTER_IMM;
     createKernel();
     ze_result_t returnValue;
-    std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    std::unique_ptr<L0::CommandList> commandList(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
     ze_group_count_t groupCount{1, 1, 1};
 
     EXPECT_LE(0u, kernel->kernelImmData->getDescriptor().kernelAttributes.slmInlineSize);

level_zero/core/test/unit_tests/gen9/test_cmdlist_create_gen9.cpp

@@ -92,7 +92,7 @@ class CommandListCreateGen9 : public DeviceFixture, public testing::Test {
 
 GEN9TEST_F(CommandListCreateGen9, WhenGettingCommandListPreemptionModeThenMatchesDevicePreemptionMode) {
     ze_result_t returnValue;
-    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
 
     auto result = commandList->close();
     ASSERT_EQ(ZE_RESULT_SUCCESS, result);
@@ -113,7 +113,7 @@ GEN9TEST_F(CommandListCreateGen9, GivenDisabledMidThreadPreemptionWhenLaunchingK
     initializeKernel(kernelThreadGroup, kernelInfoThreadGroupData, device);
 
     ze_result_t returnValue;
-    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
     EXPECT_EQ(NEO::PreemptionMode::MidThread, commandList->getCommandListPreemptionMode());
 
     CmdListKernelLaunchParams launchParams = {};
@@ -143,7 +143,7 @@ GEN9TEST_F(CommandListCreateGen9, GivenUsesFencesForReadWriteImagesWhenLaunching
     initializeKernel(kernelMidBatch, kernelInfoMidBatchData, device);
 
     ze_result_t returnValue;
-    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
     EXPECT_EQ(NEO::PreemptionMode::MidThread, commandList->getCommandListPreemptionMode());
 
     CmdListKernelLaunchParams launchParams = {};
@@ -178,7 +178,7 @@ GEN9TEST_F(CommandListCreateGen9, WhenCommandListHasLowerPreemptionLevelThenDoNo
     initializeKernel(kernelMidThread, kernelInfoMidThreadData, device);
 
     ze_result_t returnValue;
-    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue));
+    auto commandList = whiteboxCast(CommandList::create(productFamily, device, NEO::EngineGroupType::renderCompute, 0u, returnValue, false));
     EXPECT_EQ(NEO::PreemptionMode::MidThread, commandList->getCommandListPreemptionMode());
 
     CmdListKernelLaunchParams launchParams = {};