Skip to content

Commit b81a78d

Browse files
LukaszJobczykCompute-Runtime-Automation
LukaszJobczyk
authored and
Compute-Runtime-Automation
committed
Evict USM cpu allocation after migration
Related-To: NEO-5007 Change-Id: I3c91af3ca22cb6233d530b252cc0c75d8fc2f8b5 Signed-off-by: Lukasz Jobczyk <lukasz.jobczyk@intel.com>
1 parent 6ae5824 commit b81a78d

File tree

13 files changed

+116
-0
lines changed

13 files changed

+116
-0
lines changed

‎level_zero/core/source/memory/cpu_page_fault_memory_manager.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,7 @@ void PageFaultManager::transferToGpu(void *ptr, void *device) {
3636
allocData->cpuAllocation,
3737
allocData->size, false);
3838
UNRECOVERABLE_IF(ret);
39+
40+
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, deviceImp->getNEODevice());
3941
}
4042
} // namespace NEO

‎opencl/source/memory_manager/cpu_page_fault_manager_memory_sync.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,5 +24,8 @@ void PageFaultManager::transferToGpu(void *ptr, void *cmdQ) {
2424
UNRECOVERABLE_IF(retVal);
2525
retVal = commandQueue->finish();
2626
UNRECOVERABLE_IF(retVal);
27+
28+
auto allocData = memoryData[ptr].unifiedMemoryManager->getSVMAlloc(ptr);
29+
this->evictMemoryAfterImplCopy(allocData->cpuAllocation, &commandQueue->getDevice());
2730
}
2831
} // namespace NEO

‎opencl/test/unit_test/aub_tests/fixtures/aub_fixture.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "opencl/source/platform/platform.h"
2020
#include "opencl/test/unit_test/command_queue/command_queue_fixture.h"
2121
#include "opencl/test/unit_test/mocks/mock_cl_device.h"
22+
#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h"
2223
#include "opencl/test/unit_test/mocks/mock_platform.h"
2324

2425
#include "gtest/gtest.h"
@@ -42,6 +43,7 @@ class AUBFixture : public CommandQueueHwFixture {
4243
executionEnvironment = platform()->peekExecutionEnvironment();
4344
executionEnvironment->prepareRootDeviceEnvironments(1u);
4445
executionEnvironment->rootDeviceEnvironments[0]->setHwInfo(&hwInfo);
46+
executionEnvironment->rootDeviceEnvironments[0]->memoryOperationsInterface = std::make_unique<MockMemoryOperationsHandler>();
4547

4648
device = std::make_unique<MockClDevice>(MockDevice::create<MockDevice>(executionEnvironment, rootDeviceIndex));
4749

‎opencl/test/unit_test/memory_manager/cpu_page_fault_manager_memory_sync_tests.cpp

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "shared/test/unit_test/test_macros/test_checks_shared.h"
1111

1212
#include "opencl/source/command_queue/command_queue.h"
13+
#include "opencl/test/unit_test/mocks/mock_cl_device.h"
1314
#include "opencl/test/unit_test/mocks/mock_command_queue.h"
1415
#include "opencl/test/unit_test/mocks/mock_graphics_allocation.h"
1516
#include "opencl/test/unit_test/mocks/mock_memory_manager.h"
@@ -52,7 +53,9 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSynchronizeMemoryThenEnq
5253
auto svmAllocsManager = std::make_unique<SVMAllocsManager>(memoryManager.get());
5354
void *alloc = svmAllocsManager->createSVMAlloc(mockRootDeviceIndex, 256, {}, mockDeviceBitfield);
5455

56+
auto device = std::unique_ptr<MockClDevice>(new MockClDevice{MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr)});
5557
auto cmdQ = std::make_unique<CommandQueueMock>();
58+
cmdQ->device = device.get();
5659
pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {});
5760

5861
pageFaultManager->baseCpuTransfer(alloc, 10, cmdQ.get());
@@ -66,6 +69,7 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenSynchronizeMemoryThenEnq
6669
EXPECT_EQ(cmdQ->finishCalled, 1);
6770

6871
svmAllocsManager->freeSVMAlloc(alloc);
72+
cmdQ->device = nullptr;
6973
}
7074

7175
TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThenInsertMapOperation) {
@@ -83,12 +87,15 @@ TEST_F(PageFaultManagerTest, givenUnifiedMemoryAllocWhenGpuTransferIsInvokedThen
8387
auto memoryManager = std::make_unique<MockMemoryManager>(executionEnvironment);
8488
auto svmAllocsManager = std::make_unique<MockSVMAllocsManager>(memoryManager.get());
8589
void *alloc = svmAllocsManager->createSVMAlloc(mockRootDeviceIndex, 256, {}, mockDeviceBitfield);
90+
auto device = std::unique_ptr<MockClDevice>(new MockClDevice{MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr)});
8691
auto cmdQ = std::make_unique<CommandQueueMock>();
92+
cmdQ->device = device.get();
8793
pageFaultManager->insertAllocation(alloc, 256, svmAllocsManager.get(), cmdQ.get(), {});
8894

8995
EXPECT_EQ(svmAllocsManager->insertSvmMapOperationCalled, 0);
9096
pageFaultManager->baseGpuTransfer(alloc, cmdQ.get());
9197
EXPECT_EQ(svmAllocsManager->insertSvmMapOperationCalled, 1);
9298

9399
svmAllocsManager->freeSVMAlloc(alloc);
100+
cmdQ->device = nullptr;
94101
}

‎opencl/test/unit_test/test_files/igdrcl.config

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ DirectSubmissionEnableDebugBuffer = 0
8585
DirectSubmissionDiagnosticExecutionCount = 30
8686
DirectSubmissionDisableCacheFlush = 0
8787
DirectSubmissionDisableMonitorFence = 0
88+
USMEvictAfterMigration = 1
8889
EnableNullHardware = 0
8990
ForceLinearImages = 0
9091
ForceSLML3Config = 0

‎shared/source/debug_settings/debug_variables_base.inl

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,7 @@ DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionDiagnosticExecutionCount, 30, "N
137137
DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideBlitterSupport, -1, "Overrides default blitter support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start")
138138
DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideRenderSupport, -1, "Overrides default render support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start")
139139
DECLARE_DEBUG_VARIABLE(int32_t, DirectSubmissionOverrideComputeSupport, -1, "Overrides default compute support: -1: do not override, 0: disable engine support, 1: enable engine support with init start, 2: enable engine support without init start")
140+
DECLARE_DEBUG_VARIABLE(bool, USMEvictAfterMigration, true, "Evict USM allocation after implicit migration to GPU")
140141
DECLARE_DEBUG_VARIABLE(bool, DirectSubmissionDisableCacheFlush, false, "Disable dispatching cache flush commands")
141142
DECLARE_DEBUG_VARIABLE(bool, DirectSubmissionDisableMonitorFence, false, "Disable dispatching monitor fence commands")
142143

‎shared/source/page_fault_manager/cpu_page_fault_manager.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
#include <unordered_map>
1717

1818
namespace NEO {
19+
class GraphicsAllocation;
20+
class Device;
1921
class SVMAllocsManager;
2022

2123
class PageFaultManager : public NonCopyableOrMovableClass {
@@ -46,6 +48,7 @@ class PageFaultManager : public NonCopyableOrMovableClass {
4648
virtual void allowCPUMemoryAccess(void *ptr, size_t size) = 0;
4749
virtual void protectCPUMemoryAccess(void *ptr, size_t size) = 0;
4850

51+
virtual void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) = 0;
4952
virtual void broadcastWaitSignal() = 0;
5053
MOCKABLE_VIRTUAL void waitForCopy();
5154

‎shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@
77

88
#include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h"
99

10+
#include "shared/source/debug_settings/debug_settings_manager.h"
11+
#include "shared/source/device/device.h"
1012
#include "shared/source/helpers/debug_helpers.h"
13+
#include "shared/source/memory_manager/memory_operations_handler.h"
1114

1215
#include <dirent.h>
1316
#include <sys/mman.h>
@@ -40,6 +43,9 @@ PageFaultManagerLinux::PageFaultManagerLinux() {
4043

4144
retVal = sigaction(SIGUSR1, &pageFaultManagerHandler, &previousUserSignalHandler);
4245
UNRECOVERABLE_IF(retVal != 0);
46+
47+
this->evictMemoryAfterCopy = DebugManager.flags.EnableDirectSubmission.get() &&
48+
DebugManager.flags.USMEvictAfterMigration.get();
4349
}
4450

4551
PageFaultManagerLinux::~PageFaultManagerLinux() {
@@ -113,4 +119,10 @@ void PageFaultManagerLinux::sendSignalToThread(int threadId) {
113119
syscall(SYS_tkill, threadId, SIGUSR1);
114120
}
115121

122+
void PageFaultManagerLinux::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {
123+
if (evictMemoryAfterCopy) {
124+
device->getRootDeviceEnvironment().memoryOperationsInterface->evict(device, *allocation);
125+
}
126+
};
127+
116128
} // namespace NEO

‎shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ class PageFaultManagerLinux : public PageFaultManager {
2424
void allowCPUMemoryAccess(void *ptr, size_t size) override;
2525
void protectCPUMemoryAccess(void *ptr, size_t size) override;
2626

27+
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
2728
void broadcastWaitSignal() override;
2829
MOCKABLE_VIRTUAL void sendSignalToThread(int threadId);
2930

@@ -34,5 +35,7 @@ class PageFaultManagerLinux : public PageFaultManager {
3435

3536
struct sigaction previousPageFaultHandler = {};
3637
struct sigaction previousUserSignalHandler = {};
38+
39+
bool evictMemoryAfterCopy = false;
3740
};
3841
} // namespace NEO

‎shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ void PageFaultManagerWindows::protectCPUMemoryAccess(void *ptr, size_t size) {
5151
UNRECOVERABLE_IF(!retVal);
5252
}
5353

54+
void PageFaultManagerWindows::evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) {}
55+
5456
void PageFaultManagerWindows::broadcastWaitSignal() {}
5557

5658
} // namespace NEO

‎shared/source/page_fault_manager/windows/cpu_page_fault_manager_windows.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ class PageFaultManagerWindows : public PageFaultManager {
2525
void allowCPUMemoryAccess(void *ptr, size_t size) override;
2626
void protectCPUMemoryAccess(void *ptr, size_t size) override;
2727

28+
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override;
2829
void broadcastWaitSignal() override;
2930

3031
static std::function<LONG(struct _EXCEPTION_POINTERS *exceptionInfo)> pageFaultHandler;

‎shared/test/unit_test/page_fault_manager/linux/cpu_page_fault_manager_linux_tests.cpp

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,14 @@
66
*/
77

88
#include "shared/source/page_fault_manager/linux/cpu_page_fault_manager_linux.h"
9+
#include "shared/test/unit_test/helpers/debug_manager_state_restore.h"
10+
#include "shared/test/unit_test/mocks/mock_device.h"
911
#include "shared/test/unit_test/page_fault_manager/cpu_page_fault_manager_tests_fixture.h"
1012
#include "shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h"
1113

14+
#include "opencl/test/unit_test/mocks/mock_graphics_allocation.h"
15+
#include "opencl/test/unit_test/mocks/mock_memory_operations_handler.h"
16+
1217
#include "gtest/gtest.h"
1318

1419
#include <csignal>
@@ -73,6 +78,78 @@ TEST_F(PageFaultManagerLinuxTest, whenPageFaultIsRaisedThenHandlerIsInvoked) {
7378
EXPECT_TRUE(pageFaultManager->handlerInvoked);
7479
}
7580

81+
struct MockOperationsInterface : public MockMemoryOperationsHandler {
82+
bool evictCalled = false;
83+
MemoryOperationsStatus evict(Device *device, GraphicsAllocation &gfxAllocation) override {
84+
this->evictCalled = true;
85+
return MemoryOperationsStatus::UNSUPPORTED;
86+
}
87+
};
88+
89+
TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionAndUSMEvictWaEnabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsCalled) {
90+
DebugManagerStateRestore restorer;
91+
DebugManager.flags.EnableDirectSubmission.set(true);
92+
DebugManager.flags.USMEvictAfterMigration.set(true);
93+
94+
auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
95+
std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
96+
device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
97+
auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
98+
MockGraphicsAllocation allocation;
99+
100+
EXPECT_FALSE(operationInterface->evictCalled);
101+
pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
102+
EXPECT_TRUE(operationInterface->evictCalled);
103+
}
104+
105+
TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionEnabledAndUSMEvictWaDisabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) {
106+
DebugManagerStateRestore restorer;
107+
DebugManager.flags.EnableDirectSubmission.set(true);
108+
DebugManager.flags.USMEvictAfterMigration.set(false);
109+
110+
auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
111+
std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
112+
device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
113+
auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
114+
MockGraphicsAllocation allocation;
115+
116+
EXPECT_FALSE(operationInterface->evictCalled);
117+
pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
118+
EXPECT_FALSE(operationInterface->evictCalled);
119+
}
120+
121+
TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionAndUSMEvictWaDisabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) {
122+
DebugManagerStateRestore restorer;
123+
DebugManager.flags.EnableDirectSubmission.set(false);
124+
DebugManager.flags.USMEvictAfterMigration.set(false);
125+
126+
auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
127+
std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
128+
device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
129+
auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
130+
MockGraphicsAllocation allocation;
131+
132+
EXPECT_FALSE(operationInterface->evictCalled);
133+
pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
134+
EXPECT_FALSE(operationInterface->evictCalled);
135+
}
136+
137+
TEST_F(PageFaultManagerLinuxTest, givenDirectSubmissionDisabledAndUSMEvictWaEnabledWhenEvitMemoryAfterCopyThenMemoryOperationsHandlerEvictMethodIsNotCalled) {
138+
DebugManagerStateRestore restorer;
139+
DebugManager.flags.EnableDirectSubmission.set(false);
140+
DebugManager.flags.USMEvictAfterMigration.set(true);
141+
142+
auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
143+
std::unique_ptr<Device> device(MockDevice::createWithNewExecutionEnvironment<MockDevice>(nullptr));
144+
device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface = std::make_unique<MockOperationsInterface>();
145+
auto operationInterface = static_cast<MockOperationsInterface *>(device->getExecutionEnvironment()->rootDeviceEnvironments[0u]->memoryOperationsInterface.get());
146+
MockGraphicsAllocation allocation;
147+
148+
EXPECT_FALSE(operationInterface->evictCalled);
149+
pageFaultManager->evictMemoryAfterImplCopy(&allocation, device.get());
150+
EXPECT_FALSE(operationInterface->evictCalled);
151+
}
152+
76153
TEST_F(PageFaultManagerLinuxTest, givenProtectedMemoryWhenTryingToAccessThenPageFaultIsRaisedAndMemoryIsAccessibleAfterHandling) {
77154
auto pageFaultManager = std::make_unique<MockPageFaultManagerLinux>();
78155
pageFaultManager->allowCPUMemoryAccessOnPageFault = true;

‎shared/test/unit_test/page_fault_manager/mock_cpu_page_fault_manager.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ class MockPageFaultManager : public PageFaultManager {
5050
PageFaultManager::transferToGpu(ptr, cmdQ);
5151
}
5252
void broadcastWaitSignal() override {}
53+
void evictMemoryAfterImplCopy(GraphicsAllocation *allocation, Device *device) override {}
5354

5455
int allowMemoryAccessCalled = 0;
5556
int protectMemoryCalled = 0;
@@ -69,6 +70,7 @@ template <class T>
6970
class MockPageFaultManagerHandlerInvoke : public T {
7071
public:
7172
using T::allowCPUMemoryAccess;
73+
using T::evictMemoryAfterImplCopy;
7274
using T::protectCPUMemoryAccess;
7375
using T::T;
7476

0 commit comments

Comments
 (0)