sravan/system76-coreboot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

soc/intel/xeon_sp: Add PD_TYPE_CLUSTER

Browse Source 
Add a new proximity type to represent the sub-NUMA cluster (SNC).

This patch adds necessary Xeon-SP common code level support for
SNC support. When SNC on, each SNC cluster will have a proximity
domain. DIMMs and CPU cores are attached to SNC proximity domains
instead of the processor proximity domains.

With SNC, there are 3 types of proximity domains,
PD_TYPE_PROCESSOR, PD_TYPE_GENERIC_INITIATOR and PD_TYPE_CLUSTER.
proximity domain type checks in Xeon-SP codes are updated to
correctly handle the adding of the new type.

This patch doesn't actually enable SNC. To fully enable SNC, SoC
codes need to override soc_get_cluster_count(), soc_set_cpu_node_
id() and memory_to_pd(), and call soc_set_cpu_node_id() in its
per-CPU init routine.

Change-Id: I32558983780f302ff4893901540a90baebf47add
Signed-off-by: Shuo Liu <shuo.liu@intel.com>
Co-authored-by: Ziang Wang <ziang.wang@intel.com>
Co-authored-by: Gang Chen <gang.c.chen@intel.com>
Reviewed-on: https://review.coreboot.org/c/coreboot/+/81443
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: Lean Sheng Tan <sheng.tan@9elements.com>
This commit is contained in:
Shuo Liu
2024-03-27 04:52:37 +08:00
committed by Lean Sheng Tan
parent bd33b6ab9f
commit 70de5bf9fd
5 changed files with 81 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/soc/intel/xeon_sp/include/soc/numa.h
View File

@ -19,6 +19,17 @@ enum proximity_domain_type {
* Generic Initiator domain is a CXL memory device.
*/
PD_TYPE_GENERIC_INITIATOR,
/*
* PD_TYPE_CLUSTER is for Sub-NUMA cluster (SNC). SNC is localization
* domain within a socket, composed of a set of CPU cores, last-level
* cache pieces and memory controllers, which are close to each other.
* SNC will be reported as NUMA nodes to OS so that the performance
* proximity could be fully exploited for task assignment and scheduling.
*
* For more, please refer to
* https://www.intel.com/content/www/us/en/developer/articles/technical/xeon-processor-scalable-family-technical-overview.html
*/
PD_TYPE_CLUSTER,
PD_TYPE_MAX
};
@ -40,6 +51,7 @@ struct proximity_domain {
* sockets, so we need a bitmap.
*/
uint8_t socket_bitmap;
uint8_t cluster_bitmap;
/* Relative distances (memory latency) from all domains */
uint8_t *distances;
/*
@ -68,4 +80,7 @@ uint32_t get_generic_initiator_mem_size(void);
uint32_t memory_to_pd(const struct SystemMemoryMapElement *mem);
uint32_t device_to_pd(const struct device *dev);
uint8_t soc_get_cluster_count(void);
void soc_set_cpu_node_id(struct device *cpu);
#endif /* NUMA_H */

72
src/soc/intel/xeon_sp/numa.c
View File

@ -25,6 +25,9 @@ static void dump_pds(void)
printk(BIOS_DEBUG, "\t\tbase(64MB):0x%x\n", pds.pds[i].base);
printk(BIOS_DEBUG, "\t\tsize(64MB):0x%x\n", pds.pds[i].size);
}
if (pds.pds[i].pd_type == PD_TYPE_CLUSTER) {
printk(BIOS_DEBUG, "\t\tcluster_bitmap:0x%x\n", pds.pds[i].cluster_bitmap);
}
}
}
@ -32,20 +35,25 @@ static void fill_pds(void)
{
uint8_t num_sockets = soc_get_num_cpus();
uint8_t num_cxlnodes = get_cxl_node_count();
uint8_t num_clusters = soc_get_cluster_count();
const IIO_UDS *hob = get_iio_uds();
/*
* Rules/assumptions:
* 1. Each processor has a processor proximity domain regardless whether
* 1. Each socket has a processor proximity domain regardless whether
* a processor has DIMM attached to it or not.
* 2. All system memory map elements are either from processor attached memory,
* 2. When sub-NUMA cluster (SNC) is on, soc_get_cluster_count() will return a
* non-zero value and each SNC cluster will have one proximity domain.
* For SNC case, DIMMs and CPU cores are attached to SNC proximity domains instead
* of the processor proximity domains.
* 3. All system memory map elements are either from processor attached memory,
* or from CXL memory. Each CXL node info entry has a corresponding entry
* in system memory map elements.
* 3. Each CXL device may have multiple HDMs (Host-managed Device Memory). Each
* 4. Each CXL device may have multiple HDMs (Host-managed Device Memory). Each
* HDM has one and only one CXL node info entry. Each CXL node info entry
* represents a generic initiator proximity domain.
*/
pds.num_pds = num_cxlnodes + num_sockets;
pds.num_pds = num_cxlnodes + num_sockets + num_sockets * num_clusters;
pds.pds = xmalloc(sizeof(struct proximity_domain) * pds.num_pds);
if (!pds.pds)
die("%s %d out of memory.", __FILE__, __LINE__);
@ -57,12 +65,23 @@ static void fill_pds(void)
for (uint8_t socket = 0; socket < num_sockets; socket++) {
if (!soc_cpu_is_enabled(socket))
continue;
const uint8_t socket_id = hob->PlatformData.IIO_resource[socket].SocketID;
pds.pds[i].pd_type = PD_TYPE_PROCESSOR;
pds.pds[i].socket_bitmap = 1 << hob->PlatformData.IIO_resource[socket].SocketID;
pds.pds[i].socket_bitmap = 1 << socket_id;
pds.pds[i].distances = malloc(sizeof(uint8_t) * pds.num_pds);
if (!pds.pds[i].distances)
die("%s %d out of memory.", __FILE__, __LINE__);
i++;
/* Fill in cluster domains */
for (uint8_t cluster = 0; cluster < num_clusters; cluster++) {
pds.pds[i].pd_type = PD_TYPE_CLUSTER;
pds.pds[i].socket_bitmap = 1 << socket_id;
pds.pds[i].cluster_bitmap = 1 << cluster;
pds.pds[i].distances = malloc(sizeof(uint8_t) * pds.num_pds);
if (!pds.pds[i].distances)
die("%s %d out of memory.", __FILE__, __LINE__);
i++;
}
}
/* If there are no CXL nodes, we are done */
@ -100,7 +119,7 @@ uint32_t get_generic_initiator_mem_size(void)
uint32_t size = 0;
for (i = 0; i < pds.num_pds; i++) {
if (pds.pds[i].pd_type == PD_TYPE_PROCESSOR)
if (pds.pds[i].pd_type != PD_TYPE_GENERIC_INITIATOR)
continue;
size += pds.pds[i].size;
}
@ -123,6 +142,22 @@ static uint32_t socket_to_pd(uint8_t socket)
return XEONSP_INVALID_PD_INDEX;
}
static uint32_t cluster_to_pd(uint8_t socket, uint8_t cluster)
{
for (uint8_t i = 0; i < pds.num_pds; i++) {
if (pds.pds[i].pd_type != PD_TYPE_CLUSTER)
continue;
if (pds.pds[i].socket_bitmap == (1 << socket) &&
pds.pds[i].cluster_bitmap == (1 << cluster))
return i;
}
printk(BIOS_ERR, "%s: could not find proximity domain for socket/cluster %d/%d.\n",
__func__, socket, cluster);
return XEONSP_INVALID_PD_INDEX;
}
uint32_t device_to_pd(const struct device *dev)
{
/* first to see if the dev is bound to specific pd */
@ -130,8 +165,12 @@ uint32_t device_to_pd(const struct device *dev)
if (pds.pds[i].dev == dev)
return i;
if (dev->path.type == DEVICE_PATH_APIC)
return socket_to_pd(dev->path.apic.package_id);
if (dev->path.type == DEVICE_PATH_APIC) {
if (soc_get_cluster_count())
return cluster_to_pd(dev->path.apic.package_id, dev->path.apic.node_id);
else
return socket_to_pd(dev->path.apic.package_id);
}
if ((dev->path.type == DEVICE_PATH_DOMAIN) ||
(dev->path.type == DEVICE_PATH_PCI))
@ -143,8 +182,12 @@ uint32_t device_to_pd(const struct device *dev)
return XEONSP_INVALID_PD_INDEX;
}
uint32_t memory_to_pd(const struct SystemMemoryMapElement *mem)
__weak uint32_t memory_to_pd(const struct SystemMemoryMapElement *mem)
{
/*
* TODO: For SNC case, link DRAM range to cluster id instead of socket id
* in SoC codes.
*/
return socket_to_pd(mem->SocketId);
}
@ -183,3 +226,14 @@ void setup_pds(void)
fill_pd_distances();
dump_pds();
}
__weak uint8_t soc_get_cluster_count(void)
{
//TODO: Implement in SoC codes.
return 0;
}
__weak void soc_set_cpu_node_id(struct device *cpu)
{
//TODO: Implement in SoC codes.
};

2
src/soc/intel/xeon_sp/spr/chip.c
View File

@ -176,7 +176,7 @@ static void rcec_init(struct device *dev)
uint32_t ep_bus;
uint8_t i;
for (i = 0; i < pds.num_pds; i++) {
if (pds.pds[i].pd_type == PD_TYPE_PROCESSOR)
if (pds.pds[i].pd_type != PD_TYPE_GENERIC_INITIATOR)
continue;
ep_bus = PCI_BDF(pds.pds[i].dev) >> 20;
if (ep_bus == ecrc_bus + 1)

2
src/soc/intel/xeon_sp/spr/soc_util.c
View File

@ -94,7 +94,7 @@ bool is_iio_cxl_stack_res(const STACK_RES *res)
assert(pds.num_pds);
for (uint8_t i = 0; i < pds.num_pds; i++) {
if (pds.pds[i].pd_type == PD_TYPE_PROCESSOR)
if (pds.pds[i].pd_type != PD_TYPE_GENERIC_INITIATOR)
continue;
uint32_t bus = PCI_BDF(pds.pds[i].dev) >> 20;

2
src/soc/intel/xeon_sp/uncore.c
View File

@ -284,7 +284,7 @@ static void mc_add_dram_resources(struct device *dev, int *res_count)
/* CXL Memory */
uint8_t i;
for (i = 0; i < pds.num_pds; i++) {
if (pds.pds[i].pd_type == PD_TYPE_PROCESSOR)
if (pds.pds[i].pd_type != PD_TYPE_GENERIC_INITIATOR)
continue;
unsigned long flags = IORESOURCE_CACHEABLE;