Attachment #21486 for ticket #10679

Lines 3133-3138 baseboard, node and network switch). In order to optimize resource allocations Link Here

(-)a/doc/man/man5/slurm.conf.5 (+5 lines)
3133	on such hardware, Slurm will consider each NUMA node within the socket as a	3133	on such hardware, Slurm will consider each NUMA node within the socket as a
3134	separate socket by default. Use the Ignore_NUMA option to report the correct	3134	separate socket by default. Use the Ignore_NUMA option to report the correct
3135	socket count, but \fBnot\fR optimize resource allocations on the NUMA nodes.	3135	socket count, but \fBnot\fR optimize resource allocations on the NUMA nodes.
		3136
		3137	\fBNOTE\fR: Since hwloc 2.0 NUMA Nodes are are not part of the main/CPU topology tree,
		3138	because of that if Slurm is build with hwloc 2.0 or above Slurm will treat
		3139	HWLOC_OBJ_PACKAGE as Socket, you can change this behavior using
		3140	\fBSlurmdParameters\fR=l3cache_as_socket.
3136	.TP	3141	.TP
3137	\fBinventory_interval=#\fR	3142	\fBinventory_interval=#\fR
3138	On a Cray system using Slurm on top of ALPS this limits the number of times	3143	On a Cray system using Slurm on top of ALPS this limits the number of times




	objtype[SOCKET] = HWLOC_OBJ_SOCKET;
	objtype[CORE]   = HWLOC_OBJ_CORE;
	objtype[PU]     = HWLOC_OBJ_PU;
#if HWLOC_API_VERSION >= 0x00020000
	if (xstrcasestr(slurm_conf.sched_params, "Ignore_NUMA")) {
		info("SchedulerParamaters=Ignore_NUMA not supported by hwloc v2");
	}
#else
	if (hwloc_get_type_depth(topology, HWLOC_OBJ_NODE) >
	    hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET)) {
		if (xstrcasestr(slurm_conf.sched_params, "Ignore_NUMA")) {

			objtype[SOCKET] = HWLOC_OBJ_NODE;
		}
	}
#endif

	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
#if HWLOC_API_VERSION >= 0x00020000
- 
duplication
--
src/plugins/task/cgroup/task_cgroup_cpuset.c | 18 +++---------------
src/slurmd/common/xcpuinfo.c                 | 23 ++++++++++++++---------
src/slurmd/common/xcpuinfo.h                 |  7 +++++++
3 files changed, 24 insertions(+), 24 deletions(-)




	uint32_t obj_idxs[3], cps, tpc, i, j, sock_loop, ntskip, npdist;
	bool core_cyclic, core_fcyclic, sock_fcyclic;
	bool hwloc_success = true;
	hwloc_obj_type_t socket_type = slurmd_parameter_as_socket();
#if HWLOC_API_VERSION >= 0x00020000
	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket"))
		socket_type = HWLOC_OBJ_L3CACHE;
#endif

	/*
	 * We can't trust the slurmd_conf_t *conf here as we need actual

	uint32_t core_idx;
	bool core_fcyclic, core_block;

	hwloc_obj_type_t socket_type = slurmd_parameter_as_socket();
#if HWLOC_API_VERSION >= 0x00020000
	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket"))
		socket_type = HWLOC_OBJ_L3CACHE;
#endif

	nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
						       socket_type);

	uint32_t jnpus;
	int spec_threads = 0;

	hwloc_obj_type_t socket_type = slurmd_parameter_as_socket();
#if HWLOC_API_VERSION >= 0x00020000
	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket"))
		socket_type = HWLOC_OBJ_L3CACHE;
#endif

	/* Allocate and initialize hwloc objects */
	hwloc_topology_init(&topology);




	return ret;
}

/* Return a hwloc object used as a socket */
extern hwloc_obj_type_t slurmd_parameter_as_socket(void)
{
	hwloc_obj_type_t obj_as_socket = HWLOC_OBJ_SOCKET;
	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
#if HWLOC_API_VERSION >= 0x00020000
		obj_as_socket = HWLOC_OBJ_L3CACHE;
#else
		error("SlurmdParameters=l3cache_as_socket requires hwloc v2");
#endif
	}
	return obj_as_socket;
}
/*
 * xcpuinfo_hwloc_topo_get - Return detailed cpuinfo on the whole system
 * Output: p_cpus - number of processors on the system

	 * in which case Slurm will report the correct socket count on the node,
	 * but not be able to optimize resource allocations on the NUMA nodes.
	 */
	objtype[SOCKET] = slurmd_parameter_as_socket();
	objtype[CORE]   = HWLOC_OBJ_CORE;
	objtype[PU]     = HWLOC_OBJ_PU;
#if HWLOC_API_VERSION >= 0x00020000

	}
#endif

	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
#if HWLOC_API_VERSION >= 0x00020000
		objtype[SOCKET] = HWLOC_OBJ_L3CACHE;
#else
		error("SlurmdParameters=l3cache_as_socket requires hwloc v2");
#endif
	}

	/* Groups below root obj are interpreted as boards */
	obj = hwloc_get_root_obj(topology);
	obj = hwloc_get_next_child(topology, obj, NULL);




#define XCPUINFO_ERROR    1
#define XCPUINFO_SUCCESS  0

#ifdef HAVE_HWLOC
#include <hwloc.h>
/* Return a hwloc object used as a socket */
extern hwloc_obj_type_t slurmd_parameter_as_socket(void);
#endif

extern int get_procs(uint16_t *procs);

/* read or load topology and write if needed
 * init and destroy topology must be outside this function */
extern int xcpuinfo_hwloc_topo_load(
	void *topology_in, char *topo_file, bool full);

/*
 * Get the node's cpu info.
 *
- 
--
doc/man/man5/slurm.conf.5    |  9 +++++++++
src/slurmd/common/xcpuinfo.c | 18 ++++++++++++++++++
2 files changed, 27 insertions(+)

Lines 3893-3898 Equivalent to the now deprecated FastSchedule=2 option. Link Here

(-)a/doc/man/man5/slurm.conf.5 (+9 lines)
3893	Use the hwloc l3cache as the socket count. Can be useful on certain processors	3893	Use the hwloc l3cache as the socket count. Can be useful on certain processors
3894	where the socket level is too coarse, and the l3cache may provide better	3894	where the socket level is too coarse, and the l3cache may provide better
3895	task distribution. (E.g., along CCX boundaries instead of socket boundaries.)	3895	task distribution. (E.g., along CCX boundaries instead of socket boundaries.)
		3896	Mutually exclusive with numa_node_as_socket.
		3897	Requires hwloc v2.
		3898	.TP
		3899	\fBnuma_node_as_socket\fR
		3900	Use the hwloc NUMA Node to determine main hierarchy object to be used as socket.
		3901	If the option is set Slurm will check the parent object of NUMA Noda and use it
		3902	as socket. This option may be usefull for architectures likes AMD Epyc, where
		3903	number of nodes per socket may be configured.
		3904	Mutually exclusive with l3cache_as_socket.
3896	Requires hwloc v2.	3905	Requires hwloc v2.
3897	.TP	3906	.TP
3898	\fBshutdown_on_reboot\fR	3907	\fBshutdown_on_reboot\fR




extern hwloc_obj_type_t slurmd_parameter_as_socket(void)
{
	hwloc_obj_type_t obj_as_socket = HWLOC_OBJ_SOCKET;

	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
#if HWLOC_API_VERSION >= 0x00020000
		obj_as_socket = HWLOC_OBJ_L3CACHE;
#else
		error("SlurmdParameters=l3cache_as_socket requires hwloc v2");
#endif
	} else if (xstrcasestr(slurm_conf.slurmd_params,
			       "numa_node_as_socket")) {
#if HWLOC_API_VERSION >= 0x00020000
		obj = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NODE,
						 NULL);
		if (obj != NULL && obj->parent != NULL) {
			char tmp[128];
			obj_as_socket = obj->parent->type;
			hwloc_obj_type_snprintf(tmp, sizeof(tmp), obj->parent,
						0);
			debug2("%s: numa_node_as_socket mapped to '%s'",
			       __func__, tmp);
		}
#else
		error("SlurmdParameters=numa_node_as_socket requires hwloc v2");
#endif
	}

	return obj_as_socket;
}
/*
- 

Return to ticket 10679

Lines 319-324 extern int xcpuinfo_hwloc_topo_get( Link Here

(-)a/src/slurmd/common/xcpuinfo.c (-2 / +6 lines)
319	objtype[SOCKET] = HWLOC_OBJ_SOCKET;	319	objtype[SOCKET] = HWLOC_OBJ_SOCKET;
320	objtype[CORE] = HWLOC_OBJ_CORE;	320	objtype[CORE] = HWLOC_OBJ_CORE;
321	objtype[PU] = HWLOC_OBJ_PU;	321	objtype[PU] = HWLOC_OBJ_PU;
		322	#if HWLOC_API_VERSION >= 0x00020000
		323	if (xstrcasestr(slurm_conf.sched_params, "Ignore_NUMA")) {
		324	info("SchedulerParamaters=Ignore_NUMA not supported by hwloc v2");
		325	}
		326	#else
322	if (hwloc_get_type_depth(topology, HWLOC_OBJ_NODE) >	327	if (hwloc_get_type_depth(topology, HWLOC_OBJ_NODE) >
323	hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET)) {	328	hwloc_get_type_depth(topology, HWLOC_OBJ_SOCKET)) {
324	if (xstrcasestr(slurm_conf.sched_params, "Ignore_NUMA")) {	329	if (xstrcasestr(slurm_conf.sched_params, "Ignore_NUMA")) {
Lines 328-333 extern int xcpuinfo_hwloc_topo_get( Link Here
328	objtype[SOCKET] = HWLOC_OBJ_NODE;	333	objtype[SOCKET] = HWLOC_OBJ_NODE;
329	}	334	}
330	}	335	}
		336	#endif
331		337
332	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {	338	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
333	#if HWLOC_API_VERSION >= 0x00020000	339	#if HWLOC_API_VERSION >= 0x00020000
334	-
335	duplication	340	duplication
336	--
337	src/plugins/task/cgroup/task_cgroup_cpuset.c \| 18 +++---------------	341	src/plugins/task/cgroup/task_cgroup_cpuset.c \| 18 +++---------------
338	src/slurmd/common/xcpuinfo.c \| 23 ++++++++++++++---------	342	src/slurmd/common/xcpuinfo.c \| 23 ++++++++++++++---------
339	src/slurmd/common/xcpuinfo.h \| 7 +++++++	343	src/slurmd/common/xcpuinfo.h \| 7 +++++++
340	3 files changed, 24 insertions(+), 24 deletions(-)	344	3 files changed, 24 insertions(+), 24 deletions(-)

Lines 447-457 static int _task_cgroup_cpuset_dist_cyclic( Link Here

(-)a/src/plugins/task/cgroup/task_cgroup_cpuset.c (-15 / +3 lines)
447	uint32_t obj_idxs[3], cps, tpc, i, j, sock_loop, ntskip, npdist;	447	uint32_t obj_idxs[3], cps, tpc, i, j, sock_loop, ntskip, npdist;
448	bool core_cyclic, core_fcyclic, sock_fcyclic;	448	bool core_cyclic, core_fcyclic, sock_fcyclic;
449	bool hwloc_success = true;	449	bool hwloc_success = true;
450	hwloc_obj_type_t socket_type = HWLOC_OBJ_SOCKET;	450	hwloc_obj_type_t socket_type = slurmd_parameter_as_socket();
451	#if HWLOC_API_VERSION >= 0x00020000
452	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket"))
453	socket_type = HWLOC_OBJ_L3CACHE;
454	#endif
455		451
456	/*	452	/*
457	* We can't trust the slurmd_conf_t *conf here as we need actual	453	* We can't trust the slurmd_conf_t *conf here as we need actual
Lines 703-713 static int _task_cgroup_cpuset_dist_block( Link Here
703	uint32_t core_idx;	699	uint32_t core_idx;
704	bool core_fcyclic, core_block;	700	bool core_fcyclic, core_block;
705		701
706	hwloc_obj_type_t socket_type = HWLOC_OBJ_SOCKET;	702	hwloc_obj_type_t socket_type = slurmd_parameter_as_socket();
707	#if HWLOC_API_VERSION >= 0x00020000
708	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket"))
709	socket_type = HWLOC_OBJ_L3CACHE;
710	#endif
711		703
712	nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,	704	nsockets = (uint32_t) hwloc_get_nbobjs_by_type(topology,
713	socket_type);	705	socket_type);
Lines 1022-1032 extern int task_cgroup_cpuset_set_task_affinity(stepd_step_rec_t *job, Link Here
1022	uint32_t jnpus;	1014	uint32_t jnpus;
1023	int spec_threads = 0;	1015	int spec_threads = 0;
1024		1016
1025	hwloc_obj_type_t socket_type = HWLOC_OBJ_SOCKET;	1017	hwloc_obj_type_t socket_type = slurmd_parameter_as_socket();
1026	#if HWLOC_API_VERSION >= 0x00020000
1027	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket"))
1028	socket_type = HWLOC_OBJ_L3CACHE;
1029	#endif
1030		1018
1031	/* Allocate and initialize hwloc objects */	1019	/* Allocate and initialize hwloc objects */
1032	hwloc_topology_init(&topology);	1020	hwloc_topology_init(&topology);

Lines 254-259 handle_write: Link Here

(-)a/src/slurmd/common/xcpuinfo.c (-9 / +14 lines)
254	return ret;	254	return ret;
255	}	255	}
256		256
		257	/* Return a hwloc object used as a socket */
		258	extern hwloc_obj_type_t slurmd_parameter_as_socket(void)
		259	{
		260	hwloc_obj_type_t obj_as_socket = HWLOC_OBJ_SOCKET;
		261	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
		262	#if HWLOC_API_VERSION >= 0x00020000
		263	obj_as_socket = HWLOC_OBJ_L3CACHE;
		264	#else
		265	error("SlurmdParameters=l3cache_as_socket requires hwloc v2");
		266	#endif
		267	}
		268	return obj_as_socket;
		269	}
257	/*	270	/*
258	* xcpuinfo_hwloc_topo_get - Return detailed cpuinfo on the whole system	271	* xcpuinfo_hwloc_topo_get - Return detailed cpuinfo on the whole system
259	* Output: p_cpus - number of processors on the system	272	* Output: p_cpus - number of processors on the system
Lines 316-322 extern int xcpuinfo_hwloc_topo_get( Link Here
316	* in which case Slurm will report the correct socket count on the node,	329	* in which case Slurm will report the correct socket count on the node,
317	* but not be able to optimize resource allocations on the NUMA nodes.	330	* but not be able to optimize resource allocations on the NUMA nodes.
318	*/	331	*/
319	objtype[SOCKET] = HWLOC_OBJ_SOCKET;	332	objtype[SOCKET] = slurmd_parameter_as_socket();
320	objtype[CORE] = HWLOC_OBJ_CORE;	333	objtype[CORE] = HWLOC_OBJ_CORE;
321	objtype[PU] = HWLOC_OBJ_PU;	334	objtype[PU] = HWLOC_OBJ_PU;
322	#if HWLOC_API_VERSION >= 0x00020000	335	#if HWLOC_API_VERSION >= 0x00020000
Lines 335-348 extern int xcpuinfo_hwloc_topo_get( Link Here
335	}	348	}
336	#endif	349	#endif
337		350
338	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
339	#if HWLOC_API_VERSION >= 0x00020000
340	objtype[SOCKET] = HWLOC_OBJ_L3CACHE;
341	#else
342	error("SlurmdParameters=l3cache_as_socket requires hwloc v2");
343	#endif
344	}
345
346	/* Groups below root obj are interpreted as boards */	351	/* Groups below root obj are interpreted as boards */
347	obj = hwloc_get_root_obj(topology);	352	obj = hwloc_get_root_obj(topology);
348	obj = hwloc_get_next_child(topology, obj, NULL);	353	obj = hwloc_get_next_child(topology, obj, NULL);

Lines 258-270 handle_write: Link Here

(-)a/src/slurmd/common/xcpuinfo.c (-1 / +18 lines)
258	extern hwloc_obj_type_t slurmd_parameter_as_socket(void)	258	extern hwloc_obj_type_t slurmd_parameter_as_socket(void)
259	{	259	{
260	hwloc_obj_type_t obj_as_socket = HWLOC_OBJ_SOCKET;	260	hwloc_obj_type_t obj_as_socket = HWLOC_OBJ_SOCKET;
		261
261	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {	262	if (xstrcasestr(slurm_conf.slurmd_params, "l3cache_as_socket")) {
262	#if HWLOC_API_VERSION >= 0x00020000	263	#if HWLOC_API_VERSION >= 0x00020000
263	obj_as_socket = HWLOC_OBJ_L3CACHE;	264	obj_as_socket = HWLOC_OBJ_L3CACHE;
264	#else	265	#else
265	error("SlurmdParameters=l3cache_as_socket requires hwloc v2");	266	error("SlurmdParameters=l3cache_as_socket requires hwloc v2");
		267	#endif
		268	} else if (xstrcasestr(slurm_conf.slurmd_params,
		269	"numa_node_as_socket")) {
		270	#if HWLOC_API_VERSION >= 0x00020000
		271	obj = hwloc_get_next_obj_by_type(topology, HWLOC_OBJ_NODE,
		272	NULL);
		273	if (obj != NULL && obj->parent != NULL) {
		274	char tmp[128];
		275	obj_as_socket = obj->parent->type;
		276	hwloc_obj_type_snprintf(tmp, sizeof(tmp), obj->parent,
		277	0);
		278	debug2("%s: numa_node_as_socket mapped to '%s'",
		279	__func__, tmp);
		280	}
		281	#else
		282	error("SlurmdParameters=numa_node_as_socket requires hwloc v2");
266	#endif	283	#endif
267	}	284	}
		285
268	return obj_as_socket;	286	return obj_as_socket;
269	}	287	}
270	/*	288	/*
271	-

Lines 40-51 Link Here

(-)a/src/slurmd/common/xcpuinfo.h (-2 / +7 lines)
40	#define XCPUINFO_ERROR 1	40	#define XCPUINFO_ERROR 1
41	#define XCPUINFO_SUCCESS 0	41	#define XCPUINFO_SUCCESS 0
42		42
		43	#ifdef HAVE_HWLOC
		44	#include <hwloc.h>
		45	/* Return a hwloc object used as a socket */
		46	extern hwloc_obj_type_t slurmd_parameter_as_socket(void);
		47	#endif
		48
43	extern int get_procs(uint16_t *procs);	49	extern int get_procs(uint16_t *procs);
44		50
45	/* read or load topology and write if needed	51	/* read or load topology and write if needed
46	* init and destroy topology must be outside this function */	52	* init and destroy topology must be outside this function */
47	extern int xcpuinfo_hwloc_topo_load(	53	extern int xcpuinfo_hwloc_topo_load(
48	void topology_in, char topo_file, bool full);	54	void topology_in, char topo_file, bool full);
		55
49	/*	56	/*
50	* Get the node's cpu info.	57	* Get the node's cpu info.
51	*	58	*
52	-
53	--
54	doc/man/man5/slurm.conf.5 \| 9 +++++++++	59	doc/man/man5/slurm.conf.5 \| 9 +++++++++
55	src/slurmd/common/xcpuinfo.c \| 18 ++++++++++++++++++	60	src/slurmd/common/xcpuinfo.c \| 18 ++++++++++++++++++
56	2 files changed, 27 insertions(+)	61	2 files changed, 27 insertions(+)