Attachment #9931 for ticket #4105

View | Details | Raw Unified | Return to ticket 4105
Collapse All | Expand All

Lines 62-67 typedef struct slurm_switch_ops { Link Here

(-)a/src/common/switch.c (+19 lines)
62	int (build_jobinfo) ( switch_jobinfo_t jobinfo,	62	int (build_jobinfo) ( switch_jobinfo_t jobinfo,
63	slurm_step_layout_t *step_layout,	63	slurm_step_layout_t *step_layout,
64	char *network);	64	char *network);
		65	int (duplicate_jobinfo) ( switch_jobinfo_t source,
		66	switch_jobinfo_t **dest);
65	void (free_jobinfo) ( switch_jobinfo_t jobinfo );	67	void (free_jobinfo) ( switch_jobinfo_t jobinfo );
66	int (pack_jobinfo) ( switch_jobinfo_t jobinfo,	68	int (pack_jobinfo) ( switch_jobinfo_t jobinfo,
67	Buf buffer,	69	Buf buffer,
Lines 139-144 static const char *syms[] = { Link Here
139	"switch_p_libstate_restore",	141	"switch_p_libstate_restore",
140	"switch_p_alloc_jobinfo",	142	"switch_p_alloc_jobinfo",
141	"switch_p_build_jobinfo",	143	"switch_p_build_jobinfo",
		144	"switch_p_duplicate_jobinfo",
142	"switch_p_free_jobinfo",	145	"switch_p_free_jobinfo",
143	"switch_p_pack_jobinfo",	146	"switch_p_pack_jobinfo",
144	"switch_p_unpack_jobinfo",	147	"switch_p_unpack_jobinfo",
Lines 373-378 extern int switch_g_build_jobinfo(dynamic_plugin_data_t *jobinfo, Link Here
373	return (*(ops[plugin_id].build_jobinfo))(data, step_layout, network);	376	return (*(ops[plugin_id].build_jobinfo))(data, step_layout, network);
374	}	377	}
375		378
		379	extern int switch_g_duplicate_jobinfo(dynamic_plugin_data_t *source,
		380	dynamic_plugin_data_t **dest)
		381	{
		382	dynamic_plugin_data_t *dest_ptr = NULL;
		383	uint32_t plugin_id = source->plugin_id;
		384
		385	if ( switch_init(0) < 0 )
		386	return SLURM_ERROR;
		387
		388	dest_ptr = xmalloc(sizeof(dynamic_plugin_data_t));
		389	dest_ptr->plugin_id = plugin_id;
		390	*dest = dest_ptr;
		391
		392	return ((ops[plugin_id].duplicate_jobinfo))((switch_jobinfo_t )source->data, (switch_jobinfo_t **)&dest_ptr->data);
		393	}
		394
376	extern void switch_g_free_jobinfo(dynamic_plugin_data_t *jobinfo)	395	extern void switch_g_free_jobinfo(dynamic_plugin_data_t *jobinfo)
377	{	396	{
378	if ( switch_init(0) < 0 )	397	if ( switch_init(0) < 0 )

Lines 119-124 extern int switch_g_build_jobinfo(dynamic_plugin_data_t *jobinfo, Link Here

(-)a/src/common/switch.h (+9 lines)
119	slurm_step_layout_t *step_layout,	119	slurm_step_layout_t *step_layout,
120	char *network);	120	char *network);
121		121
		122	/* duplicate a job's switch credential
		123	* IN source - storage for a switch job credential
		124	* OUT dest - pointer to NULL at beginning, will point to storage for
		125	* duplicated switch job credential
		126	* NOTE: storage must be freed using g_switch_g_free_jobinfo
		127	*/
		128	extern int switch_g_duplicate_jobinfo(dynamic_plugin_data_t *source,
		129	dynamic_plugin_data_t **dest);
		130
122	/* free storage previously allocated for a switch job credential	131	/* free storage previously allocated for a switch job credential
123	* IN jobinfo - the switch job credential to be freed	132	* IN jobinfo - the switch job credential to be freed
124	*/	133	*/

Lines 94-100 int build_alpsc_pe_info(stepd_step_rec_t *job, Link Here

(-)a/src/plugins/switch/cray/pe_info.c (-16 / +58 lines)
94	}	94	}
95		95
96	// Fill in the structure	96	// Fill in the structure
97	alpsc_pe_info->totalPEs = job->ntasks;	97	if (job->pack_jobid != NO_VAL) {
		98	alpsc_pe_info->totalPEs = job->pack_ntasks;
		99	} else {
		100	alpsc_pe_info->totalPEs = job->ntasks;
		101	}
98	alpsc_pe_info->firstPeHere = _get_first_pe(job);	102	alpsc_pe_info->firstPeHere = _get_first_pe(job);
99	alpsc_pe_info->pesHere = job->node_tasks;	103	alpsc_pe_info->pesHere = job->node_tasks;
100	alpsc_pe_info->peDepth = job->cpus_per_task;	104	alpsc_pe_info->peDepth = job->cpus_per_task;
Lines 146-152 static int _get_cmd_map(stepd_step_rec_t job) Link Here
146	int cmd_index, i, pe;	150	int cmd_index, i, pe;
147	int *cmd_map = NULL;	151	int *cmd_map = NULL;
148		152
149	size = job->ntasks * sizeof(int);	153	int nnodes = job->nnodes;
		154	int ntasks = job->ntasks;
		155	char *complete_nodelist = job->msg->complete_nodelist;
		156	uint16_t *tasks_to_launch = job->msg->tasks_to_launch;
		157	if (job->pack_jobid != NO_VAL) {
		158	nnodes = job->pack_nnodes;
		159	ntasks = job->pack_ntasks;
		160	complete_nodelist = job->pack_node_list;
		161	tasks_to_launch = job->pack_task_cnts;
		162	}
		163
		164	size = ntasks * sizeof(int);
150	cmd_map = xmalloc(size);	165	cmd_map = xmalloc(size);
151	if (job->mpmd_set) {	166	if (job->mpmd_set) {
152	// Multiple programs, fill in from mpmd_set information	167	// Multiple programs, fill in from mpmd_set information
Lines 198-223 static int _get_pe_nid_map(stepd_step_rec_t job) Link Here
198	int32_t *nodes = NULL;	213	int32_t *nodes = NULL;
199	int tasks_to_launch_sum, nid;	214	int tasks_to_launch_sum, nid;
200		215
201	size = job->ntasks * sizeof(int);	216	int nnodes = job->nnodes;
		217	int ntasks = job->ntasks;
		218	char *complete_nodelist = job->msg->complete_nodelist;
		219	uint16_t *tasks_to_launch = job->msg->tasks_to_launch;
		220	if (job->pack_jobid != NO_VAL) {
		221	nnodes = job->pack_nnodes;
		222	ntasks = job->pack_ntasks;
		223	complete_nodelist = job->pack_node_list;
		224	tasks_to_launch = job->pack_task_cnts;
		225	}
		226
		227	size = ntasks * sizeof(int);
202	pe_nid_map = xmalloc(size);	228	pe_nid_map = xmalloc(size);
203		229
204	// If we have it, just copy the mpmd set information	230	// If we have it, just copy the mpmd set information
		231	/* TODO: this is not configured for hetjob yet */
205	if (job->mpmd_set && job->mpmd_set->placement) {	232	if (job->mpmd_set && job->mpmd_set->placement) {
206	// mpmd_set->placement is an int * too so this works	233	// mpmd_set->placement is an int * too so this works
207	memcpy(pe_nid_map, job->mpmd_set->placement, size);	234	memcpy(pe_nid_map, job->mpmd_set->placement, size);
208	} else {	235	} else {
209	// Initialize to -1 so we can tell if we missed any	236	// Initialize to -1 so we can tell if we missed any
210	for (i = 0; i < job->ntasks; i++) {	237	for (i = 0; i < ntasks; i++) {
211	pe_nid_map[i] = -1;	238	pe_nid_map[i] = -1;
212	}	239	}
213		240
214	// Convert the node list to an array of nids	241	// Convert the node list to an array of nids
215	rc = list_str_to_array(job->msg->complete_nodelist, &cnt,	242	rc = list_str_to_array(complete_nodelist, &cnt,
216	&nodes);	243	&nodes);
217	if (rc < 0) {	244	if (rc < 0) {
218	xfree(pe_nid_map);	245	xfree(pe_nid_map);
219	return NULL;	246	return NULL;
220	} else if (job->nnodes != cnt) {	247	} else if (nnodes != cnt) {
221	CRAY_ERR("list_str_to_array cnt %d expected %u",	248	CRAY_ERR("list_str_to_array cnt %d expected %u",
222	cnt, job->nnodes);	249	cnt, job->nnodes);
223	xfree(pe_nid_map);	250	xfree(pe_nid_map);
Lines 227-247 static int _get_pe_nid_map(stepd_step_rec_t job) Link Here
227		254
228	// Search the task id map for the values we need	255	// Search the task id map for the values we need
229	tasks_to_launch_sum = 0;	256	tasks_to_launch_sum = 0;
230	for (i = 0; i < job->nnodes; i++) {	257	for (i = 0; i < nnodes; i++) {
231	tasks_to_launch_sum += job->msg->tasks_to_launch[i];	258	tasks_to_launch_sum += tasks_to_launch[i];
232	for (j = 0; j < job->msg->tasks_to_launch[i]; j++) {	259	info("DMJ: tasks_to_launch[%d]: %d, tasks_to_launch_sum: %d", i, tasks_to_launch[i], tasks_to_launch_sum);
233	task = job->msg->global_task_ids[i][j];	260	for (j = 0; j < tasks_to_launch[i]; j++) {
		261	/* inappropriate hack */
		262	//task = job->msg->global_task_ids[i][j];
		263	task = i * (nnodes - 1) + j;
234	pe_nid_map[task] = nodes[i];	264	pe_nid_map[task] = nodes[i];
		265	info("DMJ: setting pe_nid_map[%d] = %d, i=%d, j=%d, nnodes=%d", task, nodes[i], i, j, nnodes);
235	}	266	}
236	}	267	}
237		268
238	// If this is LAM/MPI only one task per node is launched,	269	// If this is LAM/MPI only one task per node is launched,
239	// NOT job->ntasks. So fill in the rest of the tasks	270	// NOT job->ntasks. So fill in the rest of the tasks
240	// assuming a block distribution	271	// assuming a block distribution
241	if (tasks_to_launch_sum == job->nnodes	272	if (tasks_to_launch_sum == nnodes
242	&& job->nnodes < job->ntasks) {	273	&& nnodes < ntasks) {
243	nid = nodes[0]; // failsafe value	274	nid = nodes[0]; // failsafe value
244	for (i = 0; i < job->ntasks; i++) {	275	for (i = 0; i < ntasks; i++) {
245	if (pe_nid_map[i] > -1) {	276	if (pe_nid_map[i] > -1) {
246	nid = pe_nid_map[i];	277	nid = pe_nid_map[i];
247	} else {	278	} else {
Lines 252-258 static int _get_pe_nid_map(stepd_step_rec_t job) Link Here
252	xfree(nodes);	283	xfree(nodes);
253		284
254	// Make sure we didn't miss any tasks	285	// Make sure we didn't miss any tasks
255	for (i = 0; i < job->ntasks; i++) {	286	for (i = 0; i < ntasks; i++) {
256	if (pe_nid_map[i] == -1) {	287	if (pe_nid_map[i] == -1) {
257	CRAY_ERR("No NID for PE index %d", i);	288	CRAY_ERR("No NID for PE index %d", i);
258	xfree(pe_nid_map);	289	xfree(pe_nid_map);
Lines 271-279 static int _get_node_cpu_map(stepd_step_rec_t job) Link Here
271	int *node_cpu_map;	302	int *node_cpu_map;
272	int nodeid;	303	int nodeid;
273		304
		305	int nnodes = job->nnodes;
		306	int ntasks = job->ntasks;
		307	char *complete_nodelist = job->msg->complete_nodelist;
		308	uint16_t *tasks_to_launch = job->msg->tasks_to_launch;
		309	if (job->pack_jobid != NO_VAL) {
		310	nnodes = job->pack_nnodes;
		311	ntasks = job->pack_ntasks;
		312	complete_nodelist = job->pack_node_list;
		313	tasks_to_launch = job->pack_task_cnts;
		314	}
		315
274	node_cpu_map = xmalloc(job->nnodes * sizeof(int));	316	node_cpu_map = xmalloc(job->nnodes * sizeof(int));
275	for (nodeid = 0; nodeid < job->nnodes; nodeid++) {	317	for (nodeid = 0; nodeid < nnodes; nodeid++) {
276	node_cpu_map[nodeid] = (job->msg->tasks_to_launch[nodeid]	318	node_cpu_map[nodeid] = (tasks_to_launch[nodeid]
277	* job->cpus_per_task);	319	* job->cpus_per_task);
278	}	320	}
279		321




	return SLURM_SUCCESS;
}

extern int switch_p_duplicate_jobinfo(switch_jobinfo_t *source,
				      switch_jobinfo_t **dest)
{
	slurm_cray_jobinfo_t *new;
	slurm_cray_jobinfo_t *old = (slurm_cray_jobinfo_t *) source;

	new = (slurm_cray_jobinfo_t *) xmalloc(sizeof(slurm_cray_jobinfo_t));
	memcpy(new, old, sizeof(slurm_cray_jobinfo_t));

	if (old->num_cookies != 0) {
		new->cookie_ids = xmalloc(sizeof(uint32_t) * old->num_cookies);
		memcpy(new->cookie_ids, old->cookie_ids, sizeof(uint32_t) * old->num_cookies);
		new->cookies = xmalloc(sizeof(char *) * old->num_cookies);
		for (int i = 0; i < old->num_cookies; i++)
			new->cookies[i] = xstrdup(old->cookies[i]);
	}

	if (old->num_ptags) {
		new->ptags = xmalloc(sizeof(int) * old->num_ptags);
		memcpy(new->ptags, old->ptags, sizeof(int) * old->num_ptags);
	}

	xassert(old);
	*dest = (switch_jobinfo_t *) new;
	return SLURM_SUCCESS;
}

/*
 *
 */




	return SLURM_SUCCESS;
}

int switch_p_duplicate_jobinfo(switch_jobinfo_t *source,
			       switch_jobinfo_t **dest)
{
	sw_gen_step_info_t *gen_step_info;

	if (debug_flags & DEBUG_FLAG_SWITCH)
		info("switch_p_alloc_jobinfo() starting");
	xassert(switch_job);
	gen_step_info = xmalloc(sizeof(sw_gen_step_info_t));
	gen_step_info->magic = SW_GEN_STEP_INFO_MAGIC;
	*dest = (switch_jobinfo_t *) gen_step_info;

	return SLURM_SUCCESS;
}

void switch_p_free_jobinfo(switch_jobinfo_t *switch_job)
{
	sw_gen_step_info_t *gen_step_info = (sw_gen_step_info_t *) switch_job;

Lines 126-131 int switch_p_build_jobinfo ( switch_jobinfo_t *switch_job, Link Here

(-)a/src/plugins/switch/none/switch_none.c (+5 lines)
126	return SLURM_SUCCESS;	126	return SLURM_SUCCESS;
127	}	127	}
128		128
		129	int switch_p_duplicate_jobinfo (switch_jobinfo_t *tmp,
		130	switch_jobinfo_t **dest)
		131	{
		132	return SLURM_SUCCESS;
		133	}
129	void switch_p_free_jobinfo ( switch_jobinfo_t *switch_job )	134	void switch_p_free_jobinfo ( switch_jobinfo_t *switch_job )
130	{	135	{
131	return;	136	return;




			step_layout->node_cnt = hostlist_count(hl);
			hostlist_destroy(hl);
			tmp_step_layout_used = true;
		} else {
			/* assume that job offset 0 has already run! */
			struct job_record *het_job_ptr = find_job_pack_record(job_ptr->pack_job_id, 0);
			struct step_record *het_step_ptr;
			ListIterator itr;

			itr = list_iterator_create(het_job_ptr->step_list);
			while ((het_step_ptr = list_next(itr)))
				if (het_step_ptr->step_id == step_ptr->step_id)
					break;

			if (het_step_ptr)
				switch_g_duplicate_jobinfo(het_step_ptr->switch_job, &step_ptr->switch_job);

			/* prevent switch_g_build_jobinfo from getting a new
			 * cookie below */
			step_layout = NULL;
		}
	} else {
		step_layout = step_ptr->step_layout;
		jobid = job_ptr->job_id;

Return to ticket 4105

Lines 415-420 int switch_p_build_jobinfo(switch_jobinfo_t *switch_job, Link Here

(-)a/src/plugins/switch/generic/switch_generic.c (+15 lines)
415	return SLURM_SUCCESS;	415	return SLURM_SUCCESS;
416	}	416	}
417		417
		418	int switch_p_duplicate_jobinfo(switch_jobinfo_t *source,
		419	switch_jobinfo_t **dest)
		420	{
		421	sw_gen_step_info_t *gen_step_info;
		422
		423	if (debug_flags & DEBUG_FLAG_SWITCH)
		424	info("switch_p_alloc_jobinfo() starting");
		425	xassert(switch_job);
		426	gen_step_info = xmalloc(sizeof(sw_gen_step_info_t));
		427	gen_step_info->magic = SW_GEN_STEP_INFO_MAGIC;
		428	dest = (switch_jobinfo_t ) gen_step_info;
		429
		430	return SLURM_SUCCESS;
		431	}
		432
418	void switch_p_free_jobinfo(switch_jobinfo_t *switch_job)	433	void switch_p_free_jobinfo(switch_jobinfo_t *switch_job)
419	{	434	{
420	sw_gen_step_info_t gen_step_info = (sw_gen_step_info_t ) switch_job;	435	sw_gen_step_info_t gen_step_info = (sw_gen_step_info_t ) switch_job;

Lines 2801-2808 step_create(job_step_create_request_msg_t *step_specs, Link Here

(-)a/src/slurmctld/step_mgr.c (-1 / +17 lines)
2801	step_layout->node_cnt = hostlist_count(hl);	2801	step_layout->node_cnt = hostlist_count(hl);
2802	hostlist_destroy(hl);	2802	hostlist_destroy(hl);
2803	tmp_step_layout_used = true;	2803	tmp_step_layout_used = true;
2804	} else	2804	} else {
		2805	/* assume that job offset 0 has already run! */
		2806	struct job_record *het_job_ptr = find_job_pack_record(job_ptr->pack_job_id, 0);
		2807	struct step_record *het_step_ptr;
		2808	ListIterator itr;
		2809
		2810	itr = list_iterator_create(het_job_ptr->step_list);
		2811	while ((het_step_ptr = list_next(itr)))
		2812	if (het_step_ptr->step_id == step_ptr->step_id)
		2813	break;
		2814
		2815	if (het_step_ptr)
		2816	switch_g_duplicate_jobinfo(het_step_ptr->switch_job, &step_ptr->switch_job);
		2817
		2818	/* prevent switch_g_build_jobinfo from getting a new
		2819	* cookie below */
2805	step_layout = NULL;	2820	step_layout = NULL;
		2821	}
2806	} else {	2822	} else {
2807	step_layout = step_ptr->step_layout;	2823	step_layout = step_ptr->step_layout;
2808	jobid = job_ptr->job_id;	2824	jobid = job_ptr->job_id;

Lines 229-234 extern int switch_p_build_jobinfo(switch_jobinfo_t *switch_job, Link Here

(-)a/src/plugins/switch/cray/switch_cray.c (+27 lines)
229	return SLURM_SUCCESS;	229	return SLURM_SUCCESS;
230	}	230	}
231		231
		232	extern int switch_p_duplicate_jobinfo(switch_jobinfo_t *source,
		233	switch_jobinfo_t **dest)
		234	{
		235	slurm_cray_jobinfo_t *new;
		236	slurm_cray_jobinfo_t old = (slurm_cray_jobinfo_t ) source;
		237
		238	new = (slurm_cray_jobinfo_t *) xmalloc(sizeof(slurm_cray_jobinfo_t));
		239	memcpy(new, old, sizeof(slurm_cray_jobinfo_t));
		240
		241	if (old->num_cookies != 0) {
		242	new->cookie_ids = xmalloc(sizeof(uint32_t) * old->num_cookies);
		243	memcpy(new->cookie_ids, old->cookie_ids, sizeof(uint32_t) * old->num_cookies);
		244	new->cookies = xmalloc(sizeof(char ) old->num_cookies);
		245	for (int i = 0; i < old->num_cookies; i++)
		246	new->cookies[i] = xstrdup(old->cookies[i]);
		247	}
		248
		249	if (old->num_ptags) {
		250	new->ptags = xmalloc(sizeof(int) * old->num_ptags);
		251	memcpy(new->ptags, old->ptags, sizeof(int) * old->num_ptags);
		252	}
		253
		254	xassert(old);
		255	dest = (switch_jobinfo_t ) new;
		256	return SLURM_SUCCESS;
		257	}
		258
232	/*	259	/*
233	*	260	*
234	*/	261	*/