Attachment #2999 for ticket #2634




value.
The default value is zero, which will reserve resources for any pending job
and delay initiation of lower priority jobs.
Also see bf_min_prio_reserve.
.TP
\fBbf_min_prio_reserve=#\fR
The backfill and main scheduling logic will not reserve resources for pending
jobs unless they have a priority equal to or higher than the specified value.
In addition, jobs with a lower priority will not prevent a newly submitted job
from starting immediately, even if the newly submitted job has a lower priority.
This can be valuable if one wished to maximum system utilization without regard
for job priority below a certain threshold.
The default value is zero, which will reserve resources for any pending job
and delay initiation of lower priority jobs.
Also see bf_min_age_reserve.
.TP
\fBbf_resolution=#\fR
The number of seconds in the resolution of data maintained about when jobs

Lines 1011-1016 enum ctx_keys { Link Here

(-)a/slurm/slurm.h.in (+1 lines)
1011	#define NO_KILL_INV_DEP 0x00000002	1011	#define NO_KILL_INV_DEP 0x00000002
1012	#define HAS_STATE_DIR 0x00000004	1012	#define HAS_STATE_DIR 0x00000004
1013	#define BACKFILL_TEST 0x00000008 /* Backfill test in progress */	1013	#define BACKFILL_TEST 0x00000008 /* Backfill test in progress */
		1014	#define TEST_NOW_ONLY 0x00000020 /* Test for immediately start only */
1014		1015
1015	/*****************************************************************************\	1016	/*****************************************************************************\
1016	* SLURM HOSTLIST FUNCTIONS	1017	* SLURM HOSTLIST FUNCTIONS




static int backfill_window = BACKFILL_WINDOW;
static int bf_max_job_array_resv = BF_MAX_JOB_ARRAY_RESV;
static int bf_min_age_reserve = 0;
static uint32_t bf_min_prio_reserve = 0;
static int max_backfill_job_cnt = 100;
static int max_backfill_job_per_part = 0;
static int max_backfill_job_per_user = 0;

		bf_min_age_reserve = 0;
	}

	bf_min_prio_reserve = 0;
	if (sched_params &&
	    (tmp_ptr = strstr(sched_params, "bf_min_prio_reserve="))) {
		int64_t min_prio = (int64_t) atoll(tmp_ptr + 20);
		if (min_prio < 0) {
			error("Invalid SchedulerParameters bf_min_prio_reserve: %"PRIi64,
			      min_prio);
		} else {
			bf_min_prio_reserve = (uint32_t) min_prio;
		}
	}

	/* bf_continue makes backfill continue where it was if interrupted
	 */
	if (sched_params && (strstr(sched_params, "bf_continue"))) {

	struct timeval start_tv;
	uint32_t test_array_job_id = 0;
	uint32_t test_array_count = 0;
	uint32_t acct_max_nodes, wait_reason = 0, job_no_reserve;
	bool resv_overlap = false;

	bf_sleep_usec = 0;


		part_ptr = job_queue_rec->part_ptr;
		job_ptr->part_ptr = part_ptr;
		job_ptr->priority = job_queue_rec->priority;
		if (job_ptr->state_reason == FAIL_ACCOUNT) {
			slurmdb_assoc_rec_t assoc_rec;
			memset(&assoc_rec, 0, sizeof(slurmdb_assoc_rec_t));

		    !acct_policy_job_runnable_pre_select(job_ptr))
			continue;

		job_no_reserve = 0;
		if (bf_min_prio_reserve &&
		    (job_ptr->priority < bf_min_prio_reserve)) {
			job_no_reserve = TEST_NOW_ONLY;
		} else if (bf_min_age_reserve && job_ptr->details->begin_time) {
			pend_time = difftime(time(NULL),
					     job_ptr->details->begin_time);
			if (pend_time < bf_min_age_reserve)
				job_no_reserve = TEST_NOW_ONLY;
		}

		orig_start_time = job_ptr->start_time;
		orig_time_limit = job_ptr->time_limit;
		xfree(job_queue_rec);

		if (debug_flags & DEBUG_FLAG_BACKFILL_MAP)
			_dump_job_test(job_ptr, avail_bitmap, start_res);
		job_ptr->bit_flags |= BACKFILL_TEST;
		job_ptr->bit_flags |= job_no_reserve;	/* 0 or TEST_NOW_ONLY */
		j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes,
			       req_nodes, exc_core_bitmap);
		job_ptr->bit_flags &= ~BACKFILL_TEST;
		job_ptr->bit_flags &= ~TEST_NOW_ONLY;

		now = time(NULL);
		if (j != SLURM_SUCCESS) {

			_set_job_time_limit(job_ptr, orig_time_limit);
		}

		if ((job_ptr->start_time > now) && (job_no_reserve != 0))
			continue;
			pend_time = difftime(time(NULL),
					     job_ptr->details->begin_time);
			if (pend_time < bf_min_age_reserve)
				continue;
		}

		if (later_start && (job_ptr->start_time > later_start)) {
			/* Try later when some nodes currently reserved for

Lines 1845-1851 static int _will_run_test(struct job_record job_ptr, bitstr_t bitmap, Link Here

(-)a/src/plugins/select/cons_res/select_cons_res.c (-1 / +2 lines)
1845		1845
1846	/* Remove the running jobs one at a time from exp_node_cr and try	1846	/* Remove the running jobs one at a time from exp_node_cr and try
1847	* scheduling the pending job after each one. */	1847	* scheduling the pending job after each one. */
1848	if (rc != SLURM_SUCCESS) {	1848	if ((rc != SLURM_SUCCESS) &&
		1849	((job_ptr->bit_flags & TEST_NOW_ONLY) == 0)) {
1849	list_sort(cr_job_list, _cr_job_list_sort);	1850	list_sort(cr_job_list, _cr_job_list_sort);
1850	job_iterator = list_iterator_create(cr_job_list);	1851	job_iterator = list_iterator_create(cr_job_list);
1851	while ((tmp_job_ptr = list_next(job_iterator))) {	1852	while ((tmp_job_ptr = list_next(job_iterator))) {

Lines 3392-3398 static int _will_run_test(struct job_record job_ptr, bitstr_t bitmap, Link Here

(-)a/src/plugins/select/linear/select_linear.c (-1 / +2 lines)
3392		3392
3393	/* Remove the running jobs one at a time from exp_node_cr and try	3393	/* Remove the running jobs one at a time from exp_node_cr and try
3394	* scheduling the pending job after each one */	3394	* scheduling the pending job after each one */
3395	if (rc != SLURM_SUCCESS) {	3395	if ((rc != SLURM_SUCCESS) &&
		3396	((job_ptr->bit_flags & TEST_NOW_ONLY) == 0)) {
3396	list_sort(cr_job_list, _cr_job_list_sort);	3397	list_sort(cr_job_list, _cr_job_list_sort);
3397	job_iterator = list_iterator_create(cr_job_list);	3398	job_iterator = list_iterator_create(cr_job_list);
3398	while ((tmp_job_ptr = (struct job_record *)	3399	while ((tmp_job_ptr = (struct job_record *)

Lines 1291-1297 static int _will_run_test(struct job_record job_ptr, bitstr_t bitmap, Link Here

(-)a/src/plugins/select/serial/select_serial.c (-1 / +2 lines)
1291		1291
1292	/* Remove the running jobs one at a time from exp_node_cr and try	1292	/* Remove the running jobs one at a time from exp_node_cr and try
1293	* scheduling the pending job after each one. */	1293	* scheduling the pending job after each one. */
1294	if (rc != SLURM_SUCCESS) {	1294	if ((rc != SLURM_SUCCESS) &&
		1295	((job_ptr->bit_flags & TEST_NOW_ONLY) == 0)) {
1295	list_sort(cr_job_list, _cr_job_list_sort);	1296	list_sort(cr_job_list, _cr_job_list_sort);
1296	job_iterator = list_iterator_create(cr_job_list);	1297	job_iterator = list_iterator_create(cr_job_list);
1297	while ((tmp_job_ptr = list_next(job_iterator))) {	1298	while ((tmp_job_ptr = list_next(job_iterator))) {

Return to ticket 2634

Lines 2482-2487 This can be valuable if jobs lack time limits or all time limits have the same Link Here

(-)a/doc/man/man5/slurm.conf.5 (+12 lines)
2482	value.	2482	value.
2483	The default value is zero, which will reserve resources for any pending job	2483	The default value is zero, which will reserve resources for any pending job
2484	and delay initiation of lower priority jobs.	2484	and delay initiation of lower priority jobs.
		2485	Also see bf_min_prio_reserve.
		2486	.TP
		2487	\fBbf_min_prio_reserve=#\fR
		2488	The backfill and main scheduling logic will not reserve resources for pending
		2489	jobs unless they have a priority equal to or higher than the specified value.
		2490	In addition, jobs with a lower priority will not prevent a newly submitted job
		2491	from starting immediately, even if the newly submitted job has a lower priority.
		2492	This can be valuable if one wished to maximum system utilization without regard
		2493	for job priority below a certain threshold.
		2494	The default value is zero, which will reserve resources for any pending job
		2495	and delay initiation of lower priority jobs.
		2496	Also see bf_min_age_reserve.
2485	.TP	2497	.TP
2486	\fBbf_resolution=#\fR	2498	\fBbf_resolution=#\fR
2487	The number of seconds in the resolution of data maintained about when jobs	2499	The number of seconds in the resolution of data maintained about when jobs

Lines 127-132 static int backfill_resolution = BACKFILL_RESOLUTION; Link Here

(-)a/src/plugins/sched/backfill/backfill.c (-8 / +30 lines)
127	static int backfill_window = BACKFILL_WINDOW;	127	static int backfill_window = BACKFILL_WINDOW;
128	static int bf_max_job_array_resv = BF_MAX_JOB_ARRAY_RESV;	128	static int bf_max_job_array_resv = BF_MAX_JOB_ARRAY_RESV;
129	static int bf_min_age_reserve = 0;	129	static int bf_min_age_reserve = 0;
		130	static uint32_t bf_min_prio_reserve = 0;
130	static int max_backfill_job_cnt = 100;	131	static int max_backfill_job_cnt = 100;
131	static int max_backfill_job_per_part = 0;	132	static int max_backfill_job_per_part = 0;
132	static int max_backfill_job_per_user = 0;	133	static int max_backfill_job_per_user = 0;
Lines 583-588 static void _load_config(void) Link Here
583	bf_min_age_reserve = 0;	584	bf_min_age_reserve = 0;
584	}	585	}
585		586
		587	bf_min_prio_reserve = 0;
		588	if (sched_params &&
		589	(tmp_ptr = strstr(sched_params, "bf_min_prio_reserve="))) {
		590	int64_t min_prio = (int64_t) atoll(tmp_ptr + 20);
		591	if (min_prio < 0) {
		592	error("Invalid SchedulerParameters bf_min_prio_reserve: %"PRIi64,
		593	min_prio);
		594	} else {
		595	bf_min_prio_reserve = (uint32_t) min_prio;
		596	}
		597	}
		598
586	/* bf_continue makes backfill continue where it was if interrupted	599	/* bf_continue makes backfill continue where it was if interrupted
587	*/	600	*/
588	if (sched_params && (strstr(sched_params, "bf_continue"))) {	601	if (sched_params && (strstr(sched_params, "bf_continue"))) {
Lines 825-831 static int _attempt_backfill(void) Link Here
825	struct timeval start_tv;	838	struct timeval start_tv;
826	uint32_t test_array_job_id = 0;	839	uint32_t test_array_job_id = 0;
827	uint32_t test_array_count = 0;	840	uint32_t test_array_count = 0;
828	uint32_t acct_max_nodes, wait_reason = 0;	841	uint32_t acct_max_nodes, wait_reason = 0, job_no_reserve;
829	bool resv_overlap = false;	842	bool resv_overlap = false;
830		843
831	bf_sleep_usec = 0;	844	bf_sleep_usec = 0;
Lines 984-989 static int _attempt_backfill(void) Link Here
984		997
985	part_ptr = job_queue_rec->part_ptr;	998	part_ptr = job_queue_rec->part_ptr;
986	job_ptr->part_ptr = part_ptr;	999	job_ptr->part_ptr = part_ptr;
		1000	job_ptr->priority = job_queue_rec->priority;
987	if (job_ptr->state_reason == FAIL_ACCOUNT) {	1001	if (job_ptr->state_reason == FAIL_ACCOUNT) {
988	slurmdb_assoc_rec_t assoc_rec;	1002	slurmdb_assoc_rec_t assoc_rec;
989	memset(&assoc_rec, 0, sizeof(slurmdb_assoc_rec_t));	1003	memset(&assoc_rec, 0, sizeof(slurmdb_assoc_rec_t));
Lines 1036-1041 static int _attempt_backfill(void) Link Here
1036	!acct_policy_job_runnable_pre_select(job_ptr))	1050	!acct_policy_job_runnable_pre_select(job_ptr))
1037	continue;	1051	continue;
1038		1052
		1053	job_no_reserve = 0;
		1054	if (bf_min_prio_reserve &&
		1055	(job_ptr->priority < bf_min_prio_reserve)) {
		1056	job_no_reserve = TEST_NOW_ONLY;
		1057	} else if (bf_min_age_reserve && job_ptr->details->begin_time) {
		1058	pend_time = difftime(time(NULL),
		1059	job_ptr->details->begin_time);
		1060	if (pend_time < bf_min_age_reserve)
		1061	job_no_reserve = TEST_NOW_ONLY;
		1062	}
		1063
1039	orig_start_time = job_ptr->start_time;	1064	orig_start_time = job_ptr->start_time;
1040	orig_time_limit = job_ptr->time_limit;	1065	orig_time_limit = job_ptr->time_limit;
1041	xfree(job_queue_rec);	1066	xfree(job_queue_rec);
Lines 1357-1365 next_task: Link Here
1357	if (debug_flags & DEBUG_FLAG_BACKFILL_MAP)	1382	if (debug_flags & DEBUG_FLAG_BACKFILL_MAP)
1358	_dump_job_test(job_ptr, avail_bitmap, start_res);	1383	_dump_job_test(job_ptr, avail_bitmap, start_res);
1359	job_ptr->bit_flags \|= BACKFILL_TEST;	1384	job_ptr->bit_flags \|= BACKFILL_TEST;
		1385	job_ptr->bit_flags \|= job_no_reserve; /* 0 or TEST_NOW_ONLY */
1360	j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes,	1386	j = _try_sched(job_ptr, &avail_bitmap, min_nodes, max_nodes,
1361	req_nodes, exc_core_bitmap);	1387	req_nodes, exc_core_bitmap);
1362	job_ptr->bit_flags &= ~BACKFILL_TEST;	1388	job_ptr->bit_flags &= ~BACKFILL_TEST;
		1389	job_ptr->bit_flags &= ~TEST_NOW_ONLY;
1363		1390
1364	now = time(NULL);	1391	now = time(NULL);
1365	if (j != SLURM_SUCCESS) {	1392	if (j != SLURM_SUCCESS) {
Lines 1512-1524 next_task: Link Here
1512	_set_job_time_limit(job_ptr, orig_time_limit);	1539	_set_job_time_limit(job_ptr, orig_time_limit);
1513	}	1540	}
1514		1541
1515	if ((job_ptr->start_time > now) &&	1542	if ((job_ptr->start_time > now) && (job_no_reserve != 0))
1516	(bf_min_age_reserve && job_ptr->details->begin_time)) {	1543	continue;
1517	pend_time = difftime(time(NULL),
1518	job_ptr->details->begin_time);
1519	if (pend_time < bf_min_age_reserve)
1520	continue;
1521	}
1522		1544
1523	if (later_start && (job_ptr->start_time > later_start)) {	1545	if (later_start && (job_ptr->start_time > later_start)) {
1524	/* Try later when some nodes currently reserved for	1546	/* Try later when some nodes currently reserved for