Attachment #509 for ticket #478

Lines 140-145 EXTRA_DIST = \ Link Here

(-)a/testsuite/expect/Makefile.am (+3 lines)
140	test2.18 \	140	test2.18 \
141	test2.19 \	141	test2.19 \
142	test2.20 \	142	test2.20 \
		143	test2.21 \
		144	test2.22 \
		145	test2.23 \
143	test3.1 \	146	test3.1 \
144	test3.2 \	147	test3.2 \
145	test3.3 \	148	test3.3 \

Lines 520-525 EXTRA_DIST = \ Link Here

(-)a/testsuite/expect/Makefile.in (+3 lines)
520	test2.18 \	520	test2.18 \
521	test2.19 \	521	test2.19 \
522	test2.20 \	522	test2.20 \
		523	test2.21 \
		524	test2.22 \
		525	test2.23 \
523	test3.1 \	526	test3.1 \
524	test3.2 \	527	test3.2 \
525	test3.3 \	528	test3.3 \

Lines 236-241 test2.17 Validate scontrol displays and updates Allow/Deny Qos. Link Here

(-)a/testsuite/expect/README (+5 lines)
236	test2.18 Validate that Allow/Deny accounts are enforced.	236	test2.18 Validate that Allow/Deny accounts are enforced.
237	test2.19 Validate that Allow/Deny Qos are enforced.	237	test2.19 Validate that Allow/Deny Qos are enforced.
238	test2.20 Validate scontrol show hostnames.	238	test2.20 Validate scontrol show hostnames.
		239	test2.21 Validate that scontrol requeue requeues a completed of failed job.
		240	test2.22 Validate that scontrol requeuehold requeues a completed job and
		241	holds the job.
		242	test2.23 Validate that scontrol requeuehold State option requeues the job
		243	to the specified state and holds the job.
239		244
240		245
241	test3.# Testing of scontrol options (best run as SlurmUser or root).	246	test3.# Testing of scontrol options (best run as SlurmUser or root).




#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Validate that the scontrol requeue option requeues a
#          failed or completed job.
#
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2013 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id            "2.21"
set complete_script    "test$test_id\.complete"
set fail_script        "test$test_id\.fail"
set job_id             0
set exit_code          0

print_header $test_id

make_bash_script $complete_script "
sleep 20
"

make_bash_script $fail_script "
slep 20
"

proc check_state { id } {

	global scontrol exit_code

	set job_state 0
	spawn $scontrol show job $id
	expect {
		-re "JobState=PENDING" {
			set job_state 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: scontrol is not responding\n"
			set exit_code 0
		}
		eof {
			wait
		}
	}

	if {$job_state != 1} {
		send_user "\nFAILURE: Job $id state was not PENDING after it was requeued\n"
		set exit_code 1
	}

}

proc requeue_job { id } {

	global scontrol exit_code

	spawn $scontrol requeue $id
	expect {
		timeout {
			send_user "\nFAILURE: scontrol is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}
}

#
# Run a job that will complete
#
spawn $sbatch -N1 -o /dev/null -e /dev/null -t1 $complete_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sbatch is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	send_user "\nFAILURE: sbatch did not submit job\n"
	set exit_code 1
}

# Wait for the job to be in the complete state
wait_for_job $job_id DONE

# Requeue the job when it is complete
requeue_job $job_id

# Check to see if the job state is PENDING after the requeue
check_state $job_id

cancel_job $job_id

#
# Run a job that will fail
#
spawn $sbatch -N1 -o /dev/null -e /dev/null -t 1 $fail_script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sbatch is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	send_user "\nFAILURE: sbatch did not submit job\n"
	set exit_code 1
}

# Requeue the job when it is complete
requeue_job $job_id

# Check to see if the job state is PENDING after the requeue
check_state $job_id

cancel_job $job_id

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
	exec $bin_rm $complete_script $fail_script
}

exit $exit_code




#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Validate that the scontrol requeuehold option requeues a
#          job back to pending state and puts it on hold.
#
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2013 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id            "2.22"
set script             "test$test_id.bash"
set job_id             0
set exit_code          0

print_header $test_id

# Remove any vestigial scripts
exec $bin_rm -f $script

make_bash_script $script "
sleep 20
"

proc check_hold { job } {

	global scontrol exit_code

	set hold 0
	spawn $scontrol show job $job
	expect {
		-re "Priority=0" {
			set hold 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE scontrol is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if { $hold != 1 } {
		send_user "\nFAILURE scontrol did not hold job after it was requeued\n"
		set exit_code 1
	}

}

proc check_release { job } {

	global scontrol number exit_code

	set priority 0
	spawn $scontrol show job $job
	expect {
		-re "Priority=($number)" {
			set priority $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: scontrol is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if { $priority == 0 } {
		send_user "\nFAILURE: priority was not set to a non zero value after it was released\n"
		set exit_code 1
	}

}

proc check_state { job } {

	global scontrol exit_code

	set job_state 0
	spawn $scontrol show job $job
	expect {
		-re "JobState=PENDING" {
			set job_state 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: scontrol is not responding\n"
			set exit_code 0
		}
		eof {
			wait
		}
	}

	if {$job_state != 1} {
		send_user "\nFAILURE: Job $job state was not PENDING after it was requeued\n"
		set exit_code 1
	}

}

spawn $sbatch -N1 -t1 -o /dev/null -e /dev/null $script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE sbatch is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

if {$job_id == 0} {
	send_user "\nFAILURE: sbatch did not submit job\n"
	set exit_code 1
}

wait_for_job $job_id DONE

spawn $scontrol requeuehold $job_id
expect {
	timeout {
		send_user "\nFAILURE scontrol is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

# Check if the job is in hold
check_hold $job_id

# Check that the job state is pending after released
check_state $job_id

spawn $scontrol release $job_id
expect {
	timeout {
		send_user "\nFAILURE: scontrol is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

check_release $job_id

cancel_job $job_id

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
	exec $bin_rm $script
}

exit $exit_code




#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Validate that the scontrol requeuehold State=SpecialExit
#          requeues a job to JobState=Special_Exit and is on hold.
#          When the job is released the job should start to run again.
#
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2013 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://slurm.schedmd.com/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id            "2.23"
set script             "test$test_id\.bash"
set job_id             0
set exit_code          0

print_header $test_id

# Remove any vestigial files
exec $bin_rm -f $script

make_bash_script $script "
sleep 20
"

proc check_state { id } {

	global scontrol exit_code

	set chk_state 0
	spawn $scontrol show job $id
	expect {
		-re "JobState=SPECIAL_EXIT" {
			set chk_state 1
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: scontrol is nor responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if { $chk_state != 1 } {
		send_user "\nFAILURE: Job $id state was not set to SPECIAL_EXIT\n"
		set exit_code
	}

}

proc check_release { job } {

	global scontrol number exit_code

	set priority 0
	spawn $scontrol show job $job
	expect {
		-re "Priority=($number)" {
			set priority $expect_out(1,string)
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: scontrol is not responding\n"
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if { $priority == 0 } {
		send_user "\nFAILURE: priority was not set to a non zero value after it was released\n"
		set exit_code 1
	}

}

spawn $sbatch -N1 -t1 -o /dev/null -e /dev/null $script
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: sbatch is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

if { $job_id == 0 } {
	send_user "\nFAILURE: sbatch did not submit job\n"
	set exit_code 1
}

wait_for_job $job_id DONE

spawn $scontrol requeuehold State=SpecialExit $job_id
expect {
	timeout {
		send_user "\nFAILURE: scontrol is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

check_state $job_id

spawn $scontrol release $job_id
expect {
	timeout {
		send_user "\nFAILURE: scontrol is not responding\n"
		set exit_code 1
	}
	eof {
		wait
	}
}

check_release $job_id

cancel_job $job_id

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
	exec $bin_rm $script
}

exit $exit_code

Return to ticket 478

Line 0 Link Here

(-)a/testsuite/expect/test2.21 (+164 lines)
		1	#!/usr/bin/expect
		2	############################################################################
		3	# Purpose: Test of SLURM functionality
		4	# Validate that the scontrol requeue option requeues a
		5	# failed or completed job.
		6	#
		7	#
		8	# Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR
		9	# "FAILURE: ..." otherwise with an explanation of the failure, OR
		10	# anything else indicates a failure mode that must be investigated.
		11	############################################################################
		12	# Copyright (C) 2013 SchedMD LLC
		13	# Written by Nathan Yee <nyee32@schedmd.com>
		14	#
		15	# This file is part of SLURM, a resource management program.
		16	# For details, see <http://slurm.schedmd.com/>.
		17	# Please also read the included file: DISCLAIMER.
		18	#
		19	# SLURM is free software; you can redistribute it and/or modify it under
		20	# the terms of the GNU General Public License as published by the Free
		21	# Software Foundation; either version 2 of the License, or (at your option)
		22	# any later version.
		23	#
		24	# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
		25	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		26	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		27	# details.
		28	#
		29	# You should have received a copy of the GNU General Public License along
		30	# with SLURM; if not, write to the Free Software Foundation, Inc.,
		31	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		32	############################################################################
		33	source ./globals
		34
		35	set test_id "2.21"
		36	set complete_script "test$test_id\.complete"
		37	set fail_script "test$test_id\.fail"
		38	set job_id 0
		39	set exit_code 0
		40
		41	print_header $test_id
		42
		43	make_bash_script $complete_script "
		44	sleep 20
		45	"
		46
		47	make_bash_script $fail_script "
		48	slep 20
		49	"
		50
		51	proc check_state { id } {
		52
		53	global scontrol exit_code
		54
		55	set job_state 0
		56	spawn $scontrol show job $id
		57	expect {
		58	-re "JobState=PENDING" {
		59	set job_state 1
		60	exp_continue
		61	}
		62	timeout {
		63	send_user "\nFAILURE: scontrol is not responding\n"
		64	set exit_code 0
65	}
66	eof {
67	wait
68	}
69	}
70
71	if {$job_state != 1} {
72	send_user "\nFAILURE: Job $id state was not PENDING after it was requeued\n"
73	set exit_code 1
74	}
75
76	}
77
78	proc requeue_job { id } {
79
80	global scontrol exit_code
81
82	spawn $scontrol requeue $id
83	expect {
84	timeout {
85	send_user "\nFAILURE: scontrol is not responding\n"
86	set exit_code 1
87	}
88	eof {
89	wait
90	}
91	}
92	}
93
94	#
95	# Run a job that will complete
96	#
97	spawn $sbatch -N1 -o /dev/null -e /dev/null -t1 $complete_script
98	expect {
99	-re "Submitted batch job ($number)" {
100	set job_id $expect_out(1,string)
101	exp_continue
102	}
103	timeout {
104	send_user "\nFAILURE: sbatch is not responding\n"
105	set exit_code 1
106	}
107	eof {
108	wait
109	}
110	}
111
112	if { $job_id == 0 } {
113	send_user "\nFAILURE: sbatch did not submit job\n"
114	set exit_code 1
115	}
116
117	# Wait for the job to be in the complete state
118	wait_for_job $job_id DONE
119
120	# Requeue the job when it is complete
121	requeue_job $job_id
122
123	# Check to see if the job state is PENDING after the requeue
124	check_state $job_id
125
126	cancel_job $job_id
127
128	#
129	# Run a job that will fail
130	#
131	spawn $sbatch -N1 -o /dev/null -e /dev/null -t 1 $fail_script
132	expect {
133	-re "Submitted batch job ($number)" {
134	set job_id $expect_out(1,string)
135	exp_continue
136	}
137	timeout {
138	send_user "\nFAILURE: sbatch is not responding\n"
139	set exit_code 1
140	}
141	eof {
142	wait
143	}
144	}
145
146	if { $job_id == 0 } {
147	send_user "\nFAILURE: sbatch did not submit job\n"
148	set exit_code 1
149	}
150
151	# Requeue the job when it is complete
152	requeue_job $job_id
153
154	# Check to see if the job state is PENDING after the requeue
155	check_state $job_id
156
157	cancel_job $job_id
158
159	if {$exit_code == 0} {
160	send_user "\nSUCCESS\n"
161	exec $bin_rm $complete_script $fail_script
162	}
163
164	exit $exit_code

Line 0 Link Here

(-)a/testsuite/expect/test2.22 (+189 lines)
		1	#!/usr/bin/expect
		2	############################################################################
		3	# Purpose: Test of SLURM functionality
		4	# Validate that the scontrol requeuehold option requeues a
		5	# job back to pending state and puts it on hold.
		6	#
		7	#
		8	# Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR
		9	# "FAILURE: ..." otherwise with an explanation of the failure, OR
		10	# anything else indicates a failure mode that must be investigated.
		11	############################################################################
		12	# Copyright (C) 2013 SchedMD LLC
		13	# Written by Nathan Yee <nyee32@schedmd.com>
		14	#
		15	# This file is part of SLURM, a resource management program.
		16	# For details, see <http://slurm.schedmd.com/>.
		17	# Please also read the included file: DISCLAIMER.
		18	#
		19	# SLURM is free software; you can redistribute it and/or modify it under
		20	# the terms of the GNU General Public License as published by the Free
		21	# Software Foundation; either version 2 of the License, or (at your option)
		22	# any later version.
		23	#
		24	# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
		25	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		26	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		27	# details.
		28	#
		29	# You should have received a copy of the GNU General Public License along
		30	# with SLURM; if not, write to the Free Software Foundation, Inc.,
		31	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		32	############################################################################
		33	source ./globals
		34
		35	set test_id "2.22"
		36	set script "test$test_id.bash"
		37	set job_id 0
		38	set exit_code 0
		39
		40	print_header $test_id
		41
		42	# Remove any vestigial scripts
		43	exec $bin_rm -f $script
		44
		45	make_bash_script $script "
		46	sleep 20
		47	"
		48
		49	proc check_hold { job } {
		50
		51	global scontrol exit_code
		52
		53	set hold 0
		54	spawn $scontrol show job $job
		55	expect {
		56	-re "Priority=0" {
		57	set hold 1
		58	exp_continue
		59	}
		60	timeout {
		61	send_user "\nFAILURE scontrol is not responding\n"
		62	set exit_code 1
		63	}
		64	eof {
65	wait
66	}
67	}
68
69	if { $hold != 1 } {
70	send_user "\nFAILURE scontrol did not hold job after it was requeued\n"
71	set exit_code 1
72	}
73
74	}
75
76	proc check_release { job } {
77
78	global scontrol number exit_code
79
80	set priority 0
81	spawn $scontrol show job $job
82	expect {
83	-re "Priority=($number)" {
84	set priority $expect_out(1,string)
85	exp_continue
86	}
87	timeout {
88	send_user "\nFAILURE: scontrol is not responding\n"
89	set exit_code 1
90	}
91	eof {
92	wait
93	}
94	}
95
96	if { $priority == 0 } {
97	send_user "\nFAILURE: priority was not set to a non zero value after it was released\n"
98	set exit_code 1
99	}
100
101	}
102
103	proc check_state { job } {
104
105	global scontrol exit_code
106
107	set job_state 0
108	spawn $scontrol show job $job
109	expect {
110	-re "JobState=PENDING" {
111	set job_state 1
112	exp_continue
113	}
114	timeout {
115	send_user "\nFAILURE: scontrol is not responding\n"
116	set exit_code 0
117	}
118	eof {
119	wait
120	}
121	}
122
123	if {$job_state != 1} {
124	send_user "\nFAILURE: Job $job state was not PENDING after it was requeued\n"
125	set exit_code 1
126	}
127
128	}
129
130	spawn $sbatch -N1 -t1 -o /dev/null -e /dev/null $script
131	expect {
132	-re "Submitted batch job ($number)" {
133	set job_id $expect_out(1,string)
134	exp_continue
135	}
136	timeout {
137	send_user "\nFAILURE sbatch is not responding\n"
138	set exit_code 1
139	}
140	eof {
141	wait
142	}
143	}
144
145	if {$job_id == 0} {
146	send_user "\nFAILURE: sbatch did not submit job\n"
147	set exit_code 1
148	}
149
150	wait_for_job $job_id DONE
151
152	spawn $scontrol requeuehold $job_id
153	expect {
154	timeout {
155	send_user "\nFAILURE scontrol is not responding\n"
156	set exit_code 1
157	}
158	eof {
159	wait
160	}
161	}
162
163	# Check if the job is in hold
164	check_hold $job_id
165
166	# Check that the job state is pending after released
167	check_state $job_id
168
169	spawn $scontrol release $job_id
170	expect {
171	timeout {
172	send_user "\nFAILURE: scontrol is not responding\n"
173	set exit_code 1
174	}
175	eof {
176	wait
177	}
178	}
179
180	check_release $job_id
181
182	cancel_job $job_id
183
184	if {$exit_code == 0} {
185	send_user "\nSUCCESS\n"
186	exec $bin_rm $script
187	}
188
189	exit $exit_code

Line 0 Link Here

(-)a/testsuite/expect/test2.23 (+159 lines)
		1	#!/usr/bin/expect
		2	############################################################################
		3	# Purpose: Test of SLURM functionality
		4	# Validate that the scontrol requeuehold State=SpecialExit
		5	# requeues a job to JobState=Special_Exit and is on hold.
		6	# When the job is released the job should start to run again.
		7	#
		8	#
		9	# Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR
		10	# "FAILURE: ..." otherwise with an explanation of the failure, OR
		11	# anything else indicates a failure mode that must be investigated.
		12	############################################################################
		13	# Copyright (C) 2013 SchedMD LLC
		14	# Written by Nathan Yee <nyee32@schedmd.com>
		15	#
		16	# This file is part of SLURM, a resource management program.
		17	# For details, see <http://slurm.schedmd.com/>.
		18	# Please also read the included file: DISCLAIMER.
		19	#
		20	# SLURM is free software; you can redistribute it and/or modify it under
		21	# the terms of the GNU General Public License as published by the Free
		22	# Software Foundation; either version 2 of the License, or (at your option)
		23	# any later version.
		24	#
		25	# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
		26	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		27	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		28	# details.
		29	#
		30	# You should have received a copy of the GNU General Public License along
		31	# with SLURM; if not, write to the Free Software Foundation, Inc.,
		32	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		33	############################################################################
		34	source ./globals
		35
		36	set test_id "2.23"
		37	set script "test$test_id\.bash"
		38	set job_id 0
		39	set exit_code 0
		40
		41	print_header $test_id
		42
		43	# Remove any vestigial files
		44	exec $bin_rm -f $script
		45
		46	make_bash_script $script "
		47	sleep 20
		48	"
		49
		50	proc check_state { id } {
		51
		52	global scontrol exit_code
		53
		54	set chk_state 0
		55	spawn $scontrol show job $id
		56	expect {
		57	-re "JobState=SPECIAL_EXIT" {
		58	set chk_state 1
		59	exp_continue
		60	}
		61	timeout {
		62	send_user "\nFAILURE: scontrol is nor responding\n"
		63	set exit_code 1
		64	}
65	eof {
66	wait
67	}
68	}
69
70	if { $chk_state != 1 } {
71	send_user "\nFAILURE: Job $id state was not set to SPECIAL_EXIT\n"
72	set exit_code
73	}
74
75	}
76
77	proc check_release { job } {
78
79	global scontrol number exit_code
80
81	set priority 0
82	spawn $scontrol show job $job
83	expect {
84	-re "Priority=($number)" {
85	set priority $expect_out(1,string)
86	exp_continue
87	}
88	timeout {
89	send_user "\nFAILURE: scontrol is not responding\n"
90	set exit_code 1
91	}
92	eof {
93	wait
94	}
95	}
96
97	if { $priority == 0 } {
98	send_user "\nFAILURE: priority was not set to a non zero value after it was released\n"
99	set exit_code 1
100	}
101
102	}
103
104	spawn $sbatch -N1 -t1 -o /dev/null -e /dev/null $script
105	expect {
106	-re "Submitted batch job ($number)" {
107	set job_id $expect_out(1,string)
108	exp_continue
109	}
110	timeout {
111	send_user "\nFAILURE: sbatch is not responding\n"
112	set exit_code 1
113	}
114	eof {
115	wait
116	}
117	}
118
119	if { $job_id == 0 } {
120	send_user "\nFAILURE: sbatch did not submit job\n"
121	set exit_code 1
122	}
123
124	wait_for_job $job_id DONE
125
126	spawn $scontrol requeuehold State=SpecialExit $job_id
127	expect {
128	timeout {
129	send_user "\nFAILURE: scontrol is not responding\n"
130	set exit_code 1
131	}
132	eof {
133	wait
134	}
135	}
136
137	check_state $job_id
138
139	spawn $scontrol release $job_id
140	expect {
141	timeout {
142	send_user "\nFAILURE: scontrol is not responding\n"
143	set exit_code 1
144	}
145	eof {
146	wait
147	}
148	}
149
150	check_release $job_id
151
152	cancel_job $job_id
153
154	if {$exit_code == 0} {
155	send_user "\nSUCCESS\n"
156	exec $bin_rm $script
157	}
158
159	exit $exit_code