Attachment #146 for ticket #154

View | Details | Raw Unified | Return to ticket 154
Collapse All | Expand All




	send_user "\n"
	send_user "SUCCESS: test$test_id\n"
}

###############################################################
#
#Proc: change_subbp_state
#
#Purpose: Set sub mid plane state
#
#Returns SUCCESS if state of mid plane is changed
#
#
###############################################################

proc change_subbp_state { node ionodes state } {
	global scontrol smap

	set exit_code 0

	set my_pid [spawn $scontrol update subbpname=$node\[$ionodes\] state=$state]
	expect {
		-re "slurm_update error:" {
			set exit_code 1
			exp_continue
		}
		-re "Unable to contact" {
			send_user "\nFAILURE: slurm appears to be down\n"
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: scontrol not responding\n"
			slow_kill $my_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if { $exit_code } {
		return $exit_code
	}

	set match 0
	set my_pid [spawn $smap -Db -c -h -n $node -I $ionodes]
	expect {
		-nocase -re "$state" {
			incr match
			exp_continue
		}
		-re "$node" {
			incr match
			exp_continue
		}
		-re "Unable to contact" {
			send_user "\nFAILURE: slurm appears to be down\n"
			exp_continue
		}
		timeout {
			send_user "\nFAILURE: smap not responding\n"
			slow_kill $my_pid
			set exit_code 1
		}
		eof {
			wait
		}
	}

	if {$match != 2} {
		send_user "\nFAILURE: Subbp did not go into $state state. $match\n"
		set exit_code 1
	}

	return $exit_code
}





#!/usr/bin/expect
############################################################################
# Purpose: Test of SLURM functionality
#          Test if the job dies when the one of the pass over nodes are
#          set to an error state.
#
#
# Output:  "TEST: #.#" followed by "SUCCESS" if test was successful, OR
#          "FAILURE: ..." otherwise with an explanation of the failure, OR
#          anything else indicates a failure mode that must be investigated.
############################################################################
# Copyright (C) 2011 SchedMD LLC
# Written by Nathan Yee <nyee32@schedmd.com>
#
# This file is part of SLURM, a resource management program.
# For details, see <http://www.schedmd.com/slurmdocs/>.
# Please also read the included file: DISCLAIMER.
#
# SLURM is free software; you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
# details.
#
# You should have received a copy of the GNU General Public License along
# with SLURM; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
############################################################################
source ./globals

set test_id	"8.9"
set exit_code	0
set file_in 	"test$test_id.input"
set job_id      0
set  geom       ""
set a_size      ""
set x_size      ""
set y_size      ""
set z_size      ""
set job_fini1   ""
set job_fini2   ""
set job_fini3   ""
set job_fini4   ""
set a_flag      0
set x_flag      0
set y_flag      0
set z_flag      0
set tmp         0
set block       0

print_header $test_id


if {([test_bluegene] == 0) || [string compare [get_bluegene_type] "Q"]} {
	send_user "\nWARNING: This test is only compatible with bluegene systems\n"
	exit $exit_code
}

set type [get_bluegene_type]
if {$type == 0} {
	send_user "\nFAILURE: No blugene type found \n"
	exit 1
}

spawn $scontrol show node
expect {
	-re "NodeName=($alpha_numeric).($digit)($digit)($digit)($digit)" {
		set a_size $expect_out(2,string)
		set x_size $expect_out(3,string)
		set y_size $expect_out(4,string)
		set z_size $expect_out(5,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		exit 1
	}
	eof {
		wait
	}
}

if {$a_size>2} {
	set geom "2x1x1x1"
	set a_flag 1
} elseif {$x_size > 2} {
	set geom "1x2x1x1"
	set x_flag 1
} elseif {$y_size >2} {
	set geom "1x1x2x1"
	set y_flag 1
} elseif {$z_size >2} {
	set geom "1x1x1x2"
	set z_flag 1
} else {
	send_user "\nFAILURE: not enough dimenstions\n"
	exit 1
}

make_bash_script $file_in "$bin_sleep 500"

spawn $sbatch  --geometry=$geom --conn-type=T,T,T,T --output=/dev/null --error=/dev/null $file_in
expect {
	-re "Submitted batch job ($number)" {
		set job_id $expect_out(1,string)
		exp_continue
	}
	-re "error" {
		send_user "\nFAILURE: Job was not submitted\n"
		exp_continue
		exit 1
	}
	timeout {
		send_user "\nFAILURE: sbatch is not responding\n"
		exit 1
	}
	eof {
		wait
	}
}

wait_for_job $job_id "RUNNING"

spawn $scontrol show job $job_id
expect {
	-re "MidplaneList=($alpha_numeric).($alpha_numeric)x($digit)($digit)($digit)($digit)" {
		set job_fini1 $expect_out(3,string)
		set job_fini2 $expect_out(4,string)
		set job_fini3 $expect_out(5,string)
		set job_fini4 $expect_out(6,string)
		exp_continue

	}
	-re "Block_ID=($alpha_numeric)" {
		set block $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		exit 1
	}
	eof {
		wait
	}

}

if {$a_flag==1} {
	set last_node "bgq$a_size$job_fini2$job_fini3$job_fini4"
} elseif {$x_flag==1} {
	set last_node "bgq$job_fini1$x_size$job_fini3$job_fini4"
} elseif {$y_flag==1} {
	set last_node "bgq$job_fini1$job_fini2$y_size$job_fini4"
} elseif {$z_flag==1} {
	set last_node "bgq$job_fini1$job_fini2$job_fini3$z_size"
}

change_subbp_state $last_node "0" "error"

wait_for_job $job_id "DONE"

spawn $squeue --state=failed -o%i
expect {
	-re "($alpha_numeric)" {
		set tmp $expect_out(1,string)
		exp_continue
	}
	timeout {
		send_user "\nFAILURE:squeue is not responding\n"
		exit 1
	}
	eof {
		wait
	}
}

if {$tmp!=$job_id} {
	send_user "\nFAILURE: Job $job_id did not fail\n"
	exit 1
}

send_user "\nblock=$block\n"

spawn $scontrol show block $block
expect {
	-re "not found" {
		send_user "\nBlock was not found, this is expected.\n"
		exp_continue
	}
	timeout {
		send_user "\nFAILURE: scontrol not responding\n"
		exit 1
	}
	eof {
		wait
	}
}

#Sets block back to free
change_subbp_state $last_node "0" "free"

if {$exit_code == 0} {
	send_user "\nSUCCESS\n"
} else {
	send_user "\nFAILURE\n"
}

exit $exit_code

Return to ticket 154

Lines 2269-2271 proc print_success { test_id } { Link Here

(-)a/testsuite/expect/globals (+74 lines)
2269	send_user "\n"	2269	send_user "\n"
2270	send_user "SUCCESS: test$test_id\n"	2270	send_user "SUCCESS: test$test_id\n"
2271	}	2271	}
		2272
		2273	###############################################################
		2274	#
		2275	#Proc: change_subbp_state
		2276	#
		2277	#Purpose: Set sub mid plane state
		2278	#
		2279	#Returns SUCCESS if state of mid plane is changed
		2280	#
		2281	#
		2282	###############################################################
		2283
		2284	proc change_subbp_state { node ionodes state } {
		2285	global scontrol smap
		2286
		2287	set exit_code 0
		2288
		2289	set my_pid [spawn $scontrol update subbpname=$node\[$ionodes\] state=$state]
		2290	expect {
		2291	-re "slurm_update error:" {
		2292	set exit_code 1
		2293	exp_continue
		2294	}
		2295	-re "Unable to contact" {
		2296	send_user "\nFAILURE: slurm appears to be down\n"
		2297	exp_continue
		2298	}
		2299	timeout {
		2300	send_user "\nFAILURE: scontrol not responding\n"
		2301	slow_kill $my_pid
		2302	set exit_code 1
		2303	}
		2304	eof {
		2305	wait
		2306	}
		2307	}
		2308
		2309	if { $exit_code } {
		2310	return $exit_code
		2311	}
		2312
		2313	set match 0
		2314	set my_pid [spawn $smap -Db -c -h -n $node -I $ionodes]
		2315	expect {
		2316	-nocase -re "$state" {
		2317	incr match
		2318	exp_continue
		2319	}
		2320	-re "$node" {
		2321	incr match
		2322	exp_continue
		2323	}
		2324	-re "Unable to contact" {
		2325	send_user "\nFAILURE: slurm appears to be down\n"
		2326	exp_continue
		2327	}
		2328	timeout {
		2329	send_user "\nFAILURE: smap not responding\n"
		2330	slow_kill $my_pid
		2331	set exit_code 1
		2332	}
		2333	eof {
		2334	wait
		2335	}
2336	}
2337
2338	if {$match != 2} {
2339	send_user "\nFAILURE: Subbp did not go into $state state. $match\n"
2340	set exit_code 1
2341	}
2342
2343	return $exit_code
2344	}
2345

Line 0 Link Here

(-)a/testsuite/expect/test8.9 (+212 lines)
		1	#!/usr/bin/expect
		2	############################################################################
		3	# Purpose: Test of SLURM functionality
		4	# Test if the job dies when the one of the pass over nodes are
		5	# set to an error state.
		6	#
		7	#
		8	# Output: "TEST: #.#" followed by "SUCCESS" if test was successful, OR
		9	# "FAILURE: ..." otherwise with an explanation of the failure, OR
		10	# anything else indicates a failure mode that must be investigated.
		11	############################################################################
		12	# Copyright (C) 2011 SchedMD LLC
		13	# Written by Nathan Yee <nyee32@schedmd.com>
		14	#
		15	# This file is part of SLURM, a resource management program.
		16	# For details, see <http://www.schedmd.com/slurmdocs/>.
		17	# Please also read the included file: DISCLAIMER.
		18	#
		19	# SLURM is free software; you can redistribute it and/or modify it under
		20	# the terms of the GNU General Public License as published by the Free
		21	# Software Foundation; either version 2 of the License, or (at your option)
		22	# any later version.
		23	#
		24	# SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
		25	# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		26	# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		27	# details.
		28	#
		29	# You should have received a copy of the GNU General Public License along
		30	# with SLURM; if not, write to the Free Software Foundation, Inc.,
		31	# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
		32	############################################################################
		33	source ./globals
		34
		35	set test_id "8.9"
		36	set exit_code 0
		37	set file_in "test$test_id.input"
		38	set job_id 0
		39	set geom ""
		40	set a_size ""
		41	set x_size ""
		42	set y_size ""
		43	set z_size ""
		44	set job_fini1 ""
		45	set job_fini2 ""
		46	set job_fini3 ""
		47	set job_fini4 ""
		48	set a_flag 0
		49	set x_flag 0
		50	set y_flag 0
		51	set z_flag 0
		52	set tmp 0
		53	set block 0
		54
		55	print_header $test_id
		56
		57
		58	if {([test_bluegene] == 0) \|\| [string compare [get_bluegene_type] "Q"]} {
		59	send_user "\nWARNING: This test is only compatible with bluegene systems\n"
		60	exit $exit_code
		61	}
		62
		63	set type [get_bluegene_type]
		64	if {$type == 0} {
65	send_user "\nFAILURE: No blugene type found \n"
66	exit 1
67	}
68
69	spawn $scontrol show node
70	expect {
71	-re "NodeName=($alpha_numeric).($digit)($digit)($digit)($digit)" {
72	set a_size $expect_out(2,string)
73	set x_size $expect_out(3,string)
74	set y_size $expect_out(4,string)
75	set z_size $expect_out(5,string)
76	exp_continue
77	}
78	timeout {
79	send_user "\nFAILURE: scontrol not responding\n"
80	exit 1
81	}
82	eof {
83	wait
84	}
85	}
86
87	if {$a_size>2} {
88	set geom "2x1x1x1"
89	set a_flag 1
90	} elseif {$x_size > 2} {
91	set geom "1x2x1x1"
92	set x_flag 1
93	} elseif {$y_size >2} {
94	set geom "1x1x2x1"
95	set y_flag 1
96	} elseif {$z_size >2} {
97	set geom "1x1x1x2"
98	set z_flag 1
99	} else {
100	send_user "\nFAILURE: not enough dimenstions\n"
101	exit 1
102	}
103
104	make_bash_script $file_in "$bin_sleep 500"
105
106	spawn $sbatch --geometry=$geom --conn-type=T,T,T,T --output=/dev/null --error=/dev/null $file_in
107	expect {
108	-re "Submitted batch job ($number)" {
109	set job_id $expect_out(1,string)
110	exp_continue
111	}
112	-re "error" {
113	send_user "\nFAILURE: Job was not submitted\n"
114	exp_continue
115	exit 1
116	}
117	timeout {
118	send_user "\nFAILURE: sbatch is not responding\n"
119	exit 1
120	}
121	eof {
122	wait
123	}
124	}
125
126	wait_for_job $job_id "RUNNING"
127
128	spawn $scontrol show job $job_id
129	expect {
130	-re "MidplaneList=($alpha_numeric).($alpha_numeric)x($digit)($digit)($digit)($digit)" {
131	set job_fini1 $expect_out(3,string)
132	set job_fini2 $expect_out(4,string)
133	set job_fini3 $expect_out(5,string)
134	set job_fini4 $expect_out(6,string)
135	exp_continue
136
137	}
138	-re "Block_ID=($alpha_numeric)" {
139	set block $expect_out(1,string)
140	exp_continue
141	}
142	timeout {
143	send_user "\nFAILURE: scontrol not responding\n"
144	exit 1
145	}
146	eof {
147	wait
148	}
149
150	}
151
152	if {$a_flag==1} {
153	set last_node "bgq$a_size$job_fini2$job_fini3$job_fini4"
154	} elseif {$x_flag==1} {
155	set last_node "bgq$job_fini1$x_size$job_fini3$job_fini4"
156	} elseif {$y_flag==1} {
157	set last_node "bgq$job_fini1$job_fini2$y_size$job_fini4"
158	} elseif {$z_flag==1} {
159	set last_node "bgq$job_fini1$job_fini2$job_fini3$z_size"
160	}
161
162	change_subbp_state $last_node "0" "error"
163
164	wait_for_job $job_id "DONE"
165
166	spawn $squeue --state=failed -o%i
167	expect {
168	-re "($alpha_numeric)" {
169	set tmp $expect_out(1,string)
170	exp_continue
171	}
172	timeout {
173	send_user "\nFAILURE:squeue is not responding\n"
174	exit 1
175	}
176	eof {
177	wait
178	}
179	}
180
181	if {$tmp!=$job_id} {
182	send_user "\nFAILURE: Job $job_id did not fail\n"
183	exit 1
184	}
185
186	send_user "\nblock=$block\n"
187
188	spawn $scontrol show block $block
189	expect {
190	-re "not found" {
191	send_user "\nBlock was not found, this is expected.\n"
192	exp_continue
193	}
194	timeout {
195	send_user "\nFAILURE: scontrol not responding\n"
196	exit 1
197	}
198	eof {
199	wait
200	}
201	}
202
203	#Sets block back to free
204	change_subbp_state $last_node "0" "free"
205
206	if {$exit_code == 0} {
207	send_user "\nSUCCESS\n"
208	} else {
209	send_user "\nFAILURE\n"
210	}
211
212	exit $exit_code