Attachment #7195 for ticket #5361

View | Details | Raw Unified | Return to ticket 5361
Collapse All | Expand All





.TP
\fBAllowedDevicesFile\fR=<path_to_allowed_devices_file>
If the ConstrainDevices field is set to "yes" then this file can be used to declare
devices that need to be allowed or denied for all jobs. The default value is
"/etc/slurm/cgroup_allowed_devices_file.conf". The current implementation of the
cgroup devices subsystem works as a default behavior and a set of exceptions to that
default. The syntax of the file accepts comment lines using the hash (#) symbol and
a single device file or file pattern per line.  Each such line can be prefixed with
a plus (+) or minus (-) to denote allow versus deny of the device (or devices), respectively,
and an optional sequence of permission bits for the exception (r = read, w = write, m = mknod).
For example, the line "/dev/sda*" is equivalent to "+rwm /dev/sda*" in the extended syntax.  The
special file name "all" or "*" can be used to indicate all devices (e.g. "- all" or "-rwm *" to
deny all access to all devices before selectively adding-back the ones that should be available).
See also an example of this file in etc/cgroup_allowed_devices_file.conf.example.

.TP




#
# The cgroup_allowed_devices_file.conf is used to allow/deny device files when
# cgroups device constraints are enabled.
#
# The devices cgroup has a default disposition of allowing all devices; only
# deny exceptions can augment that behavior.  To allow only a specific list of
# devices the default behavior must be changed to deny, augmented by allow
# exceptions.  In most cases leaving this file empty should suffice:  Slurm
# will add deny exceptions for GRES devices that are not associated with
# a job.
#
# SYNTAX
#
# Comment lines like this begin with a hash (#) symbol.
#
# One file path or path pattern can be specified per line.  The patterns follow
# glob(7) syntax rules, e.g.
#
#	/dev/sda?
#	/dev/pts/*
#
# The original syntax of this file has been extended in this version to allow
# the specification of the disposition and permissions for each exception to
# be added.  A leading plus (+) or minus (-) is used to indicate allow or deny
# disposition, respectively.  Suffixing the disposition with the characters
# [r]ead, [w]rite, or [m]knod indicate the permissions associated with the
# exception; all three are selected by default.  All three of the following
# exception lines are equivalent:
#
#	/dev/sda?
#	+ /dev/sda?
#	+rwm /dev/sda?
#
# Besides a file path or path pattern, the word "all" or "*" can be used to
# represent all devices for the sake of allowing or denying all devices on the
# system.  For example, to change the default disposition to deny and add
# allow exceptions:
#
#	# Change default behavior to deny:
#	- all
#	# Add back devices needed by all jobs:
#	+ /dev/null
#	+ /dev/urandom
#	+ /dev/zero
#	+ /dev/sda*
#	+ /dev/cpu/*/*
#	+ /dev/pts/*
#	# For nVidia UVM and query capabilities:
#	+ /dev/nvidia-uvm*
#	+ /dev/nvidiactl
#
# In this case, Slurm will add allow exceptions for the GRES devices that are
# allocated to the job and the other GRES devices will remain denied.
#






#define _GNU_SOURCE
#include <glob.h>
#include <ctype.h>
#include <limits.h>
#include <sched.h>
#include <sys/stat.h>

static xcgroup_t job_devices_cg;
static xcgroup_t step_devices_cg;

static int _process_allowed_devices_file(xcgroup_t *cg);
			       char *dev_major[PATH_MAX],
			       int lines);

static int _read_allowed_devices_file(char *allowed_devices[PATH_MAX]);

extern int task_cgroup_devices_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
{


extern int task_cgroup_devices_create(stepd_step_rec_t *job)
{
	int rc;
	int fstatus = SLURM_ERROR;
	char *allowed_devices[PATH_MAX], *allowed_dev_major[PATH_MAX];
	xcgroup_t devices_cg;
	uint32_t jobid = job->jobid;
	uint32_t stepid = job->stepid;

	}

	/*
         * process the allowed devices file to alter which devices are
         * allowed/denied by default (for all jobs):
         */
	if ( _process_allowed_devices_file(&job_devices_cg) != SLURM_SUCCESS )
		goto error;
	for (k = 0; k < allow_lines; k++)
		xfree(allowed_devices[k]);

	/*
	 * with the current cgroup devices subsystem design (whitelist only
	 * supported) we need to allow all different devices that are supposed
	 * to be allowed by* default.
	 */
	for (k = 0; k < allow_lines; k++) {
		debug2("Default access allowed to device %s for job",
		       allowed_dev_major[k]);
		xcgroup_set_param(&job_devices_cg, "devices.allow",
				  allowed_dev_major[k]);
	}

	/*
         * allow or deny access to devices according to job GRES permissions

	if ((job->stepid != SLURM_BATCH_SCRIPT) &&
	    (job->stepid != SLURM_EXTERN_CONT)) {
		/*
		 * the step cgroup will inherit the configuration of the job
		 * cgroup, so all devices from the allowed devices config file
		 * will already be present in this cgroup.  So we'll just repeat
		 * the allow/deny of GRES devices.
		for (k = 0; k < allow_lines; k++) {
			debug2("Default access allowed to device %s for step",
			     allowed_dev_major[k]);
			xcgroup_set_param(&step_devices_cg, "devices.allow",
					  allowed_dev_major[k]);
		}

		/*
		 * allow or deny access to devices according to GRES permissions
		 * for the step
		 */
		device_list = gres_plugin_get_allocated_devices(
			step_gres_list, false);

error:
	xcgroup_unlock(&devices_cg);
	xcgroup_destroy(&devices_cg);
	for (k = 0; k < allow_lines; k++) {
		xfree(allowed_dev_major[k]);
	}
	xfree(allowed_dev_major);

	return fstatus;
}

	return fstatus;
}


static int _snprint_device_exception(
	char      *buffer,
	size_t    buffer_len,
	mode_t    mode,
	unsigned  dev_major,
	unsigned  dev_minor,
	bool      perm_r,
	bool      perm_w,
	bool      perm_m
)
{
	int       slen;
	char      type = 'a';

	if ( ! perm_r && ! perm_w && ! perm_m ) return SLURM_ERROR;

	if ( mode != 0 ) {
		if ( S_ISCHR(mode) ) {
			type = 'c';
		} else if ( S_ISBLK(mode) ) {
			type = 'b';
		} else {
			return SLURM_ERROR;
		}
	}
	*buffer++ = type; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
	*buffer++ = ' '; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;

	if ( dev_major == ~0 ) {
		*buffer++ = '*'; buffer_len--;
	} else {
		slen = snprintf(buffer, buffer_len, "%u", dev_major);
		if ( slen <= 0 ) return SLURM_ERROR;
		buffer_len -= slen;
		buffer += slen;
	}
	if ( buffer_len == 0 ) return SLURM_ERROR;
	*buffer++ = ':'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;

	if ( dev_minor == ~0 ) {
		*buffer++ = '*'; buffer_len--;
	} else {
		slen = snprintf(buffer, buffer_len, "%u", dev_minor);
		if ( slen <= 0 ) return SLURM_ERROR;
		buffer_len -= slen;
		buffer += slen;
	}
	if ( buffer_len == 0 ) return SLURM_ERROR;
	*buffer++ = ' '; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;

	if ( perm_r ) {
		*buffer++ = 'r'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
	}
	if ( perm_w ) {
		*buffer++ = 'w'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
	}
	if ( perm_m ) {
		*buffer++ = 'm'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
	}
	*buffer = '\0';
	return SLURM_SUCCESS;
}

/***

The cgroup_allowed_devices_file.conf format has been augmented to allow for
finer control over what gets written to the devices.allow and devices.deny
files.

1. Hash delimited comments run to the end of a line; the previous code would
   attempt to glob such lines.

2. Blank lines are ignored.

3. Old-style lines, e.g. "/dev/nvidia*" function as before:  they equate to
   allowing all matching device paths in the read-write-mknod mode.

4. Lines may now start with a plus ("+") or minus ("-") to indicate their
   being written to the devices.allow or devices.deny file, respectively.

5. Lines using the allow/deny syntax may also include the affected modes,
   [r]ead, [w]rite, or [m]knod

The line thus may start with the following regular expression leading up to
the path pattern to be globbed:

	\s*([-+]([rwm]+)?\s*)?

(Regular expression is mentioned only to describe the syntax; regex'es are
not used to validate or extract the parameters.)

 ***/

static int _process_allowed_devices_file(xcgroup_t *cg)
{
	FILE *file = fopen(cgroup_allowed_devices_file, "r");
  	int   rc = SLURM_SUCCESS;

	if ( file != NULL ) {
		char line[PATH_MAX];
		int  line_num = 0;

		while ( (rc == SLURM_SUCCESS) && (fgets(line, sizeof(line), file) != NULL) ) {
			bool	allow = true;
			bool	perm_r = true;
			bool	perm_w = true;
			bool	perm_m = true;
			char	*s = line, *actual_start;

			/* Increment line number counter: */
			line_num++;

			/* Drop leading whitespace: */
			while ( *s && isspace(*s) ) s++;
			actual_start = s;

			/* Allow/deny signifier? */
			if ( (*s == '+') || (*s == '-') ) {
				allow = (*s == '+') ? true : false;
				s++;
				/* Permissions bits? */
				if ( (*s == 'r') || (*s == 'w') || (*s == 'm') ) {
					/* Reset to all not present: */
					perm_r = perm_w = perm_m = false;

					/* Accumulate permissions: */
					while ( *s && ((*s == 'r') || (*s == 'w') || (*s == 'm')) ) {
						switch ( *s ) {
							case 'r':
								perm_r = true;
								break;
							case 'w':
								perm_w = true;
								break;
							case 'm':
								perm_m = true;
								break;
						}
						s++;
					}
				}
				/* Drop trailing whitespace: */
				while ( *s && isspace(*s) ) s++;
			}

			/* Classify the kind of line: */
			switch ( *s ) {
				/* empty and comment lines */
				case '\0':
				case '#': {
					if ( s > actual_start ) {
						/* Remove newline from end of string before printing: */
						while ( *s && (*s != '\n') ) s++;
						*s = '\0';
						debug("Line number %d in allowed devices file is invalid: %s", line_num, actual_start);
					}
					break;
				}

				/* all devices (maybe) */
				case 'a':
				case 'A': {
					/* Check to see if it's "all" */
					if ( xstrncasecmp(s, "all", 3) == 0 ) {
						/* ...it starts with "all" */
						s += 3;
					} else {
						/* Remove NUL termination: */
						while ( *s && (*s != '\n') ) s++;
						*s = '\0';
						debug3("Error on line %d in allowed devices file: %s", line_num, line);
						break;
					}
					/* Error condition breaks out of the switch block, otherwise we want to
					   cascade into the '*' case -- which also means "all devices" */
				}
				case '*': {
					/* Skip past whitespace: */
					while ( *s && isspace(*s) ) s++;
					switch ( *s ) {
						/* blank or comment makes this line okay */
						case '\0':
						case '#': {
							char	exception[24]; /* "T #:# rwm" = 2 + 5 + 1 + 5 + 1 + 3 + 1 = 18*/

							if ( _snprint_device_exception(exception, sizeof(exception), 0, ~0, ~0, perm_r, perm_w, perm_m) == SLURM_SUCCESS ) {
								if ( xcgroup_set_param(cg, (allow ? "devices.allow" : "devices.deny"), exception) != XCGROUP_SUCCESS ) {
									rc = SLURM_ERROR;
									error("task/cgroup: unable to %s all devices", (allow ? "allow" : "deny"));
								}
							} else {
								rc = SLURM_ERROR;
								error("task/cgroup: failed creating %s exception string for all devices", (allow ? "allow" : "deny"));
							}
							break;
						}

						default: {
							/* Remove newline from end of string before printing: */
							while ( *s && (*s != '\n') ) s++;
							*s = '\0';
							debug3("Error on line %d in allowed devices file: %s", line_num, actual_start);
							break;
						}
					}
					break;
				}

				/* device line will start with "/" ... */
				case '/': {
					glob_t	globbuf;
					char	*path_pattern = s;

					/* Remove newline from end of string before globbing: */
					while ( *s && (*s != '\n') ) s++;
					*s = '\0';

					/* expand glob pattern and walk the list of paths: */
					if ( glob(path_pattern, GLOB_NOSORT, NULL, &globbuf) != 0 ) {
						debug3("No files matching pattern '%s'", path_pattern);
					} else {
						struct stat finfo;
						int         i;

						for ( i=0; ((rc == SLURM_SUCCESS) && (i < globbuf.gl_pathc)); i++ ) {
							/* get stat for the path: */
							if ( stat(globbuf.gl_pathv[i], &finfo) != 0 ) {
								debug("task/cgroup: unable to stat(%s)", globbuf.gl_pathv[i]);
							} else {
								char	exception[24]; /* "T #:# rwm" = 2 + 5 + 1 + 5 + 1 + 3 + 1 = 18*/

								if ( _snprint_device_exception(exception, sizeof(exception), finfo.st_mode, major(finfo.st_rdev), minor(finfo.st_rdev), perm_r, perm_w, perm_m) == SLURM_SUCCESS ) {
									if ( xcgroup_set_param(cg, (allow ? "devices.allow" : "devices.deny"), exception) != XCGROUP_SUCCESS ) {
										rc = SLURM_ERROR;
										error("task/cgroup: unable to %s device %s", (allow ? "allow" : "deny"), globbuf.gl_pathv[i]);
									}
								} else {
									rc = SLURM_ERROR;
									error("task/cgroup: failed creating %s exception string for device %s", (allow ? "allow" : "deny"), globbuf.gl_pathv[i]);
								}
							}
						}
					}
					break;
				}

				default: {
					/* Remove NUL termination: */
					while ( *s && (*s != '\n') ) s++;
					*s = '\0';
					debug3("Error on line %d in allowed devices file: %s", line_num, line);
					break;
				}
				globfree(&globbuf);
			}
		}
		fclose(file);
	} else {
		rc = SLURM_ERROR;
		fatal("%s: %s does not exist, please create this file.",
		      __func__, cgroup_allowed_devices_file);
	}
	return rc;
}



Return to ticket 5361

Lines 49-61 of this particular plugin: Link Here

(-)a/doc/man/man5/cgroup.conf.5 (-7 / +11 lines)
49		49
50	.TP	50	.TP
51	\fBAllowedDevicesFile\fR=<path_to_allowed_devices_file>	51	\fBAllowedDevicesFile\fR=<path_to_allowed_devices_file>
52	If the ConstrainDevices field is set to "yes" then this file has to be used to declare	52	If the ConstrainDevices field is set to "yes" then this file can be used to declare
53	the devices that need to be allowed by default for all the jobs. The current implementation	53	devices that need to be allowed or denied for all jobs. The default value is
54	of cgroup devices subsystem works as a whitelist of entries, which means that in order to	54	"/etc/slurm/cgroup_allowed_devices_file.conf". The current implementation of the
55	isolate the access of a job upon particular devices we need to allow the access on all	55	cgroup devices subsystem works as a default behavior and a set of exceptions to that
56	the devices, supported by default and then deny on those that the job does not have the	56	default. The syntax of the file accepts comment lines using the hash (#) symbol and
57	permission to use. The default value is "/etc/slurm/cgroup_allowed_devices_file.conf". The syntax of	57	a single device file or file pattern per line. Each such line can be prefixed with
58	the file accepts one device per line and it permits lines like /dev/sda* or /dev/cpu//.	58	a plus (+) or minus (-) to denote allow versus deny of the device (or devices), respectively,
		59	and an optional sequence of permission bits for the exception (r = read, w = write, m = mknod).
		60	For example, the line "/dev/sda" is equivalent to "+rwm /dev/sda" in the extended syntax. The
		61	special file name "all" or "" can be used to indicate all devices (e.g. "- all" or "-rwm " to
		62	deny all access to all devices before selectively adding-back the ones that should be available).
59	See also an example of this file in etc/cgroup_allowed_devices_file.conf.example.	63	See also an example of this file in etc/cgroup_allowed_devices_file.conf.example.
60		64
61	.TP	65	.TP

Lines 1-6 Link Here

(-)a/etc/cgroup_allowed_devices_file.conf.example (-6 / +55 lines)
1	/dev/null	1	#
2	/dev/urandom	2	# The cgroup_allowed_devices_file.conf is used to allow/deny device files when
3	/dev/zero	3	# cgroups device constraints are enabled.
4	/dev/sda*	4	#
5	/dev/cpu//	5	# The devices cgroup has a default disposition of allowing all devices; only
6	/dev/pts/*	6	# deny exceptions can augment that behavior. To allow only a specific list of
		7	# devices the default behavior must be changed to deny, augmented by allow
		8	# exceptions. In most cases leaving this file empty should suffice: Slurm
		9	# will add deny exceptions for GRES devices that are not associated with
		10	# a job.
		11	#
		12	# SYNTAX
		13	#
		14	# Comment lines like this begin with a hash (#) symbol.
		15	#
		16	# One file path or path pattern can be specified per line. The patterns follow
		17	# glob(7) syntax rules, e.g.
		18	#
		19	# /dev/sda?
		20	# /dev/pts/*
		21	#
		22	# The original syntax of this file has been extended in this version to allow
		23	# the specification of the disposition and permissions for each exception to
		24	# be added. A leading plus (+) or minus (-) is used to indicate allow or deny
		25	# disposition, respectively. Suffixing the disposition with the characters
		26	# [r]ead, [w]rite, or [m]knod indicate the permissions associated with the
		27	# exception; all three are selected by default. All three of the following
		28	# exception lines are equivalent:
		29	#
		30	# /dev/sda?
		31	# + /dev/sda?
		32	# +rwm /dev/sda?
		33	#
		34	# Besides a file path or path pattern, the word "all" or "*" can be used to
		35	# represent all devices for the sake of allowing or denying all devices on the
		36	# system. For example, to change the default disposition to deny and add
		37	# allow exceptions:
		38	#
		39	# # Change default behavior to deny:
		40	# - all
		41	# # Add back devices needed by all jobs:
		42	# + /dev/null
		43	# + /dev/urandom
		44	# + /dev/zero
		45	# + /dev/sda*
		46	# + /dev/cpu//
		47	# + /dev/pts/*
		48	# # For nVidia UVM and query capabilities:
		49	# + /dev/nvidia-uvm*
		50	# + /dev/nvidiactl
		51	#
		52	# In this case, Slurm will add allow exceptions for the GRES devices that are
		53	# allocated to the job and the other GRES devices will remain denied.
		54	#
		55

Lines 38-43 Link Here

(-)a/src/plugins/task/cgroup/task_cgroup_devices.c (-76 / +256 lines)
38		38
39	#define _GNU_SOURCE	39	#define _GNU_SOURCE
40	#include <glob.h>	40	#include <glob.h>
		41	#include <ctype.h>
41	#include <limits.h>	42	#include <limits.h>
42	#include <sched.h>	43	#include <sched.h>
43	#include <sys/stat.h>	44	#include <sys/stat.h>
Lines 73-83 static xcgroup_t user_devices_cg; Link Here
73	static xcgroup_t job_devices_cg;	74	static xcgroup_t job_devices_cg;
74	static xcgroup_t step_devices_cg;	75	static xcgroup_t step_devices_cg;
75		76
76	static void _calc_device_major(char *dev_path[PATH_MAX],	77	static int _process_allowed_devices_file(xcgroup_t *cg);
77	char *dev_major[PATH_MAX],
78	int lines);
79
80	static int _read_allowed_devices_file(char *allowed_devices[PATH_MAX]);
81		78
82	extern int task_cgroup_devices_init(slurm_cgroup_conf_t *slurm_cgroup_conf)	79	extern int task_cgroup_devices_init(slurm_cgroup_conf_t *slurm_cgroup_conf)
83	{	80	{
Lines 209-217 extern int task_cgroup_devices_fini(slurm_cgroup_conf_t *slurm_cgroup_conf) Link Here
209		206
210	extern int task_cgroup_devices_create(stepd_step_rec_t *job)	207	extern int task_cgroup_devices_create(stepd_step_rec_t *job)
211	{	208	{
212	int k, rc, allow_lines = 0;	209	int rc;
213	int fstatus = SLURM_ERROR;	210	int fstatus = SLURM_ERROR;
214	char allowed_devices[PATH_MAX], allowed_dev_major[PATH_MAX];
215	xcgroup_t devices_cg;	211	xcgroup_t devices_cg;
216	uint32_t jobid = job->jobid;	212	uint32_t jobid = job->jobid;
217	uint32_t stepid = job->stepid;	213	uint32_t stepid = job->stepid;
Lines 333-357 extern int task_cgroup_devices_create(stepd_step_rec_t *job) Link Here
333	}	329	}
334		330
335	/*	331	/*
336	* create the entry with major minor for the default allowed devices	332	* process the allowed devices file to alter which devices are
337	* read from the file	333	* allowed/denied by default (for all jobs):
338	*/	334	*/
339	allow_lines = _read_allowed_devices_file(allowed_devices);	335	if ( _process_allowed_devices_file(&job_devices_cg) != SLURM_SUCCESS )
340	_calc_device_major(allowed_devices, allowed_dev_major, allow_lines);	336	goto error;
341	for (k = 0; k < allow_lines; k++)
342	xfree(allowed_devices[k]);
343
344	/*
345	* with the current cgroup devices subsystem design (whitelist only
346	* supported) we need to allow all different devices that are supposed
347	* to be allowed by* default.
348	*/
349	for (k = 0; k < allow_lines; k++) {
350	debug2("Default access allowed to device %s for job",
351	allowed_dev_major[k]);
352	xcgroup_set_param(&job_devices_cg, "devices.allow",
353	allowed_dev_major[k]);
354	}
355		337
356	/*	338	/*
357	* allow or deny access to devices according to job GRES permissions	339	* allow or deny access to devices according to job GRES permissions
Lines 403-422 extern int task_cgroup_devices_create(stepd_step_rec_t *job) Link Here
403	if ((job->stepid != SLURM_BATCH_SCRIPT) &&	385	if ((job->stepid != SLURM_BATCH_SCRIPT) &&
404	(job->stepid != SLURM_EXTERN_CONT)) {	386	(job->stepid != SLURM_EXTERN_CONT)) {
405	/*	387	/*
406	* with the current cgroup devices subsystem design (whitelist	388	* the step cgroup will inherit the configuration of the job
407	* only supported) we need to allow all different devices that	389	* cgroup, so all devices from the allowed devices config file
408	* are supposed to be allowed by default.	390	* will already be present in this cgroup. So we'll just repeat
409	*/	391	* the allow/deny of GRES devices.
410	for (k = 0; k < allow_lines; k++) {
411	debug2("Default access allowed to device %s for step",
412	allowed_dev_major[k]);
413	xcgroup_set_param(&step_devices_cg, "devices.allow",
414	allowed_dev_major[k]);
415	}
416
417	/*
418	* allow or deny access to devices according to GRES permissions
419	* for the step
420	*/	392	*/
421	device_list = gres_plugin_get_allocated_devices(	393	device_list = gres_plugin_get_allocated_devices(
422	step_gres_list, false);	394	step_gres_list, false);
Lines 458-467 extern int task_cgroup_devices_create(stepd_step_rec_t *job) Link Here
458	error:	430	error:
459	xcgroup_unlock(&devices_cg);	431	xcgroup_unlock(&devices_cg);
460	xcgroup_destroy(&devices_cg);	432	xcgroup_destroy(&devices_cg);
461	for (k = 0; k < allow_lines; k++) {
462	xfree(allowed_dev_major[k]);
463	}
464	xfree(allowed_dev_major);
465		433
466	return fstatus;	434	return fstatus;
467	}	435	}
Lines 476-532 extern int task_cgroup_devices_attach_task(stepd_step_rec_t *job) Link Here
476	return fstatus;	444	return fstatus;
477	}	445	}
478		446
479	static void _calc_device_major(char *dev_path[PATH_MAX],	447
480	char *dev_major[PATH_MAX],	448	static int _snprint_device_exception(
481	int lines)	449	char *buffer,
		450	size_t buffer_len,
		451	mode_t mode,
		452	unsigned dev_major,
		453	unsigned dev_minor,
		454	bool perm_r,
		455	bool perm_w,
		456	bool perm_m
		457	)
482	{	458	{
		459	int slen;
		460	char type = 'a';
		461
		462	if ( ! perm_r && ! perm_w && ! perm_m ) return SLURM_ERROR;
		463
		464	if ( mode != 0 ) {
		465	if ( S_ISCHR(mode) ) {
		466	type = 'c';
		467	} else if ( S_ISBLK(mode) ) {
		468	type = 'b';
		469	} else {
		470	return SLURM_ERROR;
		471	}
		472	}
		473	*buffer++ = type; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
		474	*buffer++ = ' '; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
		475
		476	if ( dev_major == ~0 ) {
		477	buffer++ = ''; buffer_len--;
		478	} else {
		479	slen = snprintf(buffer, buffer_len, "%u", dev_major);
		480	if ( slen <= 0 ) return SLURM_ERROR;
		481	buffer_len -= slen;
		482	buffer += slen;
		483	}
		484	if ( buffer_len == 0 ) return SLURM_ERROR;
		485	*buffer++ = ':'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
483		486
484	int k;	487	if ( dev_minor == ~0 ) {
		488	buffer++ = ''; buffer_len--;
		489	} else {
		490	slen = snprintf(buffer, buffer_len, "%u", dev_minor);
		491	if ( slen <= 0 ) return SLURM_ERROR;
		492	buffer_len -= slen;
		493	buffer += slen;
		494	}
		495	if ( buffer_len == 0 ) return SLURM_ERROR;
		496	*buffer++ = ' '; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
485		497
486	if (lines > PATH_MAX) {	498	if ( perm_r ) {
487	error("task/cgroup: more devices configured than table size "	499	*buffer++ = 'r'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
488	"(%d > %d)", lines, PATH_MAX);	500	}
489	lines = PATH_MAX;	501	if ( perm_w ) {
		502	*buffer++ = 'w'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
		503	}
		504	if ( perm_m ) {
		505	*buffer++ = 'm'; buffer_len--; if ( buffer_len == 0 ) return SLURM_ERROR;
490	}	506	}
491	for (k = 0; k < lines; k++)	507	*buffer = '\0';
492	dev_major[k] = gres_device_major(dev_path[k]);	508	return SLURM_SUCCESS;
493	}	509	}
494		510
		511	/***
495		512
496	static int _read_allowed_devices_file(char **allowed_devices)	513	The cgroup_allowed_devices_file.conf format has been augmented to allow for
497	{	514	finer control over what gets written to the devices.allow and devices.deny
		515	files.
		516
		517	1. Hash delimited comments run to the end of a line; the previous code would
		518	attempt to glob such lines.
498		519
		520	2. Blank lines are ignored.
		521
		522	3. Old-style lines, e.g. "/dev/nvidia*" function as before: they equate to
		523	allowing all matching device paths in the read-write-mknod mode.
		524
		525	4. Lines may now start with a plus ("+") or minus ("-") to indicate their
		526	being written to the devices.allow or devices.deny file, respectively.
		527
		528	5. Lines using the allow/deny syntax may also include the affected modes,
		529	[r]ead, [w]rite, or [m]knod
		530
		531	The line thus may start with the following regular expression leading up to
		532	the path pattern to be globbed:
		533
		534	\s([-+]([rwm]+)?\s)?
		535
		536	(Regular expression is mentioned only to describe the syntax; regex'es are
		537	not used to validate or extract the parameters.)
		538
		539	***/
		540
		541	static int _process_allowed_devices_file(xcgroup_t *cg)
		542	{
499	FILE *file = fopen(cgroup_allowed_devices_file, "r");	543	FILE *file = fopen(cgroup_allowed_devices_file, "r");
500	int i, l, num_lines = 0;	544	int rc = SLURM_SUCCESS;
501	char line[256];	545
502	glob_t globbuf;	546	if ( file != NULL ) {
		547	char line[PATH_MAX];
		548	int line_num = 0;
		549
		550	while ( (rc == SLURM_SUCCESS) && (fgets(line, sizeof(line), file) != NULL) ) {
		551	bool allow = true;
		552	bool perm_r = true;
		553	bool perm_w = true;
		554	bool perm_m = true;
		555	char s = line, actual_start;
		556
		557	/* Increment line number counter: */
		558	line_num++;
		559
		560	/* Drop leading whitespace: */
		561	while ( s && isspace(s) ) s++;
		562	actual_start = s;
		563
		564	/* Allow/deny signifier? */
		565	if ( (s == '+') \|\| (s == '-') ) {
		566	allow = (*s == '+') ? true : false;
		567	s++;
		568	/* Permissions bits? */
		569	if ( (s == 'r') \|\| (s == 'w') \|\| (*s == 'm') ) {
		570	/* Reset to all not present: */
		571	perm_r = perm_w = perm_m = false;
		572
		573	/* Accumulate permissions: */
		574	while ( s && ((s == 'r') \|\| (s == 'w') \|\| (s == 'm')) ) {
		575	switch ( *s ) {
		576	case 'r':
		577	perm_r = true;
		578	break;
		579	case 'w':
		580	perm_w = true;
		581	break;
		582	case 'm':
		583	perm_m = true;
		584	break;
		585	}
		586	s++;
		587	}
		588	}
		589	/* Drop trailing whitespace: */
		590	while ( s && isspace(s) ) s++;
		591	}
503		592
504	for( i=0; i<256; i++ )	593	/* Classify the kind of line: */
505	line[i] = '\0';	594	switch ( *s ) {
		595	/* empty and comment lines */
		596	case '\0':
		597	case '#': {
		598	if ( s > actual_start ) {
		599	/* Remove newline from end of string before printing: */
		600	while ( s && (s != '\n') ) s++;
		601	*s = '\0';
		602	debug("Line number %d in allowed devices file is invalid: %s", line_num, actual_start);
		603	}
		604	break;
		605	}
506		606
507	if ( file != NULL ){	607	/* all devices (maybe) */
508	while (fgets(line, sizeof(line), file)) {	608	case 'a':
509	line[strlen(line)-1] = '\0';	609	case 'A': {
		610	/* Check to see if it's "all" */
		611	if ( xstrncasecmp(s, "all", 3) == 0 ) {
		612	/* ...it starts with "all" */
		613	s += 3;
		614	} else {
		615	/* Remove NUL termination: */
		616	while ( s && (s != '\n') ) s++;
		617	*s = '\0';
		618	debug3("Error on line %d in allowed devices file: %s", line_num, line);
		619	break;
		620	}
		621	/* Error condition breaks out of the switch block, otherwise we want to
		622	cascade into the '' case -- which also means "all devices" /
		623	}
		624	case '*': {
		625	/* Skip past whitespace: */
		626	while ( s && isspace(s) ) s++;
		627	switch ( *s ) {
		628	/* blank or comment makes this line okay */
		629	case '\0':
		630	case '#': {
		631	char exception[24]; /* "T #:# rwm" = 2 + 5 + 1 + 5 + 1 + 3 + 1 = 18*/
		632
		633	if ( _snprint_device_exception(exception, sizeof(exception), 0, ~0, ~0, perm_r, perm_w, perm_m) == SLURM_SUCCESS ) {
		634	if ( xcgroup_set_param(cg, (allow ? "devices.allow" : "devices.deny"), exception) != XCGROUP_SUCCESS ) {
		635	rc = SLURM_ERROR;
		636	error("task/cgroup: unable to %s all devices", (allow ? "allow" : "deny"));
		637	}
		638	} else {
		639	rc = SLURM_ERROR;
		640	error("task/cgroup: failed creating %s exception string for all devices", (allow ? "allow" : "deny"));
		641	}
		642	break;
		643	}
		644
		645	default: {
		646	/* Remove newline from end of string before printing: */
		647	while ( s && (s != '\n') ) s++;
		648	*s = '\0';
		649	debug3("Error on line %d in allowed devices file: %s", line_num, actual_start);
		650	break;
		651	}
		652	}
		653	break;
		654	}
510		655
511	/* global pattern matching and return the list of matches*/	656	/* device line will start with "/" ... */
512	if (glob(line, GLOB_NOSORT, NULL, &globbuf)) {	657	case '/': {
513	debug3("Device %s does not exist", line);	658	glob_t globbuf;
514	} else {	659	char *path_pattern = s;
515	for (l=0; l < globbuf.gl_pathc; l++) {	660
516	allowed_devices[num_lines] =	661	/* Remove newline from end of string before globbing: */
517	xstrdup(globbuf.gl_pathv[l]);	662	while ( s && (s != '\n') ) s++;
518	num_lines++;	663	*s = '\0';
		664
		665	/* expand glob pattern and walk the list of paths: */
		666	if ( glob(path_pattern, GLOB_NOSORT, NULL, &globbuf) != 0 ) {
		667	debug3("No files matching pattern '%s'", path_pattern);
		668	} else {
		669	struct stat finfo;
		670	int i;
		671
		672	for ( i=0; ((rc == SLURM_SUCCESS) && (i < globbuf.gl_pathc)); i++ ) {
		673	/* get stat for the path: */
		674	if ( stat(globbuf.gl_pathv[i], &finfo) != 0 ) {
		675	debug("task/cgroup: unable to stat(%s)", globbuf.gl_pathv[i]);
		676	} else {
		677	char exception[24]; /* "T #:# rwm" = 2 + 5 + 1 + 5 + 1 + 3 + 1 = 18*/
		678
		679	if ( _snprint_device_exception(exception, sizeof(exception), finfo.st_mode, major(finfo.st_rdev), minor(finfo.st_rdev), perm_r, perm_w, perm_m) == SLURM_SUCCESS ) {
		680	if ( xcgroup_set_param(cg, (allow ? "devices.allow" : "devices.deny"), exception) != XCGROUP_SUCCESS ) {
		681	rc = SLURM_ERROR;
		682	error("task/cgroup: unable to %s device %s", (allow ? "allow" : "deny"), globbuf.gl_pathv[i]);
		683	}
		684	} else {
		685	rc = SLURM_ERROR;
		686	error("task/cgroup: failed creating %s exception string for device %s", (allow ? "allow" : "deny"), globbuf.gl_pathv[i]);
		687	}
		688	}
		689	}
		690	}
		691	break;
		692	}
		693
		694	default: {
		695	/* Remove NUL termination: */
		696	while ( s && (s != '\n') ) s++;
		697	*s = '\0';
		698	debug3("Error on line %d in allowed devices file: %s", line_num, line);
		699	break;
519	}	700	}
520	globfree(&globbuf);
521	}	701	}
522	}	702	}
523	fclose(file);	703	fclose(file);
524	}	704	} else {
525	else	705	rc = SLURM_ERROR;
526	fatal("%s: %s does not exist, please create this file.",	706	fatal("%s: %s does not exist, please create this file.",
527	__func__, cgroup_allowed_devices_file);	707	__func__, cgroup_allowed_devices_file);
528		708	}
529	return num_lines;	709	return rc;
530	}	710	}
531		711
532		712