mmap: check if mapping a character device is actually possible

kernel-src/fs/devfs.c

@@ -176,6 +176,11 @@ int devfs_mmap(vnode_t *node, void *addr, uintmax_t offset, int flags, cred_t *c
 	if (devnode->devops->mmap == NULL)
 		return ENODEV;
 
+	// if addr is this magic value, the caller is only
+	// testing for mmap support from the device
+	if (addr == VOP_MMAP_ADDRESS_MMAP_SUPPORTED)
+		return 0;
+
 	return devnode->devops->mmap(devnode->attr.rdevminor, addr, offset, flags);
 }
 

kernel-src/include/kernel/vfs.h

@@ -171,6 +171,8 @@ typedef struct vops_t {
 #define VOP_LOCK(v) (v)->ops->lock(v)
 #define VOP_UNLOCK(v) (v)->ops->unlock(v)
 
+#define VOP_MMAP_ADDRESS_MMAP_SUPPORTED (void *)-1
+
 #define VOP_OPEN(v, f, c) (*v)->ops->open(v, f, c)
 #define VOP_CLOSE(v, f, c) (v)->ops->close(v, f, c)
 #define VOP_READ(v, b, s, o, f, r, c) (v)->ops->read(v, b, s, o, f, r, c)

kernel-src/sys/syscalls/mmap.c

@@ -68,6 +68,8 @@ syscallret_t syscall_mmap(context_t *context, void *hint, size_t len, int prot,
 			return ret;
 		}
 
+		// check if the file is of a type that can be mapped
+		// and if the fd was opened with the required permissions
 		int type = file->vnode->type;
 		if ((type != V_TYPE_REGULAR && type != V_TYPE_BLKDEV && type != V_TYPE_CHDEV)
 			|| ((prot & PROT_READ) && (file->flags & FILE_READ) == 0)
@@ -76,6 +78,15 @@ syscallret_t syscall_mmap(context_t *context, void *hint, size_t len, int prot,
 			goto cleanup;
 		}
 
+		// not all charcter devices support mapping, so check if its supported
+		// (regular files and block devices go through the page cache)
+		// TODO move this check to vmm_map once the return value of mmap is fixed to return an errno
+		if (type == V_TYPE_CHDEV) {
+			ret.errno = VOP_MMAP(file->vnode, VOP_MMAP_ADDRESS_MMAP_SUPPORTED, 0, 0, NULL);
+			if (ret.errno)
+				goto cleanup;
+		}
+
 		vfd.node = file->vnode;
 		vfd.offset = offset;
 	}