diff --git a/dev/block/bio.c b/dev/block/bio.c new file mode 100644 index 0000000..9005702 --- /dev/null +++ b/dev/block/bio.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include + +struct bio* bio_alloc(enum bio_op op, size_t num_segs) { + struct bio* bio = mem_alloc(sizeof(struct bio) + num_segs * sizeof(struct bio_seg), 0); + if (!bio) + return nullptr; + bio->op = op; + bio->num_segs = num_segs; + event_init(&bio->event); + atomic_init(&bio->pending, 0); + atomic_init(&bio->short_offset, UINT64_MAX); + atomic_init(&bio->child_status, 0); + return bio; +} + +void bio_free(struct bio* bio) { + mem_free(bio); +} + +static void record_short(struct bio* parent, uint64_t offset, errno_t status) { + uint64_t current = atomic_load(&parent->short_offset, ATOMIC_RELAXED); + while (offset < current) { + if (atomic_compare_exchange_weak(&parent->short_offset, ¤t, offset, ATOMIC_RELAXED, ATOMIC_RELAXED)) + break; + } + if (status) { + errno_t expected = 0; + atomic_compare_exchange(&parent->child_status, &expected, status, ATOMIC_RELAXED, ATOMIC_RELAXED); + } +} + +static void parent_put(struct bio* parent) { + if (atomic_fetch_sub(&parent->pending, 1, ATOMIC_ACQ_REL) != 1) + return; + const uint64_t offset = atomic_load(&parent->short_offset, ATOMIC_RELAXED); + const size_t done = offset == UINT64_MAX ? parent->num_lbas : offset; + const errno_t status = atomic_load(&parent->child_status, ATOMIC_RELAXED); + bio_complete(parent, done == 0 ? status : 0, done); +} + +void bio_complete(struct bio* bio, errno_t status, size_t done_lbas) { + bio->status = status; + bio->done_lbas = done_lbas; + + struct bio* parent = bio->parent; + if (parent) { + if (status || done_lbas < bio->num_lbas) + record_short(parent, bio->parent_offset + done_lbas, status); + bio_free(bio); + parent_put(parent); + return; + } + + if (bio->end) + bio->end(bio); + else + event_set(&bio->event); +} + +errno_t bio_wait(struct bio* bio, size_t* out_lbas) { + event_wait(&bio->event); + if (out_lbas) + *out_lbas = bio->done_lbas; + return bio->status; +} + +// Submits one bio and completes early if it failed. +static void submit_one(struct blockdev* dev, struct bio* bio) { + errno_t status = dev->ops->submit(dev, bio); + if (status) + bio_complete(bio, status, 0); +} + +void bio_submit(struct blockdev* dev, struct bio* bio) { + bio->dev = dev; + if (bio->op == BIO_FLUSH) { + submit_one(dev, bio); + return; + } + + if (bio->op == BIO_READ) { + if (bio->lba >= dev->lba_count) { + bio_complete(bio, 0, 0); + return; + } + bio->num_lbas = MIN(bio->num_lbas, dev->lba_count - bio->lba); + } + if (bio->num_lbas == 0) { + bio_complete(bio, 0, 0); + return; + } + + const size_t lba_size = dev->lba_size; + const size_t lbas_per_page = MAX(arch_mem_page_size() / lba_size, 1); + size_t max_lbas = dev->limits.max_lbas ? dev->limits.max_lbas : SIZE_MAX; + if (max_lbas > lbas_per_page) + max_lbas -= max_lbas % lbas_per_page; + const size_t max_segs = dev->limits.max_segs ? dev->limits.max_segs : SIZE_MAX; + + if (bio->num_lbas <= max_lbas && bio->num_segs <= max_segs) { + submit_one(dev, bio); + return; + } + + atomic_store(&bio->pending, 1, ATOMIC_RELAXED); + atomic_store(&bio->short_offset, UINT64_MAX, ATOMIC_RELAXED); + atomic_store(&bio->child_status, 0, ATOMIC_RELAXED); + + size_t seg = 0; + size_t seg_off = 0; + uint64_t consumed = 0; + while (consumed < bio->num_lbas) { + const size_t cap = MIN(max_segs, bio->num_segs - seg); + struct bio* child = cap ? bio_alloc(bio->op, cap) : nullptr; + if (!child) { + record_short(bio, consumed, cap ? ENOMEM : EINVAL); + break; + } + + size_t count = 0; + size_t lbas = 0; + while (lbas < max_lbas && count < cap && consumed + lbas < bio->num_lbas) { + const struct bio_seg* src = &bio->segs[seg]; + const size_t take = + MIN(MIN(src->len - seg_off, (max_lbas - lbas) * lba_size), + (bio->num_lbas - consumed - lbas) * lba_size); + if (take < lba_size) + break; + child->segs[count++] = (struct bio_seg){.addr = src->addr + seg_off, .len = take}; + lbas += take / lba_size; + seg_off += take; + if (seg_off == src->len) { + seg++; + seg_off = 0; + } + } + + if (lbas == 0) { + bio_free(child); + record_short(bio, consumed, EINVAL); + break; + } + + child->num_segs = count; + child->num_lbas = lbas; + child->lba = bio->lba + consumed; + child->parent = bio; + child->parent_offset = consumed; + child->dev = dev; + consumed += lbas; + atomic_fetch_add(&bio->pending, 1, ATOMIC_RELAXED); + submit_one(dev, child); + } + + parent_put(bio); +} diff --git a/dev/block/blockdev.c b/dev/block/blockdev.c new file mode 100644 index 0000000..f8d6766 --- /dev/null +++ b/dev/block/blockdev.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: ISC + +#define pr_fmt(fmt) "block: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static size_t lbas_per_page(struct blockdev* dev) { + return arch_mem_page_size() / dev->lba_size; +} + +static errno_t cache_get_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count, size_t* out_valid) { + struct blockdev* dev = containerof(vmo, struct blockdev, cache); + const size_t page_size = arch_mem_page_size(); + const size_t per_page = lbas_per_page(dev); + const uint64_t lba = index * per_page; + + if (lba >= dev->lba_count) { + *out_valid = 0; + return 0; + } + + const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba); + size_t done; + errno_t status = blockdev_rw_pages(dev, BIO_READ, lba, num_lbas, pages, count, &done); + if (status) + return status; + + const size_t valid = done / per_page + (done % per_page != 0); + if (done % per_page != 0) { + const size_t tail = (done % per_page) * dev->lba_size; + memset((uint8_t*)HHDM_PTR(vm_page_addr(pages[valid - 1])) + tail, 0, page_size - tail); + } + *out_valid = valid; + return valid == 0 && num_lbas > 0 ? EIO : 0; +} + +static errno_t cache_put_pages(struct vmo* vmo, uint64_t index, struct page** pages, size_t count) { + struct blockdev* dev = containerof(vmo, struct blockdev, cache); + const size_t per_page = lbas_per_page(dev); + const uint64_t lba = index * per_page; + if (lba >= dev->lba_count) + return 0; + + const size_t num_lbas = MIN((uint64_t)count * per_page, dev->lba_count - lba); + size_t done; + errno_t status = blockdev_rw_pages(dev, BIO_WRITE, lba, num_lbas, pages, count, &done); + if (status) + return status; + return done < num_lbas ? EIO : 0; +} + +static void cache_destroy(struct vmo* vmo) { + struct blockdev* dev = containerof(vmo, struct blockdev, cache); + if (dev->ops->destroy) + dev->ops->destroy(dev); +} + +static const struct vmo_pager_ops cache_pager = { + .get_pages = cache_get_pages, + .put_pages = cache_put_pages, + .max_cluster = 64, + .destroy = cache_destroy, +}; + +void blockdev_init( + struct blockdev* dev, + const struct blockdev_ops* ops, + uint32_t major, + uint32_t minor, + size_t lba_size, + uint64_t lba_count +) { + vmo_init(&dev->cache, &cache_pager); + dev->ops = ops; + dev->dev = dev_make(major, minor); + dev->lba_size = lba_size; + dev->lba_count = lba_count; + atomic_init(&dev->claimed, false); +} + +void blockdev_ref(struct blockdev* dev) { + vmo_ref(&dev->cache); +} + +void blockdev_unref(struct blockdev* dev) { + vmo_unref(&dev->cache); +} + +errno_t blockdev_rw( + struct blockdev* dev, + enum bio_op op, + uint64_t lba, + size_t num_lbas, + const struct bio_seg* segs, + size_t num_segs, + size_t* out_lbas +) { + struct bio* bio = bio_alloc(op, num_segs); + if (!bio) + return ENOMEM; + memcpy(bio->segs, segs, num_segs * sizeof(struct bio_seg)); + bio->lba = lba; + bio->num_lbas = num_lbas; + bio_submit(dev, bio); + errno_t status = bio_wait(bio, out_lbas); + bio_free(bio); + return status; +} + +errno_t blockdev_rw_pages( + struct blockdev* dev, + enum bio_op op, + uint64_t lba, + size_t num_lbas, + struct page** pages, + size_t count, + size_t* out_lbas +) { + const size_t page_size = arch_mem_page_size(); + size_t bytes = num_lbas * dev->lba_size; + if (bytes > count * page_size) + return EINVAL; + + struct bio* bio = bio_alloc(op, count); + if (!bio) + return ENOMEM; + + size_t num_segs = 0; + for (size_t i = 0; i < count && bytes > 0; i++) { + const size_t len = MIN(bytes, page_size); + bio->segs[num_segs++] = (struct bio_seg){.addr = vm_page_addr(pages[i]), .len = len}; + bytes -= len; + } + bio->num_segs = num_segs; + bio->lba = lba; + bio->num_lbas = num_lbas; + bio_submit(dev, bio); + errno_t status = bio_wait(bio, out_lbas); + bio_free(bio); + return status; +} + +errno_t blockdev_flush(struct blockdev* dev) { + struct bio* bio = bio_alloc(BIO_FLUSH, 0); + if (!bio) + return ENOMEM; + bio_submit(dev, bio); + errno_t status = bio_wait(bio, nullptr); + bio_free(bio); + return status; +} + +static struct blockdev* file_dev(struct vfs_file* file) { + return file->priv; +} + +static uint64_t dev_size(struct blockdev* dev) { + return dev->lba_count * dev->lba_size; +} + +static errno_t blockdev_file_read(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_read) { + struct blockdev* dev = file_dev(file); + *out_read = 0; + if (offset >= dev_size(dev)) + return 0; + + size_t progress; + errno_t status = + vmo_read_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress); + *out_read = progress; + return progress > 0 ? 0 : status; +} + +static errno_t blockdev_file_write(struct vfs_file* file, iovec_iter_t* buffer, uint64_t offset, ssize_t* out_written) { + struct blockdev* dev = file_dev(file); + *out_written = 0; + if (iovec_iter_len(buffer) == 0) + return 0; + if (offset >= dev_size(dev)) + return ENOSPC; + + size_t progress; + errno_t status = + vmo_write_iter(&dev->cache, offset, buffer, MIN(iovec_iter_len(buffer), dev_size(dev) - offset), &progress); + *out_written = progress; + return progress > 0 ? 0 : status; +} + +static errno_t blockdev_file_ioctl(struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out_result) { + struct blockdev* dev = file_dev(file); + switch (request) { + case __BLKGETSIZE64: { + const uint64_t size = dev_size(dev); + if (!usercopy_write((__user void*)arg, &size, sizeof(size))) + return EFAULT; + *out_result = 0; + return 0; + } + case __BLKSSZGET: { + const unsigned int lba_size = dev->lba_size; + if (!usercopy_write((__user void*)arg, &lba_size, sizeof(lba_size))) + return EFAULT; + *out_result = 0; + return 0; + } + case __BLKRRPART: { + errno_t status = blockdev_rescan(dev); + if (!status) + *out_result = 0; + return status; + } + default: + if (dev->ops->ioctl) + return dev->ops->ioctl(dev, file, request, arg, out_result); + return ENOTTY; + } +} + +static errno_t blockdev_file_sync(struct vfs_file* file, bool data_only) { + struct blockdev* dev = file_dev(file); + errno_t status = vmo_sync(&dev->cache, 0, UINT64_MAX); + if (status) + return status; + return blockdev_flush(dev); +} + +static errno_t blockdev_file_mmap( + struct vfs_file* file, + struct vma* vma, + uintptr_t addr, + size_t len, + enum prot_flags prot, + enum map_flags flags, + off_t offset, + uintptr_t* out_addr +) { + errno_t status = + vma_map(vma, &file_dev(file)->cache, addr, len, prot, (flags & MAP_SHARED) ? VMA_SHARED : 0, offset); + if (status) + return status; + *out_addr = addr; + return 0; +} + +static void blockdev_file_release(struct vfs_file* file) { + blockdev_unref(file_dev(file)); +} + +static const struct vfs_file_ops blockdev_file_ops = { + .read = blockdev_file_read, + .write = blockdev_file_write, + .ioctl = blockdev_file_ioctl, + .sync = blockdev_file_sync, + .mmap = blockdev_file_mmap, + .release = blockdev_file_release, +}; + +static errno_t blockdev_node_open(struct vfs_node* node, struct vfs_file* file) { + struct blockdev* dev = node->priv; + blockdev_ref(dev); + file->priv = dev; + file->ops = &blockdev_file_ops; + return 0; +} + +static void blockdev_node_destroy(struct vfs_node* node) { + blockdev_unref(node->priv); +} + +static const struct vfs_node_ops blockdev_node_ops = { + .destroy = blockdev_node_destroy, + .device = { + .open = blockdev_node_open, + }, +}; + +errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode) { + struct vfs_mknod_target target = { + .type = VNODE_BLOCKDEV, + .ops = &blockdev_node_ops, + .priv = dev, + .rdev = dev->dev, + }; + + blockdev_ref(dev); + struct vfs_path root = devtmpfs_root(); + errno_t status = vfs_mknod(root, root, name, mode, &target, ident_kernel(), nullptr); + vfs_path_drop(root); + if (status) + blockdev_unref(dev); + return status; +} + +errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out) { + if (node->type != VNODE_BLOCKDEV || node->ops != &blockdev_node_ops) + return ENOTBLK; + + struct blockdev* dev = node->priv; + blockdev_ref(dev); + *out = dev; + return 0; +} + +errno_t blockdev_claim(struct blockdev* dev) { + bool expected = false; + if (!atomic_compare_exchange(&dev->claimed, &expected, true, ATOMIC_ACQUIRE, ATOMIC_RELAXED)) + return EBUSY; + return 0; +} + +void blockdev_release(struct blockdev* dev) { + atomic_store(&dev->claimed, false, ATOMIC_RELEASE); +} + +struct disk_node { + char name[64]; + struct blockdev* dev; +}; + +struct disk { + char name[32]; + struct blockdev* dev; + vec(struct disk_node) nodes; + tailq_link(struct disk) link; +}; + +static tailq_head(struct disk) disks; +static mutex_t disks_lock; + +static void add_node(struct disk* disk, const char* name, struct blockdev* dev) { + struct disk_node node = {.dev = dev}; + ksnprintf(node.name, sizeof(node.name), "%s", name); + if (vec_push(&disk->nodes, node) && dev) + blockdev_unref(dev); +} + +static void scan_partitions(struct disk* disk) { + struct gpt_partition* parts; + size_t count; + if (gpt_scan(disk->dev, &parts, &count)) + return; + + for (size_t i = 0; i < count; i++) { + const struct gpt_partition* part = &parts[i]; + if (part->end_lba < part->start_lba || part->end_lba >= disk->dev->lba_count) { + log_warn("Ignoring partition %zu of \"%s\" with invalid range\n", i + 1, disk->name); + continue; + } + + char part_name[48]; + ksnprintf(part_name, sizeof(part_name), "%sp%zu", disk->name, i + 1); + + struct blockdev* dev; + errno_t status = partition_new(disk->dev, part->start_lba, part->end_lba - part->start_lba + 1, &dev); + if (status) { + log_warn("Unable to create partition \"%s\" (%i)\n", part_name, status); + continue; + } + + char path[64]; + ksnprintf(path, sizeof(path), "block/%s", part_name); + status = devtmpfs_add_blockdev(path, dev, 0660); + if (status) { + blockdev_unref(dev); + log_warn("Unable to register \"%s\" (%i)\n", path, status); + continue; + } + add_node(disk, path, dev); + + char type_str[GUID_STR_LEN]; + char uuid_str[GUID_STR_LEN]; + guid_format(&part->type_guid, type_str); + guid_format(&part->unique_guid, uuid_str); + + ksnprintf(path, sizeof(path), "block/parttype-%s", type_str); + status = devtmpfs_symlink(path, part_name); + if (!status) + add_node(disk, path, nullptr); + else if (status == EEXIST) + log("Partition type %s already claimed, no \"%s\" link for \"%s\"\n", type_str, path, part_name); + + ksnprintf(path, sizeof(path), "block/partuuid-%s", uuid_str); + status = devtmpfs_symlink(path, part_name); + if (!status) + add_node(disk, path, nullptr); + + log("Partition %zu: \"%s\" Type: %s UUID: %s\n", i + 1, part_name, type_str, uuid_str); + } + + mem_free(parts); +} + +errno_t blockdev_register(const char* name, struct blockdev* dev) { + struct disk* disk = mem_alloc(sizeof(struct disk), 0); + if (!disk) + return ENOMEM; + ksnprintf(disk->name, sizeof(disk->name), "%s", name); + vec_init(&disk->nodes); + + char path[64]; + ksnprintf(path, sizeof(path), "block/%s", name); + errno_t status = devtmpfs_add_blockdev(path, dev, 0660); + if (status) { + mem_free(disk); + return status; + } + + blockdev_ref(dev); + disk->dev = dev; + log("Registered block device \"%s\"\n", name); + + mutex_lock(&disks_lock); + tailq_insert_tail(&disks, disk, link); + scan_partitions(disk); + mutex_unlock(&disks_lock); + return 0; +} + +errno_t blockdev_rescan(struct blockdev* dev) { + mutex_lock(&disks_lock); + struct disk* disk; + for (disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) { + if (disk->dev == dev) + break; + } + if (!disk) { + mutex_unlock(&disks_lock); + return ENOTTY; + } + + for (size_t i = 0; i < vec_length(&disk->nodes); i++) { + struct disk_node* node = &vec_elem_get(&disk->nodes, i); + errno_t status = devtmpfs_remove(node->name); + if (status) + log_warn("Unable to remove stale node \"%s\" (%i)\n", node->name, status); + if (node->dev) + blockdev_unref(node->dev); + } + disk->nodes.length = 0; + + scan_partitions(disk); + mutex_unlock(&disks_lock); + return 0; +} + +errno_t blockdev_sync_all() { + errno_t result = 0; + mutex_lock(&disks_lock); + for (struct disk* disk = tailq_first(&disks); disk; disk = tailq_next(disk, link)) { + for (size_t i = 0; i < vec_length(&disk->nodes); i++) { + struct blockdev* part = vec_elem_get(&disk->nodes, i).dev; + errno_t status = part ? vmo_sync(&part->cache, 0, UINT64_MAX) : 0; + if (status && !result) + result = status; + } + errno_t status = vmo_sync(&disk->dev->cache, 0, UINT64_MAX); + if (!status) + status = blockdev_flush(disk->dev); + if (status && !result) + result = status; + } + mutex_unlock(&disks_lock); + return result; +} + +static void block_stage_init(struct boot_info* info) { + tailq_init(&disks); + mutex_init(&disks_lock); + errno_t status = devtmpfs_mkdir("block", 0755); + if (status && status != EEXIST) + log_warn("Unable to create /dev/block (%i)\n", status); +} + +INIT_TASK(block_stage, "dev.block", block_stage_init); +INIT_DEPENDS(block_stage, devtmpfs_stage); diff --git a/dev/block/gpt.c b/dev/block/gpt.c new file mode 100644 index 0000000..8e6a009 --- /dev/null +++ b/dev/block/gpt.c @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include + +struct [[__packed]] gpt_header { + char signature[8]; + uint32_t revision; + uint32_t header_size; + uint32_t header_crc32; + uint32_t reserved; + uint64_t my_lba; + uint64_t alternate_lba; + uint64_t first_usable_lba; + uint64_t last_usable_lba; + struct guid disk_guid; + uint64_t partition_entry_lba; + uint32_t num_partition_entries; + uint32_t partition_entry_size; + uint32_t partition_entry_array_crc32; +}; + +struct [[__packed]] gpt_entry { + struct guid type_guid; + struct guid unique_guid; + uint64_t start_lba; + uint64_t end_lba; + uint64_t attributes; + uint16_t name[36]; +}; + +constexpr size_t GPT_MAX_ENTRIES = 1024; + +errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count) { + struct gpt_header header; + size_t read; + errno_t status = vmo_read(&dev->cache, dev->lba_size, &header, sizeof(header), &read); + if (status) + return status; + if (read != sizeof(header) || memcmp(header.signature, "EFI PART", 8) != 0) + return ENODATA; + if (header.partition_entry_size < sizeof(struct gpt_entry) || header.num_partition_entries > GPT_MAX_ENTRIES) + return EINVAL; + + struct gpt_partition* parts = mem_alloc(MAX(header.num_partition_entries, 1) * sizeof(struct gpt_partition), 0); + if (!parts) + return ENOMEM; + + size_t count = 0; + const uint64_t base = header.partition_entry_lba * dev->lba_size; + for (size_t i = 0; i < header.num_partition_entries; i++) { + struct gpt_entry entry; + status = vmo_read(&dev->cache, base + i * header.partition_entry_size, &entry, sizeof(entry), &read); + if (status || read != sizeof(entry)) + break; + if (guid_is_zero(&entry.type_guid)) + continue; + + struct gpt_partition* part = &parts[count++]; + part->type_guid = entry.type_guid; + part->unique_guid = entry.unique_guid; + part->start_lba = entry.start_lba; + part->end_lba = entry.end_lba; + } + + *out = parts; + *out_count = count; + return 0; +} diff --git a/dev/block/meson.build b/dev/block/meson.build index d99fcc7..636b21b 100644 --- a/dev/block/meson.build +++ b/dev/block/meson.build @@ -1,3 +1,11 @@ +kernel_sources += files( + 'bio.c', + 'blockdev.c', + 'gpt.c', + 'partition.c', + 'ram.c', +) + if kconfig.get('CONFIG_NVME_CORE', 'n') != 'n' subdir('nvme') endif diff --git a/dev/block/partition.c b/dev/block/partition.c new file mode 100644 index 0000000..874ce30 --- /dev/null +++ b/dev/block/partition.c @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include + +struct partition { + struct blockdev blk; + struct blockdev* parent; + uint64_t start; +}; + +static _Atomic(uint32_t) next_minor; + +static errno_t partition_submit(struct blockdev* blk, struct bio* bio) { + struct partition* part = containerof(blk, struct partition, blk); + struct blockdev* parent = part->parent; + + if (bio->op != BIO_FLUSH) { + if (bio->lba >= blk->lba_count) { + bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0); + return 0; + } + if (bio->op == BIO_WRITE && bio->num_lbas > blk->lba_count - bio->lba) { + bio_complete(bio, ENOSPC, 0); + return 0; + } + bio->lba += part->start; + } + return parent->ops->submit(parent, bio); +} + +static errno_t partition_ioctl( + struct blockdev* blk, + struct vfs_file* file, + uintptr_t request, + uintptr_t arg, + uintptr_t* out +) { + struct blockdev* parent = containerof(blk, struct partition, blk)->parent; + if (!parent->ops->ioctl) + return ENOTTY; + return parent->ops->ioctl(parent, file, request, arg, out); +} + +static void partition_destroy(struct blockdev* blk) { + struct partition* part = containerof(blk, struct partition, blk); + blockdev_unref(part->parent); + mem_free(part); +} + +static const struct blockdev_ops partition_ops = { + .submit = partition_submit, + .ioctl = partition_ioctl, + .destroy = partition_destroy, +}; + +errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out) { + struct partition* part = mem_alloc(sizeof(struct partition), 0); + if (!part) + return ENOMEM; + + blockdev_init( + &part->blk, + &partition_ops, + 259, + atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED), + parent->lba_size, + lba_count + ); + part->blk.limits = parent->limits; + blockdev_ref(parent); + part->parent = parent; + part->start = start_lba; + *out = &part->blk; + return 0; +} diff --git a/dev/block/ram.c b/dev/block/ram.c new file mode 100644 index 0000000..99de36c --- /dev/null +++ b/dev/block/ram.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include + +constexpr uint32_t RAM_MAJOR = 1; +constexpr size_t RAM_LBA_SIZE = 512; + +struct ramdisk { + struct blockdev blk; + phys_t base; +}; + +static atomic_uint next_minor; + +static errno_t ramdisk_submit(struct blockdev* blk, struct bio* bio) { + struct ramdisk* ram = containerof(blk, struct ramdisk, blk); + + if (bio->op == BIO_FLUSH) { + bio_complete(bio, 0, 0); + return 0; + } + if (bio->lba >= blk->lba_count || bio->num_lbas > blk->lba_count - bio->lba) { + bio_complete(bio, bio->op == BIO_READ ? 0 : ENOSPC, 0); + return 0; + } + + uint64_t lba = bio->lba; + for (size_t i = 0; i < bio->num_segs; i++) { + const struct bio_seg* seg = &bio->segs[i]; + void* store = HHDM_PTR(ram->base + lba * RAM_LBA_SIZE); + void* buffer = HHDM_PTR(seg->addr); + if (bio->op == BIO_READ) + memcpy(buffer, store, seg->len); + else + memcpy(store, buffer, seg->len); + lba += seg->len / RAM_LBA_SIZE; + } + + bio_complete(bio, 0, bio->num_lbas); + return 0; +} + +static void ramdisk_destroy(struct blockdev* blk) { + mem_free(containerof(blk, struct ramdisk, blk)); +} + +static const struct blockdev_ops ramdisk_ops = { + .submit = ramdisk_submit, + .destroy = ramdisk_destroy, +}; + +errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out) { + struct ramdisk* ram = mem_alloc(sizeof(struct ramdisk), 0); + if (!ram) + return ENOMEM; + + blockdev_init( + &ram->blk, + &ramdisk_ops, + RAM_MAJOR, + atomic_fetch_add(&next_minor, 1, ATOMIC_RELAXED), + RAM_LBA_SIZE, + length / RAM_LBA_SIZE + ); + ram->base = base; + *out = &ram->blk; + return 0; +} diff --git a/fs/initramfs.c b/fs/initramfs.c new file mode 100644 index 0000000..4eee2c4 --- /dev/null +++ b/fs/initramfs.c @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: ISC + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum ustar_type : uint8_t { + USTAR_REGULAR = 0, + USTAR_NORMAL = '0', + USTAR_SYM_LINK = '2', + USTAR_DIRECTORY = '5', + USTAR_CONTIGUOUS = '7', + USTAR_LONG_LINK = 'L', +}; + +struct ustar_header { + char name[100]; + char mode[8]; + char uid[8]; + char gid[8]; + char size[12]; + char mtime[12]; + char checksum[8]; + enum ustar_type type; + char linkname[100]; + char signature[6]; + char version[2]; + char owner[32]; + char group[32]; + char devmajor[8]; + char devminor[8]; + char prefix[155]; + char pad[12]; +}; + +static_assert(sizeof(struct ustar_header) == 512, "USTAR header must be 512 bytes"); + +static size_t oct2bin(const char* str, size_t len) { + size_t value = 0; + for (size_t i = 0; i < len; i++) { + if (str[i] == '\0' || str[i] == ' ') + break; + value *= 8; + value += (size_t)(str[i] - '0'); + } + return value; +} + +static void copy_field(char* out, const char* field, size_t field_len, size_t out_len) { + size_t i = 0; + while (i < field_len && field[i] != '\0' && i + 1 < out_len) { + out[i] = field[i]; + i++; + } + out[i] = '\0'; +} + +static errno_t create_dirs( + struct vfs_path root, + struct vfs_path at, + const char* path, + mode_t mode, + struct vfs_path* out_dir, + char* name_out, + size_t name_out_len +) { + size_t path_len = strlen(path); + + size_t dir_len = 0; + const char* name = path; + for (size_t i = path_len; i > 0; i--) { + if (path[i - 1] == '/') { + dir_len = i - 1; + name = path + i; + break; + } + } + + strncpy(name_out, name, name_out_len - 1); + name_out[name_out_len - 1] = '\0'; + + struct vfs_path current = vfs_path_clone(at); + + size_t index = 0; + while (index < dir_len) { + while (index < dir_len && path[index] == '/') + index++; + if (index >= dir_len) + break; + + size_t start = index; + while (index < dir_len && path[index] != '/') + index++; + + char component[__NAME_MAX + 1]; + size_t length = MIN(index - start, __NAME_MAX); + memcpy(component, path + start, length); + component[length] = '\0'; + + errno_t status = vfs_mkdir(root, current, component, mode, ident_kernel()); + if (status && status != EEXIST) { + vfs_path_drop(current); + return status; + } + + struct vfs_file* dir; + status = vfs_file_open(root, current, component, O_DIRECTORY, 0, ident_kernel(), &dir); + if (status) { + vfs_path_drop(current); + return status; + } + + struct vfs_path next = vfs_path_clone(dir->path); + vfs_file_unref(dir); + vfs_path_drop(current); + current = next; + } + + *out_dir = current; + return 0; +} + +bool initramfs_is_valid(const void* data, size_t len) { + const struct ustar_header* header = data; + return len >= sizeof(*header) && !memcmp(header->signature, "ustar", 5); +} + +// Sets all timestamps of the node behind `file` to `mtime`. +static void set_times(struct vfs_file* file, size_t mtime) { + struct vfs_node* node = file->vnode; + timespec_t time = time_from_sec(mtime); + + mutex_lock(&node->lock); + node->atime = time; + node->mtime = time; + node->ctime = time; + mutex_unlock(&node->lock); +} + +errno_t initramfs_load(struct vfs_path root, struct vfs_path target, const void* data, size_t len) { + const uint8_t* base = data; + size_t offset = 0; + size_t files_loaded = 0; + errno_t status = 0; + + // Paths can be much longer than what fits on the stack. + char* override = mem_alloc(__PATH_MAX + 1, ALLOC_NOZERO); + char* name = mem_alloc(__PATH_MAX + 1, ALLOC_NOZERO); + if (!override || !name) { + status = ENOMEM; + goto out; + } + bool has_override = false; + + while (offset + sizeof(struct ustar_header) <= len) { + const struct ustar_header* header = (const struct ustar_header*)(base + offset); + if (memcmp(header->signature, "ustar", 5) != 0) + break; + + mode_t file_mode = (mode_t)oct2bin(header->mode, sizeof(header->mode)); + size_t file_size = oct2bin(header->size, sizeof(header->size)); + size_t mtime = oct2bin(header->mtime, sizeof(header->mtime)); + + size_t data_offset = offset + sizeof(struct ustar_header); + if (file_size > len - data_offset) { + log_warn("vfs: Truncated initramfs entry at offset %zu\n", offset); + status = EINVAL; + goto out; + } + + if (has_override) { + memcpy(name, override, __PATH_MAX + 1); + has_override = false; + } else { + copy_field(name, header->name, sizeof(header->name), __PATH_MAX + 1); + } + + // Directory entries usually end with a slash, which would leave us with an empty file name. + size_t name_len = strlen(name); + while (name_len > 1 && name[name_len - 1] == '/') + name[--name_len] = '\0'; + + struct vfs_path dir; + char file_name[__NAME_MAX + 1]; + + switch (header->type) { + case USTAR_REGULAR: + case USTAR_NORMAL: + case USTAR_CONTIGUOUS: { + status = create_dirs(root, target, name, file_mode, &dir, file_name, sizeof(file_name)); + if (status) + goto out; + + struct vfs_file* file; + status = vfs_file_open(root, dir, file_name, O_CREAT, file_mode, ident_kernel(), &file); + vfs_path_drop(dir); + if (status) + goto out; + + if (file_size > 0) { + iovec_t vec = { + .base = (uintptr_t)(base + data_offset), + .len = file_size, + }; + iovec_iter_t iter = iovec_iter_kernel(&vec, 1); + ssize_t written = 0; + status = vfs_file_pwrite(file, &iter, 0, &written); + if (status) { + vfs_file_unref(file); + goto out; + } + } + + set_times(file, mtime); + vfs_file_unref(file); + files_loaded++; + break; + } + case USTAR_SYM_LINK: { + status = create_dirs(root, target, name, file_mode, &dir, file_name, sizeof(file_name)); + if (status) + goto out; + + char link_target[101]; + copy_field(link_target, header->linkname, sizeof(header->linkname), sizeof(link_target)); + + status = vfs_symlink(root, dir, file_name, link_target, ident_kernel()); + vfs_path_drop(dir); + if (status) + goto out; + + files_loaded++; + break; + } + case USTAR_DIRECTORY: { + status = create_dirs(root, target, name, file_mode, &dir, file_name, sizeof(file_name)); + if (status) + goto out; + + status = vfs_mkdir(root, dir, file_name, file_mode, ident_kernel()); + vfs_path_drop(dir); + if (status && status != EEXIST) + goto out; + status = 0; + + files_loaded++; + break; + } + case USTAR_LONG_LINK: { + size_t copy_len = MIN(file_size, __PATH_MAX); + memcpy(override, base + data_offset, copy_len); + override[copy_len] = '\0'; + has_override = true; + break; + } + default: + break; + } + + offset += sizeof(struct ustar_header) + ALIGN_UP(file_size, 512); + } + + log("vfs: Loaded %zu files from initramfs\n", files_loaded); + +out: + mem_free(override); + mem_free(name); + return status; +} + +void initramfs_load_boot_modules(struct boot_info* info) { + struct vfs_path root = vfs_get_root(); + size_t ramdisk_index = 0; + + for (size_t i = 0; i < info->num_files; i++) { + struct boot_file* file = &info->files[i]; + const void* data = HHDM_PTR(file->data); + + if (!initramfs_is_valid(data, file->length)) { + char name[16]; + ksnprintf(name, sizeof(name), "ram%zu", ramdisk_index); + struct blockdev* disk; + errno_t status = ramdisk_new(file->data, file->length, &disk); + if (!status) { + status = blockdev_register(name, disk); + blockdev_unref(disk); + } + if (status) { + log_warn("initramfs: Unable to expose module \"%s\" as a RAM disk (%i)\n", file->path, status); + continue; + } + log("initramfs: Exposed module \"%s\" as RAM disk /dev/block/%s\n", file->path, name); + ramdisk_index++; + continue; + } + + errno_t status = initramfs_load(root, root, data, file->length); + assert(status == 0, "Failed to load initramfs \"%s\" (%i)", file->path, status); + } + + vfs_path_drop(root); +} diff --git a/fs/meson.build b/fs/meson.build index 7f544c1..524b404 100644 --- a/fs/meson.build +++ b/fs/meson.build @@ -5,6 +5,7 @@ kernel_sources += files( 'file.c', 'ident.c', 'init.c', + 'initramfs.c', 'mount.c', 'ops.c', 'pipe.c', diff --git a/include/zinnia/blockdev.h b/include/zinnia/blockdev.h new file mode 100644 index 0000000..7a52c91 --- /dev/null +++ b/include/zinnia/blockdev.h @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum bio_op : uint8_t { + BIO_READ, + BIO_WRITE, + BIO_FLUSH, +}; + +struct bio_seg { + phys_t addr; + size_t len; +}; + +struct blockdev; + +// A block IO operation. +struct bio { + enum bio_op op; // What to do. + uint64_t lba; // Starting block. + size_t num_lbas; // Amount of blocks to read. + struct blockdev* dev; // Device to use for this bio. + errno_t status; // Result of this operation. + size_t done_lbas; // Current progress. + void (*end)(struct bio* bio); // Callback which runs when the bio completes. + void* priv; // Driver data. + struct bio* parent; // For nested bios. + uint64_t parent_offset; + _Atomic(size_t) pending; + _Atomic(uint64_t) short_offset; + _Atomic(errno_t) child_status; + struct event event; // Event for poll. + size_t num_segs; // Counter for `segs`. + struct bio_seg segs[]; +}; + +struct blockdev_limits { + size_t max_lbas; + size_t max_segs; +}; + +struct blockdev_ops { + // Submit a bio for execution on this block device. + errno_t (*submit)(struct blockdev* dev, struct bio* bio); + // Handles an ioctl for the block device. + errno_t (*ioctl)(struct blockdev* dev, struct vfs_file* file, uintptr_t request, uintptr_t arg, uintptr_t* out); + void (*destroy)(struct blockdev* dev); +}; + +struct blockdev { + struct vmo cache; + const struct blockdev_ops* ops; + dev_t dev; + size_t lba_size; + uint64_t lba_count; + struct blockdev_limits limits; + _Atomic(bool) claimed; +}; + +struct bio* bio_alloc(enum bio_op op, size_t num_segs); + +void bio_free(struct bio* bio); + +// Submit a bio to run on `dev`. +void bio_submit(struct blockdev* dev, struct bio* bio); + +void bio_complete(struct bio* bio, errno_t status, size_t done_lbas); + +errno_t bio_wait(struct bio* bio, size_t* out_lbas); + +void blockdev_init( + struct blockdev* dev, + const struct blockdev_ops* ops, + uint32_t major, + uint32_t minor, + size_t lba_size, + uint64_t lba_count +); +void blockdev_ref(struct blockdev* dev); +void blockdev_unref(struct blockdev* dev); + +// Performs IO on a block device using `bio_seg` segments. +errno_t blockdev_rw( + struct blockdev* dev, + enum bio_op op, + uint64_t lba, + size_t num_lbas, + const struct bio_seg* segs, + size_t num_segs, + size_t* out_lbas +); + +// Performs IO on a block device using pages. +errno_t blockdev_rw_pages( + struct blockdev* dev, + enum bio_op op, + uint64_t lba, + size_t num_lbas, + struct page** pages, + size_t count, + size_t* out_lbas +); + +// Flushes out all outstanding bios on `dev`. +errno_t blockdev_flush(struct blockdev* dev); + +errno_t blockdev_from_node(struct vfs_node* node, struct blockdev** out); +errno_t blockdev_claim(struct blockdev* dev); +void blockdev_release(struct blockdev* dev); + +// Adds a new block device to the list of known devices. +// Will also add a node in `/dev/block`. +errno_t blockdev_register(const char* name, struct blockdev* dev); + +// Rescans `dev` for partitions or file systems. +errno_t blockdev_rescan(struct blockdev* dev); + +// Syncs all outstanding requests blockdevs +errno_t blockdev_sync_all(); + +errno_t devtmpfs_add_blockdev(const char* name, struct blockdev* dev, mode_t mode); + +errno_t partition_new(struct blockdev* parent, uint64_t start_lba, uint64_t lba_count, struct blockdev** out); + +struct gpt_partition { + struct guid type_guid; + struct guid unique_guid; + uint64_t start_lba; + uint64_t end_lba; +}; + +errno_t gpt_scan(struct blockdev* dev, struct gpt_partition** out, size_t* out_count); + +errno_t ramdisk_new(phys_t base, size_t length, struct blockdev** out); + +INIT_EXTERN(block_stage); diff --git a/include/zinnia/initramfs.h b/include/zinnia/initramfs.h new file mode 100644 index 0000000..dc147cc --- /dev/null +++ b/include/zinnia/initramfs.h @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: ISC + +#pragma once + +#include +#include +#include + +struct boot_info; + +bool initramfs_is_valid(const void* data, size_t len); + +errno_t initramfs_load(struct vfs_path root, struct vfs_path target, const void* data, size_t len); + +void initramfs_load_boot_modules(struct boot_info* info);