From 9697ee777a5cba5f06ac2fcbd1e4d2bb01cfab9f Mon Sep 17 00:00:00 2001 From: Matheus Date: Tue, 5 May 2026 00:35:11 -0300 Subject: [PATCH] kernel: initial xhci and usb implementation --- kernel-src/arch/x86-64/kernel.ld | 13 +- kernel-src/include/kernel/interrupt.h | 1 + kernel-src/include/kernel/pci.h | 3 + kernel-src/include/kernel/usb.h | 387 +++++++++++++ kernel-src/include/kernel/xhci.h | 450 +++++++++++++++ kernel-src/io/usb/core/hub.c | 477 ++++++++++++++++ kernel-src/io/usb/core/usb.c | 60 ++ kernel-src/io/usb/ctrl/xhci.c | 762 ++++++++++++++++++++++++++ kernel-src/io/usb/ctrl/xhci_ctrl.c | 583 ++++++++++++++++++++ kernel-src/io/usb/ctrl/xhci_hub.c | 153 ++++++ kernel-src/io/usb/ctrl/xhci_ring.c | 130 +++++ kernel-src/io/usb/ctrl/xhci_xfer.c | 288 ++++++++++ 12 files changed, 3304 insertions(+), 3 deletions(-) create mode 100644 kernel-src/include/kernel/usb.h create mode 100644 kernel-src/include/kernel/xhci.h create mode 100644 kernel-src/io/usb/core/hub.c create mode 100644 kernel-src/io/usb/core/usb.c create mode 100644 kernel-src/io/usb/ctrl/xhci.c create mode 100644 kernel-src/io/usb/ctrl/xhci_ctrl.c create mode 100644 kernel-src/io/usb/ctrl/xhci_hub.c create mode 100644 kernel-src/io/usb/ctrl/xhci_ring.c create mode 100644 kernel-src/io/usb/ctrl/xhci_xfer.c diff --git a/kernel-src/arch/x86-64/kernel.ld b/kernel-src/arch/x86-64/kernel.ld index 6f3f0c48..86c7921d 100644 --- a/kernel-src/arch/x86-64/kernel.ld +++ b/kernel-src/arch/x86-64/kernel.ld @@ -20,7 +20,7 @@ SECTIONS } :text . = ALIGN(4K); - _text_end = .; + _text_end = .; _rodata_start = .; .rodata : { *(.rodata .rodata.*); @@ -32,7 +32,14 @@ SECTIONS *(.timekeeper_sources); timekeeper_sources_end = .; } :rodata - + + . = ALIGN(8); + .usb_class_drivers : { + usb_class_drivers = .; + *(.usb_class_drivers); + usb_class_drivers_end = .; + } :rodata + . = ALIGN(8); .init_routines : { init_routines = .; @@ -60,7 +67,7 @@ SECTIONS *(COMMON) *(.bss .bss.*) } :data - + . = ALIGN(4K); _data_end = .; } diff --git a/kernel-src/include/kernel/interrupt.h b/kernel-src/include/kernel/interrupt.h index 30805a54..403736a9 100644 --- a/kernel-src/include/kernel/interrupt.h +++ b/kernel-src/include/kernel/interrupt.h @@ -19,6 +19,7 @@ #define IPL_DISK 500 #define IPL_SERIAL 550 #define IPL_INPUT 600 +#define IPL_USB 675 #define IPL_ACPI 700 #define IPL_DPC 900 #define IPL_NORMAL 1000 diff --git a/kernel-src/include/kernel/pci.h b/kernel-src/include/kernel/pci.h index 1d03c7a7..0c9e9129 100644 --- a/kernel-src/include/kernel/pci.h +++ b/kernel-src/include/kernel/pci.h @@ -22,11 +22,14 @@ #define PCI_CONFIG_REVISION 0x8 #define PCI_CONFIG_PROGIF 0x9 #define PCI_PROGIF_STORAGE_SATA_AHCI 0x1 + #define PCI_PROGIF_USB_XHCI 0x30 #define PCI_CONFIG_SUBCLASS 0xa #define PCI_SUBCLASS_STORAGE_NVM 0x8 #define PCI_SUBCLASS_STORAGE_SATA 0x6 + #define PCI_SUBCLASS_SERIAL_BUS_USB 0x3 #define PCI_CONFIG_CLASS 0xb #define PCI_CLASS_STORAGE 0x1 + #define PCI_CLASS_SERIAL_BUS 0xc #define PCI_CONFIG_HEADERTYPE 0xe #define PCI_HEADERTYPE_MULTIFUNCTION_MASK 0x80 #define PCI_HEADERTYPE_MASK 0x7f diff --git a/kernel-src/include/kernel/usb.h b/kernel-src/include/kernel/usb.h new file mode 100644 index 00000000..076e7b9e --- /dev/null +++ b/kernel-src/include/kernel/usb.h @@ -0,0 +1,387 @@ +#ifndef _USB_H +#define _USB_H + +#include +#include +#include + +#include +#include + +typedef struct usb_hub_port usb_hub_port_t; +typedef struct usb_hub usb_hub_t; +typedef struct usb_device usb_device_t; +typedef struct usb_interface usb_interface_t; +typedef struct usb_endpoint usb_endpoint_t; +typedef struct usb_xfer usb_xfer_t; +typedef struct usb_ctrl usb_ctrl_t; +typedef struct usb_class_driver usb_class_driver_t; + +#define USB_DESCRIPTOR_TYPE_DEVICE 1 +#define USB_DESCRIPTOR_TYPE_CONFIG 2 +#define USB_DESCRIPTOR_TYPE_STRING 3 +#define USB_DESCRIPTOR_TYPE_INTERFACE 4 +#define USB_DESCRIPTOR_TYPE_ENDPOINT 5 +#define USB_DESCRIPTOR_TYPE_SS_EP_COMPANION 48 + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; +} usb_desc_hdr_t; + +#define USB_MAX_CONFIG_DESC_SIZE 4096 + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint16_t bcdUSB; + uint8_t bDeviceClass; + uint8_t bDeviceSubClass; + uint8_t bDeviceProtocol; + uint8_t bMaxPacketSize0; + uint16_t idVendor; + uint16_t idProduct; + uint16_t bcdDevice; + uint8_t iManufacturer; + uint8_t iProduct; + uint8_t iSerialNumber; + uint8_t bNumConfigurations; +} __attribute__((packed)) usb_device_desc_t; + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint16_t wTotalLength; + uint8_t bNumInterfaces; + uint8_t bConfigurationValue; + uint8_t iConfiguration; + uint8_t bmAttributes; + uint8_t bMaxPower; +} __attribute__((packed)) usb_config_desc_t; + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bInterfaceNumber; + uint8_t bAlternateSetting; + uint8_t bNumEndpoints; + uint8_t bInterfaceClass; + uint8_t bInterfaceSubClass; + uint8_t bInterfaceProtocol; + uint8_t iInterface; +} __attribute__((packed)) usb_interface_desc_t; + +#define USB_ENDPOINT_ADDRESS_NUM_MASK 0x0f +#define USB_ENDPOINT_ADDRESS_DIR_IN 0x80 + +#define USB_ENDPOINT_ATTRIB_TYPE_MASK 0x03 +#define USB_ENDPOINT_ATTRIB_TYPE_CONTROL 0x00 +#define USB_ENDPOINT_ATTRIB_TYPE_ISOCH 0x01 +#define USB_ENDPOINT_ATTRIB_TYPE_BULK 0x02 +#define USB_ENDPOINT_ATTRIB_TYPE_INTR 0x03 + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bEndpointAddress; + uint8_t bmAttributes; + uint16_t wMaxPacketSize; + uint8_t bInterval; +} __attribute__((packed)) usb_endpoint_desc_t; + +#define USB_ENDPOINT_MAX_PACKET_SIZE_MASK 0x7ff + +static inline uint16_t usb_endpoint_max_packet_size(usb_endpoint_desc_t *desc) { + return desc->wMaxPacketSize & USB_ENDPOINT_MAX_PACKET_SIZE_MASK; +} + +typedef struct { + uint8_t bLength; + uint8_t bDescriptorType; + uint8_t bMaxBurst; + uint8_t bmAttributes; + uint16_t wBytesPerInterval; +} __attribute__((packed)) usb_ss_ep_companion_desc_t; + +#define USB_REQUEST_RECIP_DEVICE 0x00 +#define USB_REQUEST_RECIP_INTERFACE 0x01 +#define USB_REQUEST_RECIP_ENDPOINT 0x02 +#define USB_REQUEST_RECIP_OTHER 0x03 + +#define USB_REQUEST_STANDARD 0x00 +#define USB_REQUEST_CLASS 0x20 +#define USB_REQUEST_VENDOR 0x40 + +#define USB_REQUEST_DIR_TO_DEVICE 0x00 +#define USB_REQUEST_DIR_TO_HOST 0x80 + +#define USB_REQUEST_GET_DESCRIPTOR 6 +#define USB_REQUEST_SET_CONFIGURATION 9 +#define USB_REQUEST_SET_INTERFACE 11 + +typedef struct { + uint8_t bmRequestType; + uint8_t bRequest; + uint16_t wValue; + uint16_t wIndex; + uint16_t wLength; +} __attribute__((packed)) usb_setup_t; + +typedef enum { + USB_HUB_PORT_DISCONNECTED, + USB_HUB_PORT_RESETTING, + USB_HUB_PORT_ENABLED, +} usb_hub_port_status_t; + +struct usb_hub_port { + usb_device_t *device; + uint32_t generation; +}; + +#define USB_HUB_PORT_STATUS_PORT_CONNECTION (1 << 0) +#define USB_HUB_PORT_STATUS_PORT_ENABLE (1 << 1) +#define USB_HUB_PORT_STATUS_PORT_OVER_CURRENT (1 << 3) +#define USB_HUB_PORT_STATUS_PORT_RESET (1 << 4) +#define USB_HUB_PORT_STATUS_PORT_POWER (1 << 8) +#define USB_HUB_PORT_STATUS_LOW_SPEED (1 << 9) +#define USB_HUB_PORT_STATUS_HIGH_SPEED (1 << 10) + +#define USB_HUB_PORT_CHANGE_PORT_CONNECTION (1 << 0) +#define USB_HUB_PORT_CHANGE_PORT_ENABLE (1 << 1) +#define USB_HUB_PORT_CHANGE_PORT_OVER_CURRENT (1 << 3) +#define USB_HUB_PORT_CHANGE_PORT_RESET (1 << 4) + +#define USB_HUB_FEATURE_PORT_CONNECTION 0 +#define USB_HUB_FEATURE_PORT_ENABLE 1 +#define USB_HUB_FEATURE_PORT_RESET 4 +#define USB_HUB_FEATURE_PORT_POWER 8 +#define USB_HUB_FEATURE_PORT_LOW_SPEED 9 +#define USB_HUB_FEATURE_C_PORT_CONNECTION 16 +#define USB_HUB_FEATURE_C_PORT_ENABLE 17 +#define USB_HUB_FEATURE_C_PORT_OVER_CURRENT 19 +#define USB_HUB_FEATURE_C_PORT_RESET 20 + +typedef struct { + int (*get_port_status)(usb_hub_t *, uint8_t port, uint16_t *status, uint16_t *change); + int (*set_port_feature)(usb_hub_t *, uint8_t port, uint16_t feature); + int (*clear_port_feature)(usb_hub_t *, uint8_t port, uint16_t feature); +} usb_hub_ops_t; + +struct usb_hub { + char name[32]; + + // List node for linking hubs in the controller's hub list. + list_node_t node; + // Controller the hub is connected to. + usb_ctrl_t *ctrl; + // Operations for managing the hub. + usb_hub_ops_t *ops; + // Pointer to the device representing the hub itself. + // NULL if the hub is a root hub. + usb_device_t *device; + // Array of ports on the hub. + // Each port can have a pointer to the connected device. + usb_hub_port_t *ports; + uint32_t port_count; +}; + +struct usb_endpoint { + usb_endpoint_desc_t *desc; + usb_ss_ep_companion_desc_t *ss_companion; +}; + +struct usb_interface { + usb_interface_desc_t *desc; + usb_endpoint_t *endpoints; + // Pointer to the attached class driver (for cleanup on disconnect). + usb_class_driver_t *driver; + // Private data for the attached class driver. + void *driver_data; +}; + +typedef enum { + USB_SPEED_UNKNOWN, + USB_SPEED_LOW, + USB_SPEED_FULL, + USB_SPEED_HIGH, + USB_SPEED_SUPER, + USB_SPEED_SUPER_PLUS, +} usb_speed_t; + +static inline bool usb_device_desc_valid(usb_device_desc_t *desc, usb_speed_t speed, size_t transferred) { + if (transferred < sizeof(usb_device_desc_t)) + return false; + + if (desc->bLength != sizeof(usb_device_desc_t)) + return false; + + if (desc->bDescriptorType != USB_DESCRIPTOR_TYPE_DEVICE) + return false; + + if (desc->bNumConfigurations == 0) + return false; + + if (speed == USB_SPEED_SUPER || speed == USB_SPEED_SUPER_PLUS) + return desc->bMaxPacketSize0 == 9; + + if (speed == USB_SPEED_LOW) + return desc->bMaxPacketSize0 == 8; + + if (speed == USB_SPEED_HIGH) + return desc->bMaxPacketSize0 == 64; + + if (speed == USB_SPEED_FULL) { + return desc->bMaxPacketSize0 == 8 || + desc->bMaxPacketSize0 == 16 || + desc->bMaxPacketSize0 == 32 || + desc->bMaxPacketSize0 == 64; + } + + return false; +} + +struct usb_device { + // Pointer to the parent hub. + usb_hub_t *hub; + // Descriptors for device + usb_device_desc_t desc; + usb_config_desc_t *config_desc; + // Port number on the parent hub. + uint8_t port_number; + // Hub-port generation captured when enumeration started. + uint32_t port_generation; + // Read only USB address assigned to the device. + uint8_t address; + // Device speed. + usb_speed_t speed; + // Interfaces + usb_interface_t *interfaces; +}; + +typedef enum { + USB_XFER_FLAG_TO_DEVICE = (1 << 0), + USB_XFER_FLAG_TO_HOST = (1 << 1), + USB_XFER_FLAG_IOVEC = (1 << 2), + USB_XFER_FLAG_BUFFER_PHYSICAL = (1 << 3), +} usb_xfer_flag_t; + +typedef enum { + USB_XFER_TYPE_CONTROL, + USB_XFER_TYPE_BULK, + USB_XFER_TYPE_INTERRUPT, +} usb_xfer_type_t; + +typedef enum { + USB_STATUS_SUCCESS, + USB_STATUS_SHORT_PACKET, + USB_STATUS_ERROR, + USB_STATUS_STALL, + USB_STATUS_FLOW_ERROR, +} usb_status_t; + +typedef void (*usb_completion_callback_t)(usb_device_t *device, void *completion_ctx, usb_status_t status, size_t transferred); + +// Represents a USB control or data transfer. +struct usb_xfer { + // Target endpoint for the transfer. + // Should be NULL and will be ignored for control transfers. + usb_endpoint_t *ep; + + // Flags for the transfer. + usb_xfer_flag_t flags; + usb_xfer_type_t type; + + // Only valid for control transfers. + usb_setup_t *setup; + + // Buffer for the transfer. + union { + iovec_iterator_t *iov; + struct { + void *data; + size_t data_length; + }; + }; + + // Asynchronous completion callback. + usb_completion_callback_t completion; + usb_device_t *device; + void *completion_ctx; +}; + +typedef void (*usb_address_device_callback_t)(usb_hub_t *hub, uint8_t port, usb_device_t *device); + +typedef struct { + // Allocate, enable and address a USB device. + int (*address_device)(usb_ctrl_t *, usb_hub_t *, uint8_t port, usb_address_device_callback_t callback); + // Deaddress, disable and free a USB device + void (*deaddress_device)(usb_ctrl_t *, usb_device_t *); + // Configure an endpoint. + int (*configure_ep)(usb_ctrl_t *, usb_device_t *, usb_endpoint_t *, usb_completion_callback_t callback, void *callback_ctx); + // Execute a control or data transfer. + int (*xfer)(usb_ctrl_t *, usb_device_t *, usb_xfer_t *); +} usb_ctrl_ops_t; + +struct usb_ctrl { + usb_ctrl_ops_t *ops; +}; + +int usb_submit_xfer(usb_device_t *dev, usb_xfer_t *xfer); +int usb_control_xfer(usb_device_t *dev, usb_setup_t *setup, void *buffer, usb_completion_callback_t callback, void *ctx); + +int usb_get_device_descriptor(usb_device_t *dev, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx); +int usb_get_interface_descriptor(usb_device_t *dev, uint16_t interface_number, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx); +int usb_get_endpoint_descriptor(usb_device_t *dev, uint16_t endpoint_address, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx); +int usb_set_configuration(usb_device_t *dev, uint8_t config_value, usb_completion_callback_t callback); + +void usb_hub_event_connect(usb_hub_t *hub, int port); +void usb_hub_event_disconnect(usb_hub_t *hub, int port); +void usb_hub_event_reset(usb_hub_t *hub, int port); + +#define USB_DRIVER_SCORE_NONE 0 +#define USB_DRIVER_SCORE_GENERIC 10 +#define USB_DRIVER_SCORE_CLASS_MATCH 30 +#define USB_DRIVER_SCORE_SUBCLASS_MATCH 50 +#define USB_DRIVER_SCORE_PROTOCOL_MATCH 60 +#define USB_DRIVER_SCORE_VENDOR_MATCH 80 +#define USB_DRIVER_SCORE_PRODUCT_MATCH 90 +#define USB_DRIVER_SCORE_EXACT_MATCH 100 + +#define usb_for_each_descriptor(desc, item) \ + for (usb_desc_hdr_t *item = (usb_desc_hdr_t *)((void *)(desc) + desc->bLength); \ + (void *)item < (void *)(desc) + (desc)->wTotalLength; \ + item = (usb_desc_hdr_t *)((void *)item + item->bLength)) + +static inline bool usb_descriptor_malformed(usb_config_desc_t *desc, usb_desc_hdr_t *item) { + uint8_t *top = (uint8_t *)desc + desc->wTotalLength; + uint8_t *start = (uint8_t *)item; + + if (start + sizeof(*item) > top) + return true; + + if (item->bLength < sizeof(*item)) + return true; + + return start + item->bLength > top; +} + +#define USB_FOR_EACH_DESCRIPTOR_CHECK(desc, item) \ + if (usb_descriptor_malformed((desc), (item))) + +struct usb_class_driver { + const char *name; + // Probe function, returns score - (0 = won't handle, higher = better match). + int (*probe)(usb_device_t *dev, usb_interface_t *interface); + // Called when this driver is selected for the interface. + int (*attach)(usb_device_t *dev, usb_interface_t *interface); + // Called when device is disconnected. + void (*detach)(usb_device_t *dev, usb_interface_t *interface); +}; + +#define DEFINE_USB_CLASS_DRIVER(name, ...) \ + static usb_class_driver_t __usb_class_driver_##name = { \ + __VA_ARGS__ \ + }; \ + __attribute__((section(".usb_class_drivers"), used)) static usb_class_driver_t *usb_class_driver_##name = &__usb_class_driver_##name; + +#endif diff --git a/kernel-src/include/kernel/xhci.h b/kernel-src/include/kernel/xhci.h new file mode 100644 index 00000000..45648a18 --- /dev/null +++ b/kernel-src/include/kernel/xhci.h @@ -0,0 +1,450 @@ +#ifndef _XHCI_H +#define _XHCI_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define XHCI_CALLBACK_RING_ENTRIES 256 + +typedef struct { + union { + uint64_t parameters; + + struct { + uint32_t dw0; + uint32_t dw1; + }; + }; + + uint32_t dw2; + uint32_t dw3; +} xhci_trb_t; + +typedef struct { + uint8_t caplength; + uint8_t reserved1; + uint16_t hciversion; + uint32_t hcsparams1; + uint32_t hcsparams2; + uint32_t hcsparams3; + uint32_t hccparams1; + uint32_t dboff; + uint32_t rtsoff; + uint32_t hccparams2; +} xhci_caps_t; + +typedef struct { + uint32_t portsc; + uint32_t portpmsc; + uint32_t portli; + uint32_t porthlpmc; +} xhci_port_regs_t; + +typedef struct { + uint32_t usbcmd; + uint32_t usbsts; + uint32_t pagesize; + uint32_t reserved1; + uint32_t reserved2; + uint32_t dnctrl; + uint64_t crcr; + uint32_t reserved3[4]; + uint64_t dcbaap; + uint32_t config; +} xhci_opregs_t; + +typedef struct { + uint32_t iman; + uint32_t imod; + uint32_t erstsz; + uint32_t reserved; + uint64_t erstba; + uint64_t erdp; +} xhci_ir_t; + +typedef struct { + uint32_t mfindex; + uint32_t reserved[7]; + xhci_ir_t ir[1024]; +} xhci_rtregs_t; + +typedef struct { + uint64_t ring_segment; + uint16_t ring_segment_size; + uint16_t reserved1; + uint32_t reserved2; +} xhci_erst_entry_t; + +typedef struct { + struct { + uint32_t route_string : 20; + uint8_t speed : 4; + uint8_t reserved1 : 1; + uint8_t mtt : 1; + uint8_t hub : 1; + uint8_t ctx_entries : 5; + } dw0; + + struct { + uint16_t max_exit_latency; + uint8_t root_hub_port_number; + uint8_t number_of_ports; + } dw1; + + struct { + uint8_t tt_hub_slot_id; + uint8_t tt_port_number; + uint16_t ttt : 2; + uint16_t reserved1 : 4; + uint16_t interrupter_target : 10; + } dw2; + + struct { + uint8_t usb_device_address; + uint8_t reserved1[2]; + uint8_t reserved2 : 3; + uint8_t slot_state : 5; + } dw3; + + uint32_t reserved[4]; +} xhci_slot_ctx_t; + +typedef struct { + struct { + uint8_t ep_state : 3; + uint8_t reserved1 : 5; + uint8_t mult : 2; + uint8_t max_pstreams : 5; + uint8_t lsa : 1; + uint8_t interval; + uint8_t max_esit_payload_hi; + } dw0; + + struct { + uint8_t reserved1 : 1; + uint8_t cerr : 2; + uint8_t ep_type : 3; + uint8_t reserved2 : 1; + uint8_t hid : 1; + uint8_t max_burst_size; + uint16_t max_packet_size; + } dw1; + + struct { + uint32_t dcs : 1; + uint32_t reserved1 : 3; + uint32_t tr_dequeue_pointer_lo : 28; + } dw2; + + struct { + uint32_t tr_dequeue_pointer_hi; + } dw3; + + struct { + uint16_t max_esit_payload_lo; + } dw4; + + uint32_t reserved[3]; +} xhci_ep_ctx_t; + +typedef struct { + uint32_t d; + uint32_t a; + uint32_t reserved[5]; + + struct { + uint8_t configuration_value; + uint8_t interface_number; + uint8_t alternate_setting; + uint8_t reserved1; + } dw7; +} xhci_input_ctx_t; + +#define XHCI_CONTEXT_SIZE 32 +_Static_assert(sizeof(xhci_slot_ctx_t) == XHCI_CONTEXT_SIZE, "xHCI slot context must be 32 bytes"); +_Static_assert(sizeof(xhci_ep_ctx_t) == XHCI_CONTEXT_SIZE, "xHCI endpoint context must be 32 bytes"); +_Static_assert(sizeof(xhci_input_ctx_t) == XHCI_CONTEXT_SIZE, "xHCI input control context must be 32 bytes"); + +#define XHCI_HCCPARAMS1_AC64 (1 << 0) +#define XHCI_HCCPARAMS1_CSZ (1 << 2) + +#define XHCI_USBCMD_RS (1 << 0) +#define XHCI_USBCMD_HCRST (1 << 1) +#define XHCI_USBCMD_INTE (1 << 2) + +#define XHCI_USBSTS_HCH (1 << 0) +#define XHCI_USBSTS_EINT (1 << 3) +#define XHCI_USBSTS_CNR (1 << 11) +#define XHCI_USBSTS_HCE (1 << 12) + +#define XHCI_IMAN_IP (1 << 0) +#define XHCI_IMAN_IE (1 << 1) + +#define XHCI_CRCR_RCS (1 << 0) + +#define XHCI_ERDP_EHB (1 << 3) + +#define XHCI_PORTSC_CCS (1 << 0) // current connection status +#define XHCI_PORTSC_PED (1 << 1) // port enabled +#define XHCI_PORTSC_OCA (1 << 3) // over-current active +#define XHCI_PORTSC_PR (1 << 4) // port reset +#define XHCI_PORTSC_PP (1 << 9) // port power +#define XHCI_PORTSC_CSC (1 << 17) // connect status change +#define XHCI_PORTSC_PEC (1 << 18) // port enabled change +#define XHCI_PORTSC_OCC (1 << 20) // over current change +#define XHCI_PORTSC_PRC (1 << 21) // port reset change + +#define XHCI_PORTSC_CHANGE_BITS (XHCI_PORTSC_CSC | XHCI_PORTSC_PEC | XHCI_PORTSC_OCC | XHCI_PORTSC_PRC) +#define XHCI_PORTSC_PRESERVE_BITS XHCI_PORTSC_PP + +#define XHCI_TRB_DW2_TR_LEN(LEN) (uint32_t)((LEN) & 0x1ffff) +#define XHCI_TRB_DW2_TD_SIZE(SIZE) ((uint32_t)((SIZE) & 0x1f) << 17) + +#define XHCI_TRB_DW3_C (1 << 0) +#define XHCI_TRB_DW3_TC (1 << 1) +#define XHCI_TRB_DW3_ISP (1 << 2) +#define XHCI_TRB_DW3_CH (1 << 4) +#define XHCI_TRB_DW3_IOC (1 << 5) +#define XHCI_TRB_DW3_IDT (1 << 6) +#define XHCI_TRB_DW3_TYPE_MASK (0x3f << 10) +#define XHCI_TRB_DW3_DIR (1 << 16) + +#define XHCI_TRB_DW3_TYPE(TYPE) (((TYPE) & 0x3f) << 10) +#define XHCI_TRB_DW3_TRT(TRT) (((TRT) & 0x3) << 16) + +typedef enum { + TRB_NORMAL = 1, + TRB_SETUP_STAGE = 2, + TRB_DATA_STAGE = 3, + TRB_STATUS_STAGE = 4, + TRB_LINK = 6, + TRB_ENABLE_SLOT = 9, + TRB_DISABLE_SLOT = 10, + TRB_ADDRESS_DEVICE = 11, + TRB_CONFIGURE_EP = 12, + TRB_EVALUATE_CTX = 13, + TRB_RESET_EP = 14, + TRB_STOP_EP = 15, + TRB_XFER_COMPLETION_EVENT = 32, + TRB_COMMAND_COMPLETION_EVENT = 33, + TRB_PORT_STATUS_CHANGE_EVENT = 34, +} xhci_trb_type_t; + +typedef enum { + TRB_INVALID = 0, + TRB_SUCCESS = 1, + TRB_DATA_BUFFER_ERROR = 2, + TRB_BABBLE_DETECTED = 3, + TRB_TRANSACTION_ERROR = 4, + TRB_TRB_ERROR = 5, + TRB_STALL = 6, + TRB_RESOURCE_ERROR = 7, + TRB_BANDWIDTH_ERROR = 8, + TRB_NO_SLOTS_AVAILABLE = 9, + TRB_INVALID_STREAM_TYPE = 10, + TRB_SLOT_NOT_ENABLED = 11, + TRB_ENDPOINT_NOT_ENABLED = 12, + TRB_SHORT_PACKET = 13, + TRB_RING_UNDERRUN = 14, + TRB_RING_OVERRUN = 15, + TRB_VF_EVENT_RING_FULL = 16, + TRB_PARAMETER_ERROR = 17, + TRB_BANDWIDTH_OVERRUN_ERROR = 18, + TRB_CONTEXT_STATE_ERROR = 19, + TRB_NO_PING_RESPONSE = 20, + TRB_EVENT_RING_FULL = 21, + TRB_INCOMPATIBLE_DEVICE_ERROR = 22, + TRB_MISSED_SERVICE = 23, + TRB_COMMAND_RING_STOPPED = 24, + TRB_COMMAND_RING_ABORTED = 25, + TRB_STOPPED = 26, + TRB_STOPPED_LEN = 27, + TRB_STOPPED_SHORT = 28, + TRB_LATENCY_TOO_LARGE = 29, + TRB_ERROR_RESERVED = 30, + TRB_BUFFER_OVERRUN = 31, + TRB_EVENT_LOST = 32, + TRB_ERROR_UNDEFINED = 33, + TRB_INVALID_STREAM_ID = 34, + TRB_SECONDARY_BANDWIDTH_ERROR = 35, + TRB_SPLIT_TRANSACTION_ERROR = 36, + + TRB_VENDOR_ERROR_START = 192, + TRB_VENDOR_ERROR_END = 223, + TRB_VENDOR_INFO_START = 224, + TRB_VENDOR_INFO_END = 255, +} xhci_trb_status_t; + +typedef enum { + XHCI_PORT_SPEED_FULL = 1, + XHCI_PORT_SPEED_LOW = 2, + XHCI_PORT_SPEED_HIGH = 3, + XHCI_PORT_SPEED_SUPER = 4, + XHCI_PORT_SPEED_SS_2X1 = 5, + XHCI_PORT_SPEED_SS_1X2 = 6, + XHCI_PORT_SPEED_SS_2X2 = 7, +} xhci_port_speed_t; + +typedef enum { + XHCI_EP_TYPE_ISOCH_OUT = 1, + XHCI_EP_TYPE_BULK_OUT = 2, + XHCI_EP_TYPE_INTR_OUT = 3, + XHCI_EP_TYPE_CTRL = 4, + XHCI_EP_TYPE_ISOCH_IN = 5, + XHCI_EP_TYPE_BULK_IN = 6, + XHCI_EP_TYPE_INTR_IN = 7, +} xhci_ep_type_t; + +typedef struct xhci_submission xhci_submission_t; +typedef struct xhci_device xhci_device_t; + +typedef struct { + uint8_t ver_major; + uint8_t ver_minor; +} xhci_port_protocol_t; + +struct xhci_submission { + list_node_t list_node; + // Set whenever a submission is queued + bool valid; + struct xhci_ring *ring; + + // The data associated with this submission + usb_device_t *device; + usb_completion_callback_t callback; + void *callback_ctx; + size_t data_length; + size_t data_offset; + xhci_submission_t *next; + xhci_submission_t *td_head; + + // event_trb is filled from the event ring; cmd_trb is initialized on submission. + xhci_trb_t event_trb; + xhci_trb_t cmd_trb; + usb_status_t usb_status; +}; + +typedef struct xhci_ring { + void *ring_phys; + xhci_trb_t *ring; + + // Mutex to protect the ring structure. + mutex_t lock; + + // Back-pointer to the controller. + struct xhci_ctrl *ctrl; + + // List of submissions on this ring. Will be NULL for event rings. + xhci_submission_t *submissions; + resource_allocator_t trb_allocator; + + // Ring size in TRBs. + size_t size; + + // For event rings, the index of the next event to process. + // For command and transfer rings, the index of the next free TRB. + size_t index; + + // Cycle bit for the ring. + bool cycle; +} xhci_ring_t; + +typedef struct { + usb_device_t *device; + usb_completion_callback_t callback; + void *callback_ctx; + usb_status_t status; + size_t transferred; +} xhci_completion_t; + +typedef struct xhci_ctrl { + usb_ctrl_t ctrl; + pcienum_t *pci_enum; + + list_t hubs; + + uint32_t port_count; + uint32_t slot_count; + + xhci_port_protocol_t *ports; + xhci_device_t **slots; + + xhci_ring_t command_ring; + xhci_ring_t event_ring; + + semaphore_t event_sem; + semaphore_t callback_sem; + semaphore_t callback_space_sem; + spinlock_t event_lock; + mutex_t callback_mutex; + bitmap_t port_change_bitmap; + list_t completion_list; + ringbuffer_t callback_ring; + dpc_t dpc; + + uint32_t ctx_stride; + + volatile xhci_caps_t *caps; + volatile xhci_opregs_t *opregs; + volatile xhci_rtregs_t *rtregs; + volatile xhci_port_regs_t *portregs; + + volatile uint64_t *dcbaa; + volatile uint32_t *dbs; +} xhci_ctrl_t; + +typedef struct { + usb_hub_t hub; + uint8_t port_offset; + list_node_t list_node; +} xhci_root_hub_t; + +struct xhci_device { + usb_device_t device; + xhci_ring_t ep_rings[31]; + + uint32_t route_string; + uint8_t device_tier; + uint8_t slot_id; + uint8_t ctrl_speed; + uint8_t root_port_number; + + void *device_ctx_phys; + void *device_ctx; + void *input_ctx_phys; + void *input_ctx; +}; + +void xhci_handle_port_change_event(xhci_ctrl_t *xhci, int global_port); +int xhci_data_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, xhci_submission_t *sub); +int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev, usb_xfer_t *xfer, xhci_submission_t *sub); +int xhci_control_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, xhci_submission_t *sub); +xhci_trb_t *xhci_ring_dequeue(xhci_ring_t *r); +void xhci_ring_reserve(xhci_ring_t *r, size_t needed); +void xhci_ring_unreserve(xhci_ring_t *r, size_t needed); +void xhci_ring_submit(xhci_ring_t *r, xhci_trb_t *trb, xhci_submission_t *sub); +xhci_submission_t *xhci_ring_submit_locked(xhci_ring_t *r, xhci_trb_t *trb, xhci_submission_t *sub); +int xhci_alloc_ring(xhci_ctrl_t *ctrl, xhci_ring_t *r, bool event_ring); +void xhci_free_ring(xhci_ring_t *ring); + +static inline void xhci_ring_command_doorbell(xhci_ctrl_t *ctrl) { + ctrl->dbs[0] = 0; +} + +static inline void xhci_ring_endpoint_doorbell(xhci_ctrl_t *ctrl, xhci_device_t *device, int endpoint) { + ctrl->dbs[device->slot_id] = endpoint; +} + +extern usb_hub_ops_t xhci_root_hub_ops; +extern usb_ctrl_ops_t xhci_ops; + +#endif diff --git a/kernel-src/io/usb/core/hub.c b/kernel-src/io/usb/core/hub.c new file mode 100644 index 00000000..f9253c08 --- /dev/null +++ b/kernel-src/io/usb/core/hub.c @@ -0,0 +1,477 @@ +#include +#include +#include + +// TODO support using other configurations (for simplicity, we will just pick the first one for now) + +extern usb_class_driver_t *usb_class_drivers; +extern usb_class_driver_t *usb_class_drivers_end; + +static void endpoint_configured(usb_device_t *dev, void *ctx, usb_status_t status, size_t transferred); + +static bool endpoint_desc_valid(usb_device_t *dev, usb_endpoint_desc_t *desc) { + uint8_t ep_num = desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_NUM_MASK; + uint8_t ep_type = desc->bmAttributes & USB_ENDPOINT_ATTRIB_TYPE_MASK; + uint16_t max_packet_size = usb_endpoint_max_packet_size(desc); + uint8_t high_speed_transactions = (desc->wMaxPacketSize >> 11) & 0x3; + + if (ep_num == 0) + return false; + + if (max_packet_size == 0) + return false; + + if (ep_type == USB_ENDPOINT_ATTRIB_TYPE_BULK) { + if (dev->speed == USB_SPEED_LOW) + return false; + if (dev->speed == USB_SPEED_HIGH && high_speed_transactions != 0) + return false; + } else if (ep_type == USB_ENDPOINT_ATTRIB_TYPE_INTR) { + if (desc->bInterval == 0) + return false; + if ((dev->speed == USB_SPEED_HIGH || + dev->speed == USB_SPEED_SUPER || + dev->speed == USB_SPEED_SUPER_PLUS) && + desc->bInterval > 16) + return false; + if (dev->speed == USB_SPEED_HIGH && high_speed_transactions == 3) + return false; + } else { + return false; + } + + return true; +} + +static bool endpoint_requires_ss_companion(usb_device_t *dev, usb_endpoint_t *endpoint) { + if (dev->speed != USB_SPEED_SUPER && dev->speed != USB_SPEED_SUPER_PLUS) + return false; + + return (endpoint->desc->bmAttributes & USB_ENDPOINT_ATTRIB_TYPE_MASK) != USB_ENDPOINT_ATTRIB_TYPE_CONTROL; +} + +static bool device_port_stale(usb_device_t *dev) { + return dev->hub->ports[dev->port_number].generation != dev->port_generation; +} + +static int attach_interface_to_driver(usb_device_t *device, usb_interface_t *interface) { + int best_score = 0; + usb_class_driver_t *best_driver = NULL; + + for (usb_class_driver_t **it = &usb_class_drivers; it < &usb_class_drivers_end; ++it) { + usb_class_driver_t *driver = *it; + int score = driver->probe(device, interface); + + if (score > best_score) { + best_score = score; + best_driver = driver; + } + } + + if (best_driver == NULL) { + printf("usb: no driver found for interface %u\n", interface->desc->bInterfaceNumber); + return 0; + } + + if (best_driver->attach(device, interface)) { + printf("usb: attaching %s port %u addr %u interface %u to driver '%s' failed\n", + device->hub->name, + device->port_number + 1, + device->address, + interface->desc->bInterfaceNumber, + best_driver->name); + return 0; + } + + interface->driver = best_driver; + printf("usb: %s port %u addr %u device %04x:%04x interface %u attached to driver '%s' (score: %d)\n", + device->hub->name, + device->port_number + 1, + device->address, + device->desc.idVendor, + device->desc.idProduct, + interface->desc->bInterfaceNumber, + best_driver->name, + best_score); + return 0; +} + +// device plug ininitialization is complete, pass control for each interface to a driver +static int attach_to_drivers(usb_device_t *device) { + if (device_port_stale(device)) + return ENODEV; + + device->hub->ports[device->port_number].device = device; + + for (int i = 0; i < device->config_desc->bNumInterfaces; ++i) { + usb_interface_t *interface = &device->interfaces[i]; + + if (interface->desc == NULL) + return EINVAL; + + if (attach_interface_to_driver(device, interface)) + return EINVAL; + } + + return 0; +} + +static int configure_next_endpoint(usb_device_t *dev, uint8_t interface_idx, uint8_t endpoint_idx) { + if (device_port_stale(dev)) + return ENODEV; + + for (; interface_idx < dev->config_desc->bNumInterfaces; interface_idx++, endpoint_idx = 0) { + usb_interface_t *interface = &dev->interfaces[interface_idx]; + if (interface->desc == NULL) + return EINVAL; + + if (endpoint_idx < interface->desc->bNumEndpoints) { + usb_endpoint_t *endpoint = &interface->endpoints[endpoint_idx]; + if (endpoint->desc == NULL) + return EINVAL; + + uintptr_t id = endpoint_idx | (interface_idx << 8); + return dev->hub->ctrl->ops->configure_ep(dev->hub->ctrl, dev, endpoint, endpoint_configured, (void *)id); + } + } + + return attach_to_drivers(dev); +} + +static void post_config_cleanup(usb_device_t *dev) { + for (int i = 0; i < dev->config_desc->bNumInterfaces; ++i) { + if (dev->interfaces[i].endpoints) + free(dev->interfaces[i].endpoints); + } + + free(dev->interfaces); + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); +} + +// configure all the endpoints and attach to driver if the last endpoint was configured +static void endpoint_configured(usb_device_t *dev, void *ctx, usb_status_t status, size_t transferred) { + if (status != USB_STATUS_SUCCESS) + goto error; + + if (device_port_stale(dev)) + goto error; + + uintptr_t id = (uintptr_t)ctx; + uint8_t endpoint_idx = (id & 0xff) + 1; + uint8_t interface_idx = (id >> 8) & 0xff; + + if (configure_next_endpoint(dev, interface_idx, endpoint_idx) == 0) + return; // success, wait for endpoint to be configured. + + error: + printf("usb: endpoint config failed\n"); + post_config_cleanup(dev); +} + +// allocate and populate interface and endpoint datastructures, then start configuring endpoints +static void set_configuration_done(usb_device_t *dev, void *, usb_status_t status, size_t transferred) { + if (status != USB_STATUS_SUCCESS) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: failed to set configuration\n"); + return; + } + + if (device_port_stale(dev)) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + return; + } + + dev->interfaces = alloc(dev->config_desc->bNumInterfaces * sizeof(usb_interface_t)); + if (dev->interfaces == NULL) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: out of memory\n"); + return; + } + + // set up interface data + usb_for_each_descriptor(dev->config_desc, desc) { + // since this is the first pass over the configure descriptor, + // we will verify if it is valid and fail the enumeration if not + USB_FOR_EACH_DESCRIPTOR_CHECK(dev->config_desc, desc) { + printf("usb: malformed config description\n"); + goto interface_setup_error; + } + + if (desc->bDescriptorType != USB_DESCRIPTOR_TYPE_INTERFACE) + continue; + + if (desc->bLength < sizeof(usb_interface_desc_t)) { + printf("usb: malformed interface description\n"); + goto interface_setup_error; + } + + usb_interface_desc_t *interface_desc = (usb_interface_desc_t *)desc; + if (interface_desc->bAlternateSetting) + continue; + + if (interface_desc->bInterfaceNumber >= dev->config_desc->bNumInterfaces) { + printf("usb: malformed interface description\n"); + goto interface_setup_error; + } + + usb_interface_t *interface = &dev->interfaces[interface_desc->bInterfaceNumber]; + if (interface->desc != NULL) { + printf("usb: duplicate interface description\n"); + goto interface_setup_error; + } + + interface->desc = interface_desc; + if (interface_desc->bNumEndpoints > 0) { + interface->endpoints = alloc(interface_desc->bNumEndpoints * sizeof(usb_endpoint_t)); + if (interface->endpoints == NULL) { + printf("usb: out of memory\n"); + goto interface_setup_error; + } + } + } + + // check for missing interfaces + for (int i = 0; i < dev->config_desc->bNumInterfaces; ++i) { + if (dev->interfaces[i].desc == NULL) { + printf("usb: missing interface description\n"); + goto interface_setup_error; + } + } + + // set up endpoint data + usb_interface_t *current_interface = NULL; + size_t current_endpoint = 0; + usb_for_each_descriptor(dev->config_desc, desc) { + USB_FOR_EACH_DESCRIPTOR_CHECK(dev->config_desc, desc) { + printf("usb: malformed config description\n"); + goto interface_setup_error; + } + + if (desc->bDescriptorType == USB_DESCRIPTOR_TYPE_INTERFACE) { + if (desc->bLength < sizeof(usb_interface_desc_t)) { + printf("usb: malformed interface description\n"); + goto interface_setup_error; + } + + usb_interface_desc_t *interface_desc = (usb_interface_desc_t *)desc; + if (interface_desc->bInterfaceNumber >= dev->config_desc->bNumInterfaces) { + printf("usb: malformed interface description\n"); + goto interface_setup_error; + } + + if (interface_desc->bAlternateSetting) { + current_endpoint = 0; + current_interface = NULL; + continue; + } + + current_endpoint = 0; + current_interface = &dev->interfaces[interface_desc->bInterfaceNumber]; + continue; + } + + if (desc->bDescriptorType == USB_DESCRIPTOR_TYPE_ENDPOINT) { + // skip any endpoints on an alternate setting + if (current_interface == NULL) + continue; + + if (desc->bLength < sizeof(usb_endpoint_desc_t)) { + printf("usb: malformed endpoint description\n"); + goto interface_setup_error; + } + + if (current_endpoint >= current_interface->desc->bNumEndpoints) { + printf("usb: malformed interface description\n"); + goto interface_setup_error; + } + + usb_endpoint_desc_t *endpoint_desc = (usb_endpoint_desc_t *)desc; + if (!endpoint_desc_valid(dev, endpoint_desc)) { + printf("usb: malformed endpoint description\n"); + goto interface_setup_error; + } + + current_interface->endpoints[current_endpoint++].desc = endpoint_desc; + } else if (desc->bDescriptorType == USB_DESCRIPTOR_TYPE_SS_EP_COMPANION) { + if (current_interface == NULL || current_endpoint == 0 || desc->bLength < sizeof(usb_ss_ep_companion_desc_t)) { + printf("usb: malformed endpoint companion description\n"); + goto interface_setup_error; + } + + current_interface->endpoints[current_endpoint - 1].ss_companion = (usb_ss_ep_companion_desc_t *)desc; + } + } + + // verify if we got all advertised endpoints + for (int i = 0; i < dev->config_desc->bNumInterfaces; ++i) { + usb_interface_t *interface = &dev->interfaces[i]; + for (int j = 0; j < interface->desc->bNumEndpoints; ++j) { + if (interface->endpoints[j].desc == NULL) { + printf("usb: missing endpoint description\n"); + goto interface_setup_error; + } + + if (endpoint_requires_ss_companion(dev, &interface->endpoints[j]) && interface->endpoints[j].ss_companion == NULL) { + printf("usb: missing endpoint companion description\n"); + goto interface_setup_error; + } + } + } + + if (configure_next_endpoint(dev, 0, 0) == 0) + return; // success, wait for endpoint to be configured. + +interface_setup_error: + post_config_cleanup(dev); +} + +// set default configuration +static void get_configuration_done(usb_device_t *dev, void *, usb_status_t status, size_t transferred) { + if (status != USB_STATUS_SUCCESS) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: failed to get config descriptor\n"); + return; + } + + if (device_port_stale(dev)) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + return; + } + + if (usb_set_configuration(dev, dev->config_desc->bConfigurationValue, set_configuration_done)) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: usb_set_configuration failed\n"); + } +} + +static bool config_desc_header_valid(usb_config_desc_t *desc, size_t transferred) { + if (transferred < sizeof(usb_config_desc_t)) + return false; + + if (desc->bLength != sizeof(usb_config_desc_t)) + return false; + + if (desc->bDescriptorType != USB_DESCRIPTOR_TYPE_CONFIG) + return false; + + if (desc->wTotalLength < desc->bLength) + return false; + + if (desc->wTotalLength > USB_MAX_CONFIG_DESC_SIZE) + return false; + + return true; +} + +// get the rest of configuration +static void get_configuration_header_done(usb_device_t *dev, void *, usb_status_t status, size_t transferred) { + if (status != USB_STATUS_SUCCESS) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: usb_set_configuration failed\n"); + return; + } + + if (device_port_stale(dev)) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + return; + } + + if (!config_desc_header_valid(dev->config_desc, transferred)) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: malformed config descriptor header\n"); + return; + } + + void *p = realloc(dev->config_desc, dev->config_desc->wTotalLength); + if (p == NULL) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: out of memory to get configuration\n"); + return; + } + dev->config_desc = p; + + if (usb_get_device_descriptor(dev, USB_DESCRIPTOR_TYPE_CONFIG, 0, dev->config_desc, dev->config_desc->wTotalLength, get_configuration_done, NULL)) { + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + printf("usb: usb_get_device_descriptor failed\n"); + } +} + +// get configuration descriptor size +static void address_done(usb_hub_t *hub, uint8_t port, usb_device_t *dev) { + if (dev == NULL) { + printf("usb: failed to address device in plug-in path\n"); + return; + } + + if (device_port_stale(dev)) { + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + return; + } + + dev->config_desc = alloc(sizeof(usb_config_desc_t)); + if (dev->config_desc == NULL) { + printf("usb: out of memory to get configuration\n"); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + return; + } + + if (usb_get_device_descriptor(dev, USB_DESCRIPTOR_TYPE_CONFIG, 0, dev->config_desc, sizeof(usb_config_desc_t), get_configuration_header_done, NULL)) { + printf("usb: usb_get_device_descriptor failed\n"); + free(dev->config_desc); + dev->hub->ctrl->ops->deaddress_device(dev->hub->ctrl, dev); + } +} + +// address device +void usb_hub_event_reset(usb_hub_t *hub, int port) { + if (hub->ctrl->ops->address_device(hub->ctrl, hub, port, address_done)) { + printf("usb_hub %s: failed to address device\n", hub->name); + } +} + +// reset device +void usb_hub_event_connect(usb_hub_t *hub, int port) { + __assert(port < hub->port_count); + hub->ports[port].generation++; + + if (hub->ops->set_port_feature(hub, port, USB_HUB_FEATURE_PORT_RESET)) { + printf("usb_hub %s: port reset failed\n", hub->name); + } +} + +void usb_hub_event_disconnect(usb_hub_t* hub, int port) { + __assert(port < hub->port_count); + hub->ports[port].generation++; + + usb_device_t *device = hub->ports[port].device; + if (device == NULL) { + printf("usb: unplugged device during device init\n"); + return; + } + + hub->ports[port].device = NULL; + + // TODO in the case of a hub driver this would also cause a sort of "recursive disconnect" from there + for (int i = 0; i < device->config_desc->bNumInterfaces; ++i) { + usb_interface_t *interface = &device->interfaces[i]; + if (interface->driver && interface->driver->detach) + interface->driver->detach(device, interface); + } + + // TODO block new transfers + // TODO stop transfers + + post_config_cleanup(device); +} diff --git a/kernel-src/io/usb/core/usb.c b/kernel-src/io/usb/core/usb.c new file mode 100644 index 00000000..ee701dee --- /dev/null +++ b/kernel-src/io/usb/core/usb.c @@ -0,0 +1,60 @@ +#include +#include +#include +#include + +extern usb_class_driver_t *usb_class_drivers; +extern usb_class_driver_t *usb_class_drivers_end; + +int usb_submit_xfer(usb_device_t *dev, usb_xfer_t *xfer) { + usb_ctrl_t *ctrl = dev->hub->ctrl; + xfer->device = dev; + return ctrl->ops->xfer(ctrl, dev, xfer); +} + +int usb_control_xfer(usb_device_t *dev, usb_setup_t *setup, void *buffer, usb_completion_callback_t callback, void *ctx) { + usb_xfer_t xfer = {0}; + xfer.flags = (setup->bmRequestType & USB_REQUEST_DIR_TO_HOST) ? USB_XFER_FLAG_TO_HOST : USB_XFER_FLAG_TO_DEVICE; + xfer.type = USB_XFER_TYPE_CONTROL; + xfer.setup = setup; + xfer.data = buffer; + xfer.data_length = setup->wLength; + xfer.completion = callback; + xfer.completion_ctx = ctx; + + return usb_submit_xfer(dev, &xfer); +} + +static int usb_get_descriptor_for_recipient(usb_device_t *dev, uint8_t recipient, uint16_t index, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx) { + usb_setup_t setup = {0}; + setup.bmRequestType = recipient | USB_REQUEST_STANDARD | USB_REQUEST_DIR_TO_HOST; + setup.bRequest = USB_REQUEST_GET_DESCRIPTOR; + setup.wValue = ((uint16_t)desc_type << 8) | desc_index; + setup.wIndex = index; + setup.wLength = length; + + return usb_control_xfer(dev, &setup, buffer, callback, callback_ctx ? callback_ctx : buffer); +} + +int usb_get_device_descriptor(usb_device_t *dev, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx) { + return usb_get_descriptor_for_recipient(dev, USB_REQUEST_RECIP_DEVICE, 0, desc_type, desc_index, buffer, length, callback, callback_ctx); +} + +int usb_get_interface_descriptor(usb_device_t *dev, uint16_t interface_number, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx) { + return usb_get_descriptor_for_recipient(dev, USB_REQUEST_RECIP_INTERFACE, interface_number, desc_type, desc_index, buffer, length, callback, callback_ctx); +} + +int usb_get_endpoint_descriptor(usb_device_t *dev, uint16_t endpoint_address, uint8_t desc_type, uint8_t desc_index, void *buffer, uint16_t length, usb_completion_callback_t callback, void *callback_ctx) { + return usb_get_descriptor_for_recipient(dev, USB_REQUEST_RECIP_ENDPOINT, endpoint_address, desc_type, desc_index, buffer, length, callback, callback_ctx); +} + +int usb_set_configuration(usb_device_t *dev, uint8_t config_value, usb_completion_callback_t callback) { + usb_setup_t setup = {0}; + setup.bmRequestType = USB_REQUEST_RECIP_DEVICE | USB_REQUEST_STANDARD | USB_REQUEST_DIR_TO_DEVICE; + setup.bRequest = USB_REQUEST_SET_CONFIGURATION; + setup.wValue = config_value; + setup.wIndex = 0; + setup.wLength = 0; + + return usb_control_xfer(dev, &setup, NULL, callback, NULL); +} diff --git a/kernel-src/io/usb/ctrl/xhci.c b/kernel-src/io/usb/ctrl/xhci.c new file mode 100644 index 00000000..18dd63da --- /dev/null +++ b/kernel-src/io/usb/ctrl/xhci.c @@ -0,0 +1,762 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static uint32_t xhci_trb_type(xhci_trb_t *trb) { + return (trb->dw3 >> 10) & 0x3f; +} + +static void xhci_signal_event_thread(xhci_ctrl_t *xhci) { + semaphore_signal_limit(&xhci->event_sem, 1); +} + +static void xhci_signal_callback_thread(xhci_ctrl_t *xhci) { + semaphore_signal_limit(&xhci->callback_sem, 1); +} + +static void xhci_release_submission_page(xhci_submission_t *submission) { + uint32_t cmd_type = xhci_trb_type(&submission->cmd_trb); + + if ((cmd_type == TRB_NORMAL || cmd_type == TRB_DATA_STAGE) && submission->cmd_trb.parameters != 0) + pmm_release((void *)(submission->cmd_trb.parameters & ~(PAGE_SIZE - 1))); +} + +static bool xhci_reclaim_submission_slot(xhci_submission_t *slot, xhci_submission_t *submission) { + bool valid = __atomic_exchange_n(&slot->valid, false, __ATOMIC_SEQ_CST); + __assert(valid); + + xhci_release_submission_page(submission); + xhci_ring_unreserve(submission->ring, 1); + return true; +} + +static void xhci_reclaim_submission_chain(xhci_submission_t *submissionp) { + while (submissionp != NULL) { + if (!__atomic_load_n(&submissionp->valid, __ATOMIC_SEQ_CST)) + return; + + xhci_submission_t submission = *submissionp; + xhci_submission_t *next = submission.next; + xhci_reclaim_submission_slot(submissionp, &submission); + submissionp = next; + } +} + +static void xhci_enqueue_completion(xhci_ctrl_t *xhci, xhci_completion_t *completion) { + semaphore_wait(&xhci->callback_space_sem, false); + + MUTEX_ACQUIRE(&xhci->callback_mutex); + size_t written = ringbuffer_write(&xhci->callback_ring, completion, sizeof(*completion)); + __assert(written == sizeof(*completion)); + MUTEX_RELEASE(&xhci->callback_mutex); + + xhci_signal_callback_thread(xhci); +} + +static void handle_submission_complete(xhci_ctrl_t *xhci, xhci_submission_t *submissionp) { + xhci_submission_t submission = *submissionp; + + uint32_t cmd_type = xhci_trb_type(&submission.cmd_trb); + if (cmd_type == TRB_SETUP_STAGE) { + if (submission.usb_status == USB_STATUS_SUCCESS) + return; + } else if (cmd_type == TRB_NORMAL || cmd_type == TRB_DATA_STAGE) { + // Check if we need to wait for more TRBs to complete. + if (submission.usb_status == USB_STATUS_SUCCESS && (cmd_type == TRB_DATA_STAGE || (submission.cmd_trb.dw3 & XHCI_TRB_DW3_CH) != 0)) + return; + } + + // Complete the transfer. + size_t xfer_len; + uint32_t status = (submission.event_trb.dw2 >> 24) & 0xff; + if (status == TRB_SUCCESS) { + xfer_len = submission.data_offset + submission.data_length; + } else if (status == TRB_SHORT_PACKET) { + size_t remainder = submission.event_trb.dw2 & 0xffffff; + size_t trb_xfer_len = submission.data_length - min(remainder, submission.data_length); + xfer_len = submission.data_offset + trb_xfer_len; + } else { + xfer_len = 0; + } + + if (cmd_type == TRB_DATA_STAGE && submission.usb_status == USB_STATUS_SHORT_PACKET) { + // surgically edit the status stage trb to return a short packet status with the final length + __assert(submission.next != NULL); + submission.next->usb_status = USB_STATUS_SHORT_PACKET; + submission.next->data_offset = xfer_len; + return; + } + + // for ENABLE_SLOT commands, we use xfer_len for the slot id + if ((submission.cmd_trb.dw3 & XHCI_TRB_DW3_TYPE_MASK) == XHCI_TRB_DW3_TYPE(TRB_ENABLE_SLOT)) + xfer_len = (submission.event_trb.dw3 >> 24) & 0xff; + + xhci_submission_t *td_head = submission.td_head != NULL ? submission.td_head : submissionp; + xhci_reclaim_submission_chain(td_head); + + if (submission.callback) { + xhci_completion_t completion = { + .device = submission.device, + .callback = submission.callback, + .callback_ctx = submission.callback_ctx, + .status = submission.usb_status, + .transferred = xfer_len, + }; + + xhci_enqueue_completion(xhci, &completion); + } +} + +static xhci_submission_t *xhci_pop_completed_submission(xhci_ctrl_t *xhci) { + long ipl = spinlock_acquire_raise_ipl(&xhci->event_lock, IPL_USB); + xhci_submission_t *submission = (xhci_submission_t *)list_pop_front(&xhci->completion_list); + spinlock_release_lower_ipl(&xhci->event_lock, ipl); + return submission; +} + +static int xhci_pop_port_change(xhci_ctrl_t *xhci) { + long ipl = spinlock_acquire_raise_ipl(&xhci->event_lock, IPL_USB); + + int port = bitmap_find_first_set(&xhci->port_change_bitmap); + if (port != -1) + bitmap_set(&xhci->port_change_bitmap, port, 0); + + spinlock_release_lower_ipl(&xhci->event_lock, ipl); + return port; +} + +static bool xhci_pop_completion(xhci_ctrl_t *xhci, xhci_completion_t *completion) { + bool ret = false; + + MUTEX_ACQUIRE(&xhci->callback_mutex); + if (RINGBUFFER_DATACOUNT(&xhci->callback_ring) >= sizeof(*completion)) { + size_t read = ringbuffer_read(&xhci->callback_ring, completion, sizeof(*completion)); + __assert(read == sizeof(*completion)); + ret = true; + } + MUTEX_RELEASE(&xhci->callback_mutex); + + if (ret) + semaphore_signal(&xhci->callback_space_sem); + + return ret; +} + +static void xhci_event_thread(void) { + xhci_ctrl_t *xhci = current_thread()->kernelarg; + + for (;;) { + semaphore_wait(&xhci->event_sem, false); + + for (;;) { + xhci_submission_t *submission = xhci_pop_completed_submission(xhci); + if (submission == NULL) + break; + + handle_submission_complete(xhci, submission); + } + } +} + +static void xhci_callback_thread(void) { + xhci_ctrl_t *xhci = current_thread()->kernelarg; + + for (;;) { + semaphore_wait(&xhci->callback_sem, false); + + for (;;) { + bool did_work = false; + + for (;;) { + int port = xhci_pop_port_change(xhci); + if (port == -1) + break; + + xhci_handle_port_change_event(xhci, port + 1); + did_work = true; + } + + for (;;) { + xhci_completion_t completion; + if (!xhci_pop_completion(xhci, &completion)) + break; + + completion.callback(completion.device, completion.callback_ctx, completion.status, completion.transferred); + did_work = true; + } + + if (!did_work) + break; + } + } +} + + +static void xhci_dpc(context_t *, dpcarg_t dpcarg) { + xhci_ctrl_t *xhci = dpcarg; + volatile xhci_ir_t *ir = &xhci->rtregs->ir[0]; + + // Process events from the event ring. + bool signal_event_thread = false; + bool signal_callback_thread = false; + xhci_trb_t *r_trb; + while ((r_trb = xhci_ring_dequeue(&xhci->event_ring)) != NULL) { + uint32_t type = (r_trb->dw3 >> 10) & 0x3f; + if (type == TRB_COMMAND_COMPLETION_EVENT || type == TRB_XFER_COMPLETION_EVENT) { + uint32_t status = (r_trb->dw2 >> 24) & 0xff; + + usb_status_t usb_status; + switch (status) { + case TRB_SUCCESS: + usb_status = USB_STATUS_SUCCESS; + break; + case TRB_SHORT_PACKET: + usb_status = USB_STATUS_SHORT_PACKET; + break; + case TRB_DATA_BUFFER_ERROR: + case TRB_BABBLE_DETECTED: + case TRB_TRANSACTION_ERROR: + case TRB_RESOURCE_ERROR: + case TRB_BANDWIDTH_ERROR: + case TRB_NO_SLOTS_AVAILABLE: + case TRB_INVALID_STREAM_TYPE: + case TRB_SLOT_NOT_ENABLED: + case TRB_ENDPOINT_NOT_ENABLED: + case TRB_BANDWIDTH_OVERRUN_ERROR: + case TRB_CONTEXT_STATE_ERROR: + case TRB_NO_PING_RESPONSE: + case TRB_INCOMPATIBLE_DEVICE_ERROR: + case TRB_COMMAND_RING_STOPPED: + case TRB_COMMAND_RING_ABORTED: + case TRB_STOPPED: + case TRB_STOPPED_LEN: + case TRB_STOPPED_SHORT: + case TRB_LATENCY_TOO_LARGE: + case TRB_ERROR_RESERVED: + case TRB_ERROR_UNDEFINED: + case TRB_INVALID_STREAM_ID: + case TRB_SECONDARY_BANDWIDTH_ERROR: + case TRB_SPLIT_TRANSACTION_ERROR: + usb_status = USB_STATUS_ERROR; + break; + case TRB_STALL: + usb_status = USB_STATUS_STALL; + break; + case TRB_BUFFER_OVERRUN: + case TRB_RING_UNDERRUN: + case TRB_RING_OVERRUN: + case TRB_MISSED_SERVICE: + usb_status = USB_STATUS_FLOW_ERROR; + break; + default: + printf("xhci: bad event: %lu (driver bug!)\n", status); + _panic("Driver bug", NULL); + break; + } + + xhci_ring_t *ring; + if (type == TRB_COMMAND_COMPLETION_EVENT) { + ring = &xhci->command_ring; + } else { + uint32_t slot_id = (r_trb->dw3 >> 24) & 0xff; + __assert(slot_id > 0 && slot_id <= xhci->slot_count); + xhci_device_t *dev = xhci->slots[slot_id - 1]; + __assert(dev); + + uint8_t ep_index = (r_trb->dw3 >> 16) & 0x1f; + __assert(ep_index > 0); + ring = &dev->ep_rings[ep_index - 1]; + } + + xhci_trb_t *trb = MAKE_HHDM(r_trb->parameters); + uintptr_t trb_addr = (uintptr_t)trb; + uintptr_t ring_start = (uintptr_t)ring->ring; + uintptr_t ring_end = (uintptr_t)&ring->ring[ring->size - 1]; + __assert(trb_addr >= ring_start && trb_addr < ring_end); + __assert((trb_addr - ring_start) % sizeof(xhci_trb_t) == 0); + size_t trb_index = (trb_addr - ring_start) / sizeof(xhci_trb_t); + xhci_submission_t *sub = &ring->submissions[trb_index]; + __assert(__atomic_load_n(&sub->valid, __ATOMIC_SEQ_CST)); + + memcpy(&sub->event_trb, r_trb, sizeof(xhci_trb_t)); + + if (sub->usb_status == USB_STATUS_SHORT_PACKET) { + // this is a status stage trb and the data stage returned a short packet. + // the offset has been surgically edited to the shorter length as to + // return the proper transferred length to the callback. + uint32_t original_cmd = xhci_trb_type(&sub->cmd_trb); + __assert(original_cmd == TRB_STATUS_STAGE); + } else { + sub->usb_status = usb_status; + } + + spinlock_acquire(&xhci->event_lock); + list_push_back(&xhci->completion_list, &sub->list_node); + spinlock_release(&xhci->event_lock); + + signal_event_thread = true; + } else if (type == TRB_PORT_STATUS_CHANGE_EVENT) { + uint8_t port_id = (r_trb->dw0 >> 24) & 0xff; + + spinlock_acquire(&xhci->event_lock); + bitmap_set(&xhci->port_change_bitmap, port_id - 1, 1); + spinlock_release(&xhci->event_lock); + signal_callback_thread = true; + } + + // Update the Event Ring Dequeue Pointer. + uint64_t event_ring_phys = (uint64_t)xhci->event_ring.ring_phys; + event_ring_phys += xhci->event_ring.index * sizeof(xhci_trb_t); + ir->erdp = event_ring_phys | XHCI_ERDP_EHB; + } + + if (signal_event_thread) + xhci_signal_event_thread(xhci); + if (signal_callback_thread) + xhci_signal_callback_thread(xhci); +} + +static void xhci_isr(isr_t *isr, context_t *ctx) { + xhci_ctrl_t *xhci = isr->priv; + + // Check if there is an interrupt pending. + volatile xhci_ir_t *ir = &xhci->rtregs->ir[0]; + uint32_t iman = ir->iman; + uint32_t usbsts = xhci->opregs->usbsts; + uint64_t erdp = ir->erdp; + + if ((erdp & XHCI_ERDP_EHB) == 0 && (iman & XHCI_IMAN_IP) == 0 && (usbsts & XHCI_USBSTS_EINT) == 0) + return; + + // Clear the interrupt pending bit. + if (iman & XHCI_IMAN_IP) + ir->iman |= XHCI_IMAN_IP; + if (usbsts & XHCI_USBSTS_EINT) + xhci->opregs->usbsts = XHCI_USBSTS_EINT; + + dpc_enqueue(&xhci->dpc, xhci); +} + +static int xhci_halt(xhci_ctrl_t *ctrl) { + if ((ctrl->opregs->usbcmd & XHCI_USBCMD_RS) == 0) + return 0; + + ctrl->opregs->usbcmd &= ~XHCI_USBCMD_RS; + + timespec_t start = timekeeper_timefromboot(); + while ((ctrl->opregs->usbsts & XHCI_USBSTS_HCH) == 0) { + sched_sleep_us(10 * 1000); // 10ms + + timespec_t now = timekeeper_timefromboot(); + if (timespec_diffms(now, start) > 1000) + return -1; + } + + return 0; +} + +static int xhci_reset(xhci_ctrl_t *ctrl) { + ctrl->opregs->usbcmd |= XHCI_USBCMD_HCRST; + + timespec_t start = timekeeper_timefromboot(); + while ((ctrl->opregs->usbcmd & XHCI_USBCMD_HCRST) != 0 || (ctrl->opregs->usbsts & XHCI_USBSTS_CNR) != 0) { + sched_sleep_us(10 * 1000); // 10ms + + timespec_t now = timekeeper_timefromboot(); + if (timespec_diffms(now, start) > 1000) + return -1; + } + + return 0; +} + +static int xhci_run(xhci_ctrl_t *ctrl) { + if ((ctrl->opregs->usbcmd & XHCI_USBCMD_RS)) + return 0; + + ctrl->opregs->usbcmd |= XHCI_USBCMD_RS | XHCI_USBCMD_INTE; + + timespec_t start = timekeeper_timefromboot(); + while ((ctrl->opregs->usbsts & XHCI_USBSTS_HCH) != 0 || (ctrl->opregs->usbsts & XHCI_USBSTS_CNR) != 0) { + sched_sleep_us(10 * 1000); // 10ms + + timespec_t now = timekeeper_timefromboot(); + if (timespec_diffms(now, start) > 1000) + return -1; + } + + return 0; +} + + +static volatile uint32_t *xhci_ext_caps(pcibar_t bar0) { + volatile xhci_caps_t *caps = (xhci_caps_t *)bar0.address; + uint32_t xecp = (caps->hccparams1 >> 16) & 0xffff; + if (xecp == 0) + return NULL; + return (uint32_t *)(bar0.address + (xecp << 2)); +} + +static int xhci_handoff(pcibar_t bar0) { + volatile uint32_t *ext_caps = xhci_ext_caps(bar0); + if (ext_caps == NULL) + return 0; + + for (;;) { + uint8_t cap_id = *ext_caps & 0xff; + uint8_t next_cap_off = (*ext_caps >> 8) & 0xff; + + if (cap_id == 1) { + printf("xhci: found BIOS/OS ownership capability\n"); + + bool is_bios_owned = (*ext_caps >> 16) & 0x1; + if (is_bios_owned) { + printf("xhci: attempting to take ownership from BIOS\n"); + + // Set OS Owned Semaphore. + *ext_caps |= (1 << 24); + + timespec_t start = timekeeper_timefromboot(); + while (is_bios_owned) { + sched_sleep_us(500000); // 500ms + is_bios_owned = (*ext_caps >> 16) & 0x1; + timespec_t now = timekeeper_timefromboot(); + if (timespec_diffms(now, start) > 5000) { + printf("xhci: BIOS ownership handoff timeout\n"); + return ETIMEDOUT; + } + } + + printf("xhci: took ownership of controller from BIOS\n"); + } + } + + if (next_cap_off == 0) + break; + ext_caps += next_cap_off; + } + + return 0; +} + + +static int xhci_ctrl_start(usb_ctrl_t *ctrl) { + int ret = 0; + xhci_ctrl_t *xhci = container_of(ctrl, xhci_ctrl_t, ctrl); + pcibar_t bar0 = pci_getbar(xhci->pci_enum, 0); + + ret = xhci_halt(xhci); + if (ret != 0) { + printf("xhci: controller halt timeout\n"); + return ret; + } + + ret = xhci_reset(xhci); + if (ret != 0) { + printf("xhci: controller reset timeout\n"); + return ret; + } + + // Enable MSI interrupts. + size_t intcount; + if (xhci->pci_enum->msix.exists) { + intcount = pci_initmsix(xhci->pci_enum); + } else if (xhci->pci_enum->msi.exists) { + intcount = pci_initmsi(xhci->pci_enum, 1); + } else { + printf("xhci: controller does not support MSI or MSI-X\n"); + return ENODEV; + } + + __assert(intcount > 0); + + isr_t *isr = interrupt_allocate(xhci_isr, ARCH_EOI, IPL_USB); + __assert(isr != NULL); + isr->priv = ctrl; + + if (xhci->pci_enum->msix.exists) { + pci_msixadd(xhci->pci_enum, 0, INTERRUPT_IDTOVECTOR(isr->id), 1, 0); + pci_msixsetmask(xhci->pci_enum, 0); + } else { + pci_msisetbase(xhci->pci_enum, INTERRUPT_IDTOVECTOR(isr->id), 1, 0); + pci_msisetmask(xhci->pci_enum, 0, 0); + } + + uint32_t max_slots = xhci->caps->hcsparams1 & 0xff; + uint32_t max_ports = (xhci->caps->hcsparams1 >> 24) & 0xff; + + printf("xhci: controller supports %u slots and %u ports\n", max_slots, max_ports); + + xhci->port_count = max_ports; + xhci->ports = alloc(sizeof(xhci_port_protocol_t) * max_ports); + __assert(xhci->ports); + __assert(bitmap_init(&xhci->port_change_bitmap, max_ports) == 0); + + for (uint32_t i = 0; i < max_ports; i++) { + xhci->ports[i].ver_major = 0; + xhci->ports[i].ver_minor = 0; + } + + xhci->slot_count = max_slots; + xhci->slots = alloc(sizeof(xhci_device_t *) * max_slots); + __assert(xhci->slots); + + xhci->ctx_stride = (xhci->caps->hccparams1 & XHCI_HCCPARAMS1_CSZ) != 0 ? 64 : 32; + + volatile uint32_t *ext_caps = xhci_ext_caps(bar0); + if (ext_caps == NULL) { + printf("xhci: controller has no extended capabilities\n"); + return ENODEV; + } + + for (;;) { + uint8_t cap_id = *ext_caps & 0xff; + uint8_t next_cap_off = (*ext_caps >> 8) & 0xff; + + if (cap_id == 2) { + // map ports to usb types + uint8_t ver_major = (*ext_caps >> 24) & 0xff; + uint8_t ver_minor = (*ext_caps >> 16) & 0xff; + + uint32_t raw_port_start = ext_caps[2] & 0xff; + uint32_t port_count = (ext_caps[2] >> 8) & 0xff; + + if (raw_port_start == 0 || port_count == 0 || raw_port_start - 1 >= max_ports || port_count > max_ports - (raw_port_start - 1)) { + printf("xhci: invalid usb %u.%u port range start=%u count=%u max=%u\n", + ver_major, + ver_minor, + raw_port_start, + port_count, + max_ports); + return EINVAL; + } + + uint32_t port_start = raw_port_start - 1; + + printf("xhci: %u usb %u.%u ports starting at port %u\n", + port_count, ver_major, ver_minor, port_start + 1); + + for (uint32_t i = 0; i < port_count; i++) { + xhci->ports[port_start + i].ver_major = ver_major; + xhci->ports[port_start + i].ver_minor = ver_minor; + } + + xhci_root_hub_t *hub = alloc(sizeof(xhci_root_hub_t)); + __assert(hub); + + snprintf(hub->hub.name, sizeof(hub->hub.name), "xhci-root-hub-%u.%u", ver_major, ver_minor); + + hub->hub.ctrl = ctrl; + hub->hub.ops = &xhci_root_hub_ops; + hub->hub.device = NULL; + + hub->hub.ports = alloc(sizeof(usb_hub_port_t) * port_count); + __assert(hub->hub.ports); + hub->hub.port_count = port_count; + + hub->port_offset = port_start; + + list_push_back(&xhci->hubs, &hub->list_node); + } + + if (next_cap_off == 0) + break; + + ext_caps += next_cap_off; + } + + __assert(xhci_alloc_ring(xhci, &xhci->command_ring, false) == 0); + __assert(xhci_alloc_ring(xhci, &xhci->event_ring, true) == 0); + + // Set up MaxSlotsEn field. + xhci->opregs->config = max_slots; + + // Set up the Device Context Base Address Array. + void *dcbaa_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + __assert(dcbaa_phys); + + uint64_t *dcbaa_virt = MAKE_HHDM(dcbaa_phys); + memset(dcbaa_virt, 0, PAGE_SIZE); + + // Set up scratchpad buffers if requested. + uint32_t max_scratchpads_hi = (xhci->caps->hcsparams2 >> 21) & 0x1f; + uint32_t max_scratchpads_lo = (xhci->caps->hcsparams2 >> 27) & 0x1f; + uint32_t max_scratchpads = max_scratchpads_lo | (max_scratchpads_hi << 5); + + if (max_scratchpads > 0) { + printf("xhci: setting up %u scratchpad buffers\n", max_scratchpads); + __assert(max_scratchpads < PAGE_SIZE / sizeof(uint64_t)); + + void *scratchpad_array_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + __assert(scratchpad_array_phys); + uint64_t *scratchpad_array_virt = MAKE_HHDM(scratchpad_array_phys); + + for (uint32_t i = 0; i < max_scratchpads; i++) { + void *scratchpad_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + __assert(scratchpad_phys); + memset(MAKE_HHDM(scratchpad_phys), 0, PAGE_SIZE); + scratchpad_array_virt[i] = (uint64_t)scratchpad_phys; + } + + dcbaa_virt[0] = (uint64_t)scratchpad_array_phys; + } + + xhci->dcbaa = dcbaa_virt; + xhci->opregs->dcbaap = (uint64_t)dcbaa_phys; + + // Set up command rings. + xhci->opregs->crcr = (uint64_t)xhci->command_ring.ring_phys | XHCI_CRCR_RCS; + + // Set up Event Ring Segment Table. + void *erst_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + __assert(erst_phys); + + xhci_erst_entry_t *erst_virt = MAKE_HHDM(erst_phys); + memset(erst_virt, 0, PAGE_SIZE); + + erst_virt[0].ring_segment = (uint64_t)xhci->event_ring.ring_phys; + erst_virt[0].ring_segment_size = xhci->event_ring.size; + + // Set up the interrupter. + volatile xhci_ir_t *ir = &xhci->rtregs->ir[0]; + ir->iman = XHCI_IMAN_IE; + ir->erstsz = 1; + ir->erstba = (uint64_t)erst_phys; + ir->erdp = (uint64_t)xhci->event_ring.ring_phys | XHCI_ERDP_EHB; + + // Start the controller. + ret = xhci_run(xhci); + if (ret != 0) { + printf("xhci: controller run timeout\n"); + return ret; + } + + // power on all ports + bool debounce_wait = false; + list_for_each (&xhci->hubs, node) { + xhci_root_hub_t *rh = container_of(node, xhci_root_hub_t, list_node); + usb_hub_t *hub = &rh->hub; + + for (uint8_t port = 0; port < hub->port_count; port++) { + volatile xhci_port_regs_t *portregs = &xhci->portregs[rh->port_offset + port]; + uint32_t portsc = portregs->portsc; + portregs->portsc = (portsc & XHCI_PORTSC_PRESERVE_BITS) | (portsc & XHCI_PORTSC_CHANGE_BITS); + if ((portsc & XHCI_PORTSC_PP) == 0) { + portregs->portsc = (portsc & XHCI_PORTSC_CHANGE_BITS) | XHCI_PORTSC_PP; + debounce_wait = true; + } + } + } + + if (debounce_wait) + sched_sleep_us(100000); // 100 ms + + // reset all conected ports + list_for_each (&xhci->hubs, node) { + xhci_root_hub_t *rh = container_of(node, xhci_root_hub_t, list_node); + usb_hub_t *hub = &rh->hub; + + for (uint8_t port = 0; port < hub->port_count; port++) { + volatile xhci_port_regs_t *portregs = &xhci->portregs[rh->port_offset + port]; + // NOTE: while USB3 ports don't need this, we will still do it to simplify the path + uint32_t portsc = portregs->portsc; + if (portsc & XHCI_PORTSC_CCS) + portregs->portsc = (portsc & XHCI_PORTSC_PRESERVE_BITS) | XHCI_PORTSC_PR; + } + } + + return 0; +} + + +static void init_ctrl(pcienum_t *e) { + printf("xhci: found controller at %02x:%02x.%x\n", e->bus, e->device, e->function); + + pci_setcommand(e, PCI_COMMAND_MMIO, 1); + + // perform handoff before changing anything else + pcibar_t bar0 = pci_getbar(e, 0); + if (xhci_handoff(bar0) != 0) + return; + + pci_setcommand(e, PCI_COMMAND_IO, 0); + pci_setcommand(e, PCI_COMMAND_IRQDISABLE, 1); + pci_setcommand(e, PCI_COMMAND_BUSMASTER, 1); + + volatile xhci_caps_t *caps = (xhci_caps_t *)bar0.address; + volatile xhci_opregs_t *opregs = (xhci_opregs_t *)(bar0.address + caps->caplength); + volatile xhci_rtregs_t *rtregs = (xhci_rtregs_t *)(bar0.address + caps->rtsoff); + + // Only 64-bit-addressing controllers are supported for simplicity. + if ((caps->hccparams1 & XHCI_HCCPARAMS1_AC64) == 0) { + printf("xhci: controller does not support 64-bit addressing\n"); + return; + } + + // Make sure the controller supports 4K pages + if ((opregs->pagesize & (1 << 0)) == 0) { + printf("xhci: controller does not support 4K page size\n"); + return; + } + + // Register the controller with the USB subsystem + xhci_ctrl_t *ctrl = alloc(sizeof(xhci_ctrl_t)); + __assert(ctrl); + + ctrl->ctrl.ops = &xhci_ops; + + SEMAPHORE_INIT(&ctrl->event_sem, 0); + SEMAPHORE_INIT(&ctrl->callback_sem, 0); + SEMAPHORE_INIT(&ctrl->callback_space_sem, XHCI_CALLBACK_RING_ENTRIES); + SPINLOCK_INIT(ctrl->event_lock); + MUTEX_INIT(&ctrl->callback_mutex); + list_init(&ctrl->completion_list); + __assert(ringbuffer_init(&ctrl->callback_ring, sizeof(xhci_completion_t) * XHCI_CALLBACK_RING_ENTRIES) == 0); + dpc_prepare(&ctrl->dpc, xhci_dpc); + + list_init(&ctrl->hubs); + + ctrl->pci_enum = e; + ctrl->caps = caps; + ctrl->opregs = opregs; + ctrl->rtregs = rtregs; + ctrl->portregs = (xhci_port_regs_t *)(bar0.address + caps->caplength + 0x400); + ctrl->dbs = (uint32_t *)(bar0.address + caps->dboff); + + int res = xhci_ctrl_start(&ctrl->ctrl); + if (res) { + printf("xhci: controller init returned an error: %s\n", strerror(res)); + return; + } + + struct thread_t *event_thread = sched_newthread(xhci_event_thread, PAGE_SIZE * 4, 0, NULL, NULL); + __assert(event_thread != NULL); + event_thread->kernelarg = ctrl; + sched_queue(event_thread); + + struct thread_t *callback_thread = sched_newthread(xhci_callback_thread, PAGE_SIZE * 4, 0, NULL, NULL); + __assert(callback_thread != NULL); + callback_thread->kernelarg = ctrl; + sched_queue(callback_thread); +} + +void xhci_init(void) { + for (int i = 0;; ++i) { + pcienum_t *e = pci_getenum(PCI_CLASS_SERIAL_BUS, PCI_SUBCLASS_SERIAL_BUS_USB, PCI_PROGIF_USB_XHCI, -1, -1, -1, i); + if (e == NULL) + break; + init_ctrl(e); + } +} + +INIT_ROUTINE_DEFINE(xhci, INIT_ROUTINE_FLAGS_NONE, xhci_init, acpi); diff --git a/kernel-src/io/usb/ctrl/xhci_ctrl.c b/kernel-src/io/usb/ctrl/xhci_ctrl.c new file mode 100644 index 00000000..76a53e70 --- /dev/null +++ b/kernel-src/io/usb/ctrl/xhci_ctrl.c @@ -0,0 +1,583 @@ +#include +#include +#include +#include +#include +#include + +static bool usb_speed_to_xhci_speed(usb_speed_t speed, uint8_t *out) { + switch (speed) { + case USB_SPEED_FULL: + *out = XHCI_PORT_SPEED_FULL; + return true; + case USB_SPEED_LOW: + *out = XHCI_PORT_SPEED_LOW; + return true; + case USB_SPEED_HIGH: + *out = XHCI_PORT_SPEED_HIGH; + return true; + case USB_SPEED_SUPER: + *out = XHCI_PORT_SPEED_SUPER; + return true; + default: + return false; + } +} + +static usb_speed_t xhci_speed_to_usb_speed(uint8_t speed) { + switch (speed) { + case XHCI_PORT_SPEED_FULL: + return USB_SPEED_FULL; + case XHCI_PORT_SPEED_LOW: + return USB_SPEED_LOW; + case XHCI_PORT_SPEED_HIGH: + return USB_SPEED_HIGH; + case XHCI_PORT_SPEED_SUPER: + return USB_SPEED_SUPER; + case XHCI_PORT_SPEED_SS_2X1: + case XHCI_PORT_SPEED_SS_1X2: + case XHCI_PORT_SPEED_SS_2X2: + return USB_SPEED_SUPER_PLUS; + default: + return USB_SPEED_UNKNOWN; + } +} + +static uint16_t xhci_ep0_max_packet_size(usb_device_t *device) { + if (device->speed == USB_SPEED_SUPER || device->speed == USB_SPEED_SUPER_PLUS) + return 1 << device->desc.bMaxPacketSize0; + return device->desc.bMaxPacketSize0; +} + +#define CTX_PTR(CTRL, CTX, INDEX) ((uintptr_t)(CTX) + ((CTRL)->ctx_stride * (INDEX))) + +static volatile xhci_input_ctx_t *xhci_get_input_ctrl_ctx(xhci_ctrl_t *xhci, xhci_device_t *dev) { + return (volatile xhci_input_ctx_t *)CTX_PTR(xhci, dev->input_ctx, 0); +} + +static volatile xhci_slot_ctx_t *xhci_get_input_slot_ctx(xhci_ctrl_t *xhci, xhci_device_t *dev) { + return (volatile xhci_slot_ctx_t *)CTX_PTR(xhci, dev->input_ctx, 1); +} + +static volatile xhci_slot_ctx_t *xhci_get_device_slot_ctx(xhci_ctrl_t *xhci, xhci_device_t *dev) { + return (volatile xhci_slot_ctx_t *)CTX_PTR(xhci, dev->device_ctx, 0); +} + +static volatile xhci_ep_ctx_t *xhci_get_input_ep_ctx(xhci_ctrl_t *xhci, xhci_device_t *dev, uint8_t ep) { + return (volatile xhci_ep_ctx_t *)CTX_PTR(xhci, dev->input_ctx, ep + 2); +} + +typedef struct { + usb_completion_callback_t callback; + void *callback_ctx; + uint32_t ep_index; +} xhci_configure_ep_ctx_t; + +static void xhci_release_ep_ring(xhci_device_t *dev, uint32_t ep_index) { + __assert(ep_index > 0 && ep_index <= 31); + xhci_ring_t *ring = &dev->ep_rings[ep_index - 1]; + xhci_free_ring(ring); + memset(ring, 0, sizeof(*ring)); +} + +static void xhci_update_input_context(xhci_ctrl_t *ctrl, xhci_device_t *dev) { + uint8_t *input_ctx = dev->input_ctx; + uint8_t *device_ctx = dev->device_ctx; + + // Copy over the slot context and all endpoint contexts. + memcpy(input_ctx + ctrl->ctx_stride, device_ctx, ctrl->ctx_stride * (1 + 31)); + // Clear input context header. + memset(input_ctx, 0, sizeof(uint32_t) * 2); +} + +static void xhci_release_device_resources(xhci_ctrl_t *xhci, xhci_device_t *dev) { + for (int i = 0; i < 31; ++i) { + if (dev->ep_rings[i].ring != NULL) + xhci_release_ep_ring(dev, i + 1); + } + + if (dev->slot_id > 0 && dev->slot_id <= xhci->slot_count) { + xhci->dcbaa[dev->slot_id] = 0; + if (xhci->slots[dev->slot_id - 1] == dev) + xhci->slots[dev->slot_id - 1] = NULL; + } + + if (dev->input_ctx_phys) + pmm_release(dev->input_ctx_phys); + if (dev->device_ctx_phys) + pmm_release(dev->device_ctx_phys); + + free(dev); +} + +static int xhci_disable_slot(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_completion_callback_t callback, void *callback_ctx) { + xhci_submission_t xhci_submission = { + .callback = callback, + .callback_ctx = callback_ctx, + .device = &dev->device, + }; + + xhci_trb_t trb = { + .dw3 = XHCI_TRB_DW3_TYPE(TRB_DISABLE_SLOT) | (dev->slot_id << 24) + }; + + xhci_ring_submit(&xhci->command_ring, &trb, &xhci_submission); + xhci_ring_command_doorbell(xhci); + return 0; +} + +static void address_device_failed_cleanup_callback(usb_device_t *device, void *ctx, usb_status_t status, size_t) { + usb_address_device_callback_t callback = ctx; + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + usb_hub_t *hub = device->hub; + uint8_t port = device->port_number; + + if (status != USB_STATUS_SUCCESS) { + printf("xhci: disable slot %u failed during address cleanup\n", xhci_dev->slot_id); + // If Disable Slot fails, the controller may still reference this device's + // contexts or transfer rings. Leak them for now instead of freeing memory + // that hardware may still own. + } else { + xhci_release_device_resources(xhci, xhci_dev); + } + + callback(hub, port, NULL); +} + +static void address_device_failed_cleanup(usb_device_t *device, usb_address_device_callback_t callback) { + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + + xhci_disable_slot(xhci, xhci_dev, address_device_failed_cleanup_callback, callback); +} + +static void address_device_failed_before_slot(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_address_device_callback_t callback) { + usb_hub_t *hub = dev->device.hub; + uint8_t port = dev->device.port_number; + + xhci_release_device_resources(xhci, dev); + callback(hub, port, NULL); +} + +static void address_device_evaluate_ctx_callback(usb_device_t *device, void *ctx, usb_status_t status, size_t) { + usb_address_device_callback_t callback = ctx; + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + + if (status != USB_STATUS_SUCCESS) { + printf("xhci: evaluate context failed on slot %u\n", xhci_dev->slot_id); + address_device_failed_cleanup(device, callback); + return; + } + // Update the input context again. + xhci_update_input_context(xhci, xhci_dev); + + // Tell the original caller that the device has been addresses + callback(xhci_dev->device.hub, xhci_dev->device.port_number, &xhci_dev->device); +} + +static void address_device_get_descriptor_callback(usb_device_t *device, void *ctx, usb_status_t status, size_t transferred) { + usb_address_device_callback_t callback = ctx; + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + + if (status != USB_STATUS_SUCCESS) { + printf("xhci: failed to get device descriptor on slot %u\n", xhci_dev->slot_id); + address_device_failed_cleanup(device, callback); + return; + } + + if (!usb_device_desc_valid(&device->desc, device->speed, transferred)) { + printf("xhci: malformed device descriptor\n"); + address_device_failed_cleanup(device, callback); + return; + } + + // Update EP0 context with correct max packet size. + volatile xhci_ep_ctx_t *ep0_ctx = xhci_get_input_ep_ctx(xhci, xhci_dev, 0); + volatile xhci_input_ctx_t *input_ctx = xhci_get_input_ctrl_ctx(xhci, xhci_dev); + + ep0_ctx->dw1.max_packet_size = xhci_ep0_max_packet_size(device); + + input_ctx->a |= (1 << 1); + + // Submit an Evaluate Context command. + xhci_submission_t xhci_submission = { + .callback = address_device_evaluate_ctx_callback, + .callback_ctx = callback, + .device = &xhci_dev->device, + }; + + xhci_trb_t trb = { + .parameters = (uint64_t)xhci_dev->input_ctx_phys, + .dw3 = XHCI_TRB_DW3_TYPE(TRB_EVALUATE_CTX) | (xhci_dev->slot_id << 24) + }; + + xhci_ring_submit(&xhci->command_ring, &trb, &xhci_submission); + xhci_ring_command_doorbell(xhci); +} + +static void address_device_addressed_callback(usb_device_t *device, void *ctx, usb_status_t status, size_t) { + usb_address_device_callback_t callback = ctx; + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + + if (status != USB_STATUS_SUCCESS) { + printf("xhci: address device failed on slot %d failed\n", xhci_dev->slot_id); + address_device_failed_cleanup(device, callback); + return; + } + + printf("xhci: addressed device on slot %u\n", xhci_dev->slot_id); + + volatile xhci_slot_ctx_t *slot_ctx = xhci_get_device_slot_ctx(xhci, xhci_dev); + + // Update the input context and set the device address. + xhci_dev->device.address = slot_ctx->dw3.usb_device_address; + xhci_update_input_context(xhci, xhci_dev); + + if (usb_get_device_descriptor( + &xhci_dev->device, + USB_DESCRIPTOR_TYPE_DEVICE, + 0, + &xhci_dev->device.desc, + sizeof(xhci_dev->device.desc), + address_device_get_descriptor_callback, + callback + )) { + printf("xhci: usb_get_device_descriptor failed\n"); + address_device_failed_cleanup(device, callback); + } +} + +static void address_device_create_slot_callback(usb_device_t *device, void *ctx, usb_status_t status, size_t slot_id) { + usb_address_device_callback_t callback = ctx; + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + + if (status != USB_STATUS_SUCCESS) { + printf("xhci: create slot failed\n"); + address_device_failed_before_slot(xhci, xhci_dev, callback); + return; + } + + xhci_dev->slot_id = slot_id; + __assert(xhci_dev->slot_id > 0 && xhci_dev->slot_id <= xhci->slot_count); + printf("xhci: allocated slot %u for device %s:%u\n", xhci_dev->slot_id, xhci_dev->device.hub->name, xhci_dev->device.port_number); + + // Set up the Device Context Base Address Array entry. + xhci->dcbaa[xhci_dev->slot_id] = (uint64_t)xhci_dev->device_ctx_phys; + xhci->slots[xhci_dev->slot_id - 1] = xhci_dev; + + // Set up the input context and address the device. + volatile xhci_input_ctx_t *input_ctx = xhci_get_input_ctrl_ctx(xhci, xhci_dev); + volatile xhci_slot_ctx_t *slot_ctx = xhci_get_input_slot_ctx(xhci, xhci_dev); + volatile xhci_ep_ctx_t *ep0_ctx = xhci_get_input_ep_ctx(xhci, xhci_dev, 0); + + // Enable slot and EP0 contexts. + input_ctx->a |= (1 << 0) | (1 << 1); + + // Set up slot context. + + slot_ctx->dw0.route_string = xhci_dev->route_string; + slot_ctx->dw0.speed = xhci_dev->ctrl_speed; + slot_ctx->dw0.ctx_entries = 1; + slot_ctx->dw1.root_hub_port_number = xhci_dev->root_port_number; + if (device->hub->device != NULL && + (device->speed == USB_SPEED_LOW || device->speed == USB_SPEED_FULL) && + device->hub->device->speed == USB_SPEED_HIGH) { + xhci_device_t *parent = container_of(device->hub->device, xhci_device_t, device); + slot_ctx->dw2.tt_hub_slot_id = parent->slot_id; + slot_ctx->dw2.tt_port_number = device->port_number + 1; + slot_ctx->dw2.ttt = 0; + } + + // Set up EP0 context. + ep0_ctx->dw1.ep_type = XHCI_EP_TYPE_CTRL; + ep0_ctx->dw1.cerr = 3; + + if (xhci_dev->ctrl_speed == XHCI_PORT_SPEED_LOW || xhci_dev->ctrl_speed == XHCI_PORT_SPEED_FULL) + ep0_ctx->dw1.max_packet_size = 8; + else if (xhci_dev->ctrl_speed == XHCI_PORT_SPEED_HIGH) + ep0_ctx->dw1.max_packet_size = 64; + else + ep0_ctx->dw1.max_packet_size = 512; + + uint64_t ep0_ring_phys = (uint64_t)xhci_dev->ep_rings[0].ring_phys; + ep0_ctx->dw2.dcs = xhci_dev->ep_rings[0].cycle; + ep0_ctx->dw2.tr_dequeue_pointer_lo = (uint32_t)((ep0_ring_phys >> 4) & 0xffffffff); + ep0_ctx->dw3.tr_dequeue_pointer_hi = (uint32_t)(ep0_ring_phys >> 32); + + xhci_submission_t xhci_submission = { + .callback = address_device_addressed_callback, + .callback_ctx = callback, + .device = &xhci_dev->device + }; + + xhci_trb_t trb = { + .parameters = (uint64_t)xhci_dev->input_ctx_phys, + .dw3 = XHCI_TRB_DW3_TYPE(TRB_ADDRESS_DEVICE) | (xhci_dev->slot_id << 24) + }; + + xhci_ring_submit(&xhci->command_ring, &trb, &xhci_submission); + xhci_ring_command_doorbell(xhci); +} + +static int xhci_ctrl_address_device(usb_ctrl_t *ctrl, usb_hub_t *hub, uint8_t port, usb_address_device_callback_t callback) { + ++port; // convert to the format xhci expects + + xhci_ctrl_t *xhci = container_of(ctrl, xhci_ctrl_t, ctrl); + + xhci_device_t *xhci_dev = alloc(sizeof(xhci_device_t)); + if (xhci_dev == NULL) + return ENOMEM; + + void *device_ctx_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + if (device_ctx_phys == NULL) { + free(xhci_dev); + return ENOMEM; + } + + void *input_ctx_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + if (input_ctx_phys == NULL) { + pmm_release(device_ctx_phys); + free(xhci_dev); + return ENOMEM; + } + + if (xhci_alloc_ring(xhci, &xhci_dev->ep_rings[0], false)) { + pmm_release(input_ctx_phys); + pmm_release(device_ctx_phys); + free(xhci_dev); + return ENOMEM; + } + + xhci_dev->device_ctx_phys = device_ctx_phys; + xhci_dev->device_ctx = MAKE_HHDM(device_ctx_phys); + memset(xhci_dev->device_ctx, 0, PAGE_SIZE); + + xhci_dev->input_ctx_phys = input_ctx_phys; + xhci_dev->input_ctx = MAKE_HHDM(input_ctx_phys); + memset(xhci_dev->input_ctx, 0, PAGE_SIZE); + + xhci_dev->device.hub = hub; + xhci_dev->device.port_number = port - 1; + xhci_dev->device.port_generation = hub->ports[port - 1].generation; + + if (hub->device) { + xhci_device_t *parent = container_of(hub->device, xhci_device_t, device); + xhci_dev->device_tier = parent->device_tier + 1; + xhci_dev->route_string = parent->route_string | (port << (parent->device_tier * 4)); + + xhci_device_t *xhci_hub_device = (xhci_device_t *)hub->device; + xhci_dev->root_port_number = xhci_hub_device->root_port_number; + } else { + xhci_dev->device_tier = 0; + xhci_dev->route_string = 0; + xhci_dev->root_port_number = port; + } + + // TODO the device speed is gonna come from elsewhere and not from the (just now allocated) usb device + if (hub->device == NULL) { + // this is a root hub! we can just take the speed directly from the portsc register + xhci_root_hub_t *rh = container_of(hub, xhci_root_hub_t, hub); + uint32_t portsc = xhci->portregs[rh->port_offset + port - 1].portsc; + uint32_t port_speed = (portsc >> 10) & 0xf; + xhci_dev->ctrl_speed = port_speed; + xhci_dev->device.speed = xhci_speed_to_usb_speed(port_speed); + } else if (!usb_speed_to_xhci_speed(xhci_dev->device.speed, &xhci_dev->ctrl_speed)) { + // this is a super speed plus device behind a hub. + // we can't know its speed in advance, so set it as super speed and figure it out later. + xhci_dev->ctrl_speed = XHCI_PORT_SPEED_SUPER; + } + + xhci_submission_t xhci_submission = { + .callback = address_device_create_slot_callback, + .callback_ctx = callback, + .device = &xhci_dev->device + }; + + xhci_trb_t trb = { + .dw3 = XHCI_TRB_DW3_TYPE(TRB_ENABLE_SLOT) + }; + + xhci_ring_submit(&xhci->command_ring, &trb, &xhci_submission); + xhci_ring_command_doorbell(xhci); + return 0; +} + +static void configure_ep_callback(usb_device_t *dev, void *ctx, usb_status_t status, size_t transferred) { + xhci_configure_ep_ctx_t *configure_ctx = ctx; + usb_completion_callback_t callback = configure_ctx->callback; + void *callback_ctx = configure_ctx->callback_ctx; + + if (status != USB_STATUS_SUCCESS) { + xhci_device_t *xhci_dev = container_of(dev, xhci_device_t, device); + xhci_release_ep_ring(xhci_dev, configure_ctx->ep_index); + } + + free(configure_ctx); + + if (callback) + callback(dev, callback_ctx, status, transferred); +} + +static int xhci_ctrl_configure_ep(usb_ctrl_t *ctrl, usb_device_t *dev, usb_endpoint_t *ep, usb_completion_callback_t callback, void *callback_ctx) { + xhci_ctrl_t *xhci = container_of(ctrl, xhci_ctrl_t, ctrl); + xhci_device_t *xhci_dev = container_of(dev, xhci_device_t, device); + + bool is_in = (ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_DIR_IN) != 0; + + uint32_t ep_num = ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_NUM_MASK; + uint32_t ep_index = ep_num ? ((ep_num << 1) | (is_in ? 1 : 0)) : 0; + + if (ep_index == 0 || ep_index > 31) + return EINVAL; + + if (xhci_dev->ep_rings[ep_index - 1].ring != NULL) { + printf("xhci: duplicate endpoint address 0x%02x\n", ep->desc->bEndpointAddress); + return EINVAL; + } + + xhci_configure_ep_ctx_t *configure_ctx = alloc(sizeof(xhci_configure_ep_ctx_t)); + if (configure_ctx == NULL) + return ENOMEM; + + if (xhci_alloc_ring(xhci, &xhci_dev->ep_rings[ep_index - 1], false)) { + free(configure_ctx); + return ENOMEM; + } + + xhci_update_input_context(xhci, xhci_dev); + + volatile xhci_input_ctx_t *input_ctx = xhci_get_input_ctrl_ctx(xhci, xhci_dev); + volatile xhci_slot_ctx_t *slot_ctx = xhci_get_input_slot_ctx(xhci, xhci_dev); + volatile xhci_ep_ctx_t *ep_ctx = xhci_get_input_ep_ctx(xhci, xhci_dev, ep_index - 1); + + memset((void *)ep_ctx, 0, sizeof(xhci_ep_ctx_t)); + + input_ctx->a |= 1 << 0; + input_ctx->a |= 1 << ep_index; + + if (slot_ctx->dw0.ctx_entries < ep_index) + slot_ctx->dw0.ctx_entries = ep_index; + + // Set up the endpoint context. + uint8_t ep_type = ep->desc->bmAttributes & USB_ENDPOINT_ATTRIB_TYPE_MASK; + uint16_t max_packet_size = usb_endpoint_max_packet_size(ep->desc); + uint8_t max_burst_size = 0; + uint16_t max_esit_payload = max_packet_size; + + if ((dev->speed == USB_SPEED_SUPER || dev->speed == USB_SPEED_SUPER_PLUS) && ep->ss_companion == NULL) { + free(configure_ctx); + xhci_release_ep_ring(xhci_dev, ep_index); + return EINVAL; + } + + if (ep->ss_companion != NULL) { + max_burst_size = ep->ss_companion->bMaxBurst; + max_esit_payload = ep->ss_companion->wBytesPerInterval; + } else if (dev->speed == USB_SPEED_HIGH) { + uint8_t transactions = ((ep->desc->wMaxPacketSize >> 11) & 0x3) + 1; + max_esit_payload = max_packet_size * transactions; + } + + if (ep_type == USB_ENDPOINT_ATTRIB_TYPE_BULK) { + ep_ctx->dw1.ep_type = is_in ? XHCI_EP_TYPE_BULK_IN : XHCI_EP_TYPE_BULK_OUT; + } else if (ep_type == USB_ENDPOINT_ATTRIB_TYPE_INTR) { + if (ep->desc->bInterval == 0) { + free(configure_ctx); + xhci_release_ep_ring(xhci_dev, ep_index); + return EINVAL; + } + + if (dev->speed == USB_SPEED_LOW || dev->speed == USB_SPEED_FULL) + ep_ctx->dw0.interval = ceil_log2(ep->desc->bInterval * 8); + else + ep_ctx->dw0.interval = ep->desc->bInterval - 1; + + ep_ctx->dw1.ep_type = is_in ? XHCI_EP_TYPE_INTR_IN : XHCI_EP_TYPE_INTR_OUT; + ep_ctx->dw4.max_esit_payload_lo = max_esit_payload; + } else { + free(configure_ctx); + xhci_release_ep_ring(xhci_dev, ep_index); + return EINVAL; + } + + ep_ctx->dw1.cerr = 3; + ep_ctx->dw1.max_packet_size = max_packet_size; + ep_ctx->dw1.max_burst_size = max_burst_size; + + // Set up the ring dequeue pointer. + uint64_t ep_ring_phys = (uint64_t)xhci_dev->ep_rings[ep_index - 1].ring_phys; + ep_ctx->dw2.dcs = xhci_dev->ep_rings[ep_index - 1].cycle; + ep_ctx->dw2.tr_dequeue_pointer_lo = (uint32_t)((ep_ring_phys >> 4) & 0xffffffff); + ep_ctx->dw3.tr_dequeue_pointer_hi = (uint32_t)(ep_ring_phys >> 32); + + configure_ctx->callback = callback; + configure_ctx->callback_ctx = callback_ctx; + configure_ctx->ep_index = ep_index; + + xhci_submission_t xhci_submission = { + .callback = configure_ep_callback, + .callback_ctx = configure_ctx, + .device = &xhci_dev->device, + }; + + xhci_trb_t trb = { + .parameters = (uint64_t)xhci_dev->input_ctx_phys, + .dw3 = XHCI_TRB_DW3_TYPE(TRB_CONFIGURE_EP) | (xhci_dev->slot_id << 24) + }; + + xhci_ring_submit(&xhci->command_ring, &trb, &xhci_submission); + xhci_ring_command_doorbell(xhci); + return 0; +} + +static int xhci_ctrl_xfer(usb_ctrl_t *ctrl, usb_device_t *dev, usb_xfer_t *xfer) { + xhci_ctrl_t *xhci = container_of(ctrl, xhci_ctrl_t, ctrl); + xhci_device_t *xhci_dev = container_of(dev, xhci_device_t, device); + + int error = 0; + xhci_submission_t sub = { + .callback = xfer->completion, + .callback_ctx = xfer->completion_ctx, + .device = xfer->device, + }; + + if (xfer->type == USB_XFER_TYPE_CONTROL) + error = xhci_control_xfer(xhci, xhci_dev, xfer, &sub); + else + error = xhci_data_xfer(xhci, xhci_dev, xfer, &sub); + + return error; +} + +static void deaddress_device_callback(usb_device_t *device, void *, usb_status_t status, size_t) { + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + xhci_ctrl_t *xhci = (xhci_ctrl_t *)xhci_dev->device.hub->ctrl; + + if (status != USB_STATUS_SUCCESS) { + printf("xhci: disable slot %u failed during deaddress\n", xhci_dev->slot_id); + // If Disable Slot fails, the controller may still reference this device's + // contexts or transfer rings. Leak them for now instead of freeing memory + // that hardware may still own. + } else { + xhci_release_device_resources(xhci, xhci_dev); + } +} + +static void xhci_deaddress_device(usb_ctrl_t *ctrl, usb_device_t *device) { + xhci_ctrl_t *xhci = container_of(ctrl, xhci_ctrl_t, ctrl); + xhci_device_t *xhci_dev = container_of(device, xhci_device_t, device); + + xhci_disable_slot(xhci, xhci_dev, deaddress_device_callback, NULL); +} + +usb_ctrl_ops_t xhci_ops = { + .address_device = xhci_ctrl_address_device, + .configure_ep = xhci_ctrl_configure_ep, + .xfer = xhci_ctrl_xfer, + .deaddress_device = xhci_deaddress_device +}; diff --git a/kernel-src/io/usb/ctrl/xhci_hub.c b/kernel-src/io/usb/ctrl/xhci_hub.c new file mode 100644 index 00000000..cceb92b6 --- /dev/null +++ b/kernel-src/io/usb/ctrl/xhci_hub.c @@ -0,0 +1,153 @@ +#include +#include + +void xhci_handle_port_change_event(xhci_ctrl_t *xhci, int global_port) { + int local_port = -1; + xhci_root_hub_t *hub = NULL; + list_for_each(&xhci->hubs, i) { + hub = container_of(i, xhci_root_hub_t, list_node); + int port = global_port - 1 - hub->port_offset; + if (port >= 0 && port < hub->hub.port_count) { + local_port = port; + break; + } + } + + // getting an event on a non-existant hub is either a hardware bug or a driver bug + // either way, we probably don't want to continue + __assert(local_port != -1); + + uint32_t portsc = xhci->portregs[global_port - 1].portsc; + bool reset_event = false; + bool connect_event = false; + bool disconnect_event = false; + + if (portsc & XHCI_PORTSC_PRC) + reset_event = true; + + if (portsc & XHCI_PORTSC_CSC) { + connect_event = portsc & XHCI_PORTSC_CCS; + disconnect_event = !connect_event; + } + + if (disconnect_event) + usb_hub_event_disconnect(&hub->hub, local_port); + else if (connect_event) { + if (hub->hub.ports[local_port].device != NULL) + usb_hub_event_disconnect(&hub->hub, local_port); + usb_hub_event_connect(&hub->hub, local_port); + } else if (reset_event) { + usb_hub_event_reset(&hub->hub, local_port); + } + + xhci->portregs[global_port - 1].portsc = (portsc & XHCI_PORTSC_PRESERVE_BITS) | + (portsc & XHCI_PORTSC_CHANGE_BITS); +} + +static int xhci_root_hub_get_port_status(usb_hub_t *hub, uint8_t port, uint16_t *status, uint16_t *change) { + xhci_root_hub_t *rh = container_of(hub, xhci_root_hub_t, hub); + xhci_ctrl_t *ctrl = container_of(hub->ctrl, xhci_ctrl_t, ctrl); + ++port; + + if (port == 0 || port > hub->port_count) + return EINVAL; + + uint32_t portsc = ctrl->portregs[rh->port_offset + port - 1].portsc; + uint32_t port_speed = (portsc >> 10) & 0xf; + + *status = 0; + *change = 0; + + if (port_speed == XHCI_PORT_SPEED_LOW) + *status |= USB_HUB_PORT_STATUS_LOW_SPEED; + else if (port_speed == XHCI_PORT_SPEED_HIGH) + *status |= USB_HUB_PORT_STATUS_HIGH_SPEED; + + if (portsc & XHCI_PORTSC_CCS) + *status |= USB_HUB_PORT_STATUS_PORT_CONNECTION; + if (portsc & XHCI_PORTSC_PED) + *status |= USB_HUB_PORT_STATUS_PORT_ENABLE; + if (portsc & XHCI_PORTSC_OCA) + *status |= USB_HUB_PORT_STATUS_PORT_OVER_CURRENT; + if (portsc & XHCI_PORTSC_PR) + *status |= USB_HUB_PORT_STATUS_PORT_RESET; + if (portsc & XHCI_PORTSC_PP) + *status |= USB_HUB_PORT_STATUS_PORT_POWER; + + if (portsc & XHCI_PORTSC_CSC) + *change |= USB_HUB_PORT_CHANGE_PORT_CONNECTION; + if (portsc & XHCI_PORTSC_PEC) + *change |= USB_HUB_PORT_CHANGE_PORT_ENABLE; + if (portsc & XHCI_PORTSC_OCC) + *change |= USB_HUB_PORT_CHANGE_PORT_OVER_CURRENT; + if (portsc & XHCI_PORTSC_PRC) + *change |= USB_HUB_PORT_CHANGE_PORT_RESET; + + return 0; +} + +static int xhci_root_hub_set_port_feature(usb_hub_t *hub, uint8_t port, uint16_t feature) { + xhci_root_hub_t *rh = container_of(hub, xhci_root_hub_t, hub); + xhci_ctrl_t *ctrl = container_of(hub->ctrl, xhci_ctrl_t, ctrl); + ++port; + + if (port == 0 || port > hub->port_count) + return EINVAL; + + volatile xhci_port_regs_t *portregs = &ctrl->portregs[rh->port_offset + port - 1]; + + uint32_t portsc = portregs->portsc; + portsc &= ~(XHCI_PORTSC_CCS | XHCI_PORTSC_PED | XHCI_PORTSC_OCA | XHCI_PORTSC_PR); + portsc &= ~XHCI_PORTSC_CHANGE_BITS; + + if (feature == USB_HUB_FEATURE_PORT_RESET) + portsc |= XHCI_PORTSC_PR; + else if (feature == USB_HUB_FEATURE_PORT_POWER) + portsc |= XHCI_PORTSC_PP; + else + return EINVAL; + + portregs->portsc = portsc; + return 0; +} + +static int xhci_root_hub_clear_port_feature(usb_hub_t *hub, uint8_t port, uint16_t feature) { + xhci_root_hub_t *rh = container_of(hub, xhci_root_hub_t, hub); + xhci_ctrl_t *ctrl = container_of(hub->ctrl, xhci_ctrl_t, ctrl); + ++port; + + if (port == 0 || port > hub->port_count) + return EINVAL; + + volatile xhci_port_regs_t *portregs = &ctrl->portregs[rh->port_offset + port - 1]; + + uint32_t portsc = portregs->portsc; + portsc &= ~(XHCI_PORTSC_CCS | XHCI_PORTSC_PED | XHCI_PORTSC_OCA | XHCI_PORTSC_PR); + portsc &= ~XHCI_PORTSC_CHANGE_BITS; + + if (feature == USB_HUB_FEATURE_PORT_ENABLE) + portsc &= ~XHCI_PORTSC_PED; + else if (feature == USB_HUB_FEATURE_PORT_RESET) + portsc &= ~XHCI_PORTSC_PR; + else if (feature == USB_HUB_FEATURE_PORT_POWER) + portsc &= ~XHCI_PORTSC_PP; + else if (feature == USB_HUB_FEATURE_C_PORT_CONNECTION) + portsc |= XHCI_PORTSC_CSC; + else if (feature == USB_HUB_FEATURE_C_PORT_ENABLE) + portsc |= XHCI_PORTSC_PEC; + else if (feature == USB_HUB_FEATURE_C_PORT_OVER_CURRENT) + portsc |= XHCI_PORTSC_OCC; + else if (feature == USB_HUB_FEATURE_C_PORT_RESET) + portsc |= XHCI_PORTSC_PRC; + else + return EINVAL; + + portregs->portsc = portsc; + return 0; +} + +usb_hub_ops_t xhci_root_hub_ops = { + .get_port_status = xhci_root_hub_get_port_status, + .set_port_feature = xhci_root_hub_set_port_feature, + .clear_port_feature = xhci_root_hub_clear_port_feature, +}; diff --git a/kernel-src/io/usb/ctrl/xhci_ring.c b/kernel-src/io/usb/ctrl/xhci_ring.c new file mode 100644 index 00000000..ff5e811d --- /dev/null +++ b/kernel-src/io/usb/ctrl/xhci_ring.c @@ -0,0 +1,130 @@ +#include +#include +#include +#include +#include + +int xhci_alloc_ring(xhci_ctrl_t *ctrl, xhci_ring_t *r, bool event_ring) { + void *ring_phys = pmm_allocpage(PMM_SECTION_DEFAULT); + if (ring_phys == NULL) + return ENOMEM; + + xhci_trb_t *ring = MAKE_HHDM(ring_phys); + memset(ring, 0, PAGE_SIZE); + + r->ring_phys = ring_phys; + r->ring = ring; + r->ctrl = ctrl; + r->size = PAGE_SIZE / sizeof(xhci_trb_t); + r->index = 0; + r->cycle = true; + + if (event_ring) { + r->submissions = NULL; + } else { + r->submissions = alloc(sizeof(xhci_submission_t) * r->size); + if (r->submissions == NULL) { + pmm_release(ring_phys); + return ENOMEM; + } + + resource_allocator_init(&r->trb_allocator, r->size - 1, r->size - 1); + } + + MUTEX_INIT(&r->lock); + + return 0; +} + +void xhci_free_ring(xhci_ring_t *ring) { + pmm_release(ring->ring_phys); + free(ring->submissions); +} + +void xhci_ring_reserve(xhci_ring_t *r, size_t needed) { + if (needed == 0) + return; + + __assert(needed <= r->size - 1) + + resource_allocate(&r->trb_allocator, needed); +} + +void xhci_ring_unreserve(xhci_ring_t *r, size_t needed) { + if (needed) + resource_free(&r->trb_allocator, needed); +} + +xhci_submission_t *xhci_ring_submit_locked(xhci_ring_t *r, xhci_trb_t *trb, xhci_submission_t *sub) { + // Cycle bit must be 0 in the TRB being submitted. + __assert((trb->dw3 & XHCI_TRB_DW3_C) == 0); + + if (r->index == r->size - 1) { + // If we're at the end of the ring, we need to set up a link TRB. + xhci_trb_t *link_trb = &r->ring[r->index]; + link_trb->parameters = (uint64_t)r->ring_phys; + link_trb->dw2 = 0; + link_trb->dw3 = XHCI_TRB_DW3_TYPE(TRB_LINK) | XHCI_TRB_DW3_TC | (r->cycle ? XHCI_TRB_DW3_C : 0); + + // Now wrap around to start of ring and flip cycle bit. + r->index = 0; + r->cycle ^= 1; + } + + size_t idx = r->index++; + xhci_trb_t *r_trb = &r->ring[idx]; + xhci_submission_t *r_sub = NULL; + + memcpy(r_trb, trb, sizeof(xhci_trb_t)); + + if (sub) { + r_sub = &r->submissions[idx]; + + bool old_valid = __atomic_exchange_n(&r_sub->valid, true, __ATOMIC_SEQ_CST); + __assert(!old_valid); + + r_sub->ring = r; + r_sub->device = sub->device; + r_sub->callback = sub->callback; + r_sub->callback_ctx = sub->callback_ctx; + r_sub->data_length = sub->data_length; + r_sub->data_offset = sub->data_offset; + r_sub->next = NULL; + r_sub->td_head = NULL; + memcpy(&r_sub->cmd_trb, trb, sizeof(xhci_trb_t)); + r_sub->usb_status = USB_STATUS_SUCCESS; // initialize this to a known value to allow the handling of short packets on control transfers + } + + // Set cycle bit appropriately. + if (r->cycle) + r_trb->dw3 |= XHCI_TRB_DW3_C; + + return r_sub; +} + +void xhci_ring_submit(xhci_ring_t *r, xhci_trb_t *trb, xhci_submission_t *sub) { + xhci_ring_reserve(r, 1); + + MUTEX_ACQUIRE(&r->lock); + xhci_ring_submit_locked(r, trb, sub); + MUTEX_RELEASE(&r->lock); +} + +xhci_trb_t *xhci_ring_dequeue(xhci_ring_t *r) { + xhci_trb_t *r_trb = &r->ring[r->index]; + + // Make sure the cycle bit matches. + if (((r_trb->dw3 & XHCI_TRB_DW3_C) != 0) != r->cycle) { + return NULL; + } + + // Advance offset and flip cycle bit if needed. + r->index++; + + if (r->index == r->size) { + r->index = 0; + r->cycle ^= 1; + } + + return r_trb; +} diff --git a/kernel-src/io/usb/ctrl/xhci_xfer.c b/kernel-src/io/usb/ctrl/xhci_xfer.c new file mode 100644 index 00000000..2ef373b7 --- /dev/null +++ b/kernel-src/io/usb/ctrl/xhci_xfer.c @@ -0,0 +1,288 @@ +#include +#include +#include +#include +#include +#include + +typedef struct { + xhci_trb_t trb; + void *page; + size_t data_length; + size_t data_offset; +} sg_trb_t; + +static int xhci_count_sg_trbs(iovec_iterator_t *iov, size_t total_size, size_t *trb_count) { + iovec_iterator_t iter = *iov; + size_t done = 0; + size_t count = 0; + + while (done < total_size) { + size_t page_offset, page_remaining; + void *page; + + int err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page); + if (err) + return err; + if (page == NULL) + return EFAULT; + + pmm_release(page); + + done += min(page_remaining, total_size - done); + count++; + } + + *trb_count = count; + return 0; +} + +int xhci_data_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, xhci_submission_t *sub) { + __assert(xfer->ep != NULL); + __assert(xfer->setup == NULL); + + uint32_t ep_num = xfer->ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_NUM_MASK; + uint32_t ep_index = (ep_num << 1) | (xfer->ep->desc->bEndpointAddress & USB_ENDPOINT_ADDRESS_DIR_IN ? 1 : 0); + + xhci_ring_t *ring = &dev->ep_rings[ep_index - 1]; + int error = 0; + + if (xfer->flags & USB_XFER_FLAG_IOVEC) { + error = xhci_sg_data_xfer(xhci, ring, dev, xfer, sub); + } else { + if (xfer->data_length == 0) { + if (sub->callback) + sub->callback(sub->device, sub->callback_ctx, USB_STATUS_SUCCESS, 0); + return 0; + } + + xhci_ring_reserve(ring, 1); + + // Make sure the data transfer does not cross a page boundary. + uint64_t page_offset = (uint64_t)xfer->data & (PAGE_SIZE - 1); + __assert(page_offset + xfer->data_length <= PAGE_SIZE); + + xhci_trb_t trb = {0}; + if (xfer->flags & USB_XFER_FLAG_BUFFER_PHYSICAL) { + pmm_hold(xfer->data - page_offset); + trb.parameters = (uint64_t)xfer->data; + } else { + // This should never be called with user pages. Doing so is a bug. For user-fronting code, use + // the scatter-gather variant + __assert(!IS_USER_ADDRESS(xfer->data)); + void *page = vmm_getphysical(xfer->data - page_offset, VMM_GET_PHYSICAL_FLAGS_HOLD); + __assert(page != NULL); + trb.parameters = (uint64_t)page + page_offset; + } + + trb.dw2 = (uint32_t)xfer->data_length; + trb.dw3 = XHCI_TRB_DW3_TYPE(TRB_NORMAL) | XHCI_TRB_DW3_ISP | XHCI_TRB_DW3_IOC; + + sub->data_length = xfer->data_length; + sub->data_offset = 0; + + MUTEX_ACQUIRE(&ring->lock); + xhci_submission_t *current = xhci_ring_submit_locked(ring, &trb, sub); + current->td_head = current; + MUTEX_RELEASE(&ring->lock); + } + + if (error) + return error; + + // Ring the doorbell for the endpoint. + xhci_ring_endpoint_doorbell(xhci, dev, ep_index); + return error; +} + +int xhci_sg_data_xfer(xhci_ctrl_t *xhci, xhci_ring_t *ring, xhci_device_t *dev, usb_xfer_t *xfer, xhci_submission_t *sub) { + uint16_t max_packet_size = usb_endpoint_max_packet_size(xfer->ep->desc); + uint32_t td_count = ROUND_UP(xfer->iov->total_size, max_packet_size) / max_packet_size; + uint32_t done = 0; + + if (xfer->iov->total_size == 0) { + if (sub->callback) + sub->callback(sub->device, sub->callback_ctx, USB_STATUS_SUCCESS, 0); + return 0; + } + + size_t trb_count; + int err = xhci_count_sg_trbs(xfer->iov, xfer->iov->total_size, &trb_count); + if (err) + return err; + + xhci_ring_reserve(ring, trb_count); + + sg_trb_t *prepared = alloc(sizeof(sg_trb_t) * trb_count); + if (prepared == NULL) { + xhci_ring_unreserve(ring, trb_count); + return ENOMEM; + } + memset(prepared, 0, sizeof(sg_trb_t) * trb_count); + + iovec_iterator_t iter = *xfer->iov; + + size_t prepared_count = 0; + while (done < xfer->iov->total_size) { + size_t page_offset, page_remaining; + void *page; + + err = iovec_iterator_next_page(&iter, &page_offset, &page_remaining, &page); + if (err) + goto fail; + if (page == NULL) { + err = EFAULT; + goto fail; + } + + size_t to_xfer = min(page_remaining, xfer->iov->total_size - done); + + uint8_t td_size = td_count - (done + to_xfer) / max_packet_size; + if (done + to_xfer == xfer->iov->total_size) + td_size = 0; + else if (td_size > 31) + td_size = 31; + + xhci_trb_t trb = {0}; + trb.parameters = (uint64_t)page + page_offset; + trb.dw2 = XHCI_TRB_DW2_TR_LEN(to_xfer) | XHCI_TRB_DW2_TD_SIZE(td_size); + trb.dw3 = XHCI_TRB_DW3_TYPE(TRB_NORMAL) | XHCI_TRB_DW3_ISP; + + if (to_xfer + done < xfer->iov->total_size) { + // More TRBs to come in this transfer, set the CH bit. + trb.dw3 |= XHCI_TRB_DW3_CH; + } else { + trb.dw3 |= XHCI_TRB_DW3_IOC; + } + + prepared[prepared_count].trb = trb; + prepared[prepared_count].page = page; + prepared[prepared_count].data_length = to_xfer; + prepared[prepared_count].data_offset = done; + prepared_count++; + done += to_xfer; + } + + *xfer->iov = iter; + + MUTEX_ACQUIRE(&ring->lock); + xhci_submission_t *prev = NULL; + for (size_t i = 0; i < prepared_count; ++i) { + sub->data_length = prepared[i].data_length; + sub->data_offset = prepared[i].data_offset; + xhci_submission_t *current = xhci_ring_submit_locked(ring, &prepared[i].trb, sub); + if (prev == NULL) { + current->td_head = current; + } else { + current->td_head = prev->td_head; + prev->next = current; + } + + prev = current; + } + MUTEX_RELEASE(&ring->lock); + + free(prepared); + return 0; + + fail: + for (size_t i = 0; i < prepared_count; ++i) + pmm_release(prepared[i].page); + free(prepared); + xhci_ring_unreserve(ring, trb_count); + return err; +} + +int xhci_control_xfer(xhci_ctrl_t *xhci, xhci_device_t *dev, usb_xfer_t *xfer, xhci_submission_t *sub) { + __assert(xfer->ep == NULL); + __assert(xfer->setup != NULL); + + // Why would anyone want to do scatter-gather control transfers??? + __assert(!(xfer->flags & USB_XFER_FLAG_IOVEC)); + + // Make sure direction matches request type and length matches buffer length. + usb_setup_t *setup = xfer->setup; + + { + bool flags_to_host = (xfer->flags & USB_XFER_FLAG_TO_HOST) != 0; + bool req_to_host = (setup->bmRequestType & USB_REQUEST_DIR_TO_HOST) != 0; + __assert(xfer->data_length == setup->wLength); + __assert(flags_to_host == req_to_host); + } + + bool has_data_stage = xfer->data != NULL && xfer->data_length > 0; + bool data_stage_in = has_data_stage && xfer->flags & USB_XFER_FLAG_TO_HOST; + + xhci_ring_t *ring = &dev->ep_rings[0]; + + size_t trb_count = has_data_stage ? 3 : 2; + xhci_ring_reserve(ring, trb_count); + + xhci_trb_t setup_trb = {0}; + xhci_trb_t data_trb = {0}; + xhci_trb_t status_trb = {0}; + + uint32_t trt = 0; + if (has_data_stage) + // 0 = no data stage, 2 = data out, 3 = data in + trt = data_stage_in ? 3 : 2; + + setup_trb.dw0 = (uint32_t)setup->bmRequestType | ((uint32_t)setup->bRequest << 8) | ((uint32_t)setup->wValue << 16); + setup_trb.dw1 = (uint32_t)setup->wIndex | ((uint32_t)setup->wLength << 16); + setup_trb.dw2 = 8; // Always 8 bytes for setup stage. + setup_trb.dw3 = XHCI_TRB_DW3_TYPE(TRB_SETUP_STAGE) | XHCI_TRB_DW3_TRT(trt) | XHCI_TRB_DW3_IDT; + + if (has_data_stage) { + // Make sure the data transfer does not cross a page boundary. + uint64_t page_offset = (uint64_t)xfer->data & (PAGE_SIZE - 1); + __assert(page_offset + xfer->data_length <= PAGE_SIZE); + + if (xfer->data_length) { + if (xfer->flags & USB_XFER_FLAG_BUFFER_PHYSICAL) { + pmm_hold(xfer->data - page_offset); + data_trb.parameters = (uint64_t)xfer->data; + } else { + void *page = vmm_getphysical(xfer->data - page_offset, VMM_GET_PHYSICAL_FLAGS_HOLD); + if (page == NULL) { + xhci_ring_unreserve(ring, trb_count); + return EFAULT; + } + data_trb.parameters = (uint64_t)page + page_offset; + } + } + + data_trb.dw2 = (uint32_t)xfer->data_length; + data_trb.dw3 = XHCI_TRB_DW3_TYPE(TRB_DATA_STAGE) | XHCI_TRB_DW3_ISP | (data_stage_in ? XHCI_TRB_DW3_DIR : 0); + } + + status_trb.dw3 = XHCI_TRB_DW3_TYPE(TRB_STATUS_STAGE) | XHCI_TRB_DW3_IOC | (data_stage_in ? 0 : XHCI_TRB_DW3_DIR); + + MUTEX_ACQUIRE(&ring->lock); + + sub->data_length = 0; + sub->data_offset = 0; + xhci_submission_t *prev = xhci_ring_submit_locked(ring, &setup_trb, sub); + xhci_submission_t *head = prev; + prev->td_head = head; + + if (has_data_stage) { + sub->data_length = xfer->data_length; + sub->data_offset = 0; + xhci_submission_t *current = xhci_ring_submit_locked(ring, &data_trb, sub); + current->td_head = head; + prev->next = current; + prev = current; + } + + sub->data_length = 0; + sub->data_offset = has_data_stage ? xfer->data_length : 0; + xhci_submission_t *status = xhci_ring_submit_locked(ring, &status_trb, sub); + status->td_head = head; + prev->next = status; + + MUTEX_RELEASE(&ring->lock); + + // Ring the doorbell for the control endpoint. + xhci_ring_endpoint_doorbell(xhci, dev, 1); + return 0; +}