diff --git a/arch/arm/boot/dts/broadcom/bcm2711-rpi-ds.dtsi b/arch/arm/boot/dts/broadcom/bcm2711-rpi-ds.dtsi index 652062b5ba3785..e498c634fb727c 100644 --- a/arch/arm/boot/dts/broadcom/bcm2711-rpi-ds.dtsi +++ b/arch/arm/boot/dts/broadcom/bcm2711-rpi-ds.dtsi @@ -3,7 +3,7 @@ / { chosen { - bootargs = "coherent_pool=1M 8250.nr_uarts=1 snd_bcm2835.enable_headphones=0 cgroup_disable=memory numa_policy=interleave nvme.max_host_mem_size_mb=0"; + bootargs = "coherent_pool=1M 8250.nr_uarts=1 snd_bcm2835.enable_headphones=0 cgroup_disable=memory numa_policy=interleave nvme.max_host_mem_size_mb=32"; }; __overrides__ { diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi index 2fa5eb16ae08a6..b25e761dd23ff1 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi @@ -113,7 +113,7 @@ watchdog: &pm {}; / { chosen: chosen { - bootargs = "reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe cgroup_disable=memory numa_policy=interleave nvme.max_host_mem_size_mb=0"; + bootargs = "reboot=w coherent_pool=1M 8250.nr_uarts=1 pci=pcie_bus_safe cgroup_disable=memory numa_policy=interleave nvme.max_host_mem_size_mb=32"; stdout-path = "serial10:115200n8"; }; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index a64b4b4a18a194..89615cac702da8 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -183,9 +183,11 @@ struct nvme_dev { /* host memory buffer support: */ u64 host_mem_size; u32 nr_host_mem_descs; + u32 nr_sgl_ents; u32 host_mem_descs_size; dma_addr_t host_mem_descs_dma; struct nvme_host_mem_buf_desc *host_mem_descs; + struct scatterlist *host_mem_sgl; void **host_mem_desc_bufs; unsigned int nr_allocated_queues; unsigned int nr_write_queues; @@ -2300,6 +2302,13 @@ static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits) return ret; } +#if IS_ENABLED(CONFIG_ARM64) +static void nvme_free_host_mem_multi(struct nvme_dev *dev) +{ + dma_unmap_sg(dev->dev, dev->host_mem_sgl, dev->nr_host_mem_descs, DMA_FROM_DEVICE); + sgl_free(dev->host_mem_sgl); +} +#else static void nvme_free_host_mem_multi(struct nvme_dev *dev) { int i; @@ -2316,6 +2325,7 @@ static void nvme_free_host_mem_multi(struct nvme_dev *dev) kfree(dev->host_mem_desc_bufs); dev->host_mem_desc_bufs = NULL; } +#endif static void nvme_free_host_mem(struct nvme_dev *dev) { @@ -2358,6 +2368,93 @@ static int nvme_alloc_host_mem_single(struct nvme_dev *dev, u64 size) return 0; } +#if IS_ENABLED(CONFIG_ARM64) +static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred, + u32 chunk_size) +{ + struct nvme_host_mem_buf_desc *descs; + u32 max_entries, len, descs_size; + dma_addr_t descs_dma; + struct scatterlist *slist; + struct page *page; + int i = 0, mapped_nents; + u64 size, tmp; + + tmp = (preferred + chunk_size - 1); + do_div(tmp, chunk_size); + max_entries = tmp; + + if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries) + max_entries = dev->ctrl.hmmaxd; + + descs_size = max_entries * sizeof(*descs); + /* + * Allocate the descriptor table from coherent memory - + * usually occupies less than/up to a single page. + */ + descs = dma_alloc_coherent(dev->dev, descs_size, &descs_dma, + GFP_KERNEL); + if (!descs) + goto out; + + slist = kcalloc(max_entries, sizeof(struct scatterlist), GFP_KERNEL); + if (!slist) + goto out_free_descs; + + sg_init_table(slist, max_entries); + + dev_dbg(dev->dev, "Allocating HMB pref = %llu max_entries = %u\n", + preferred, max_entries); + + for (size = 0; size < preferred && i < max_entries; size += len) { + int order; + + len = min_t(u64, chunk_size, preferred - size); + order = get_order(len); + page = alloc_pages(GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN, order); + if (!page) + break; + sg_set_page(&slist[i], page, len, 0); + i++; + } + if (size < preferred) + goto out_free_sgl; + + mapped_nents = dma_map_sg(dev->dev, slist, i, DMA_FROM_DEVICE); + if (mapped_nents <= 0) + goto out_free_pages; + + /* Flush in case the CPU has cached any parts of the DMA buffers */ + dma_sync_sg_for_device(dev->dev, slist, i, DMA_FROM_DEVICE); + + i = dev->nr_host_mem_descs = mapped_nents; + + while (--i >= 0) { + descs[i].addr = sg_dma_address(&slist[i]); + WARN_ON_ONCE(sg_dma_len(&slist[i]) & (NVME_CTRL_PAGE_SIZE - 1)); + descs[i].size = sg_dma_len(&slist[i]) / NVME_CTRL_PAGE_SIZE; + } + + dev->host_mem_size = size; + dev->host_mem_descs = descs; + dev->host_mem_descs_dma = descs_dma; + dev->host_mem_descs_size = descs_size; + dev->host_mem_sgl = slist; + return 0; + +out_free_pages: + /* Don't use mapped_nents here as it could be incomplete */ + while (--i >= 0) + __free_pages(sg_page(&slist[i]), get_order(slist[i].length)); +out_free_sgl: + kfree(slist); +out_free_descs: + dma_free_coherent(dev->dev, descs_size, descs, descs_dma); +out: + dev->host_mem_descs = NULL; + return -ENOMEM; +} +#else static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred, u32 chunk_size) { @@ -2418,6 +2515,7 @@ static int nvme_alloc_host_mem_multi(struct nvme_dev *dev, u64 preferred, dev->host_mem_descs = NULL; return -ENOMEM; } +#endif static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred) {