/* $NetBSD: pci_msi_machdep.c,v 1.13 2017/07/28 14:26:50 maxv Exp $ */
/*
* Copyright (c) 2015 Internet Initiative Japan Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/*
* TODO
*
* - PBA (Pending Bit Array) support
* - HyperTransport mapping support
*/
#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.13 2017/07/28 14:26:50 maxv Exp $");
#include "opt_intrdebug.h"
#include "ioapic.h"
#include <sys/types.h>
#include <sys/param.h>
#include <sys/time.h>
#include <sys/systm.h>
#include <sys/cpu.h>
#include <sys/errno.h>
#include <sys/device.h>
#include <sys/intr.h>
#include <sys/kmem.h>
#include <dev/pci/pcivar.h>
#include <machine/i82093var.h>
#include <machine/pic.h>
#include <x86/pci/msipic.h>
#include <x86/pci/pci_msi_machdep.h>
#ifdef INTRDEBUG
#define MSIDEBUG
#endif
#ifdef MSIDEBUG
#define DPRINTF(msg) printf msg
#else
#define DPRINTF(msg)
#endif
static pci_intr_handle_t
pci_msi_calculate_handle(struct pic *msi_pic, int vector)
{
pci_intr_handle_t pih;
KASSERT(msipic_is_msi_pic(msi_pic));
pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK)
| __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK)
| APIC_INT_VIA_MSI;
if (msi_pic->pic_type == PIC_MSI)
MSI_INT_MAKE_MSI(pih);
else if (msi_pic->pic_type == PIC_MSIX)
MSI_INT_MAKE_MSIX(pih);
else
panic("%s: Unexpected pic_type: %d\n", __func__,
msi_pic->pic_type);
return pih;
}
#ifdef __HAVE_PCI_MSI_MSIX
static pci_intr_handle_t *
pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count)
{
struct intrsource *isp;
pci_intr_handle_t *vectors, pih;
int i;
const char *intrstr;
char intrstr_buf[INTRIDBUF];
vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP);
mutex_enter(&cpu_lock);
for (i = 0; i < *count; i++) {
u_int table_index;
if (table_indexes == NULL)
table_index = i;
else
table_index = table_indexes[i];
pih = pci_msi_calculate_handle(msi_pic, table_index);
intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
sizeof(intrstr_buf));
isp = intr_allocate_io_intrsource(intrstr);
if (isp == NULL) {
mutex_exit(&cpu_lock);
DPRINTF(("can't allocate io_intersource\n"));
kmem_free(vectors, sizeof(vectors[0]) * (*count));
return NULL;
}
vectors[i] = pih;
}
mutex_exit(&cpu_lock);
return vectors;
}
#endif /* __HAVE_PCI_MSI_MSIX */
static void
pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count)
{
pci_intr_handle_t pih;
int i;
const char *intrstr;
char intrstr_buf[INTRIDBUF];
mutex_enter(&cpu_lock);
for (i = 0; i < count; i++) {
pih = pci_msi_calculate_handle(msi_pic, i);
intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf,
sizeof(intrstr_buf));
intr_free_io_intrsource(intrstr);
}
mutex_exit(&cpu_lock);
kmem_free(pihs, sizeof(pihs[0]) * count);
}
#ifdef __HAVE_PCI_MSI_MSIX
static int
pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count,
const struct pci_attach_args *pa, bool exact)
{
struct pic *msi_pic;
pci_intr_handle_t *vectors;
int error, i;
#if NIOAPIC > 0
if (nioapics == 0) {
DPRINTF(("no IOAPIC.\n"));
return ENODEV;
}
#endif
if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) {
DPRINTF(("PCI host bridge does not support MSI.\n"));
return ENODEV;
}
msi_pic = msipic_construct_msi_pic(pa);
if (msi_pic == NULL) {
DPRINTF(("cannot allocate MSI pic.\n"));
return EINVAL;
}
vectors = NULL;
while (*count > 0) {
vectors = pci_msi_alloc_vectors(msi_pic, NULL, count);
if (vectors != NULL)
break;
if (exact) {
DPRINTF(("cannot allocate MSI vectors.\n"));
msipic_destruct_msi_pic(msi_pic);
return ENOMEM;
} else {
(*count) >>= 1; /* must be power of 2. */
continue;
}
}
if (vectors == NULL) {
DPRINTF(("cannot allocate MSI vectors.\n"));
msipic_destruct_msi_pic(msi_pic);
return ENOMEM;
}
for (i = 0; i < *count; i++) {
MSI_INT_MAKE_MSI(vectors[i]);
}
error = msipic_set_msi_vectors(msi_pic, NULL, *count);
if (error) {
pci_msi_free_vectors(msi_pic, vectors, *count);
msipic_destruct_msi_pic(msi_pic);
return error;
}
*ihps = vectors;
return 0;
}
#endif /* __HAVE_PCI_MSI_MSIX */
static void *
pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
int level, int (*func)(void *), void *arg, struct pic *pic,
const char *xname)
{
int irq, pin;
bool mpsafe;
KASSERT(INT_VIA_MSI(ih));
irq = -1;
pin = MSI_INT_VEC(ih);
mpsafe = ((ih & MPSAFE_MASK) != 0);
return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg,
mpsafe, xname);
}
static void
pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie)
{
intr_disestablish(cookie);
}
#ifdef __HAVE_PCI_MSI_MSIX
static int
pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes,
int *count, const struct pci_attach_args *pa, bool exact)
{
struct pic *msix_pic;
pci_intr_handle_t *vectors;
int error, i;
#if NIOAPIC > 0
if (nioapics == 0) {
DPRINTF(("no IOAPIC.\n"));
return ENODEV;
}
#endif
if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) {
DPRINTF(("PCI host bridge does not support MSI-X.\n"));
return ENODEV;
}
msix_pic = msipic_construct_msix_pic(pa);
if (msix_pic == NULL)
return EINVAL;
vectors = NULL;
while (*count > 0) {
vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count);
if (vectors != NULL)
break;
if (exact) {
DPRINTF(("cannot allocate MSI-X vectors.\n"));
msipic_destruct_msix_pic(msix_pic);
return ENOMEM;
} else {
(*count)--;
continue;
}
}
if (vectors == NULL) {
DPRINTF(("cannot allocate MSI-X vectors.\n"));
msipic_destruct_msix_pic(msix_pic);
return ENOMEM;
}
for (i = 0; i < *count; i++) {
MSI_INT_MAKE_MSIX(vectors[i]);
}
error = msipic_set_msi_vectors(msix_pic, vectors, *count);
if (error) {
pci_msi_free_vectors(msix_pic, vectors, *count);
msipic_destruct_msix_pic(msix_pic);
return error;
}
*ihps = vectors;
return 0;
}
static int
x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count,
const struct pci_attach_args *pa)
{
return pci_msi_alloc_common(ihps, count, pa, false);
}
static int
x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count,
const struct pci_attach_args *pa)
{
return pci_msi_alloc_common(ihps, &count, pa, true);
}
#endif /* __HAVE_PCI_MSI_MSIX */
static void
x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count)
{
struct pic *pic;
pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
if (pic == NULL)
return;
pci_msi_free_vectors(pic, pihs, count);
msipic_destruct_msi_pic(pic);
}
#ifdef __HAVE_PCI_MSI_MSIX
static int
x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count,
const struct pci_attach_args *pa)
{
return pci_msix_alloc_common(ihps, NULL, count, pa, false);
}
static int
x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count,
const struct pci_attach_args *pa)
{
return pci_msix_alloc_common(ihps, NULL, &count, pa, true);
}
static int
x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes,
int count, const struct pci_attach_args *pa)
{
return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true);
}
#endif /* __HAVE_PCI_MSI_MSIX */
static void
x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count)
{
struct pic *pic;
pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0]));
if (pic == NULL)
return;
pci_msi_free_vectors(pic, pihs, count);
msipic_destruct_msix_pic(pic);
}
/*****************************************************************************/
/*
* extern for MD code.
*/
/*
* Return intrid for a MSI/MSI-X device.
* "buf" must be allocated by caller.
*/
const char *
x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf,
size_t len)
{
int dev, vec;
KASSERT(INT_VIA_MSI(ih));
dev = MSI_INT_DEV(ih);
vec = MSI_INT_VEC(ih);
if (MSI_INT_IS_MSIX(ih))
snprintf(buf, len, "msix%d vec %d", dev, vec);
else
snprintf(buf, len, "msi%d vec %d", dev, vec);
return buf;
}
/*
* Release MSI handles.
*/
void
x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
{
if (count < 1)
return;
return x86_pci_msi_release_internal(pihs, count);
}
/*
* Establish a MSI handle.
* If multiple MSI handle is requied to establish, device driver must call
* this function for each handle.
*/
void *
x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
int level, int (*func)(void *), void *arg, const char *xname)
{
struct pic *pic;
pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
if (pic == NULL) {
DPRINTF(("pci_intr_handler has no msi_pic\n"));
return NULL;
}
return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
}
/*
* Disestablish a MSI handle.
* If multiple MSI handle is requied to disestablish, device driver must call
* this function for each handle.
*/
void
x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie)
{
pci_msi_common_disestablish(pc, cookie);
}
/*
* Release MSI-X handles.
*/
void
x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count)
{
if (count < 1)
return;
return x86_pci_msix_release_internal(pihs, count);
}
/*
* Establish a MSI-X handle.
* If multiple MSI-X handle is requied to establish, device driver must call
* this function for each handle.
*/
void *
x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih,
int level, int (*func)(void *), void *arg, const char *xname)
{
struct pic *pic;
pic = msipic_find_msi_pic(MSI_INT_DEV(ih));
if (pic == NULL) {
DPRINTF(("pci_intr_handler has no msi_pic\n"));
return NULL;
}
return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname);
}
/*
* Disestablish a MSI-X handle.
* If multiple MSI-X handle is requied to disestablish, device driver must call
* this function for each handle.
*/
void
x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie)
{
pci_msi_common_disestablish(pc, cookie);
}
#ifdef __HAVE_PCI_MSI_MSIX
/*****************************************************************************/
/*
* extern for MI code.
*/
/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI used instead of IRQ number.
* "count" must be power of 2.
* "count" can decrease if struct intrsource cannot be allocated.
* if count == 0, return non-zero value.
*/
int
pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
int *count)
{
int hw_max;
/* MSI vector count must be power of 2. */
KASSERT(*count > 0);
KASSERT(((*count - 1) & *count) == 0);
hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
if (hw_max == 0)
return ENODEV;
if (*count > hw_max) {
DPRINTF(("cut off MSI count to %d\n", hw_max));
*count = hw_max; /* cut off hw_max */
}
return x86_pci_msi_alloc(ihps, count, pa);
}
/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI used instead of IRQ number.
* "count" must be power of 2.
* "count" can not decrease.
* If "count" struct intrsources cannot be allocated, return non-zero value.
*/
int
pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
int count)
{
int hw_max;
/* MSI vector count must be power of 2. */
KASSERT(count > 0);
KASSERT(((count - 1) & count) == 0);
hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag);
if (hw_max == 0)
return ENODEV;
if (count > hw_max) {
DPRINTF(("over hardware max MSI count %d\n", hw_max));
return EINVAL;
}
return x86_pci_msi_alloc_exact(ihps, count, pa);
}
/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
* "count" can decrease if enough struct intrsources cannot be allocated.
* if count == 0, return non-zero value.
*/
int
pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
int *count)
{
int hw_max;
KASSERT(*count > 0);
hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
if (hw_max == 0)
return ENODEV;
if (*count > hw_max) {
DPRINTF(("cut off MSI-X count to %d\n", hw_max));
*count = hw_max; /* cut off hw_max */
}
return x86_pci_msix_alloc(ihps, count, pa);
}
/*
* This function is used by device drivers like pci_intr_map().
*
* "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
* "count" can not decrease.
* If "count" struct intrsource cannot be allocated, return non-zero value.
*/
int
pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
int count)
{
int hw_max;
KASSERT(count > 0);
hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
if (hw_max == 0)
return ENODEV;
if (count > hw_max) {
DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
return EINVAL;
}
return x86_pci_msix_alloc_exact(ihps, count, pa);
}
/*
* This function is used by device drivers like pci_intr_map().
* Futhermore, this function can map each handle to a MSI-X table index.
*
* "ihps" is the array of vector numbers which MSI-X used instead of IRQ number.
* "count" can not decrease.
* "map" size must be equal to "count".
* If "count" struct intrsource cannot be allocated, return non-zero value.
* e.g.
* If "map" = { 1, 4, 0 },
* 1st handle is bound to MSI-X index 1
* 2nd handle is bound to MSI-X index 4
* 3rd handle is bound to MSI-X index 0
*/
int
pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps,
u_int *table_indexes, int count)
{
int hw_max, i, j;
KASSERT(count > 0);
hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag);
if (hw_max == 0)
return ENODEV;
if (count > hw_max) {
DPRINTF(("over hardware max MSI-X count %d\n", hw_max));
return EINVAL;
}
/* check not to duplicate table_index */
for (i = 0; i < count; i++) {
u_int basing = table_indexes[i];
KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS);
if (basing >= hw_max) {
DPRINTF(("table index is over hardware max MSI-X index %d\n",
hw_max - 1));
return EINVAL;
}
for (j = i + 1; j < count; j++) {
if (basing == table_indexes[j]) {
DPRINTF(("MSI-X table index duplicated\n"));
return EINVAL;
}
}
}
return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa);
}
#endif /* __HAVE_PCI_MSI_MSIX */