/*-
* Copyright (c) 2018 VMware, Inc.
*
* SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
*/
/* This file implements the VMCI doorbell API. */
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/types.h>
#include "vmci_doorbell.h"
#include "vmci_driver.h"
#include "vmci_kernel_api.h"
#include "vmci_kernel_defs.h"
#include "vmci_resource.h"
#include "vmci_utils.h"
#define LGPFX "vmci_doorbell: "
#define VMCI_DOORBELL_INDEX_TABLE_SIZE 64
#define VMCI_DOORBELL_HASH(_idx) \
vmci_hash_id((_idx), VMCI_DOORBELL_INDEX_TABLE_SIZE)
/* Describes a doorbell notification handle allocated by the host. */
struct vmci_doorbell_entry {
struct vmci_resource resource;
uint32_t idx;
vmci_list_item(vmci_doorbell_entry) idx_list_item;
vmci_privilege_flags priv_flags;
bool is_doorbell;
bool run_delayed;
vmci_callback notify_cb;
void *client_data;
vmci_event destroy_event;
volatile int active;
};
struct vmci_doorbell_index_table {
vmci_lock lock;
vmci_list(vmci_doorbell_entry) entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
};
/* The VMCI index table keeps track of currently registered doorbells. */
static struct vmci_doorbell_index_table vmci_doorbell_it;
/*
* The max_notify_idx is one larger than the currently known bitmap index in
* use, and is used to determine how much of the bitmap needs to be scanned.
*/
static uint32_t max_notify_idx;
/*
* The notify_idx_count is used for determining whether there are free entries
* within the bitmap (if notify_idx_count + 1 < max_notify_idx).
*/
static uint32_t notify_idx_count;
/*
* The last_notify_idx_reserved is used to track the last index handed out - in
* the case where multiple handles share a notification index, we hand out
* indexes round robin based on last_notify_idx_reserved.
*/
static uint32_t last_notify_idx_reserved;
/* This is a one entry cache used to by the index allocation. */
static uint32_t last_notify_idx_released = PAGE_SIZE;
static void vmci_doorbell_free_cb(void *client_data);
static int vmci_doorbell_release_cb(void *client_data);
static void vmci_doorbell_delayed_dispatch_cb(void *data);
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_init --
*
* General init code.
*
* Result:
* VMCI_SUCCESS on success, lock allocation error otherwise.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
int
vmci_doorbell_init(void)
{
uint32_t bucket;
for (bucket = 0; bucket < ARRAYSIZE(vmci_doorbell_it.entries);
++bucket)
vmci_list_init(&vmci_doorbell_it.entries[bucket]);
return (vmci_init_lock(&vmci_doorbell_it.lock,
"VMCI Doorbell index table lock"));
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_exit --
*
* General exit code.
*
* Result:
* None.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
void
vmci_doorbell_exit(void)
{
vmci_cleanup_lock(&vmci_doorbell_it.lock);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_free_cb --
*
* Callback to free doorbell entry structure when resource is no longer used,
* i.e. the reference count reached 0. The entry is freed in
* vmci_doorbell_destroy(), which is waiting on the signal that gets fired
* here.
*
* Result:
* None.
*
* Side effects:
* Signals VMCI event.
*
*------------------------------------------------------------------------------
*/
static void
vmci_doorbell_free_cb(void *client_data)
{
struct vmci_doorbell_entry *entry;
entry = (struct vmci_doorbell_entry *)client_data;
ASSERT(entry);
vmci_signal_event(&entry->destroy_event);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_release_cb --
*
* Callback to release the resource reference. It is called by the
* vmci_wait_on_event function before it blocks.
*
* Result:
* Always 0.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
static int
vmci_doorbell_release_cb(void *client_data)
{
struct vmci_doorbell_entry *entry;
entry = (struct vmci_doorbell_entry *)client_data;
ASSERT(entry);
vmci_resource_release(&entry->resource);
return (0);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_get_priv_flags --
*
* Utility function that retrieves the privilege flags associated with a
* given doorbell handle. For guest endpoints, the privileges are determined
* by the context ID, but for host endpoints privileges are associated with
* the complete handle. Hypervisor endpoints are not yet supported.
*
* Result:
* VMCI_SUCCESS on success,
* VMCI_ERROR_NOT_FOUND if handle isn't found,
* VMCI_ERROR_INVALID_ARGS if handle is invalid.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
int
vmci_doorbell_get_priv_flags(struct vmci_handle handle,
vmci_privilege_flags *priv_flags)
{
if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
return (VMCI_ERROR_INVALID_ARGS);
if (handle.context == VMCI_HOST_CONTEXT_ID) {
struct vmci_doorbell_entry *entry;
struct vmci_resource *resource;
resource = vmci_resource_get(handle,
VMCI_RESOURCE_TYPE_DOORBELL);
if (resource == NULL)
return (VMCI_ERROR_NOT_FOUND);
entry = RESOURCE_CONTAINER(
resource, struct vmci_doorbell_entry, resource);
*priv_flags = entry->priv_flags;
vmci_resource_release(resource);
} else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
/* Hypervisor endpoints for notifications are not supported. */
return (VMCI_ERROR_INVALID_ARGS);
} else
*priv_flags = VMCI_NO_PRIVILEGE_FLAGS;
return (VMCI_SUCCESS);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_index_table_find --
*
* Find doorbell entry by bitmap index.
*
* Results:
* Entry if found, NULL if not.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
static struct vmci_doorbell_entry *
vmci_doorbell_index_table_find(uint32_t idx)
{
struct vmci_doorbell_entry *iter;
uint32_t bucket;
bucket = VMCI_DOORBELL_HASH(idx);
vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
if (idx == iter->idx)
return (iter);
}
return (NULL);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_index_table_add --
*
* Add the given entry to the index table. This will hold() the entry's
* resource so that the entry is not deleted before it is removed from the
* table.
*
* Results:
* None.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
static void
vmci_doorbell_index_table_add(struct vmci_doorbell_entry *entry)
{
uint32_t bucket;
uint32_t new_notify_idx;
ASSERT(entry);
vmci_resource_hold(&entry->resource);
vmci_grab_lock_bh(&vmci_doorbell_it.lock);
/*
* Below we try to allocate an index in the notification bitmap with
* "not too much" sharing between resources. If we use less that the
* full bitmap, we either add to the end if there are no unused flags
* within the currently used area, or we search for unused ones. If we
* use the full bitmap, we allocate the index round robin.
*/
if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
if (last_notify_idx_released < max_notify_idx &&
!vmci_doorbell_index_table_find(last_notify_idx_released)) {
new_notify_idx = last_notify_idx_released;
last_notify_idx_released = PAGE_SIZE;
} else {
bool reused = false;
new_notify_idx = last_notify_idx_reserved;
if (notify_idx_count + 1 < max_notify_idx) {
do {
if (!vmci_doorbell_index_table_find(
new_notify_idx)) {
reused = true;
break;
}
new_notify_idx = (new_notify_idx + 1) %
max_notify_idx;
} while (new_notify_idx !=
last_notify_idx_released);
}
if (!reused) {
new_notify_idx = max_notify_idx;
max_notify_idx++;
}
}
} else {
new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
}
last_notify_idx_reserved = new_notify_idx;
notify_idx_count++;
entry->idx = new_notify_idx;
bucket = VMCI_DOORBELL_HASH(entry->idx);
vmci_list_insert(&vmci_doorbell_it.entries[bucket], entry,
idx_list_item);
vmci_release_lock_bh(&vmci_doorbell_it.lock);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_index_table_remove --
*
* Remove the given entry from the index table. This will release() the
* entry's resource.
*
* Results:
* None.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
static void
vmci_doorbell_index_table_remove(struct vmci_doorbell_entry *entry)
{
ASSERT(entry);
vmci_grab_lock_bh(&vmci_doorbell_it.lock);
vmci_list_remove(entry, idx_list_item);
notify_idx_count--;
if (entry->idx == max_notify_idx - 1) {
/*
* If we delete an entry with the maximum known notification
* index, we take the opportunity to prune the current max. As
* there might be other unused indices immediately below, we
* lower the maximum until we hit an index in use
*/
while (max_notify_idx > 0 &&
!vmci_doorbell_index_table_find(max_notify_idx - 1))
max_notify_idx--;
}
last_notify_idx_released = entry->idx;
vmci_release_lock_bh(&vmci_doorbell_it.lock);
vmci_resource_release(&entry->resource);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_link --
*
* Creates a link between the given doorbell handle and the given index in
* the bitmap in the device backend.
*
* Results:
* VMCI_SUCCESS if success, appropriate error code otherwise.
*
* Side effects:
* Notification state is created in hypervisor.
*
*------------------------------------------------------------------------------
*/
static int
vmci_doorbell_link(struct vmci_handle handle, bool is_doorbell,
uint32_t notify_idx)
{
struct vmci_doorbell_link_msg link_msg;
vmci_id resource_id;
ASSERT(!VMCI_HANDLE_INVALID(handle));
if (is_doorbell)
resource_id = VMCI_DOORBELL_LINK;
else {
ASSERT(false);
return (VMCI_ERROR_UNAVAILABLE);
}
link_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
resource_id);
link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
link_msg.handle = handle;
link_msg.notify_idx = notify_idx;
return (vmci_send_datagram((struct vmci_datagram *)&link_msg));
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_unlink --
*
* Unlinks the given doorbell handle from an index in the bitmap in the
* device backend.
*
* Results:
* VMCI_SUCCESS if success, appropriate error code otherwise.
*
* Side effects:
* Notification state is destroyed in hypervisor.
*
*------------------------------------------------------------------------------
*/
static int
vmci_doorbell_unlink(struct vmci_handle handle, bool is_doorbell)
{
struct vmci_doorbell_unlink_msg unlink_msg;
vmci_id resource_id;
ASSERT(!VMCI_HANDLE_INVALID(handle));
if (is_doorbell)
resource_id = VMCI_DOORBELL_UNLINK;
else {
ASSERT(false);
return (VMCI_ERROR_UNAVAILABLE);
}
unlink_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
resource_id);
unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
unlink_msg.handle = handle;
return (vmci_send_datagram((struct vmci_datagram *)&unlink_msg));
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_create --
*
* Creates a doorbell with the given callback. If the handle is
* VMCI_INVALID_HANDLE, a free handle will be assigned, if possible. The
* callback can be run immediately (potentially with locks held - the
* default) or delayed (in a kernel thread) by specifying the flag
* VMCI_FLAG_DELAYED_CB. If delayed execution is selected, a given callback
* may not be run if the kernel is unable to allocate memory for the delayed
* execution (highly unlikely).
*
* Results:
* VMCI_SUCCESS on success, appropriate error code otherwise.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
int
vmci_doorbell_create(struct vmci_handle *handle, uint32_t flags,
vmci_privilege_flags priv_flags, vmci_callback notify_cb, void *client_data)
{
struct vmci_doorbell_entry *entry;
struct vmci_handle new_handle;
int result;
if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
return (VMCI_ERROR_INVALID_ARGS);
entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
if (entry == NULL) {
VMCI_LOG_WARNING(LGPFX"Failed allocating memory for datagram "
"entry.\n");
return (VMCI_ERROR_NO_MEM);
}
if (!vmci_can_schedule_delayed_work() &&
(flags & VMCI_FLAG_DELAYED_CB)) {
result = VMCI_ERROR_INVALID_ARGS;
goto free_mem;
}
if (VMCI_HANDLE_INVALID(*handle)) {
vmci_id context_id;
context_id = vmci_get_context_id();
vmci_id resource_id = vmci_resource_get_id(context_id);
if (resource_id == VMCI_INVALID_ID) {
result = VMCI_ERROR_NO_HANDLE;
goto free_mem;
}
new_handle = VMCI_MAKE_HANDLE(context_id, resource_id);
} else {
if (VMCI_INVALID_ID == handle->resource) {
VMCI_LOG_DEBUG(LGPFX"Invalid argument "
"(handle=0x%x:0x%x).\n", handle->context,
handle->resource);
result = VMCI_ERROR_INVALID_ARGS;
goto free_mem;
}
new_handle = *handle;
}
entry->idx = 0;
entry->priv_flags = priv_flags;
entry->is_doorbell = true;
entry->run_delayed = (flags & VMCI_FLAG_DELAYED_CB) ? true : false;
entry->notify_cb = notify_cb;
entry->client_data = client_data;
atomic_store_int(&entry->active, 0);
vmci_create_event(&entry->destroy_event);
result = vmci_resource_add(&entry->resource,
VMCI_RESOURCE_TYPE_DOORBELL, new_handle, vmci_doorbell_free_cb,
entry);
if (result != VMCI_SUCCESS) {
VMCI_LOG_WARNING(LGPFX"Failed to add new resource "
"(handle=0x%x:0x%x).\n", new_handle.context,
new_handle.resource);
if (result == VMCI_ERROR_DUPLICATE_ENTRY)
result = VMCI_ERROR_ALREADY_EXISTS;
goto destroy;
}
vmci_doorbell_index_table_add(entry);
result = vmci_doorbell_link(new_handle, entry->is_doorbell, entry->idx);
if (VMCI_SUCCESS != result)
goto destroy_resource;
atomic_store_int(&entry->active, 1);
if (VMCI_HANDLE_INVALID(*handle))
*handle = new_handle;
return (result);
destroy_resource:
vmci_doorbell_index_table_remove(entry);
vmci_resource_remove(new_handle, VMCI_RESOURCE_TYPE_DOORBELL);
destroy:
vmci_destroy_event(&entry->destroy_event);
free_mem:
vmci_free_kernel_mem(entry, sizeof(*entry));
return (result);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_destroy --
*
* Destroys a doorbell previously created with vmci_doorbell_create. This
* operation may block waiting for a callback to finish.
*
* Results:
* VMCI_SUCCESS on success, appropriate error code otherwise.
*
* Side effects:
* May block.
*
*------------------------------------------------------------------------------
*/
int
vmci_doorbell_destroy(struct vmci_handle handle)
{
struct vmci_doorbell_entry *entry;
struct vmci_resource *resource;
int result;
if (VMCI_HANDLE_INVALID(handle))
return (VMCI_ERROR_INVALID_ARGS);
resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL);
if (resource == NULL) {
VMCI_LOG_DEBUG(LGPFX"Failed to destroy doorbell "
"(handle=0x%x:0x%x).\n", handle.context, handle.resource);
return (VMCI_ERROR_NOT_FOUND);
}
entry = RESOURCE_CONTAINER(resource, struct vmci_doorbell_entry,
resource);
vmci_doorbell_index_table_remove(entry);
result = vmci_doorbell_unlink(handle, entry->is_doorbell);
if (VMCI_SUCCESS != result) {
/*
* The only reason this should fail would be an inconsistency
* between guest and hypervisor state, where the guest believes
* it has an active registration whereas the hypervisor doesn't.
* One case where this may happen is if a doorbell is
* unregistered following a hibernation at a time where the
* doorbell state hasn't been restored on the hypervisor side
* yet. Since the handle has now been removed in the guest,
* we just print a warning and return success.
*/
VMCI_LOG_DEBUG(LGPFX"Unlink of %s (handle=0x%x:0x%x) unknown "
"by hypervisor (error=%d).\n",
entry->is_doorbell ? "doorbell" : "queuepair",
handle.context, handle.resource, result);
}
/*
* Now remove the resource from the table. It might still be in use
* after this, in a callback or still on the delayed work queue.
*/
vmci_resource_remove(handle, VMCI_RESOURCE_TYPE_DOORBELL);
/*
* We now wait on the destroyEvent and release the reference we got
* above.
*/
vmci_wait_on_event(&entry->destroy_event, vmci_doorbell_release_cb,
entry);
/*
* We know that we are now the only reference to the above entry so
* can safely free it.
*/
vmci_destroy_event(&entry->destroy_event);
vmci_free_kernel_mem(entry, sizeof(*entry));
return (VMCI_SUCCESS);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_notify_as_guest --
*
* Notify another guest or the host. We send a datagram down to the host
* via the hypervisor with the notification info.
*
* Results:
* VMCI_SUCCESS on success, appropriate error code otherwise.
*
* Side effects:
* May do a hypercall.
*
*------------------------------------------------------------------------------
*/
static int
vmci_doorbell_notify_as_guest(struct vmci_handle handle,
vmci_privilege_flags priv_flags)
{
struct vmci_doorbell_notify_msg notify_msg;
notify_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
VMCI_DOORBELL_NOTIFY);
notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
notify_msg.handle = handle;
return (vmci_send_datagram((struct vmci_datagram *)¬ify_msg));
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_notify --
*
* Generates a notification on the doorbell identified by the handle. For
* host side generation of notifications, the caller can specify what the
* privilege of the calling side is.
*
* Results:
* VMCI_SUCCESS on success, appropriate error code otherwise.
*
* Side effects:
* May do a hypercall.
*
*------------------------------------------------------------------------------
*/
int
vmci_doorbell_notify(struct vmci_handle dst, vmci_privilege_flags priv_flags)
{
struct vmci_handle src;
if (VMCI_HANDLE_INVALID(dst) ||
(priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
return (VMCI_ERROR_INVALID_ARGS);
src = VMCI_INVALID_HANDLE;
return (vmci_doorbell_notify_as_guest(dst, priv_flags));
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_delayed_dispatch_cb --
*
* Calls the specified callback in a delayed context.
*
* Results:
* None.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
static void
vmci_doorbell_delayed_dispatch_cb(void *data)
{
struct vmci_doorbell_entry *entry = (struct vmci_doorbell_entry *)data;
ASSERT(data);
entry->notify_cb(entry->client_data);
vmci_resource_release(&entry->resource);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_sync --
*
* Use this as a synchronization point when setting globals, for example,
* during device shutdown.
*
* Results:
* None.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
void
vmci_doorbell_sync(void)
{
vmci_grab_lock_bh(&vmci_doorbell_it.lock);
vmci_release_lock_bh(&vmci_doorbell_it.lock);
vmci_resource_sync();
}
/*
*------------------------------------------------------------------------------
*
* vmci_register_notification_bitmap --
*
* Register the notification bitmap with the host.
*
* Results:
* true if the bitmap is registered successfully with the device, false
* otherwise.
*
* Side effects:
* None.
*
*------------------------------------------------------------------------------
*/
bool
vmci_register_notification_bitmap(PPN bitmap_ppn)
{
struct vmci_notify_bitmap_set_msg bitmap_set_msg;
int result;
/*
* Do not ASSERT() on the guest device here. This function can get
* called during device initialization, so the ASSERT() will fail even
* though the device is (almost) up.
*/
bitmap_set_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
VMCI_SET_NOTIFY_BITMAP);
bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
bitmap_set_msg.hdr.payload_size =
sizeof(bitmap_set_msg) - VMCI_DG_HEADERSIZE;
bitmap_set_msg.bitmap_ppn = bitmap_ppn;
result = vmci_send_datagram((struct vmci_datagram *)&bitmap_set_msg);
if (result != VMCI_SUCCESS) {
VMCI_LOG_DEBUG(LGPFX"Failed to register (PPN=%u) as "
"notification bitmap (error=%d).\n",
bitmap_ppn, result);
return (false);
}
return (true);
}
/*
*------------------------------------------------------------------------------
*
* vmci_doorbell_fire_entries --
*
* Executes or schedules the handlers for a given notify index.
*
* Result:
* Notification hash entry if found. NULL otherwise.
*
* Side effects:
* Whatever the side effects of the handlers are.
*
*------------------------------------------------------------------------------
*/
static void
vmci_doorbell_fire_entries(uint32_t notify_idx)
{
struct vmci_doorbell_entry *iter;
uint32_t bucket = VMCI_DOORBELL_HASH(notify_idx);
vmci_grab_lock_bh(&vmci_doorbell_it.lock);
vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
if (iter->idx == notify_idx &&
atomic_load_int(&iter->active) == 1) {
ASSERT(iter->notify_cb);
if (iter->run_delayed) {
int err;
vmci_resource_hold(&iter->resource);
err = vmci_schedule_delayed_work(
vmci_doorbell_delayed_dispatch_cb, iter);
if (err != VMCI_SUCCESS) {
vmci_resource_release(&iter->resource);
goto out;
}
} else
iter->notify_cb(iter->client_data);
}
}
out:
vmci_release_lock_bh(&vmci_doorbell_it.lock);
}
/*
*------------------------------------------------------------------------------
*
* vmci_scan_notification_bitmap --
*
* Scans the notification bitmap, collects pending notifications, resets
* the bitmap and invokes appropriate callbacks.
*
* Results:
* None.
*
* Side effects:
* May schedule tasks, allocate memory and run callbacks.
*
*------------------------------------------------------------------------------
*/
void
vmci_scan_notification_bitmap(uint8_t *bitmap)
{
uint32_t idx;
ASSERT(bitmap);
for (idx = 0; idx < max_notify_idx; idx++) {
if (bitmap[idx] & 0x1) {
bitmap[idx] &= ~1;
vmci_doorbell_fire_entries(idx);
}
}
}