Attachment #212556 for bug #236922

Lines 3478-3483 Link Here

(-)sys/conf/files (+3 lines)
3478	dev/virtio/virtio_bus_if.m optional virtio	3478	dev/virtio/virtio_bus_if.m optional virtio
3479	dev/virtio/virtio_if.m optional virtio	3479	dev/virtio/virtio_if.m optional virtio
3480	dev/virtio/pci/virtio_pci.c optional virtio_pci	3480	dev/virtio/pci/virtio_pci.c optional virtio_pci
		3481	dev/virtio/pci/virtio_pci_if.m optional virtio_pci
		3482	dev/virtio/pci/virtio_pci_legacy.c optional virtio_pci
		3483	dev/virtio/pci/virtio_pci_modern.c optional virtio_pci
3481	dev/virtio/mmio/virtio_mmio.c optional virtio_mmio fdt	3484	dev/virtio/mmio/virtio_mmio.c optional virtio_mmio fdt
3482	dev/virtio/mmio/virtio_mmio_if.m optional virtio_mmio fdt	3485	dev/virtio/mmio/virtio_mmio_if.m optional virtio_mmio fdt
3483	dev/virtio/network/if_vtnet.c optional vtnet	3486	dev/virtio/network/if_vtnet.c optional vtnet

Lines 27-32 Link Here

(-)sys/modules/virtio/pci/Makefile (+2 lines)
27	KMOD= virtio_pci	27	KMOD= virtio_pci
28	SRCS= virtio_pci.c	28	SRCS= virtio_pci.c
		29	SRCS+= virtio_pci_legacy.c virtio_pci_modern.c
		30	SRCS+= virtio_pci_if.c virtio_pci_if.h
29	SRCS+= virtio_bus_if.h virtio_if.h	31	SRCS+= virtio_bus_if.h virtio_if.h
30	SRCS+= bus_if.h device_if.h pci_if.h	32	SRCS+= bus_if.h device_if.h pci_if.h
31		33




static struct virtio_feature_desc vtballoon_feature_desc[] = {
	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
	{ VIRTIO_BALLOON_F_DEFLATE_ON_OOM,	"DeflateOnOOM"	},

	{ 0, NULL }
};

static int	vtballoon_detach(device_t);
static int	vtballoon_config_change(device_t);

static void	vtballoon_negotiate_features(struct vtballoon_softc *);
static int	vtballoon_setup_features(struct vtballoon_softc *);
static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);

static void	vtballoon_vq_intr(void *);


static int	vtballoon_sleep(struct vtballoon_softc *);
static void	vtballoon_thread(void *);
static void	vtballoon_add_sysctl(struct vtballoon_softc *);

#define vtballoon_modern(_sc) \
    (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)

/* Features desired/implemented by this driver. */
#define VTBALLOON_FEATURES		0

/* Timeout between retries when the balloon needs inflating. */
#define VTBALLOON_LOWMEM_TIMEOUT	hz

};
static devclass_t vtballoon_devclass;

DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
    vtballoon_devclass, 0, 0);
DRIVER_MODULE(virtio_balloon, vtpcim, vtballoon_driver,
    vtballoon_devclass, 0, 0);
MODULE_VERSION(virtio_balloon, 1);
MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);



	sc = device_get_softc(dev);
	sc->vtballoon_dev = dev;
	virtio_set_feature_desc(dev, vtballoon_feature_desc);

	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
	TAILQ_INIT(&sc->vtballoon_pages);

	vtballoon_add_sysctl(sc);

	virtio_set_feature_desc(dev, vtballoon_feature_desc);
	vtballoon_negotiate_features(sc);
		device_printf(dev, "cannot setup features\n");
		goto fail;
	}

	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);

	return (1);
}

static void
vtballoon_negotiate_features(struct vtballoon_softc *sc)
{
	device_t dev;
	uint64_t features;

	dev = sc->vtballoon_dev;
	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
	sc->vtballoon_features = features;
	sc->vtballoon_features = virtio_negotiate_features(dev, features);
	return (virtio_finalize_features(dev));
}

static int
vtballoon_setup_features(struct vtballoon_softc *sc)
{
	int error;

	error = vtballoon_negotiate_features(sc);
	if (error)
		return (error);

	return (0);
}

static int
vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
{
	device_t dev;

{
	vm_page_t m;

	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
	    VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP);
	if (m != NULL)
		sc->vtballoon_current_npages++;


	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
	    offsetof(struct virtio_balloon_config, num_pages));

	return (le32toh(desired));
		return (desired);
	else
		return (le32toh(desired));
}

static void
vtballoon_update_size(struct vtballoon_softc *sc)
{
	uint32_t npages;

	npages = sc->vtballoon_current_npages;
	if (!vtballoon_modern(sc))
		npages = htole32(npages);

	virtio_write_dev_config_4(sc->vtballoon_dev,
	    offsetof(struct virtio_balloon_config, actual),
	    htole32(sc->vtballoon_current_npages));
}

static int

}

static void
vtballoon_add_sysctl(struct vtballoon_softc *sc)
{
	device_t dev;
	struct sysctl_ctx_list *ctx;




/* Feature bits. */
#define VIRTIO_BALLOON_F_MUST_TELL_HOST	0x1 /* Tell before reclaiming pages */
#define VIRTIO_BALLOON_F_STATS_VQ	0x2 /* Memory stats virtqueue */
#define VIRTIO_BALLOON_F_DEFLATE_ON_OOM	0x4 /* Deflate balloon on OOM */

/* Size of a PFN in the balloon interface. */
#define VIRTIO_BALLOON_PFN_SHIFT 12

#define VIRTIO_BALLOON_S_MINFLT   3   /* Number of minor faults */
#define VIRTIO_BALLOON_S_MEMFREE  4   /* Total amount of free memory */
#define VIRTIO_BALLOON_S_MEMTOT   5   /* Total amount of memory */
#define VIRTIO_BALLOON_S_NR       6
#define VIRTIO_BALLOON_S_CACHES   7   /* Disk caches */
#define VIRTIO_BALLOON_S_NR       8

/*
 * Memory statistics structure.
 * Driver fills an array of these structures and passes to device.
 *
 * NOTE: fields are laid out in a way that would make compiler add padding
 * between and after fields, so we have to use compiler-specific attributes to
 * pack it, to disable this padding. This also often causes compiler to
 * generate suboptimal code.
 *
 * We maintain this statistics structure format for backwards compatibility,
 * but don't follow this example.
 *
 * If implementing a similar structure, do something like the below instead:
 *     struct virtio_balloon_stat {
 *         __virtio16 tag;
 *         __u8 reserved[6];
 *         __virtio64 val;
 *     };
 *
 * In other words, add explicit reserved fields to align field and
 * structure boundaries at field size, avoiding compiler padding
 * without the packed attribute.
 */
struct virtio_balloon_stat {
	uint16_t tag;
	uint64_t val;




	uint64_t		 vtblk_features;
	uint32_t		 vtblk_flags;
#define VTBLK_FLAG_INDIRECT	0x0001
#define VTBLK_FLAG_READONLY	0x0002
#define VTBLK_FLAG_DETACH	0x0004
#define VTBLK_FLAG_SUSPEND	0x0008
#define VTBLK_FLAG_BARRIER	0x0010
#define VTBLK_FLAG_WC_CONFIG	0x0020

	struct virtqueue	*vtblk_vq;
	struct sglist		*vtblk_sglist;

	{ VIRTIO_BLK_F_RO,		"ReadOnly"	},
	{ VIRTIO_BLK_F_BLK_SIZE,	"BlockSize"	},
	{ VIRTIO_BLK_F_SCSI,		"SCSICmds"	},
	{ VIRTIO_BLK_F_WCE,		"WriteCache"	},
	{ VIRTIO_BLK_F_TOPOLOGY,	"Topology"	},
	{ VIRTIO_BLK_F_CONFIG_WCE,	"ConfigWCE"	},
	{ VIRTIO_BLK_F_MQ,		"Multiqueue"	},

	{ 0, NULL }
};

static int	vtblk_dump(void *, void *, vm_offset_t, off_t, size_t);
static void	vtblk_strategy(struct bio *);

static void	vtblk_negotiate_features(struct vtblk_softc *);
static void	vtblk_setup_features(struct vtblk_softc *);
static int	vtblk_maximum_segments(struct vtblk_softc *,
		    struct virtio_blk_config *);
static int	vtblk_alloc_virtqueue(struct vtblk_softc *);

static void	vtblk_setup_sysctl(struct vtblk_softc *);
static int	vtblk_tunable_int(struct vtblk_softc *, const char *, int);

#define vtblk_modern(_sc) (((_sc)->vtblk_features & VIRTIO_F_VERSION_1) != 0)
#define vtblk_htog16(_sc, _val)	virtio_htog16(vtblk_modern(_sc), _val)
#define vtblk_htog32(_sc, _val)	virtio_htog32(vtblk_modern(_sc), _val)
#define vtblk_htog64(_sc, _val)	virtio_htog64(vtblk_modern(_sc), _val)
#define vtblk_gtoh16(_sc, _val)	virtio_gtoh16(vtblk_modern(_sc), _val)
#define vtblk_gtoh32(_sc, _val)	virtio_gtoh32(vtblk_modern(_sc), _val)
#define vtblk_gtoh64(_sc, _val)	virtio_gtoh64(vtblk_modern(_sc), _val)

/* Tunables. */
static int vtblk_no_ident = 0;
TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);

TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);

/* Features desired/implemented by this driver. */
#define VTBLK_FEATURES \
    (VIRTIO_BLK_F_BARRIER		| \
     VIRTIO_BLK_F_SIZE_MAX		| \
     VIRTIO_BLK_F_SEG_MAX		| \
     VIRTIO_BLK_F_GEOMETRY		| \
     VIRTIO_BLK_F_RO			| \
     VIRTIO_BLK_F_BLK_SIZE		| \
     VIRTIO_BLK_F_WCE			| \
     VIRTIO_BLK_F_TOPOLOGY		| \
     VIRTIO_BLK_F_CONFIG_WCE		| \
     VIRTIO_RING_F_INDIRECT_DESC)

#define VTBLK_MODERN_FEATURES 	(VTBLK_COMMON_FEATURES)
#define VTBLK_LEGACY_FEATURES	(VIRTIO_BLK_F_BARRIER | VTBLK_COMMON_FEATURES)

#define VTBLK_MTX(_sc)		&(_sc)->vtblk_mtx
#define VTBLK_LOCK_INIT(_sc, _name) \
				mtx_init(VTBLK_MTX((_sc)), (_name), \


DRIVER_MODULE(virtio_blk, virtio_mmio, vtblk_driver, vtblk_devclass,
    vtblk_modevent, 0);
DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
    vtblk_modevent, 0);
DRIVER_MODULE(virtio_blk, vtpcim, vtblk_driver, vtblk_devclass,
    vtblk_modevent, 0);
MODULE_VERSION(virtio_blk, 1);
MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);


	struct virtio_blk_config blkcfg;
	int error;

	sc = device_get_softc(dev);
	sc->vtblk_dev = dev;
	virtio_set_feature_desc(dev, vtblk_feature_desc);

	sc = device_get_softc(dev);
	sc->vtblk_dev = dev;
	VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
	bioq_init(&sc->vtblk_bioq);
	TAILQ_INIT(&sc->vtblk_dump_queue);

	TAILQ_INIT(&sc->vtblk_req_ready);

	vtblk_setup_sysctl(sc);
	vtblk_setup_features(sc);

	error = vtblk_setup_features(sc);
	if (error) {
		device_printf(dev, "cannot setup features\n");
		goto fail;
	}

	vtblk_read_config(sc, &blkcfg);

	/*

		return;
	}

	/*
	 * Fail any write if RO. Unfortunately, there does not seem to
	 * be a better way to report our readonly'ness to GEOM above.
	 */
	if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
	    (bp->bio_cmd == BIO_WRITE || bp->bio_cmd == BIO_FLUSH)) {
		vtblk_bio_done(sc, bp, EROFS);
		return;
	}

	VTBLK_LOCK(sc);

	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {

	VTBLK_UNLOCK(sc);
}

static void
vtblk_negotiate_features(struct vtblk_softc *sc)
{
	device_t dev;
	uint64_t features;

	dev = sc->vtblk_dev;
	features = VTBLK_FEATURES;
	    VTBLK_LEGACY_FEATURES;

	sc->vtblk_features = virtio_negotiate_features(dev, features);
	return (virtio_finalize_features(dev));
}

static void
vtblk_setup_features(struct vtblk_softc *sc)
{
	device_t dev;

	
	dev = sc->vtblk_dev;

	vtblk_negotiate_features(sc);
	if (error)
		return (error);

	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		sc->vtblk_flags |= VTBLK_FLAG_INDIRECT;
	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
		sc->vtblk_flags |= VTBLK_FLAG_READONLY;
	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
		sc->vtblk_flags |= VTBLK_FLAG_BARRIER;
	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
		sc->vtblk_flags |= VTBLK_FLAG_WC_CONFIG;
}

static int

	dp->d_name = VTBLK_DISK_NAME;
	dp->d_unit = device_get_unit(dev);
	dp->d_drv1 = sc;
	dp->d_flags = DISKFLAG_CANFLUSHCACHE | DISKFLAG_UNMAPPED_BIO |
	    DISKFLAG_DIRECT_COMPLETION;
	dp->d_hba_vendor = virtio_get_vendor(dev);
	dp->d_hba_device = virtio_get_device(dev);
	dp->d_hba_subvendor = virtio_get_subvendor(dev);
	dp->d_hba_subdevice = virtio_get_subdevice(dev);

	if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0)
		dp->d_flags |= DISKFLAG_WRITE_PROTECT;
	else {
		if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH))
			dp->d_flags |= DISKFLAG_CANFLUSHCACHE;
		dp->d_dump = vtblk_dump;
	}

	/* Capacity is always in 512-byte units. */
	dp->d_mediasize = blkcfg->capacity * 512;

	bp = bioq_takefirst(bioq);
	req->vbr_bp = bp;
	req->vbr_ack = -1;
	req->vbr_hdr.ioprio = 1;

	switch (bp->bio_cmd) {
	case BIO_FLUSH:
		req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
		req->vbr_hdr.sector = 0;
		break;
	case BIO_READ:
		req->vbr_hdr.type = VIRTIO_BLK_T_IN;
		req->vbr_hdr.sector = bp->bio_offset / 512;
		break;
	case BIO_WRITE:
		req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
		req->vbr_hdr.sector = bp->bio_offset / 512;
		break;
	default:
		panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
	}

	if (bp->bio_flags & BIO_ORDERED)
		req->vbr_hdr.type |= VIRTIO_BLK_T_BARRIER;

	return (req);
}

			if (!virtqueue_empty(vq))
				return (EBUSY);
			ordered = 1;
			req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER;
				~VIRTIO_BLK_T_BARRIER);
		}
	}


static void
vtblk_drain(struct vtblk_softc *sc)
{
	struct bio_queue queue;
	struct bio_queue_head *bioq;
	struct vtblk_request *req;
	struct bio *bp;

	bioq = &sc->vtblk_bioq;
	TAILQ_INIT(&queue);

	if (sc->vtblk_vq != NULL) {
		struct bio_queue queue;

		TAILQ_INIT(&queue);
		vtblk_queue_completed(sc, &queue);
		vtblk_done_completed(sc, &queue);


	/* Read the configuration if the feature was negotiated. */
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg);
	    geometry.cylinders, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
	    geometry.heads, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
	    geometry.sectors, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
	    topology.alignment_offset, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
	    topology.min_io_size, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
	    topology.opt_io_size, blkcfg);
	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, wce, blkcfg);
}

#undef VTBLK_GET_CONFIG

		return;

	req->vbr_ack = -1;
	req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID;
	req->vbr_hdr.ioprio = 1;
	req->vbr_hdr.sector = 0;

	req->vbr_bp = &buf;


	req = &sc->vtblk_dump_request;
	req->vbr_ack = -1;
	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
	req->vbr_hdr.ioprio = 1;
	req->vbr_hdr.sector = offset / 512;

	req->vbr_bp = &buf;
	g_reset_bio(&buf);


	req = &sc->vtblk_dump_request;
	req->vbr_ack = -1;
	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
	req->vbr_hdr.ioprio = 1;
	req->vbr_hdr.sector = 0;

	req->vbr_bp = &buf;


	/* Set either writeback (1) or writethrough (0) mode. */
	virtio_write_dev_config_1(sc->vtblk_dev,
	    offsetof(struct virtio_blk_config, writeback), wc);
}

static int

{
	int wc;

	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
		wc = vtblk_tunable_int(sc, "writecache_mode",
		    vtblk_writecache_mode);
		if (wc >= 0 && wc < VTBLK_CACHE_MAX)
			vtblk_set_write_cache(sc, wc);
		else
			wc = blkcfg->writeback;
	} else
		wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);

	return (wc);
}

	error = sysctl_handle_int(oidp, &wc, 0, req);
	if (error || req->newptr == NULL)
		return (error);
	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
		return (EPERM);
	if (wc < 0 || wc >= VTBLK_CACHE_MAX)
		return (EINVAL);




#define _VIRTIO_BLK_H

/* Feature bits */
#define VIRTIO_BLK_F_BARRIER	0x0001	/* Does host support barriers? */
#define VIRTIO_BLK_F_SIZE_MAX	0x0002	/* Indicates maximum segment size */
#define VIRTIO_BLK_F_SEG_MAX	0x0004	/* Indicates maximum # of segments */
#define VIRTIO_BLK_F_GEOMETRY	0x0010	/* Legacy geometry available  */
#define VIRTIO_BLK_F_RO		0x0020	/* Disk is read-only */
#define VIRTIO_BLK_F_BLK_SIZE	0x0040	/* Block size of disk is available*/
#define VIRTIO_BLK_F_SCSI	0x0080	/* Supports scsi command passthru */
#define VIRTIO_BLK_F_WCE	0x0200	/* Writeback mode enabled after reset */
#define VIRTIO_BLK_F_TOPOLOGY	0x0400	/* Topology information is available */
#define VIRTIO_BLK_F_CONFIG_WCE 0x0800	/* Writeback mode available in config */
#define VIRTIO_BLK_F_MQ 	0x1000 	/* Support more than one vq */

/* Legacy feature bits */
#define VIRTIO_BLK_F_BARRIER	0x0001	/* Does host support barriers? */
#define VIRTIO_BLK_F_SCSI	0x0080	/* Supports scsi command passthru */

/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
#define VIRTIO_BLK_ID_BYTES	20	/* ID string length */

struct virtio_blk_config {


	/* Topology of the device (if VIRTIO_BLK_F_TOPOLOGY) */
	struct virtio_blk_topology {
		/* exponent for physical block per logical block. */
		uint8_t physical_block_exp;
		/* alignment offset in logical blocks. */
		uint8_t alignment_offset;
		/* minimum I/O size without performance penalty in logical
		 * blocks. */
		uint16_t min_io_size;
		/* optimal sustained I/O size in logical blocks. */
		uint32_t opt_io_size;
	} topology;

	/* Writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
	uint8_t writeback;
	uint8_t unused;

	/* Number of vqs, only available when VIRTIO_BLK_F_MQ is set */
	uint16_t num_queues;
} __packed;

/*

/* ID string length */
#define VIRTIO_BLK_ID_BYTES	20

/* This is the first element of the read scatter-gather list. */
 * This comes first in the read scatter-gather list.
 * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated,
 * this is the first element of the read scatter-gather list.
 */
struct virtio_blk_outhdr {
	/* VIRTIO_BLK_T* */
	uint32_t type;




static int	 vtcon_detach(device_t);
static int	 vtcon_config_change(device_t);

static void	 vtcon_setup_features(struct vtcon_softc *);
static void	 vtcon_negotiate_features(struct vtcon_softc *);
static int	 vtcon_alloc_scports(struct vtcon_softc *);
static int	 vtcon_alloc_virtqueues(struct vtcon_softc *);
static void	 vtcon_read_config(struct vtcon_softc *,

static void	 vtcon_enable_interrupts(struct vtcon_softc *);
static void	 vtcon_disable_interrupts(struct vtcon_softc *);

#define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0)
#define vtcon_htog16(_sc, _val)	virtio_htog16(vtcon_modern(_sc), _val)
#define vtcon_htog32(_sc, _val)	virtio_htog32(vtcon_modern(_sc), _val)
#define vtcon_htog64(_sc, _val)	virtio_htog64(vtcon_modern(_sc), _val)
#define vtcon_gtoh16(_sc, _val)	virtio_gtoh16(vtcon_modern(_sc), _val)
#define vtcon_gtoh32(_sc, _val)	virtio_gtoh32(vtcon_modern(_sc), _val)
#define vtcon_gtoh64(_sc, _val)	virtio_gtoh64(vtcon_modern(_sc), _val)

static int	 vtcon_pending_free;

static struct ttydevsw vtcon_tty_class = {

};
static devclass_t vtcon_devclass;

DRIVER_MODULE(virtio_console, virtio_pci, vtcon_driver, vtcon_devclass,
    vtcon_modevent, 0);
DRIVER_MODULE(virtio_console, vtpcim, vtcon_driver, vtcon_devclass,
    vtcon_modevent, 0);
MODULE_VERSION(virtio_console, 1);
MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);



	sc = device_get_softc(dev);
	sc->vtcon_dev = dev;
	virtio_set_feature_desc(dev, vtcon_feature_desc);

	mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);
	mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);

	virtio_set_feature_desc(dev, vtcon_feature_desc);
	vtcon_setup_features(sc);
		device_printf(dev, "cannot setup features\n");
		goto fail;
	}

	vtcon_read_config(sc, &concfg);
	vtcon_determine_max_ports(sc, &concfg);

	return (0);
}

static void
vtcon_negotiate_features(struct vtcon_softc *sc)
{
	device_t dev;

	features = VTCON_FEATURES;

	sc->vtcon_features = virtio_negotiate_features(dev, features);
	return (virtio_finalize_features(dev));
}

static void
vtcon_setup_features(struct vtcon_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtcon_dev;

	vtcon_negotiate_features(sc);
	if (error)
		return (error);

	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))
		sc->vtcon_flags |= VTCON_FLAG_SIZE;
	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))
		sc->vtcon_flags |= VTCON_FLAG_MULTIPORT;

	return (0);
}

#define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg)			\

    struct virtio_console_control *control, void *data, size_t data_len)
{
	device_t dev;
	int id;
	uint16_t event;

	dev = sc->vtcon_dev;
	id = control->id;
	event = vtcon_htog16(sc, control->event);

	if (id < 0 || id >= sc->vtcon_max_ports) {
		device_printf(dev, "%s: invalid port ID %d\n", __func__, id);
		    __func__, event, id);
		return;
	}

	switch (control->event) {
	case VIRTIO_CONSOLE_PORT_ADD:
		vtcon_ctrl_port_add_event(sc, id);
		break;

	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)
		return;

	control.id = portid;
	control.event = event;
	control.value = value;

	vtcon_ctrl_poll(sc, &control);
}

Lines 426-435 Link Here

(-)sys/dev/virtio.ori/mmio/virtio_mmio.c (-4 lines)
426	case VIRTIO_IVAR_VENDOR:	426	case VIRTIO_IVAR_VENDOR:
427	*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);	427	*result = vtmmio_read_config_4(sc, VIRTIO_MMIO_VENDOR_ID);
428	break;	428	break;
429	case VIRTIO_IVAR_SUBVENDOR:
430	case VIRTIO_IVAR_MODERN:
431	*result = 0;
432	break;
433	default:	429	default:
434	return (ENOENT);	430	return (ENOENT);
435	}	431	}




#include <netinet6/ip6_var.h>
#include <netinet/udp.h>
#include <netinet/tcp.h>
#include <netinet/tcp_lro.h>
#include <netinet/netdump/netdump.h>

#include <machine/bus.h>

#include "opt_inet.h"
#include "opt_inet6.h"

#if defined(INET) || defined(INET6)
#include <machine/in_cksum.h>
#endif

static int	vtnet_modevent(module_t, int, void *);

static int	vtnet_probe(device_t);

static int	vtnet_attach_completed(device_t);
static int	vtnet_config_change(device_t);

static void	vtnet_negotiate_features(struct vtnet_softc *);
static void	vtnet_setup_features(struct vtnet_softc *);
static int	vtnet_init_rxq(struct vtnet_softc *, int);
static int	vtnet_init_txq(struct vtnet_softc *, int);
static int	vtnet_alloc_rxtx_queues(struct vtnet_softc *);

static int	vtnet_alloc_rx_filters(struct vtnet_softc *);
static void	vtnet_free_rx_filters(struct vtnet_softc *);
static int	vtnet_alloc_virtqueues(struct vtnet_softc *);
static int	vtnet_alloc_interface(struct vtnet_softc *);
static int	vtnet_setup_interface(struct vtnet_softc *);
static int	vtnet_change_mtu(struct vtnet_softc *, int);
static int	vtnet_ioctl_ifflags(struct vtnet_softc *);
static int	vtnet_ioctl_multi(struct vtnet_softc *);
static int	vtnet_ioctl_ifcap(struct vtnet_softc *, struct ifreq *);
static int	vtnet_ioctl(struct ifnet *, u_long, caddr_t);
static uint64_t	vtnet_get_counter(struct ifnet *, ift_counter);


static void	vtnet_rxq_free_mbufs(struct vtnet_rxq *);
static struct mbuf *
		vtnet_rx_alloc_buf(struct vtnet_softc *, int , struct mbuf **);
static int	vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *,
		    struct mbuf *, int);
static int	vtnet_rxq_replace_buf(struct vtnet_rxq *, struct mbuf *, int);
static int	vtnet_rxq_enqueue_buf(struct vtnet_rxq *, struct mbuf *);
static int	vtnet_rxq_new_buf(struct vtnet_rxq *);
static int	vtnet_rxq_csum_needs_csum(struct vtnet_rxq *, struct mbuf *,
		     uint16_t, int, struct virtio_net_hdr *);
static int	vtnet_rxq_csum_data_valid(struct vtnet_rxq *, struct mbuf *,
		     uint16_t, int, struct virtio_net_hdr *);
static int	vtnet_rxq_csum(struct vtnet_rxq *, struct mbuf *,
		     struct virtio_net_hdr *);
static void	vtnet_rxq_discard_merged_bufs(struct vtnet_rxq *, int);

static void	vtnet_rx_vq_intr(void *);
static void	vtnet_rxq_tq_intr(void *, int);

static int	vtnet_txq_intr_threshold(struct vtnet_txq *);
static int	vtnet_txq_below_threshold(struct vtnet_txq *);
static int	vtnet_txq_notify(struct vtnet_txq *);
static void	vtnet_txq_free_mbufs(struct vtnet_txq *);

static int	vtnet_init_tx_queues(struct vtnet_softc *);
static int	vtnet_init_rxtx_queues(struct vtnet_softc *);
static void	vtnet_set_active_vq_pairs(struct vtnet_softc *);
static void	vtnet_update_rx_offloads(struct vtnet_softc *);
static int	vtnet_reinit(struct vtnet_softc *);
static void	vtnet_init_locked(struct vtnet_softc *);
static void	vtnet_init(void *);

static void	vtnet_exec_ctrl_cmd(struct vtnet_softc *, void *,
		    struct sglist *, int, int);
static int	vtnet_ctrl_mac_cmd(struct vtnet_softc *, uint8_t *);
static int	vtnet_ctrl_guest_offloads(struct vtnet_softc *, uint64_t);
static int	vtnet_ctrl_mq_cmd(struct vtnet_softc *, uint16_t);
static int	vtnet_ctrl_rx_cmd(struct vtnet_softc *, int, int);
static int	vtnet_set_promisc(struct vtnet_softc *, int);
static int	vtnet_set_allmulti(struct vtnet_softc *, int);
static void	vtnet_attach_disable_promisc(struct vtnet_softc *);
static void	vtnet_rx_filter(struct vtnet_softc *);
static void	vtnet_rx_filter_mac(struct vtnet_softc *);
static int	vtnet_exec_vlan_filter(struct vtnet_softc *, int, uint16_t);

static void	vtnet_register_vlan(void *, struct ifnet *, uint16_t);
static void	vtnet_unregister_vlan(void *, struct ifnet *, uint16_t);

static void	vtnet_update_speed_duplex(struct vtnet_softc *);
static int	vtnet_is_link_up(struct vtnet_softc *);
static void	vtnet_update_link_status(struct vtnet_softc *);
static int	vtnet_ifmedia_upd(struct ifnet *);
static void	vtnet_ifmedia_sts(struct ifnet *, struct ifmediareq *);
static void	vtnet_get_hwaddr(struct vtnet_softc *);
static void	vtnet_set_hwaddr(struct vtnet_softc *);
static void	vtnet_attached_set_macaddr(struct vtnet_softc *);
static void	vtnet_vlan_tag_remove(struct mbuf *);
static void	vtnet_set_rx_process_limit(struct vtnet_softc *);
static void	vtnet_set_tx_intr_threshold(struct vtnet_softc *);

static void	vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *,
		    struct sysctl_oid_list *, struct vtnet_rxq *);
static void	vtnet_setup_txq_sysctl(struct sysctl_ctx_list *,
		    struct sysctl_oid_list *, struct vtnet_txq *);
static void	vtnet_setup_queue_sysctl(struct vtnet_softc *);
static void	vtnet_load_tunables(struct vtnet_softc *);
static void	vtnet_setup_sysctl(struct vtnet_softc *);

static int	vtnet_rxq_enable_intr(struct vtnet_rxq *);


NETDUMP_DEFINE(vtnet);

/* Tunables. */
static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD, 0, "VNET driver parameters");
#define vtnet_htog64(_sc, _val)	virtio_htog64(vtnet_modern(_sc), _val)
#define vtnet_gtoh16(_sc, _val)	virtio_gtoh16(vtnet_modern(_sc), _val)
#define vtnet_gtoh32(_sc, _val)	virtio_gtoh32(vtnet_modern(_sc), _val)
#define vtnet_gtoh64(_sc, _val)	virtio_gtoh64(vtnet_modern(_sc), _val)

static SYSCTL_NODE(_hw, OID_AUTO, vtnet, CTLFLAG_RD, 0, "VirtIO Net driver");

static int vtnet_csum_disable = 0;
TUNABLE_INT("hw.vtnet.csum_disable", &vtnet_csum_disable);
SYSCTL_INT(_hw_vtnet, OID_AUTO, csum_disable, CTLFLAG_RDTUN,
    &vtnet_csum_disable, 0, "Disables receive and send checksum offload");

static int vtnet_fixup_needs_csum = 0;
SYSCTL_INT(_hw_vtnet, OID_AUTO, fixup_needs_csum, CTLFLAG_RDTUN,
    &vtnet_fixup_needs_csum, 0,
    "Calculate valid checksum for NEEDS_CSUM packets");

static int vtnet_tso_disable = 0;
TUNABLE_INT("hw.vtnet.tso_disable", &vtnet_tso_disable);
SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_disable, CTLFLAG_RDTUN, &vtnet_tso_disable,
    0, "Disables TCP Segmentation Offload");
static int vtnet_lro_disable = 0;
TUNABLE_INT("hw.vtnet.lro_disable", &vtnet_lro_disable);
SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_disable, CTLFLAG_RDTUN, &vtnet_lro_disable,
    0, "Disables TCP Large Receive Offload");
static int vtnet_mq_disable = 0;
TUNABLE_INT("hw.vtnet.mq_disable", &vtnet_mq_disable);
SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_disable, CTLFLAG_RDTUN, &vtnet_mq_disable,
    0, "Disables Multi Queue support");
static int vtnet_mq_max_pairs = VTNET_MAX_QUEUE_PAIRS;
TUNABLE_INT("hw.vtnet.mq_max_pairs", &vtnet_mq_max_pairs);
SYSCTL_INT(_hw_vtnet, OID_AUTO, mq_max_pairs, CTLFLAG_RDTUN,
    &vtnet_mq_max_pairs, 0, "Sets the maximum number of Multi Queue pairs");
static int vtnet_rx_process_limit = 512;
TUNABLE_INT("hw.vtnet.rx_process_limit", &vtnet_rx_process_limit);
SYSCTL_INT(_hw_vtnet, OID_AUTO, tso_maxlen, CTLFLAG_RDTUN,
    &vtnet_tso_maxlen, 0, "TSO burst limit");

static int vtnet_rx_process_limit = 1024;
SYSCTL_INT(_hw_vtnet, OID_AUTO, rx_process_limit, CTLFLAG_RDTUN,
    &vtnet_rx_process_limit, 0,
    "Limits the number RX segments processed in a single pass");

static int vtnet_lro_entry_count = 128;
SYSCTL_INT(_hw_vtnet, OID_AUTO, lro_entry_count, CTLFLAG_RDTUN,
    &vtnet_lro_entry_count, 0, "Software LRO entry count");

/* Enable sorted LRO, and the depth of the mbuf queue. */
static int vtnet_lro_mbufq_depth = 0;
SYSCTL_UINT(_hw_vtnet, OID_AUTO, lro_mbufq_depth, CTLFLAG_RDTUN,
    &vtnet_lro_mbufq_depth, 0, "Depth of software LRO mbuf queue");

static uma_zone_t vtnet_tx_header_zone;

static struct virtio_feature_desc vtnet_feature_desc[] = {
	{ VIRTIO_NET_F_CSUM,		"TxChecksum"	},
	{ VIRTIO_NET_F_GUEST_CSUM,	"RxChecksum"	},
	{ VIRTIO_NET_F_MAC,		"MacAddress"	},
	{ VIRTIO_NET_F_GSO,		"TxAllGSO"	},
	{ VIRTIO_NET_F_GUEST_TSO4,	"RxTSOv4"	},
	{ VIRTIO_NET_F_GUEST_TSO6,	"RxTSOv6"	},
	{ VIRTIO_NET_F_GUEST_ECN,	"RxECN"		},
	{ VIRTIO_NET_F_GUEST_UFO,	"RxUFO"		},
	{ VIRTIO_NET_F_HOST_TSO4,	"TxTSOv4"	},
	{ VIRTIO_NET_F_HOST_TSO6,	"TxTSOv6"	},
	{ VIRTIO_NET_F_HOST_ECN,	"TxTSOECN"	},
	{ VIRTIO_NET_F_HOST_UFO,	"TxUFO"		},
	{ VIRTIO_NET_F_MRG_RXBUF,	"MrgRxBuf"	},
	{ VIRTIO_NET_F_STATUS,		"Status"	},
	{ VIRTIO_NET_F_CTRL_VQ,		"ControlVq"	},
	{ VIRTIO_NET_F_CTRL_RX,		"RxMode"	},
	{ VIRTIO_NET_F_CTRL_VLAN,	"VLanFilter"	},
	{ VIRTIO_NET_F_CTRL_RX_EXTRA,	"RxModeExtra"	},
	{ VIRTIO_NET_F_GUEST_ANNOUNCE,	"GuestAnnounce"	},
	{ VIRTIO_NET_F_MQ,		"Multiqueue"	},
	{ VIRTIO_NET_F_CTRL_MAC_ADDR,	"SetMacAddress"	},
	{ VIRTIO_NET_F_CTRL_MAC_ADDR,		"CtrlMacAddr"		},
	{ VIRTIO_NET_F_SPEED_DUPLEX,		"SpeedDuplex"		},

	{ 0, NULL }
};


#ifdef DEV_NETMAP
#include <dev/netmap/if_vtnet_netmap.h>
#endif /* DEV_NETMAP */

static driver_t vtnet_driver = {
	"vtnet",
	vtnet_methods,
	sizeof(struct vtnet_softc)
};
static devclass_t vtnet_devclass;

DRIVER_MODULE(vtnet, virtio_mmio, vtnet_driver, vtnet_devclass,
    vtnet_modevent, 0);
DRIVER_MODULE(vtnet, virtio_pci, vtnet_driver, vtnet_devclass,
    vtnet_modevent, 0);
MODULE_VERSION(vtnet, 1);
MODULE_DEPEND(vtnet, virtio, 1, 1, 1);
#ifdef DEV_NETMAP
MODULE_DEPEND(vtnet, netmap, 1, 1, 1);
#endif /* DEV_NETMAP */

static int
vtnet_modevent(module_t mod, int type, void *unused)

	if (virtio_get_device_type(dev) != VIRTIO_ID_NETWORK)
		return (ENXIO);

	device_set_desc(dev, "VirtIO Networking Adapter");

	return (BUS_PROBE_DEFAULT);
}


	sc = device_get_softc(dev);
	sc->vtnet_dev = dev;

	/* Register our feature descriptions. */
	virtio_set_feature_desc(dev, vtnet_feature_desc);

	VTNET_CORE_LOCK_INIT(sc);
	callout_init_mtx(&sc->vtnet_tick_ch, VTNET_CORE_MTX(sc), 0);
	vtnet_load_tunables(sc);

	error = vtnet_alloc_interface(sc);
	if (error) {
		device_printf(dev, "cannot allocate interface\n");
		goto fail;
	}

	vtnet_setup_sysctl(sc);
	vtnet_setup_features(sc);

	error = vtnet_setup_features(sc);
	if (error) {
		device_printf(dev, "cannot setup features\n");
		goto fail;
	}

	error = vtnet_alloc_rx_filters(sc);
	if (error) {
		device_printf(dev, "cannot allocate Rx filters\n");


	error = virtio_setup_intr(dev, INTR_TYPE_NET);
	if (error) {
		device_printf(dev, "cannot setup virtqueue interrupts\n");
		/* BMV: This will crash if during boot! */
		ether_ifdetach(sc->vtnet_ifp);
		goto fail;
	}

#ifdef DEV_NETMAP
	vtnet_netmap_attach(sc);
#endif /* DEV_NETMAP */

	vtnet_start_taskqueues(sc);

fail:


#ifdef DEV_NETMAP
	netmap_detach(ifp);
#endif /* DEV_NETMAP */

	vtnet_free_taskqueues(sc);


static int
vtnet_shutdown(device_t dev)
{

	/*
	 * Suspend already does all of what we need to
	 * do here; we just never expect to be resumed.

static int
vtnet_attach_completed(device_t dev)
{
	struct vtnet_softc *sc;

	vtnet_attach_disable_promisc(device_get_softc(dev));

	VTNET_CORE_LOCK(sc);
	vtnet_attached_set_macaddr(sc);
	VTNET_CORE_UNLOCK(sc);

	return (0);
}


	return (0);
}

static void
vtnet_negotiate_features(struct vtnet_softc *sc)
{
	device_t dev;
	uint64_t mask, features;
	int no_csum;

	dev = sc->vtnet_dev;
	mask = 0;
	    VTNET_LEGACY_FEATURES;

	/*
	 * TSO and LRO are only available when their corresponding checksum
	 * offload feature is also negotiated.
	 */
	if (vtnet_tunable_int(sc, "csum_disable", vtnet_csum_disable)) {
		mask |= VIRTIO_NET_F_CSUM | VIRTIO_NET_F_GUEST_CSUM;
		mask |= VTNET_TSO_FEATURES | VTNET_LRO_FEATURES;
	}
	if (vtnet_tunable_int(sc, "tso_disable", vtnet_tso_disable))
		mask |= VTNET_TSO_FEATURES;
	if (vtnet_tunable_int(sc, "lro_disable", vtnet_lro_disable))
		mask |= VTNET_LRO_FEATURES;
#ifndef VTNET_LEGACY_TX
	if (vtnet_tunable_int(sc, "mq_disable", vtnet_mq_disable))
		mask |= VIRTIO_NET_F_MQ;
#else
	mask |= VIRTIO_NET_F_MQ;
#endif

	features = VTNET_FEATURES & ~mask;
	sc->vtnet_features = virtio_negotiate_features(dev, features);

	if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) {
		uint16_t mtu;

		mtu = virtio_read_dev_config_2(dev,
		    offsetof(struct virtio_net_config, mtu));
		if (mtu < VTNET_MIN_MTU /* || mtu > VTNET_MAX_MTU */) {
			device_printf(dev, "Invalid MTU value: %d. "
			    "MTU feature disabled.\n", mtu);
			features &= ~VIRTIO_NET_F_MTU;
			negotiated_features =
			    virtio_negotiate_features(dev, features);
		}
	}

	if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) {
		uint16_t npairs;

		npairs = virtio_read_dev_config_2(dev,
		    offsetof(struct virtio_net_config, max_virtqueue_pairs));
		if (npairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
		    npairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX) {
			device_printf(dev, "Invalid max_virtqueue_pairs value: "
			    "%d. Multiqueue feature disabled.\n", npairs);
			features &= ~VIRTIO_NET_F_MQ;
			negotiated_features =
			    virtio_negotiate_features(dev, features);
		}
	}

	if (virtio_with_feature(dev, VTNET_LRO_FEATURES) &&
	    virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF) == 0) {
		/*

		 */
		if (!virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC)) {
			device_printf(dev,
			    "LRO disabled due to both mergeable buffers and "
			    "indirect descriptors not negotiated\n");

			features &= ~VTNET_LRO_FEATURES;
			sc->vtnet_features =
			    virtio_negotiate_features(dev, features);
		} else
			sc->vtnet_flags |= VTNET_FLAG_LRO_NOMRG;
	}

	sc->vtnet_features = negotiated_features;
	sc->vtnet_negotiated_features = negotiated_features;

	return (virtio_finalize_features(dev));
}

static void
vtnet_setup_features(struct vtnet_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtnet_dev;

	vtnet_negotiate_features(sc);
	if (error)
		return (error);

	if (virtio_with_feature(dev, VIRTIO_F_VERSION_1))
		sc->vtnet_flags |= VTNET_FLAG_MODERN;
	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		sc->vtnet_flags |= VTNET_FLAG_INDIRECT;
	if (virtio_with_feature(dev, VIRTIO_RING_F_EVENT_IDX))

		sc->vtnet_flags |= VTNET_FLAG_MAC;
	}

	if (virtio_with_feature(dev, VIRTIO_NET_F_MTU)) {
		sc->vtnet_max_mtu = virtio_read_dev_config_2(dev,
		    offsetof(struct virtio_net_config, mtu));
	} else
		sc->vtnet_max_mtu = VTNET_MAX_MTU;

	if (virtio_with_feature(dev, VIRTIO_NET_F_MRG_RXBUF)) {
		sc->vtnet_flags |= VTNET_FLAG_MRG_RXBUFS;
		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
	} else if (vtnet_modern(sc)) {
		/* This is identical to the mergeable header. */
		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr_v1);
	} else
		sc->vtnet_hdr_size = sizeof(struct virtio_net_hdr);

	if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
		sc->vtnet_rx_nsegs = VTNET_MRG_RX_SEGS;
	else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
		sc->vtnet_rx_nsegs = VTNET_MAX_RX_SEGS;
	else
		sc->vtnet_rx_nsegs = VTNET_MIN_RX_SEGS;

	/*
	 * Favor "hardware" LRO if negotiated, but support software LRO as
	 * a fallback; there is usually little benefit (or worse) with both.
	 */
	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) == 0 &&
	    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6) == 0)
		sc->vtnet_flags |= VTNET_FLAG_SW_LRO;

	if (virtio_with_feature(dev, VIRTIO_NET_F_GSO) ||
	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4) ||
	    virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
		sc->vtnet_tx_nsegs = VTNET_MAX_TX_SEGS;
	else
		sc->vtnet_tx_nsegs = VTNET_MIN_TX_SEGS;

	sc->vtnet_req_vq_pairs = 1;
	sc->vtnet_max_vq_pairs = 1;

	if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_VQ)) {
		sc->vtnet_flags |= VTNET_FLAG_CTRL_VQ;


			sc->vtnet_flags |= VTNET_FLAG_VLAN_FILTER;
		if (virtio_with_feature(dev, VIRTIO_NET_F_CTRL_MAC_ADDR))
			sc->vtnet_flags |= VTNET_FLAG_CTRL_MAC;

		if (virtio_with_feature(dev, VIRTIO_NET_F_MQ)) {
			sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev,
			    offsetof(struct virtio_net_config,
			    max_virtqueue_pairs));
		}
	}

	if (virtio_with_feature(dev, VIRTIO_NET_F_MQ) &&
	    sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
		sc->vtnet_max_vq_pairs = virtio_read_dev_config_2(dev,
		    offsetof(struct virtio_net_config, max_virtqueue_pairs));
	} else
		sc->vtnet_max_vq_pairs = 1;

	if (sc->vtnet_max_vq_pairs > 1) {
		/*
		 * Limit the maximum number of queue pairs to the lower of
		 * the number of CPUs and the configured maximum.
		 * The actual number of queues that get used may be less.
		 */
		int max;

		max = vtnet_tunable_int(sc, "mq_max_pairs", vtnet_mq_max_pairs);
		if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN) {
			if (max > mp_ncpus)
				max = mp_ncpus;
			if (max > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
				max = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX;
			if (max > 1) {
				sc->vtnet_requested_vq_pairs = max;
				sc->vtnet_flags |= VTNET_FLAG_MULTIQ;
			}
		}
	}

	return (0);
}

static int

	if (rxq->vtnrx_sg == NULL)
		return (ENOMEM);

#if defined(INET) || defined(INET6)
	if (vtnet_software_lro(sc)) {
		if (tcp_lro_init_args(&rxq->vtnrx_lro, sc->vtnet_ifp,
		    sc->vtnet_lro_entry_count, sc->vtnet_lro_mbufq_depth) != 0)
			return (ENOMEM);
	}
#endif

	TASK_INIT(&rxq->vtnrx_intrtask, 0, vtnet_rxq_tq_intr, rxq);
	rxq->vtnrx_tq = taskqueue_create(rxq->vtnrx_name, M_NOWAIT,
	    taskqueue_thread_enqueue, &rxq->vtnrx_tq);

			return (error);
	}

	vtnet_set_rx_process_limit(sc);
	vtnet_setup_queue_sysctl(sc);

	return (0);

	rxq->vtnrx_sc = NULL;
	rxq->vtnrx_id = -1;

#if defined(INET) || defined(INET6)
	tcp_lro_free(&rxq->vtnrx_lro);
#endif

	if (rxq->vtnrx_sg != NULL) {
		sglist_free(rxq->vtnrx_sg);
		rxq->vtnrx_sg = NULL;

	if (info == NULL)
		return (ENOMEM);

	for (i = 0, idx = 0; i < sc->vtnet_max_vq_pairs; i++, idx+=2) {
		rxq = &sc->vtnet_rxqs[i];
		VQ_ALLOC_INFO_INIT(&info[idx], sc->vtnet_rx_nsegs,
		    vtnet_rx_vq_intr, rxq, &rxq->vtnrx_vq,
		    "%s-%d rx", device_get_nameunit(dev), rxq->vtnrx_id);

		txq = &sc->vtnet_txqs[i];
		VQ_ALLOC_INFO_INIT(&info[idx+1], sc->vtnet_tx_nsegs,
		    vtnet_tx_vq_intr, txq, &txq->vtntx_vq,
		    "%s-%d tx", device_get_nameunit(dev), txq->vtntx_id);
	}

	/* These queues will not be used so allocate the minimum resources. */
	for (/**/; i < sc->vtnet_max_vq_pairs; i++, idx += 2) {
		rxq = &sc->vtnet_rxqs[i];
		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, rxq, &rxq->vtnrx_vq,
		    "%s-rx%d", device_get_nameunit(dev), rxq->vtnrx_id);

		txq = &sc->vtnet_txqs[i];
		VQ_ALLOC_INFO_INIT(&info[idx+1], 0, NULL, txq, &txq->vtntx_vq,
		    "%s-tx%d", device_get_nameunit(dev), txq->vtntx_id);
	}

	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ) {
		VQ_ALLOC_INFO_INIT(&info[idx], 0, NULL, NULL,
		    &sc->vtnet_ctrl_vq, "%s ctrl", device_get_nameunit(dev));
	}

	/*
	 * Enable interrupt binding if this is multiqueue. This only matters
	 * when per-vq MSIX is available.
	 */
	if (sc->vtnet_flags & VTNET_FLAG_MULTIQ)
		flags |= 0;

	error = virtio_alloc_virtqueues(dev, flags, nvqs, info);

}

static int
vtnet_setup_interface(struct vtnet_softc *sc)
{
	device_t dev;
	struct ifnet *ifp;

	dev = sc->vtnet_dev;

	ifp = sc->vtnet_ifp = if_alloc(IFT_ETHER);
	if (ifp == NULL) {
		device_printf(dev, "cannot allocate ifnet structure\n");
		return (ENOSPC);
	}

	sc->vtnet_ifp = ifp;
	ifp->if_softc = sc;
	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
	ifp->if_baudrate = IF_Gbps(10);	/* Approx. */
	ifp->if_softc = sc;
}

static int
vtnet_setup_interface(struct vtnet_softc *sc)
{
	device_t dev;
	struct ifnet *ifp;

	dev = sc->vtnet_dev;
	ifp = sc->vtnet_ifp;

	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
	ifp->if_baudrate = IF_Gbps(10);
	ifp->if_init = vtnet_init;
	ifp->if_ioctl = vtnet_ioctl;
	ifp->if_get_counter = vtnet_get_counter;

	IFQ_SET_READY(&ifp->if_snd);
#endif

	ifmedia_init(&sc->vtnet_media, IFM_IMASK, vtnet_ifmedia_upd,
	    vtnet_ifmedia_sts);
	ifmedia_add(&sc->vtnet_media, VTNET_MEDIATYPE, 0, NULL);
	ifmedia_set(&sc->vtnet_media, VTNET_MEDIATYPE);

	/* Read (or generate) the MAC address for the adapter. */
	vtnet_get_hwaddr(sc);

	ether_ifattach(ifp, sc->vtnet_hwaddr);

	if (virtio_with_feature(dev, VIRTIO_NET_F_STATUS))
		ifp->if_capabilities |= IFCAP_LINKSTATE;

	/* Tell the upper layer(s) we support long frames. */
	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
	ifp->if_capabilities |= IFCAP_JUMBO_MTU | IFCAP_VLAN_MTU;

	if (virtio_with_feature(dev, VIRTIO_NET_F_CSUM)) {
		int gso;

		ifp->if_capabilities |= IFCAP_TXCSUM | IFCAP_TXCSUM_IPV6;

		if (virtio_with_feature(dev, VIRTIO_NET_F_GSO)) {
			ifp->if_capabilities |= IFCAP_TSO4 | IFCAP_TSO6;
			ifp->if_capabilities |= IFCAP_TSO4;
		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
			ifp->if_capabilities |= IFCAP_TSO6;
		if (gso || virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
			sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
		} else {
			if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO4))
				ifp->if_capabilities |= IFCAP_TSO4;
			if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_TSO6))
				ifp->if_capabilities |= IFCAP_TSO6;
			if (virtio_with_feature(dev, VIRTIO_NET_F_HOST_ECN))
				sc->vtnet_flags |= VTNET_FLAG_TSO_ECN;
		}

		if (ifp->if_capabilities & IFCAP_TSO)
			int tso_maxlen;

			ifp->if_capabilities |= IFCAP_VLAN_HWTSO;

			tso_maxlen = vtnet_tunable_int(sc, "tso_maxlen",
			    vtnet_tso_maxlen);
			ifp->if_hw_tsomax = tso_maxlen -
			    (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
			ifp->if_hw_tsomaxsegcount = sc->vtnet_tx_nsegs - 1;
			ifp->if_hw_tsomaxsegsize = PAGE_SIZE;
		}
	}

	if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_CSUM)) {
		ifp->if_capabilities |= IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6;
#ifdef notyet
		/* BMV: Rx checksums not distinguished between IPv4 and IPv6. */
		ifp->if_capabilities |= IFCAP_RXCSUM_IPV6;
#endif

		if (virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO4) ||
		    virtio_with_feature(dev, VIRTIO_NET_F_GUEST_TSO6))
			ifp->if_capabilities |= IFCAP_LRO;

		/* Support either "hardware" or software LRO. */
		ifp->if_capabilities |= IFCAP_LRO;
	}

	if (ifp->if_capabilities & IFCAP_HWCSUM) {
		/*
		 * VirtIO does not support VLAN tagging, but we can fake
		 * it by inserting and removing the 802.1Q header during

		    IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_HWCSUM;
	}

	ifp->if_capenable = ifp->if_capabilities;
		ifp->if_capabilities |= IFCAP_JUMBO_MTU;
	ifp->if_capabilities |= IFCAP_VLAN_MTU;

	/*
	 * Capabilities after here are not enabled by default.
	 */
	ifp->if_capenable = ifp->if_capabilities;

	if (sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER) {
		ifp->if_capabilities |= IFCAP_VLAN_HWFILTER;

		    vtnet_unregister_vlan, sc, EVENTHANDLER_PRI_FIRST);
	}

	vtnet_set_rx_process_limit(sc);
	vtnet_set_tx_intr_threshold(sc);

	/* Tell the upper layer(s) we support long frames. */
	ifp->if_hdrlen = sizeof(struct ether_vlan_header);

	NETDUMP_SET(ifp, vtnet);

	return (0);
}

static int
vtnet_change_mtu(struct vtnet_softc *sc, int new_mtu)
{
	int framesz;

	if (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS)
		return (MJUMPAGESIZE);
	else if (sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG)
		return (MCLBYTES);

	/*
	 * Try to scale the receive mbuf cluster size from the MTU. Without
	 * the GUEST_TSO[46] features, the VirtIO specification says the
	 * driver must only be able to receive ~1500 byte frames. But if
	 * jumbo frames can be transmitted then try to receive jumbo.
	 *
	 * BMV: Not quite true when F_MTU is negotiated!
	 */
	if (vtnet_modern(sc)) {
		MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr_v1));
		framesz = sizeof(struct virtio_net_hdr_v1);
	} else
		framesz = sizeof(struct vtnet_rx_header);
	framesz += sizeof(struct ether_vlan_header) + mtu;

	if (framesz <= MCLBYTES)
		return (MCLBYTES);
	else if (framesz <= MJUMPAGESIZE)
		return (MJUMPAGESIZE);
	else if (framesz <= MJUM9BYTES)
		return (MJUM9BYTES);

	/* Sane default; avoid 16KB clusters. */
	return (MCLBYTES);
}

static int
vtnet_ioctl_mtu(struct vtnet_softc *sc, int mtu)
{
	struct ifnet *ifp;
	int frame_size, clsize;

	ifp = sc->vtnet_ifp;
	VTNET_CORE_LOCK_ASSERT(sc);

	if (new_mtu < ETHERMIN || new_mtu > VTNET_MAX_MTU)
		return (0);
	else if (mtu < ETHERMIN || mtu > sc->vtnet_max_mtu)
		return (EINVAL);

	frame_size = sc->vtnet_hdr_size + sizeof(struct ether_vlan_header) +
	    new_mtu;

	/*
	 * Based on the new MTU (and hence frame size) determine which
	 * cluster size is most appropriate for the receive queues.
	 */
	if (frame_size <= MCLBYTES) {
		clsize = MCLBYTES;
	} else if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
		/* Avoid going past 9K jumbos. */
		if (frame_size > MJUM9BYTES)
			return (EINVAL);
		clsize = MJUM9BYTES;
	} else
		clsize = MJUMPAGESIZE;

	ifp->if_mtu = new_mtu;
	sc->vtnet_rx_new_clsize = clsize;

	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
{
	struct ifnet *ifp;
	int drv_running;

	ifp = sc->vtnet_ifp;
	drv_running = (ifp->if_drv_flags & IFF_DRV_RUNNING) != 0;

	VTNET_CORE_LOCK_ASSERT(sc);

	if ((ifp->if_flags & IFF_UP) == 0) {
		if (drv_running)
			vtnet_stop(sc);
		goto out;
	}

	if (!drv_running) {
		vtnet_init_locked(sc);
		goto out;
	}

	if ((ifp->if_flags ^ sc->vtnet_if_flags) &
	    (IFF_PROMISC | IFF_ALLMULTI)) {
		if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
			return (ENOTSUP);
		vtnet_rx_filter(sc);
	}

out:
	sc->vtnet_if_flags = ifp->if_flags;
	return (0);
}

static int
vtnet_ioctl_multi(struct vtnet_softc *sc)
{
	struct ifnet *ifp;

	ifp = sc->vtnet_ifp;

	VTNET_CORE_LOCK_ASSERT(sc);

	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX &&
	    ifp->if_drv_flags & IFF_DRV_RUNNING)
		vtnet_rx_filter_mac(sc);

	return (0);
}

static int
vtnet_ioctl_ifcap(struct vtnet_softc *sc, struct ifreq *ifr)
{
	struct ifnet *ifp;
	int mask, reinit, update;

	ifp = sc->vtnet_ifp;
	mask = (ifr->ifr_reqcap & ifp->if_capabilities) ^ ifp->if_capenable;
	reinit = update = 0;

	VTNET_CORE_LOCK_ASSERT(sc);

	if (mask & IFCAP_TXCSUM)
		ifp->if_capenable ^= IFCAP_TXCSUM;
	if (mask & IFCAP_TXCSUM_IPV6)
		ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
	if (mask & IFCAP_TSO4)
		ifp->if_capenable ^= IFCAP_TSO4;
	if (mask & IFCAP_TSO6)
		ifp->if_capenable ^= IFCAP_TSO6;

	if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) {
		/*
		 * These Rx features require the negotiated features to
		 * be updated. Avoid a full reinit if possible.
		 */
		if (sc->vtnet_features & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)
			update = 1;
		else
			reinit = 1;

		/* BMV: Avoid needless renegotiation for just software LRO. */
		if ((mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO)) ==
		    IFCAP_LRO && vtnet_software_lro(sc))
			reinit = update = 0;

		if (mask & IFCAP_RXCSUM)
			ifp->if_capenable ^= IFCAP_RXCSUM;
		if (mask & IFCAP_RXCSUM_IPV6)
			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
		if (mask & IFCAP_LRO)
			ifp->if_capenable ^= IFCAP_LRO;

		/*
		 * VirtIO does not distinguish between IPv4 and IPv6 checksums
		 * so treat them as a pair. Guest TSO (LRO) requires receive
		 * checksums.
		 */
		if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
			ifp->if_capenable |= IFCAP_RXCSUM;
#ifdef notyet
			ifp->if_capenable |= IFCAP_RXCSUM_IPV6;
#endif
		} else
			ifp->if_capenable &=
			    ~(IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO);
	}

	if (mask & IFCAP_VLAN_HWFILTER) {
		/* These Rx features require renegotiation. */
		reinit = 1;

		if (mask & IFCAP_VLAN_HWFILTER)
			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
	}

	if (mask & IFCAP_VLAN_HWTSO)
		ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
	if (mask & IFCAP_VLAN_HWTAGGING)
		ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;

	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
		if (reinit) {
			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
			vtnet_init_locked(sc);
		} else if (update)
			vtnet_update_rx_offloads(sc);
	}

	return (0);
}

static int
vtnet_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
{
	struct vtnet_softc *sc;
	struct ifreq *ifr;
	int reinit, mask, error;

	sc = ifp->if_softc;
	ifr = (struct ifreq *) data;


	switch (cmd) {
	case SIOCSIFMTU:
		if (ifp->if_mtu != ifr->ifr_mtu) {
			VTNET_CORE_LOCK(sc);
			error = vtnet_change_mtu(sc, ifr->ifr_mtu);
			VTNET_CORE_UNLOCK(sc);
		}
		break;

	case SIOCSIFFLAGS:
		VTNET_CORE_LOCK(sc);
		if ((ifp->if_flags & IFF_UP) == 0) {
			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
				vtnet_stop(sc);
		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
			if ((ifp->if_flags ^ sc->vtnet_if_flags) &
			    (IFF_PROMISC | IFF_ALLMULTI)) {
				if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX)
					vtnet_rx_filter(sc);
				else {
					ifp->if_flags |= IFF_PROMISC;
					if ((ifp->if_flags ^ sc->vtnet_if_flags)
					    & IFF_ALLMULTI)
						error = ENOTSUP;
				}
			}
		} else
			vtnet_init_locked(sc);

		if (error == 0)
			sc->vtnet_if_flags = ifp->if_flags;
		VTNET_CORE_UNLOCK(sc);
		break;

	case SIOCADDMULTI:
	case SIOCDELMULTI:
		if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0)
			break;
		VTNET_CORE_LOCK(sc);
		if (ifp->if_drv_flags & IFF_DRV_RUNNING)
			vtnet_rx_filter_mac(sc);
		VTNET_CORE_UNLOCK(sc);
		break;



	case SIOCSIFCAP:
		VTNET_CORE_LOCK(sc);
		mask = ifr->ifr_reqcap ^ ifp->if_capenable;

		if (mask & IFCAP_TXCSUM)
			ifp->if_capenable ^= IFCAP_TXCSUM;
		if (mask & IFCAP_TXCSUM_IPV6)
			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
		if (mask & IFCAP_TSO4)
			ifp->if_capenable ^= IFCAP_TSO4;
		if (mask & IFCAP_TSO6)
			ifp->if_capenable ^= IFCAP_TSO6;

		if (mask & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6 | IFCAP_LRO |
		    IFCAP_VLAN_HWFILTER)) {
			/* These Rx features require us to renegotiate. */
			reinit = 1;

			if (mask & IFCAP_RXCSUM)
				ifp->if_capenable ^= IFCAP_RXCSUM;
			if (mask & IFCAP_RXCSUM_IPV6)
				ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
			if (mask & IFCAP_LRO)
				ifp->if_capenable ^= IFCAP_LRO;
			if (mask & IFCAP_VLAN_HWFILTER)
				ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
		} else
			reinit = 0;

		if (mask & IFCAP_VLAN_HWTSO)
			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
		if (mask & IFCAP_VLAN_HWTAGGING)
			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;

		if (reinit && (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
			vtnet_init_locked(sc);
		}

		VTNET_CORE_UNLOCK(sc);
		VLAN_CAPABILITIES(ifp);

		break;

	default:

	struct virtqueue *vq;
	int nbufs, error;

#ifdef DEV_NETMAP
	error = vtnet_netmap_rxq_populate(rxq);
	if (error >= 0)
		return (error);
#endif  /* DEV_NETMAP */

	vq = rxq->vtnrx_vq;
	error = ENOSPC;


	struct virtqueue *vq;
	struct mbuf *m;
	int last;
#ifdef DEV_NETMAP
	int netmap_bufs = vtnet_netmap_queue_on(rxq->vtnrx_sc, NR_RX,
						rxq->vtnrx_id);
#else  /* !DEV_NETMAP */
	int netmap_bufs = 0;
#endif /* !DEV_NETMAP */

	vq = rxq->vtnrx_vq;
	last = 0;

	while ((m = virtqueue_drain(vq, &last)) != NULL) {
		if (!netmap_bufs)
			m_freem(m);
	}

	KASSERT(virtqueue_empty(vq),
	    ("%s: mbufs remaining in rx queue %p", __func__, rxq));

vtnet_rx_alloc_buf(struct vtnet_softc *sc, int nbufs, struct mbuf **m_tailp)
{
	struct mbuf *m_head, *m_tail, *m;
	int i, clsize;

	clsize = sc->vtnet_rx_clsize;
	size = sc->vtnet_rx_clustersz;

	KASSERT(nbufs == 1 || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
	    ("%s: chained mbuf %d request without LRO_NOMRG", __func__, nbufs));

	m_head = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, clsize);
	if (m_head == NULL)
		goto fail;
			sc->vtnet_stats.mbuf_alloc_failed++;
			m_freem(m_head);
			return (NULL);
		}

	m_head->m_len = clsize;
	m_tail = m_head;

	/* Allocate the rest of the chain. */
	for (i = 1; i < nbufs; i++) {
		m = m_getjcl(M_NOWAIT, MT_DATA, 0, clsize);
		if (m == NULL)
			goto fail;

		m->m_len = clsize;
		m_tail->m_next = m;
		m_tail = m;
	}

	if (m_tailp != NULL)
		*m_tailp = m_tail;

	return (m_head);

fail:
	sc->vtnet_stats.mbuf_alloc_failed++;
	m_freem(m_head);

	return (NULL);
}

/*
 * Slow path for when LRO without mergeable buffers is negotiated.
 */
static int
vtnet_rxq_replace_lro_nomgr_buf(struct vtnet_rxq *rxq, struct mbuf *m0,
    int len0)
{
	struct vtnet_softc *sc;
	struct mbuf *m, *m_prev;
	struct mbuf *m_new, *m_tail;
	int len, clsize, nreplace, error;

	sc = rxq->vtnrx_sc;
	clsize = sc->vtnet_rx_clsize;

	m_prev = NULL;
	m_tail = NULL;

	len = len0;

	/*
	 * Since these mbuf chains are so large, we avoid allocating an
	 * entire replacement chain if possible. When the received frame
	 * did not consume the entire chain, the unused mbufs are moved
	 * to the replacement chain.
	 */
	while (len > 0) {
		/*
		 * Something is seriously wrong if we received a frame
		 * larger than the chain. Drop it.
		 */
		if (m == NULL) {
			sc->vtnet_stats.rx_frame_too_large++;
			return (EMSGSIZE);
		}

		/* We always allocate the same cluster size. */
		KASSERT(m->m_len == clsize,
		    ("%s: mbuf size %d is not the cluster size %d",
		    __func__, m->m_len, clsize));
		KASSERT(m->m_len == clustersz,
		    ("%s: mbuf size %d not expected cluster size %d", __func__,
		    m->m_len, clustersz));

		m->m_len = MIN(m->m_len, len);
		len -= m->m_len;

		nreplace++;
	}

	KASSERT(nreplace <= sc->vtnet_rx_nmbufs,
	    ("%s: too many replacement mbufs %d max %d", __func__, nreplace,
	    sc->vtnet_rx_nmbufs));

	m_new = vtnet_rx_alloc_buf(sc, nreplace, &m_tail);
	if (m_new == NULL) {
		m_prev->m_len = clsize;
		return (ENOBUFS);
	}

	/*
	 * Move any unused mbufs from the received chain onto the end
	 * of the new chain.
	 */
	if (m_prev->m_next != NULL) {
		m_tail->m_next = m_prev->m_next;

	error = vtnet_rxq_enqueue_buf(rxq, m_new);
	if (error) {
		/*
		 * BAD! We could not enqueue the replacement mbuf chain. We
		 * must restore the m0 chain to the original state if it was
		 * modified so we can subsequently discard it.
		 *
		 * NOTE: The replacement is suppose to be an identical copy
		 * to the one just dequeued so this is an unexpected error.
		 */
		sc->vtnet_stats.rx_enq_replacement_failed++;

		if (m_tail->m_next != NULL) {
			m_prev->m_next = m_tail->m_next;
			m_tail->m_next = NULL;
		}

		m_prev->m_len = clsize;
		m_freem(m_new);
	}



	sc = rxq->vtnrx_sc;

	KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL,
	    ("%s: chained mbuf without LRO_NOMRG", __func__));

	if (m->m_next == NULL) {
		/* Fast-path for the common case of just one mbuf. */
		if (m->m_len < len)
			return (EINVAL);

		m_new = vtnet_rx_alloc_buf(sc, 1, NULL);
		if (m_new == NULL)
			return (ENOBUFS);

		error = vtnet_rxq_enqueue_buf(rxq, m_new);
		if (error) {
			/*
			 * The new mbuf is suppose to be an identical
			 * copy of the one just dequeued so this is an
			 * unexpected error.
			 */
			m_freem(m_new);
			sc->vtnet_stats.rx_enq_replacement_failed++;
		} else
			m->m_len = len;
	} else
		error = vtnet_rxq_replace_lro_nomgr_buf(rxq, m, len);

	return (error);
}

{
	struct vtnet_softc *sc;
	struct sglist *sg;
	struct vtnet_rx_header *rxhdr;
	uint8_t *mdata;
	int offset, error;

	sc = rxq->vtnrx_sc;
	sg = rxq->vtnrx_sg;
	mdata = mtod(m, uint8_t *);

	KASSERT(m->m_next == NULL || sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG,
	    ("%s: mbuf chain without LRO_NOMRG", __func__));
	VTNET_RXQ_LOCK_ASSERT(rxq);
	KASSERT(sc->vtnet_flags & VTNET_FLAG_LRO_NOMRG || m->m_next == NULL,
	    ("%s: chained mbuf without LRO_NOMRG", __func__));
	KASSERT(m->m_len == sc->vtnet_rx_clsize,
	    ("%s: unexpected cluster size %d/%d", __func__, m->m_len,
	     sc->vtnet_rx_clsize));

	sglist_reset(sg);
	if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
	    (sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) != 0; /* TODO: ANY_LAYOUT */

	if (header_inlined)
		error = sglist_append_mbuf(sg, m);
	else {
		struct vtnet_rx_header *rxhdr =
		    mtod(m, struct vtnet_rx_header *);
		MPASS(sc->vtnet_hdr_size == sizeof(struct virtio_net_hdr));
		rxhdr = (struct vtnet_rx_header *) mdata;
		sglist_append(sg, &rxhdr->vrh_hdr, sc->vtnet_hdr_size);
		offset = sizeof(struct vtnet_rx_header);
	} else
		offset = 0;

	sglist_append(sg, mdata + offset, m->m_len - offset);
	if (m->m_next != NULL) {
		error = sglist_append_mbuf(sg, m->m_next);
		MPASS(error == 0);
		error = sglist_append(sg, &rxhdr[1],
		    m->m_len - sizeof(struct vtnet_rx_header));
		if (error)
			return (error);

		if (m->m_next != NULL)
			error = sglist_append_mbuf(sg, m->m_next);
	}

	error = virtqueue_enqueue(rxq->vtnrx_vq, m, sg, 0, sg->sg_nseg);
		return (error);

	return (error);
}

static int

	return (error);
}

/*
 * Use the checksum offset in the VirtIO header to set the
 * correct CSUM_* flags.
 */
static int
vtnet_rxq_csum_by_offset(struct vtnet_rxq *rxq, struct mbuf *m,
    uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr)
{
	struct vtnet_softc *sc;
#if defined(INET) || defined(INET6)
	int offset = hdr->csum_start + hdr->csum_offset;
#endif

	sc = rxq->vtnrx_sc;

	/* Only do a basic sanity check on the offset. */
	switch (eth_type) {
#if defined(INET)
	case ETHERTYPE_IP:
		if (__predict_false(offset < ip_start + sizeof(struct ip)))
			return (1);
		break;
#endif
#if defined(INET6)
	case ETHERTYPE_IPV6:
		if (__predict_false(offset < ip_start + sizeof(struct ip6_hdr)))
			return (1);
		break;
#endif
	default:
		sc->vtnet_stats.rx_csum_bad_ethtype++;
		return (1);
	if ((sc->vtnet_flags & VTNET_FLAG_FIXUP_NEEDS_CSUM) == 0) {
		error = vtnet_rxq_csum_data_valid(rxq, m, etype, hoff, hdr);
		return (error);
	}

	/*
	 * Use the offset to determine the appropriate CSUM_* flags. This is
	 * a bit dirty, but we can get by with it since the checksum offsets
	 * happen to be different. We assume the host host does not do IPv4
	 * header checksum offloading.
	 */
	switch (hdr->csum_offset) {
	case offsetof(struct udphdr, uh_sum):
	case offsetof(struct tcphdr, th_sum):
	case ETHERTYPE_IPV6: {
		int csum_off, csum_end;
		uint16_t csum;

		csum_off = hdr->csum_start + hdr->csum_offset;
		csum_end = csum_off + sizeof(uint16_t);

		/* Assume checksum will be in the first mbuf. */
		if (m->m_len < csum_end || m->m_pkthdr.len < csum_end)
			return (1);

		/*
		 * Like in_delayed_cksum()/in6_delayed_cksum(), compute the
		 * checksum and write it at the specified offset. We could
		 * try to verify the packet: csum_start should probably
		 * correspond to the start of the TCP/UDP header.
		 *
		 * BMV: Need to properly handle UDP with zero checksum. Is
		 * the IPv4 header checksum implicitly validated?
		 */
		csum = in_cksum_skip(m, m->m_pkthdr.len, hdr->csum_start);
		*(uint16_t *)(mtodo(m, csum_off)) = csum;
		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
		m->m_pkthdr.csum_data = 0xFFFF;
		break;
	}
#endif
	default:
		sc->vtnet_stats.rx_csum_bad_offset++;
		return (1);
	}


}

static int
vtnet_rxq_csum_by_parse(struct vtnet_rxq *rxq, struct mbuf *m,
    uint16_t eth_type, int ip_start, struct virtio_net_hdr *hdr)
{
	struct vtnet_softc *sc;
	int offset, proto;

	sc = rxq->vtnrx_sc;

	switch (eth_type) {
#if defined(INET)
	case ETHERTYPE_IP: {
		struct ip *ip;
		if (__predict_false(m->m_len < ip_start + sizeof(struct ip)))
			return (1);
		ip = (struct ip *)(m->m_data + ip_start);
		proto = ip->ip_p;
		offset = ip_start + (ip->ip_hl << 2);
		break;
	}
#endif
#if defined(INET6)
	case ETHERTYPE_IPV6:
		if (__predict_false(m->m_len < ip_start +
		    sizeof(struct ip6_hdr)))
			return (1);
		offset = ip6_lasthdr(m, ip_start, IPPROTO_IPV6, &proto);
		if (__predict_false(offset < 0))
			return (1);
		break;
#endif
	default:
		sc->vtnet_stats.rx_csum_bad_ethtype++;
		return (1);
	}

	switch (proto) {
	case IPPROTO_TCP:
		if (__predict_false(m->m_len < offset + sizeof(struct tcphdr)))
			return (1);
		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
		m->m_pkthdr.csum_data = 0xFFFF;
		break;
	case IPPROTO_UDP:
		if (__predict_false(m->m_len < offset + sizeof(struct udphdr)))
			return (1);
		m->m_pkthdr.csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
		m->m_pkthdr.csum_data = 0xFFFF;
		break;
	default:
		/*
		 * For the remaining protocols, FreeBSD does not support
		 * checksum offloading, so the checksum will be recomputed.
		 * if the protocol is supported.
		 */
#if 0
		if_printf(sc->vtnet_ifp, "cksum offload of unsupported "
		    "protocol eth_type=%#x proto=%d csum_start=%d "
		    "csum_offset=%d\n", __func__, eth_type, proto,
		    hdr->csum_start, hdr->csum_offset);
		    hdr->csum_offset);
#endif
		break;
	}

	return (0);
}

/*
 * Set the appropriate CSUM_* flags. Unfortunately, the information
 * provided is not directly useful to us. The VirtIO header gives the
 * offset of the checksum, which is all Linux needs, but this is not
 * how FreeBSD does things. We are forced to peek inside the packet
 * a bit.
 *
 * It would be nice if VirtIO gave us the L4 protocol or if FreeBSD
 * could accept the offsets and let the stack figure it out.
 */
static int
vtnet_rxq_csum(struct vtnet_rxq *rxq, struct mbuf *m,
    struct virtio_net_hdr *hdr)
{
	struct ether_header *eh;
	struct ether_vlan_header *evh;
	uint16_t eth_type;
	int offset, error;

	eh = mtod(m, struct ether_header *);
	eth_type = ntohs(eh->ether_type);
	if (eth_type == ETHERTYPE_VLAN) {
		/* BMV: We should handle nested VLAN tags too. */
		evh = mtod(m, struct ether_vlan_header *);
		eth_type = ntohs(evh->evl_proto);
		offset = sizeof(struct ether_vlan_header);
		hoff = sizeof(struct ether_vlan_header);
	} else
		offset = sizeof(struct ether_header);

	if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
		error = vtnet_rxq_csum_by_offset(rxq, m, eth_type, offset, hdr);
	else
		error = vtnet_rxq_csum_by_parse(rxq, m, eth_type, offset, hdr);

	return (error);
}

static void

{
	struct vtnet_softc *sc;
	struct virtqueue *vq;
	struct mbuf *m, *m_tail;
	int len;

	sc = rxq->vtnrx_sc;
	vq = rxq->vtnrx_vq;
	m_tail = m_head;

	while (--nbufs > 0) {
		struct mbuf *m;
		int len;

		m = virtqueue_dequeue(vq, &len);
		if (m == NULL) {
			rxq->vtnrx_stats.vrxs_ierrors++;

	return (1);
}

#if defined(INET) || defined(INET6)
static int
vtnet_lro_rx(struct vtnet_rxq *rxq, struct mbuf *m)
{
	struct lro_ctrl *lro;

	lro = &rxq->vtnrx_lro;

	if (lro->lro_mbuf_max != 0) {
		tcp_lro_queue_mbuf(lro, m);
		return (0);
	}

	return (tcp_lro_rx(lro, m, 0));
}
#endif

static void
vtnet_rxq_input(struct vtnet_rxq *rxq, struct mbuf *m,
    struct virtio_net_hdr *hdr)
{
	struct vtnet_softc *sc;
	struct ifnet *ifp;
	struct ether_header *eh;

	sc = rxq->vtnrx_sc;
	ifp = sc->vtnet_ifp;

	if (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) {
		eh = mtod(m, struct ether_header *);
		if (eh->ether_type == htons(ETHERTYPE_VLAN)) {
			vtnet_vlan_tag_remove(m);
			/*

	m->m_pkthdr.flowid = rxq->vtnrx_id;
	M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);

	/*
	 * BMV: FreeBSD does not have the UNNECESSARY and PARTIAL checksum
	 * distinction that Linux does. Need to reevaluate if performing
	 * offloading for the NEEDS_CSUM case is really appropriate.
	 */
	if (hdr->flags & (VIRTIO_NET_HDR_F_NEEDS_CSUM |
	    VIRTIO_NET_HDR_F_DATA_VALID)) {
		if (vtnet_rxq_csum(rxq, m, hdr) == 0)
			rxq->vtnrx_stats.vrxs_csum++;
		else
			rxq->vtnrx_stats.vrxs_csum_failed++;
	}

	if (hdr->gso_size != 0) {
		switch (hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) {
		case VIRTIO_NET_HDR_GSO_TCPV4:
		case VIRTIO_NET_HDR_GSO_TCPV6:
			m->m_pkthdr.lro_nsegs =
			    howmany(m->m_pkthdr.len, hdr->gso_size);
			rxq->vtnrx_stats.vrxs_host_lro++;
			break;
		}
	}

	rxq->vtnrx_stats.vrxs_ipackets++;
	rxq->vtnrx_stats.vrxs_ibytes += m->m_pkthdr.len;

	VTNET_RXQ_UNLOCK(rxq);
	if (vtnet_software_lro(sc) && ifp->if_capenable & IFCAP_LRO) {
		if (vtnet_lro_rx(rxq, m) == 0)
			return;
	}
#endif

	(*ifp->if_input)(ifp, m);
	VTNET_RXQ_LOCK(rxq);
}

static int

	struct vtnet_softc *sc;
	struct ifnet *ifp;
	struct virtqueue *vq;
	struct mbuf *m;
	struct virtio_net_hdr_mrg_rxbuf *mhdr;
	int len, deq, nbufs, adjsz, count;

	sc = rxq->vtnrx_sc;
	vq = rxq->vtnrx_vq;
	ifp = sc->vtnet_ifp;
	hdr = &lhdr;
	deq = 0;
	count = sc->vtnet_rx_process_limit;

	VTNET_RXQ_LOCK_ASSERT(rxq);

#ifdef DEV_NETMAP
	if (netmap_rx_irq(ifp, 0, &deq))
		return (0);
#endif

	while (count-- > 0) {
		struct mbuf *m;
		int len, nbufs, adjsz;

		m = virtqueue_dequeue(vq, &len);
		if (m == NULL)
			break;

			continue;
		}

		if ((sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS) == 0) {
			struct virtio_net_hdr_mrg_rxbuf *mhdr =
			    mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
			nbufs = vtnet_htog16(sc, mhdr->num_buffers);
			adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
		} else if (vtnet_modern(sc)) {
			nbufs = 1; /* num_buffers is always 1 */
			adjsz = sizeof(struct virtio_net_hdr_v1);
		} else {
			nbufs = 1;
			adjsz = sizeof(struct vtnet_rx_header);
			/*
			 * Account for our pad inserted between the header
			 * and the actual start of the frame.
			 */
			len += VTNET_RX_HEADER_PAD;
		} else {
			mhdr = mtod(m, struct virtio_net_hdr_mrg_rxbuf *);
			nbufs = mhdr->num_buffers;
			adjsz = sizeof(struct virtio_net_hdr_mrg_rxbuf);
		}

		if (vtnet_rxq_replace_buf(rxq, m, len) != 0) {

		}

		/*
		 * Save copy of header before we strip it. For both mergeable
		 * and non-mergeable, the header is at the beginning of the
		 * mbuf data. We no longer need num_buffers, so always use a
		 * regular header.
		 *
		 * BMV: Is this memcpy() expensive? We know the mbuf data is
		 * still valid even after the m_adj().
		 */
		memcpy(hdr, mtod(m, void *), sizeof(struct virtio_net_hdr));
		lhdr.flags = hdr->flags;
		lhdr.gso_type = hdr->gso_type;
		lhdr.hdr_len = vtnet_htog16(sc, hdr->hdr_len);
		lhdr.gso_size = vtnet_htog16(sc, hdr->gso_size);
		lhdr.csum_start = vtnet_htog16(sc, hdr->csum_start);
		lhdr.csum_offset = vtnet_htog16(sc, hdr->csum_offset);
		m_adj(m, adjsz);

		vtnet_rxq_input(rxq, m, hdr);

		/* Must recheck after dropping the Rx lock. */
		if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
			break;
	}

	if (deq > 0)
#if defined(INET) || defined(INET6)
		tcp_lro_flush_all(&rxq->vtnrx_lro);
#endif
		virtqueue_notify(vq);
	}

	return (count > 0 ? 0 : EAGAIN);
}

		return;
	}

#ifdef DEV_NETMAP
	if (netmap_rx_irq(ifp, rxq->vtnrx_id, &more) != NM_IRQ_PASS)
		return;
#endif /* DEV_NETMAP */

	VTNET_RXQ_LOCK(rxq);

again:

		if (tries++ < VTNET_INTR_DISABLE_RETRIES)
			goto again;

		rxq->vtnrx_stats.vrxs_rescheduled++;
		VTNET_RXQ_UNLOCK(rxq);
		rxq->vtnrx_stats.vrxs_rescheduled++;
		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
	} else
		VTNET_RXQ_UNLOCK(rxq);

		if (!more)
			vtnet_rxq_disable_intr(rxq);
		rxq->vtnrx_stats.vrxs_rescheduled++;
		VTNET_RXQ_UNLOCK(rxq);
		taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
	}
		VTNET_RXQ_UNLOCK(rxq);
}

	VTNET_RXQ_UNLOCK(rxq);
vtnet_txq_intr_threshold(struct vtnet_txq *txq)
{
	struct vtnet_softc *sc;
	int threshold;

	sc = txq->vtntx_sc;

	/*
	 * The Tx interrupt is disabled until the queue free count falls
	 * below our threshold. Completed frames are drained from the Tx
	 * virtqueue before transmitting new frames and in the watchdog
	 * callout, so the frequency of Tx interrupts is greatly reduced,
	 * at the cost of not freeing mbufs as quickly as they otherwise
	 * would be.
	 */
	threshold = virtqueue_size(txq->vtntx_vq) / 4;

	/*
	 * Without indirect descriptors, leave enough room for the most
	 * segments we handle.
	 */
	if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 &&
	    threshold < sc->vtnet_tx_nsegs)
		threshold = sc->vtnet_tx_nsegs;

	return (threshold);
}

static int
vtnet_txq_below_threshold(struct vtnet_txq *txq)
{
	struct vtnet_softc *sc;
	struct virtqueue *vq;

	sc = txq->vtntx_sc;
	vq = txq->vtntx_vq;

	return (virtqueue_nfree(vq) <= sc->vtnet_tx_intr_thresh);
}

static int

	struct virtqueue *vq;
	struct vtnet_tx_header *txhdr;
	int last;
#ifdef DEV_NETMAP
	int netmap_bufs = vtnet_netmap_queue_on(txq->vtntx_sc, NR_TX,
						txq->vtntx_id);
#else  /* !DEV_NETMAP */
	int netmap_bufs = 0;
#endif /* !DEV_NETMAP */

	vq = txq->vtntx_vq;
	last = 0;

	while ((txhdr = virtqueue_drain(vq, &last)) != NULL) {
		if (!netmap_bufs) {
			m_freem(txhdr->vth_mbuf);
			uma_zfree(vtnet_tx_header_zone, txhdr);
		}
	}

	KASSERT(virtqueue_empty(vq),

}

/*
 * BMV: Much of this can go away once we finally have offsets in
 * the mbuf packet header. Bug andre@.
 */
static int
vtnet_txq_offload_ctx(struct vtnet_txq *txq, struct mbuf *m,
    int *etype, int *proto, int *start)
{
	struct vtnet_softc *sc;
	struct ether_vlan_header *evh;

		break;
#endif
	default:
		sc->vtnet_stats.tx_csum_bad_ethtype++;
		return (EINVAL);
	}


}

static int
vtnet_txq_offload_tso(struct vtnet_txq *txq, struct mbuf *m, int eth_type,
    int offset, struct virtio_net_hdr *hdr)
{
	static struct timeval lastecn;

	} else
		tcp = (struct tcphdr *)(m->m_data + offset);

	hdr->hdr_len = offset + (tcp->th_off << 2);
	hdr->gso_size = m->m_pkthdr.tso_segsz;
	hdr->gso_type = eth_type == ETHERTYPE_IP ? VIRTIO_NET_HDR_GSO_TCPV4 :
	    VIRTIO_NET_HDR_GSO_TCPV6;

	if (tcp->th_flags & TH_CWR) {
		/*
		 * Drop if VIRTIO_NET_F_HOST_ECN was not negotiated. In FreeBSD,
		 * ECN support is not on a per-interface basis, but globally via
		 * the net.inet.tcp.ecn.enable sysctl knob. The default is off.
		 * knob. The default is off.
		 */
		if ((sc->vtnet_flags & VTNET_FLAG_TSO_ECN) == 0) {
			if (ppsratecheck(&lastecn, &curecn, 1))

	if (error)
		goto drop;

	if ((etype == ETHERTYPE_IP && flags & VTNET_CSUM_OFFLOAD) ||
	    (etype == ETHERTYPE_IPV6 && flags & VTNET_CSUM_OFFLOAD_IPV6)) {
		/*
		 * We could compare the IP protocol vs the CSUM_ flag too,
		 * but that really should not be necessary.
		 */
			goto drop;
		}

		hdr->flags |= VIRTIO_NET_HDR_F_NEEDS_CSUM;
		hdr->csum_start = csum_start;
		hdr->csum_offset = m->m_pkthdr.csum_data;
		txq->vtntx_stats.vtxs_csum++;
	}

	if (flags & CSUM_TSO) {
		/*
		 * Sanity check the parsed mbuf IP protocol is TCP, and
		 * VirtIO TSO reqires the checksum offloading above.
		 */
		if (__predict_false(proto != IPPROTO_TCP)) {
			/* Likely failed to correctly parse the mbuf. */
			sc->vtnet_stats.tx_tso_not_tcp++;
			goto drop;
		} else if (__predict_false((hdr->flags &
		    VIRTIO_NET_HDR_F_NEEDS_CSUM) == 0)) {
			sc->vtnet_stats.tx_tso_without_csum++;
			goto drop;
		}

		KASSERT(hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM,
		    ("%s: mbuf %p TSO without checksum offload %#x",
		    __func__, m, flags));

		error = vtnet_txq_offload_tso(txq, m, etype, csum_start, hdr);
		if (error)
			goto drop;
	}


	sglist_reset(sg);
	error = sglist_append(sg, &txhdr->vth_uhdr, sc->vtnet_hdr_size);
	KASSERT(error == 0 && sg->sg_nseg == 1,
	    ("%s: error %d adding header to sglist", __func__, error));
		    __func__, error, sg->sg_nseg));
		goto fail;
	}

	error = sglist_append_mbuf(sg, m);
	if (error) {

	}

	/*
	 * Always use the non-mergeable header, regardless if the feature
	 * was negotiated. For transmit, num_buffers is always zero. The
	 * vtnet_hdr_size is used to enqueue the correct header size.
	 */
	hdr = &txhdr->vth_uhdr.hdr;


	}

	error = vtnet_txq_enqueue_buf(txq, m_head, txhdr);
	if (error == 0)
		return (0);

fail:
	uma_zfree(vtnet_tx_header_zone, txhdr);
		uma_zfree(vtnet_tx_header_zone, txhdr);

	return (error);
}

	sc = ifp->if_softc;
	npairs = sc->vtnet_act_vq_pairs;

	/* check if flowid is set */
	if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
		i = m->m_pkthdr.flowid % npairs;
	else

	deq = 0;
	VTNET_TXQ_LOCK_ASSERT(txq);

#ifdef DEV_NETMAP
	if (netmap_tx_irq(txq->vtntx_sc->vtnet_ifp, txq->vtntx_id)) {
		virtqueue_disable_intr(vq); // XXX luigi
		return (0); // XXX or 1 ?
	}
#endif

	while ((txhdr = virtqueue_dequeue(vq, NULL)) != NULL) {
		m = txhdr->vth_mbuf;
		deq++;

		return;
	}

#ifdef DEV_NETMAP
	if (netmap_tx_irq(ifp, txq->vtntx_id) != NM_IRQ_PASS)
		return;
#endif /* DEV_NETMAP */

	VTNET_TXQ_LOCK(txq);

	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {

	 * Most drivers just ignore the return value - it only fails
	 * with ENOMEM so an error is not likely.
	 */
	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
		rxq = &sc->vtnet_rxqs[i];
		error = taskqueue_start_threads(&rxq->vtnrx_tq, 1, PI_NET,
		    "%s rxq %d", device_get_nameunit(dev), rxq->vtnrx_id);

		rxq = &sc->vtnet_rxqs[i];
		if (rxq->vtnrx_tq != NULL) {
			taskqueue_free(rxq->vtnrx_tq);
			rxq->vtnrx_tq = NULL;
		}

		txq = &sc->vtnet_txqs[i];

	struct vtnet_txq *txq;
	int i;

	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
	if (nm_native_on(NA(sc->vtnet_ifp)))
		return;
#endif

	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
		rxq = &sc->vtnet_rxqs[i];
		vtnet_rxq_free_mbufs(rxq);


	struct vtnet_txq *txq;
	int i;

	VTNET_CORE_LOCK_ASSERT(sc);

	/*
	 * Lock and unlock the per-queue mutex so we known the stop
	 * state is visible. Doing only the active queues should be
	 * sufficient, but it does not cost much extra to do all the
	 * queues. Note we hold the core mutex here too.
	 */
	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
		rxq = &sc->vtnet_rxqs[i];

	virtio_stop(dev);
	vtnet_stop_rendezvous(sc);

	/* Free any mbufs left in the virtqueues. */
	vtnet_drain_rxtx_queues(sc);
	sc->vtnet_act_vq_pairs = 1;
}

static int

	device_t dev;
	struct ifnet *ifp;
	uint64_t features;
	int mask, error;

	dev = sc->vtnet_dev;
	ifp = sc->vtnet_ifp;
	features = sc->vtnet_features;

	mask = 0;
#if defined(INET)
	mask |= IFCAP_RXCSUM;
#endif
#if defined (INET6)
	mask |= IFCAP_RXCSUM_IPV6;
#endif

	/*
	 * Re-negotiate with the host, removing any disabled receive
	 * features. Transmit features are disabled only on our side
	 * via if_capenable and if_hwassist.
	 */

	if (ifp->if_capabilities & mask) {
		/*
		 * We require both IPv4 and IPv6 offloading to be enabled
		 * in order to negotiated it: VirtIO does not distinguish
		 * between the two.
		 */
		if ((ifp->if_capenable & mask) != mask)
			features &= ~VIRTIO_NET_F_GUEST_CSUM;
	}

	if (ifp->if_capabilities & IFCAP_LRO) {
		if ((ifp->if_capenable & IFCAP_LRO) == 0)
			features &= ~VTNET_LRO_FEATURES;
	}

	if (ifp->if_capabilities & IFCAP_VLAN_HWFILTER) {
		if ((ifp->if_capenable & IFCAP_VLAN_HWFILTER) == 0)
			features &= ~VIRTIO_NET_F_CTRL_VLAN;
	}

	error = virtio_reinit(dev, features);
	if (error)
		device_printf(dev, "virtio reinit error %d\n", error);
		return (error);
	}

	return (error);
	virtio_reinit_complete(dev);

	return (0);
}

static void

	ifp = sc->vtnet_ifp;

	if (sc->vtnet_flags & VTNET_FLAG_CTRL_RX) {
		/* Restore promiscuous and all-multicast modes. */
		vtnet_rx_filter(sc);
		/* Restore filtered MAC addresses. */
		vtnet_rx_filter_mac(sc);
	}


vtnet_init_rx_queues(struct vtnet_softc *sc)
{
	device_t dev;
	struct ifnet *ifp;
	struct vtnet_rxq *rxq;
	int i, clsize, error;

	dev = sc->vtnet_dev;
	ifp = sc->vtnet_ifp;

	/*
	 * Use the new cluster size if one has been set (via a MTU
	 * change). Otherwise, use the standard 2K clusters.
	 *
	 * BMV: It might make sense to use page sized clusters as
	 * the default (depending on the features negotiated).
	 */
	if (sc->vtnet_rx_new_clsize != 0) {
		clsize = sc->vtnet_rx_new_clsize;
		sc->vtnet_rx_new_clsize = 0;
	} else
		clsize = MCLBYTES;

	sc->vtnet_rx_clsize = clsize;
	sc->vtnet_rx_nmbufs = VTNET_NEEDED_RX_MBUFS(sc, clsize);
		return (0);
#endif

	KASSERT(sc->vtnet_flags & VTNET_FLAG_MRG_RXBUFS ||
	    sc->vtnet_rx_nmbufs < sc->vtnet_rx_nsegs,
	    ("%s: too many rx mbufs %d for %d segments", __func__,
	    sc->vtnet_rx_nmbufs, sc->vtnet_rx_nsegs));

	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
		rxq = &sc->vtnet_rxqs[i];


		VTNET_RXQ_UNLOCK(rxq);

		if (error) {
			device_printf(dev,
			    "cannot allocate mbufs for Rx queue %d\n", i);
			return (error);
		}
	}

	for (i = 0; i < sc->vtnet_act_vq_pairs; i++) {
		txq = &sc->vtnet_txqs[i];
		txq->vtntx_watchdog = 0;
		txq->vtntx_intr_threshold = vtnet_txq_intr_threshold(txq);
	}

	return (0);


	dev = sc->vtnet_dev;

	if ((sc->vtnet_flags & VTNET_FLAG_MULTIQ) == 0) {
		sc->vtnet_act_vq_pairs = 1;
		return;
	}

	npairs = sc->vtnet_requested_vq_pairs;

	if (vtnet_ctrl_mq_cmd(sc, npairs) != 0) {
		device_printf(dev,
		    "cannot set active queue pairs to %d\n", npairs);
		npairs = 1;
	}

	sc->vtnet_act_vq_pairs = npairs;
}

static void
vtnet_update_rx_offloads(struct vtnet_softc *sc)
{
	struct ifnet *ifp;
	uint64_t features;
	int error;

	ifp = sc->vtnet_ifp;
	features = sc->vtnet_features;

	VTNET_CORE_LOCK_ASSERT(sc);

	if (ifp->if_capabilities & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) {
		if (ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6))
			features |= VIRTIO_NET_F_GUEST_CSUM;
		else
			features &= ~VIRTIO_NET_F_GUEST_CSUM;
	}

	if (ifp->if_capabilities & IFCAP_LRO && !vtnet_software_lro(sc)) {
		if (ifp->if_capenable & IFCAP_LRO)
			features |= VTNET_LRO_FEATURES;
		else
			features &= ~VTNET_LRO_FEATURES;
	}

	error = vtnet_ctrl_guest_offloads(sc,
	    features & (VIRTIO_NET_F_GUEST_CSUM | VIRTIO_NET_F_GUEST_TSO4 |
		        VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN  |
			VIRTIO_NET_F_GUEST_UFO));
	if (error) {
		device_printf(sc->vtnet_dev,
		    "%s: cannot update Rx features\n", __func__);
		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
			ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
			vtnet_init_locked(sc);
		}
	} else
		sc->vtnet_features = features;
}

static int
vtnet_reinit(struct vtnet_softc *sc)
{
	device_t dev;
	struct ifnet *ifp;
	int error;

	dev = sc->vtnet_dev;
	ifp = sc->vtnet_ifp;

	/* Use the current MAC address. */
	bcopy(IF_LLADDR(ifp), sc->vtnet_hwaddr, ETHER_ADDR_LEN);
	vtnet_set_hwaddr(sc);

	error = vtnet_virtio_reinit(sc);
	if (error)
		return (error);

	vtnet_set_macaddr(sc);
	vtnet_set_active_vq_pairs(sc);

	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
		vtnet_init_rx_filters(sc);

	ifp->if_hwassist = 0;
	if (ifp->if_capenable & IFCAP_TXCSUM)
		ifp->if_hwassist |= VTNET_CSUM_OFFLOAD;

	if (ifp->if_capenable & IFCAP_TSO6)
		ifp->if_hwassist |= CSUM_IP6_TSO;

	if (sc->vtnet_flags & VTNET_FLAG_CTRL_VQ)
		vtnet_init_rx_filters(sc);

	error = vtnet_init_rxtx_queues(sc);
	if (error)
		return (error);

	vtnet_enable_interrupts(sc);
	ifp->if_drv_flags |= IFF_DRV_RUNNING;

	return (0);
}



	vtnet_stop(sc);

	/* Reinitialize with the host. */
	if (vtnet_virtio_reinit(sc) != 0)
		goto fail;
	}

	if (vtnet_reinit(sc) != 0)
		goto fail;

	virtio_reinit_complete(dev);

	vtnet_update_link_status(sc);
	vtnet_enable_interrupts(sc);
	callout_reset(&sc->vtnet_tick_ch, hz, vtnet_tick, sc);

	return;

fail:
	vtnet_stop(sc);
}

static void


	sc = xsc;

#ifdef DEV_NETMAP
	if (!NA(sc->vtnet_ifp)) {
		D("try to attach again");
		vtnet_netmap_attach(sc);
	}
#endif

	VTNET_CORE_LOCK(sc);
	vtnet_init_locked(sc);
	VTNET_CORE_UNLOCK(sc);

static void
vtnet_free_ctrl_vq(struct vtnet_softc *sc)
{
	struct virtqueue *vq;

	vq = sc->vtnet_ctrl_vq;

	/*
	 * The control virtqueue is only polled and therefore it should
	 * already be empty.
	 */
	KASSERT(virtqueue_empty(vq),
	    ("%s: ctrl vq %p not empty", __func__, vq));
}

static void


	vq = sc->vtnet_ctrl_vq;

	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ);
	VTNET_CORE_LOCK_ASSERT(sc);
	KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_VQ,
	    ("%s: CTRL_VQ feature not negotiated", __func__));

	if (!virtqueue_empty(vq))
		return;
	if (virtqueue_enqueue(vq, cookie, sg, readable, writable) != 0)
		return;

	/*
	 * Poll for the response, but the command is likely already
	 * done when we return from the notify.
	 */
	virtqueue_notify(vq);
	virtqueue_poll(vq, NULL);
		virtqueue_poll(vq, NULL);
	}
}

static int
vtnet_ctrl_mac_cmd(struct vtnet_softc *sc, uint8_t *hwaddr)
{
	struct virtio_net_ctrl_hdr hdr __aligned(2);
	struct sglist_seg segs[3];
	struct sglist sg;
	uint8_t ack;
		struct virtio_net_ctrl_hdr hdr __aligned(2);
		uint8_t pad1;
		uint8_t addr[ETHER_ADDR_LEN] __aligned(8);
		uint8_t pad2;
		uint8_t ack;
	} s;
	int error;

	hdr.class = VIRTIO_NET_CTRL_MAC;
	hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
	ack = VIRTIO_NET_ERR;

	sglist_init(&sg, 3, segs);
	s.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
	bcopy(hwaddr, &s.addr[0], ETHER_ADDR_LEN);
	s.ack = VIRTIO_NET_ERR;

	sglist_init(&sg, nitems(segs), segs);
	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, &s.addr[0], ETHER_ADDR_LEN);
	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
	MPASS(error == 0 && sg.sg_nseg == nitems(segs));

	if (error == 0)
		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);

	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}

static int
vtnet_ctrl_guest_offloads(struct vtnet_softc *sc, uint64_t offloads)
{
	struct sglist_seg segs[3];
	struct sglist sg;
	struct {
		struct virtio_net_ctrl_hdr hdr __aligned(2);
		uint8_t pad1;
		uint64_t offloads __aligned(8);
		uint8_t pad2;
		uint8_t ack;
	} s;
	int error;

	error = 0;
	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, hwaddr, ETHER_ADDR_LEN);
	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
	KASSERT(error == 0 && sg.sg_nseg == 3,
	    ("%s: error %d adding set MAC msg to sglist", __func__, error));

	vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);
	s.hdr.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET;
	s.offloads = vtnet_gtoh64(sc, offloads);
	s.ack = VIRTIO_NET_ERR;

	return (ack == VIRTIO_NET_OK ? 0 : EIO);
	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, &s.offloads, sizeof(uint64_t));
	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
	MPASS(error == 0 && sg.sg_nseg == nitems(segs));

	if (error == 0)
		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);

	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}

static int

	struct sglist_seg segs[3];
	struct sglist sg;
	struct {
		struct virtio_net_ctrl_hdr hdr;
		uint8_t pad1;
		struct virtio_net_ctrl_mq mq;
		uint8_t pad2;
		uint8_t ack;
	} s __aligned(2);
	int error;

	error = 0;
	MPASS(sc->vtnet_flags & VTNET_FLAG_MQ);

	s.hdr.class = VIRTIO_NET_CTRL_MQ;
	s.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
	s.mq.virtqueue_pairs = npairs;
	s.ack = VIRTIO_NET_ERR;

	sglist_init(&sg, 3, segs);
	error = 0;
	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, &s.mq, sizeof(struct virtio_net_ctrl_mq));
	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
	KASSERT(error == 0 && sg.sg_nseg == 3,
	    ("%s: error %d adding MQ message to sglist", __func__, error));

	vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);

	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}

static int
vtnet_ctrl_rx_cmd(struct vtnet_softc *sc, int cmd, int on)
{
	struct sglist_seg segs[3];
	struct sglist sg;
	struct {
		struct virtio_net_ctrl_hdr hdr;
		uint8_t pad1;
		uint8_t onoff;
		uint8_t pad2;
		uint8_t ack;
	} s __aligned(2);
	int error;

	KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
	    ("%s: CTRL_RX feature not negotiated", __func__));

	s.hdr.class = VIRTIO_NET_CTRL_RX;
	s.hdr.cmd = cmd;
	s.onoff = !!on;
	s.ack = VIRTIO_NET_ERR;

	sglist_init(&sg, 3, segs);
	error = 0;
	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, &s.onoff, sizeof(uint8_t));
	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
	KASSERT(error == 0 && sg.sg_nseg == 3,
	    ("%s: error %d adding Rx message to sglist", __func__, error));

	vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);

	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}

static int
vtnet_set_promisc(struct vtnet_softc *sc, int on)
{

	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_PROMISC, on));
}

static int
vtnet_set_allmulti(struct vtnet_softc *sc, int on)
{

	return (vtnet_ctrl_rx_cmd(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, on));
}

/*
 * The device defaults to promiscuous mode for backwards compatibility.
 * Turn it off at attach time if possible.
 */
static void
vtnet_attach_disable_promisc(struct vtnet_softc *sc)
{
	struct ifnet *ifp;

	ifp = sc->vtnet_ifp;

	VTNET_CORE_LOCK(sc);
	if ((sc->vtnet_flags & VTNET_FLAG_CTRL_RX) == 0) {
		ifp->if_flags |= IFF_PROMISC;
	} else if (vtnet_set_promisc(sc, 0) != 0) {
		ifp->if_flags |= IFF_PROMISC;
		device_printf(sc->vtnet_dev,
		    "cannot disable default promiscuous mode\n");
	}
	VTNET_CORE_UNLOCK(sc);
}

static void
vtnet_rx_filter(struct vtnet_softc *sc)
{
	device_t dev;


	VTNET_CORE_LOCK_ASSERT(sc);

	if (vtnet_set_promisc(sc, ifp->if_flags & IFF_PROMISC) != 0)
		device_printf(dev, "cannot %s promiscuous mode\n",
		    ifp->if_flags & IFF_PROMISC ? "enable" : "disable");
	}

	if (vtnet_set_allmulti(sc, ifp->if_flags & IFF_ALLMULTI) != 0)
		device_printf(dev, "cannot %s all-multicast mode\n",
		    ifp->if_flags & IFF_ALLMULTI ? "enable" : "disable");
	}
}

static void


	ifp = sc->vtnet_ifp;
	filter = sc->vtnet_mac_filter;

	ucnt = 0;
	mcnt = 0;
	promisc = 0;
	allmulti = 0;
	error = 0;

	MPASS(sc->vtnet_flags & VTNET_FLAG_CTRL_RX);
	VTNET_CORE_LOCK_ASSERT(sc);
	KASSERT(sc->vtnet_flags & VTNET_FLAG_CTRL_RX,
	    ("%s: CTRL_RX feature not negotiated", __func__));

	/* Unicast MAC addresses: */
	if_addr_rlock(ifp);

	}
	if_addr_runlock(ifp);

	if (promisc != 0) {
		filter->vmf_unicast.nentries = 0;
		if_printf(ifp, "more than %d MAC addresses assigned, "
		    "falling back to promiscuous mode\n",
		    VTNET_MAX_MAC_ENTRIES);
	} else
		filter->vmf_unicast.nentries = ucnt;

	/* Multicast MAC addresses: */
	if_maddr_rlock(ifp);
	CK_STAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {

	}
	if_maddr_runlock(ifp);

	if (promisc != 0) {
		if_printf(ifp, "cannot filter more than %d MAC addresses, "
		    "falling back to promiscuous mode\n",
		    VTNET_MAX_MAC_ENTRIES);
		ucnt = 0;
	}
	if (allmulti != 0) {
		filter->vmf_multicast.nentries = 0;
		if_printf(ifp, "more than %d multicast MAC addresses "
		    "assigned, falling back to all-multicast mode\n",
		    VTNET_MAX_MAC_ENTRIES);
	} else
		filter->vmf_multicast.nentries = mcnt;

	if (promisc != 0 && allmulti != 0)
		goto out;

	filter->vmf_unicast.nentries = vtnet_gtoh32(sc, ucnt);
	filter->vmf_multicast.nentries = vtnet_gtoh32(sc, mcnt);

	hdr.class = VIRTIO_NET_CTRL_MAC;
	hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
	ack = VIRTIO_NET_ERR;

	sglist_init(&sg, 4, segs);
	error = 0;
	error |= sglist_append(&sg, &hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, &filter->vmf_unicast,
	    sizeof(uint32_t) + filter->vmf_unicast.nentries * ETHER_ADDR_LEN);
	error |= sglist_append(&sg, &filter->vmf_multicast,
	    sizeof(uint32_t) + filter->vmf_multicast.nentries * ETHER_ADDR_LEN);
	error |= sglist_append(&sg, &ack, sizeof(uint8_t));
	KASSERT(error == 0 && sg.sg_nseg == 4,
	    ("%s: error %d adding MAC filter msg to sglist", __func__, error));

	vtnet_exec_ctrl_cmd(sc, &ack, &sg, sg.sg_nseg - 1, 1);

	if (ack != VIRTIO_NET_OK)
		if_printf(ifp, "error setting host MAC filter table\n");


	struct sglist_seg segs[3];
	struct sglist sg;
	struct {
		struct virtio_net_ctrl_hdr hdr;
		uint8_t pad1;
		uint16_t tag;
		uint8_t pad2;
		uint8_t ack;
	} s __aligned(2);
	int error;

	error = 0;
	MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);

	s.hdr.class = VIRTIO_NET_CTRL_VLAN;
	s.hdr.cmd = add ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
	s.tag = tag;
	s.ack = VIRTIO_NET_ERR;

	sglist_init(&sg, 3, segs);
	error = 0;
	error |= sglist_append(&sg, &s.hdr, sizeof(struct virtio_net_ctrl_hdr));
	error |= sglist_append(&sg, &s.tag, sizeof(uint16_t));
	error |= sglist_append(&sg, &s.ack, sizeof(uint8_t));
	KASSERT(error == 0 && sg.sg_nseg == 3,
	    ("%s: error %d adding VLAN message to sglist", __func__, error));

	vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);
		vtnet_exec_ctrl_cmd(sc, &s.ack, &sg, sg.sg_nseg - 1, 1);

	return (s.ack == VIRTIO_NET_OK ? 0 : EIO);
}

static void
vtnet_rx_filter_vlan(struct vtnet_softc *sc)
{
	int i, bit;
	uint32_t w;
	uint16_t tag;
	int i, bit;

	MPASS(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER);
	VTNET_CORE_LOCK_ASSERT(sc);
	KASSERT(sc->vtnet_flags & VTNET_FLAG_VLAN_FILTER,
	    ("%s: VLAN_FILTER feature not negotiated", __func__));

	/* Enable the filter for each configured VLAN. */
	for (i = 0; i < VTNET_VLAN_FILTER_NWORDS; i++) {

	vtnet_update_vlan_filter(arg, 0, tag);
}

static void
vtnet_update_speed_duplex(struct vtnet_softc *sc)
{
	struct ifnet *ifp;
	uint32_t speed;

	ifp = sc->vtnet_ifp;

	if ((sc->vtnet_features & VIRTIO_NET_F_SPEED_DUPLEX) == 0)
		return;

	/* BMV: Ignore duplex. */
	speed = virtio_read_dev_config_4(sc->vtnet_dev,
	    offsetof(struct virtio_net_config, speed));
	if (speed != -1)
		ifp->if_baudrate = IF_Mbps(speed);
}

static int
vtnet_is_link_up(struct vtnet_softc *sc)
{
	device_t dev;
	struct ifnet *ifp;
	uint16_t status;

	dev = sc->vtnet_dev;
	ifp = sc->vtnet_ifp;

	if ((ifp->if_capabilities & IFCAP_LINKSTATE) == 0)
		status = VIRTIO_NET_S_LINK_UP;
	else
		status = virtio_read_dev_config_2(dev,
		    offsetof(struct virtio_net_config, status));

	return ((status & VIRTIO_NET_S_LINK_UP) != 0);
}

	int link;

	ifp = sc->vtnet_ifp;

	VTNET_CORE_LOCK_ASSERT(sc);
	link = vtnet_is_link_up(sc);

	/* Notify if the link status has changed. */
	if (link != 0 && sc->vtnet_link_active == 0) {
		vtnet_update_speed_duplex(sc);
		sc->vtnet_link_active = 1;
		if_link_state_change(ifp, LINK_STATE_UP);
	} else if (link == 0 && sc->vtnet_link_active != 0) {

static int
vtnet_ifmedia_upd(struct ifnet *ifp)
{
	struct vtnet_softc *sc;
	struct ifmedia *ifm;

	sc = ifp->if_softc;
	ifm = &sc->vtnet_media;

	if (IFM_TYPE(ifm->ifm_media) != IFM_ETHER)
		return (EINVAL);

	return (0);
}

static void

	VTNET_CORE_LOCK(sc);
	if (vtnet_is_link_up(sc) != 0) {
		ifmr->ifm_status |= IFM_ACTIVE;
		ifmr->ifm_active |= VTNET_MEDIATYPE;
	} else
		ifmr->ifm_active |= IFM_NONE;
	VTNET_CORE_UNLOCK(sc);
}

static void
vtnet_set_hwaddr(struct vtnet_softc *sc)
{

	if (sc->vtnet_flags & VTNET_FLAG_MAC) {
		virtio_read_device_config_array(sc->vtnet_dev,
		    offsetof(struct virtio_net_config, mac),
		    &sc->vtnet_hwaddr[0], sizeof(uint8_t), ETHER_ADDR_LEN);
	} else {
		/* Generate a random locally administered unicast address. */
		sc->vtnet_hwaddr[0] = 0xB2;
		arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
	}
}

static void
vtnet_set_macaddr(struct vtnet_softc *sc)
{
	device_t dev;
	int i;

	dev = sc->vtnet_dev;

	if (sc->vtnet_flags & VTNET_FLAG_CTRL_MAC) {
		if (vtnet_ctrl_mac_cmd(sc, sc->vtnet_hwaddr) != 0)
		if (error)
			device_printf(dev, "unable to set MAC address\n");
	} else if (sc->vtnet_flags & VTNET_FLAG_MAC) {
		for (i = 0; i < ETHER_ADDR_LEN; i++) {

	/* MAC in config is read-only in modern VirtIO. */
	if (!vtnet_modern(sc) && sc->vtnet_flags & VTNET_FLAG_MAC) {
		for (int i = 0; i < ETHER_ADDR_LEN; i++) {
			virtio_write_dev_config_1(dev,
			    offsetof(struct virtio_net_config, mac) + i,
			    sc->vtnet_hwaddr[i]);

}

static void
vtnet_get_hwaddr(struct vtnet_softc *sc)
{
	device_t dev;
	int i;

	dev = sc->vtnet_dev;

	if ((sc->vtnet_flags & VTNET_FLAG_MAC) == 0) {
		/*
		 * Generate a random locally administered unicast address.
		 *
		 * It would be nice to generate the same MAC address across
		 * reboots, but it seems all the hosts currently available
		 * support the MAC feature, so this isn't too important.
		 */
		sc->vtnet_hwaddr[0] = 0xB2;
		arc4rand(&sc->vtnet_hwaddr[1], ETHER_ADDR_LEN - 1, 0);
		vtnet_set_hwaddr(sc);
		return;
	}

	for (i = 0; i < ETHER_ADDR_LEN; i++) {
		sc->vtnet_hwaddr[i] = virtio_read_dev_config_1(dev,
		    offsetof(struct virtio_net_config, mac) + i);
	}
}

static void

}

static void
vtnet_set_tx_intr_threshold(struct vtnet_softc *sc)
{
	int size, thresh;

	size = virtqueue_size(sc->vtnet_txqs[0].vtntx_vq);

	/*
	 * The Tx interrupt is disabled until the queue free count falls
	 * below our threshold. Completed frames are drained from the Tx
	 * virtqueue before transmitting new frames and in the watchdog
	 * callout, so the frequency of Tx interrupts is greatly reduced,
	 * at the cost of not freeing mbufs as quickly as they otherwise
	 * would be.
	 *
	 * N.B. We assume all the Tx queues are the same size.
	 */
	thresh = size / 4;

	/*
	 * Without indirect descriptors, leave enough room for the most
	 * segments we handle.
	 */
	if ((sc->vtnet_flags & VTNET_FLAG_INDIRECT) == 0 &&
	    thresh < sc->vtnet_tx_nsegs)
		thresh = sc->vtnet_tx_nsegs;

	sc->vtnet_tx_intr_thresh = thresh;
}

static void
vtnet_setup_rxq_sysctl(struct sysctl_ctx_list *ctx,
    struct sysctl_oid_list *child, struct vtnet_rxq *rxq)
{

	    &stats->vrxs_csum, "Receive checksum offloaded");
	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum_failed", CTLFLAG_RD,
	    &stats->vrxs_csum_failed, "Receive checksum offload failed");
	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "host_lro", CTLFLAG_RD,
	    &stats->vrxs_host_lro, "Receive host segmentation offloaded");
	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
	    &stats->vrxs_rescheduled,
	    "Receive interrupt handler rescheduled");

	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "csum", CTLFLAG_RD,
	    &stats->vtxs_csum, "Transmit checksum offloaded");
	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "tso", CTLFLAG_RD,
	    &stats->vtxs_tso, "Transmit segmentation offloaded");
	SYSCTL_ADD_UQUAD(ctx, list, OID_AUTO, "rescheduled", CTLFLAG_RD,
	    &stats->vtxs_rescheduled,
	    "Transmit interrupt handler rescheduled");

	tree = device_get_sysctl_tree(dev);
	child = SYSCTL_CHILDREN(tree);

	for (i = 0; i < sc->vtnet_max_vq_pairs; i++) {
		vtnet_setup_rxq_sysctl(ctx, child, &sc->vtnet_rxqs[i]);
		vtnet_setup_txq_sysctl(ctx, child, &sc->vtnet_txqs[i]);
	}

	    CTLFLAG_RD, &stats->rx_task_rescheduled,
	    "Times the receive interrupt task rescheduled itself");

	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_csum_bad_ethtype",
	    CTLFLAG_RD, &stats->tx_csum_bad_ethtype,
	    "Aborted transmit of checksum offloaded buffer with unknown "
	    "Ethernet type");
	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_bad_ethtype",
	    CTLFLAG_RD, &stats->tx_tso_bad_ethtype,
	    "Aborted transmit of TSO buffer with unknown Ethernet type");
	    "protocols");
	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_not_tcp",
	    CTLFLAG_RD, &stats->tx_tso_not_tcp,
	    "Aborted transmit of TSO buffer with non TCP protocol");
	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_tso_without_csum",
	    CTLFLAG_RD, &stats->tx_tso_without_csum,
	    "Aborted transmit of TSO buffer without TCP checksum offload");
	SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "tx_defragged",
	    CTLFLAG_RD, &stats->tx_defragged,
	    "Transmit mbufs defragged");


	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "max_vq_pairs",
	    CTLFLAG_RD, &sc->vtnet_max_vq_pairs, 0,
	    "Maximum number of supported virtqueue pairs");
	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "requested_vq_pairs",
	    CTLFLAG_RD, &sc->vtnet_requested_vq_pairs, 0,
	    "Requested number of virtqueue pairs");
	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "act_vq_pairs",
	    CTLFLAG_RD, &sc->vtnet_act_vq_pairs, 0,
	    "Number of active virtqueue pairs");

	vtnet_setup_stat_sysctl(ctx, child, sc);
}

static void
vtnet_load_tunables(struct vtnet_softc *sc)
{

	sc->vtnet_lro_entry_count = vtnet_tunable_int(sc,
	    "lro_entry_count", vtnet_lro_entry_count);
	if (sc->vtnet_lro_entry_count < TCP_LRO_ENTRIES)
		sc->vtnet_lro_entry_count = TCP_LRO_ENTRIES;

	sc->vtnet_lro_mbufq_depth = vtnet_tunable_int(sc,
	    "lro_mbufq_depeth", vtnet_lro_mbufq_depth);
}

static int
vtnet_rxq_enable_intr(struct vtnet_rxq *rxq)
{

static void
vtnet_enable_rx_interrupts(struct vtnet_softc *sc)
{
	struct vtnet_rxq *rxq;
	int i;

	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
		vtnet_rxq_enable_intr(&sc->vtnet_rxqs[i]);
		if (vtnet_rxq_enable_intr(rxq) != 0)
			taskqueue_enqueue(rxq->vtnrx_tq, &rxq->vtnrx_intrtask);
	}
}

static void

{
	int i;

	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
		vtnet_rxq_disable_intr(&sc->vtnet_rxqs[i]);
}


{
	int i;

	for (i = 0; i < sc->vtnet_act_vq_pairs; i++)
		vtnet_txq_disable_intr(&sc->vtnet_txqs[i]);
}


	sc = if_getsoftc(ifp);

	VTNET_CORE_LOCK(sc);
	*nrxr = sc->vtnet_max_vq_pairs;
	*ncl = NETDUMP_MAX_IN_FLIGHT;
	*clsize = sc->vtnet_rx_clsize;
	VTNET_CORE_UNLOCK(sc);

	/*

		return (EBUSY);

	(void)vtnet_txq_eof(&sc->vtnet_txqs[0]);
	for (i = 0; i < sc->vtnet_max_vq_pairs; i++)
		(void)vtnet_rxq_eof(&sc->vtnet_rxqs[i]);
	return (0);
}




	uint64_t	rx_csum_bad_ipproto;
	uint64_t	rx_csum_bad_offset;
	uint64_t	rx_csum_bad_proto;
	uint64_t	tx_csum_bad_ethtype;
	uint64_t	tx_tso_bad_ethtype;
	uint64_t	tx_tso_not_tcp;
	uint64_t	tx_tso_without_csum;
	uint64_t	tx_defragged;
	uint64_t	tx_defrag_failed;


	uint64_t	vrxs_ierrors;	/* if_ierrors */
	uint64_t	vrxs_csum;
	uint64_t	vrxs_csum_failed;
	uint64_t	vrxs_host_lro;
	uint64_t	vrxs_rescheduled;
};


	struct vtnet_rxq_stats	 vtnrx_stats;
	struct taskqueue	*vtnrx_tq;
	struct task		 vtnrx_intrtask;
	struct lro_ctrl		 vtnrx_lro;
#ifdef DEV_NETMAP
	struct virtio_net_hdr_mrg_rxbuf vtnrx_shrhdr;
#endif  /* DEV_NETMAP */

#endif
	int			 vtntx_id;
	int			 vtntx_watchdog;
	int			 vtntx_intr_threshold;
	struct vtnet_txq_stats	 vtntx_stats;
	struct taskqueue	*vtntx_tq;
	struct task		 vtntx_intrtask;

	struct ifnet		*vtnet_ifp;
	struct vtnet_rxq	*vtnet_rxqs;
	struct vtnet_txq	*vtnet_txqs;
	uint64_t		 vtnet_features;

	uint32_t		 vtnet_flags;
#define VTNET_FLAG_SUSPENDED	 0x0001
#define VTNET_FLAG_MAC		 0x0002
#define VTNET_FLAG_CTRL_VQ	 0x0004
#define VTNET_FLAG_CTRL_RX	 0x0008

#define VTNET_FLAG_TSO_ECN	 0x0040
#define VTNET_FLAG_MRG_RXBUFS	 0x0080
#define VTNET_FLAG_LRO_NOMRG	 0x0100
#define VTNET_FLAG_MULTIQ	 0x0200
#define VTNET_FLAG_INDIRECT	 0x0400
#define VTNET_FLAG_EVENT_IDX	 0x0800
#define VTNET_FLAG_SUSPENDED	 0x1000
#define VTNET_FLAG_FIXUP_NEEDS_CSUM 0x2000
#define VTNET_FLAG_SW_LRO	 0x4000

	int			 vtnet_link_active;
	int			 vtnet_hdr_size;
	int			 vtnet_rx_nmbufs;
	int			 vtnet_rx_clustersz;
	int			 vtnet_rx_nsegs;
	int			 vtnet_rx_process_limit;
	int			 vtnet_rx_nsegs;
	int			 vtnet_rx_nmbufs;
	int			 vtnet_rx_clsize;
	int			 vtnet_rx_new_clsize;
	int			 vtnet_tx_intr_thresh;
	int			 vtnet_tx_nsegs;
	int			 vtnet_if_flags;
	int			 vtnet_act_vq_pairs;
	int			 vtnet_max_vq_pairs;
	int			 vtnet_requested_vq_pairs;

	struct virtqueue	*vtnet_ctrl_vq;
	struct vtnet_mac_filter	*vtnet_mac_filter;
	uint32_t		*vtnet_vlan_filter;

	uint64_t		 vtnet_features;
	struct vtnet_statistics	 vtnet_stats;
	struct callout		 vtnet_tick_ch;
	struct ifmedia		 vtnet_media;

	char			 vtnet_hwaddr[ETHER_ADDR_LEN];
};

static bool
vtnet_modern(struct vtnet_softc *sc)
{
	return ((sc->vtnet_flags & VTNET_FLAG_MODERN) != 0);
}

static bool
vtnet_software_lro(struct vtnet_softc *sc)
{
	return ((sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0);
}

/*
 * Maximum number of queue pairs we will autoconfigure to.
 */
#define VTNET_MAX_QUEUE_PAIRS	8

/*
 * Additional completed entries can appear in a virtqueue before we can

#define VTNET_NOTIFY_RETRIES		4

/*
 * Fake the media type. The host does not provide us with any real media
 * information.
 */
#define VTNET_MEDIATYPE		 (IFM_ETHER | IFM_10G_T | IFM_FDX)

/*
 * Number of words to allocate for the VLAN shadow table. There is one
 * bit for each VLAN.
 */
#define VTNET_VLAN_FILTER_NWORDS	(4096 / 32)

/* We depend on these being the same size (and same layout). */
CTASSERT(sizeof(struct virtio_net_hdr_mrg_rxbuf) ==
    sizeof(struct virtio_net_hdr_v1));

/*
 * When mergeable buffers are not negotiated, the vtnet_rx_header structure
 * below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
 * both keep the VirtIO header and the data non-contiguous and to keep the
 * frame's payload 4 byte aligned.
 * VIRTIO_F_ANY_LAYOUT feature was negotiated (but we don't support that yet).
 *
 * When mergeable buffers are negotiated, the host puts the VirtIO header in
 * the beginning of the first mbuf's data.
 */
#define VTNET_RX_HEADER_PAD	4
struct vtnet_rx_header {

	union {
		struct virtio_net_hdr		hdr;
		struct virtio_net_hdr_mrg_rxbuf	mhdr;
		struct virtio_net_hdr_v1	v1hdr;
	} vth_uhdr;

	struct mbuf *vth_mbuf;

 */
#define VTNET_MAX_MAC_ENTRIES	128

/*
 * The driver version of struct virtio_net_ctrl_mac but with our predefined
 * number of MAC addresses allocated. This structure is shared with the host,
 * so nentries field is in the correct VirtIO endianness.
 */
struct vtnet_mac_table {
	uint32_t	nentries;
	uint8_t		macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];

    (VTNET_CSUM_OFFLOAD | VTNET_CSUM_OFFLOAD_IPV6 | CSUM_TSO)

/* Features desired/implemented by this driver. */
#define VTNET_FEATURES \
    (VIRTIO_NET_F_MAC			| \
     VIRTIO_NET_F_STATUS		| \
     VIRTIO_NET_F_CTRL_GUEST_OFFLOADS	| \
     VIRTIO_NET_F_MTU			| \
     VIRTIO_NET_F_CTRL_VQ		| \
     VIRTIO_NET_F_CTRL_RX		| \
     VIRTIO_NET_F_CTRL_MAC_ADDR		| \
     VIRTIO_NET_F_CTRL_VLAN		| \
     VIRTIO_NET_F_CSUM			| \
     VIRTIO_NET_F_GSO			| \
     VIRTIO_NET_F_HOST_TSO4		| \
     VIRTIO_NET_F_HOST_TSO6		| \
     VIRTIO_NET_F_HOST_ECN		| \

     VIRTIO_NET_F_GUEST_ECN		| \
     VIRTIO_NET_F_MRG_RXBUF		| \
     VIRTIO_NET_F_MQ			| \
     VIRTIO_NET_F_SPEED_DUPLEX		| \
     VIRTIO_RING_F_EVENT_IDX		| \
     VIRTIO_RING_F_INDIRECT_DESC)

#define VTNET_MODERN_FEATURES (VTNET_COMMON_FEATURES)
#define VTNET_LEGACY_FEATURES (VTNET_COMMON_FEATURES | VIRTIO_NET_F_GSO)

/*
 * The VIRTIO_NET_F_HOST_TSO[46] features permit us to send the host
 * frames larger than 1514 bytes.


/*
 * The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
 * frames larger than 1514 bytes. We do not yet support software LRO
 * via tcp_lro_rx().
 */
#define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 | \
    VIRTIO_NET_F_GUEST_TSO6 | VIRTIO_NET_F_GUEST_ECN)

#define VTNET_MIN_MTU		68
#define VTNET_MAX_MTU		65536
#define VTNET_MAX_RX_SIZE	65550

/*
 * Used to preallocate the Vq indirect descriptors. The first segment
 * is reserved for the header, except for mergeable buffers since the
 * header is placed inline with the data.
 */
#define VTNET_MRG_RX_SEGS	1
#define VTNET_MIN_RX_SEGS	2
#define VTNET_MAX_RX_SEGS	34
#define VTNET_MIN_TX_SEGS	32
#define VTNET_MAX_TX_SEGS	64

/*
 * Assert we can receive and transmit the maximum with regular
 * size clusters.
 */
CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);

/*
 * Number of slots in the Tx bufrings. This value matches most other
 * multiqueue drivers.
 */
#define VTNET_DEFAULT_BUFRING_SIZE	4096

/*
 * Determine how many mbufs are in each receive buffer. For LRO without
 * mergeable buffers, we must allocate an mbuf chain large enough to
 * hold both the vtnet_rx_header and the maximum receivable data.
 */
#define VTNET_NEEDED_RX_MBUFS(_sc, _clsize)				\
	((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 :		\
	    howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE,	\
	        (_clsize))

#define VTNET_CORE_MTX(_sc)		&(_sc)->vtnet_mtx
#define VTNET_CORE_LOCK(_sc)		mtx_lock(VTNET_CORE_MTX((_sc)))




#define _VIRTIO_NET_H

/* The feature bitmap for virtio net */
#define VIRTIO_NET_F_CSUM	0x00001 /* Host handles pkts w/ partial csum */
#define VIRTIO_NET_F_GUEST_CSUM 0x00002 /* Guest handles pkts w/ partial csum*/
#define VIRTIO_NET_F_MAC	0x00020 /* Host has given MAC address. */
#define VIRTIO_NET_F_GSO	0x00040 /* Host handles pkts w/ any GSO type */
#define VIRTIO_NET_F_GUEST_TSO4	0x00080 /* Guest can handle TSOv4 in. */
#define VIRTIO_NET_F_GUEST_TSO6	0x00100 /* Guest can handle TSOv6 in. */
#define VIRTIO_NET_F_GUEST_ECN	0x00200 /* Guest can handle TSO[6] w/ ECN in.*/
#define VIRTIO_NET_F_GUEST_UFO	0x00400 /* Guest can handle UFO in. */
#define VIRTIO_NET_F_HOST_TSO4	0x00800 /* Host can handle TSOv4 in. */
#define VIRTIO_NET_F_HOST_TSO6	0x01000 /* Host can handle TSOv6 in. */
#define VIRTIO_NET_F_HOST_ECN	0x02000 /* Host can handle TSO[6] w/ ECN in. */
#define VIRTIO_NET_F_HOST_UFO	0x04000 /* Host can handle UFO in. */
#define VIRTIO_NET_F_MRG_RXBUF	0x08000 /* Host can merge receive buffers. */
#define VIRTIO_NET_F_STATUS	0x10000 /* virtio_net_config.status available*/
#define VIRTIO_NET_F_CTRL_VQ	0x20000 /* Control channel available */
#define VIRTIO_NET_F_CTRL_RX	0x40000 /* Control channel RX mode support */
#define VIRTIO_NET_F_CTRL_VLAN	0x80000 /* Control channel VLAN filtering */
#define VIRTIO_NET_F_CTRL_RX_EXTRA 0x100000 /* Extra RX mode control support */
#define VIRTIO_NET_F_GUEST_ANNOUNCE 0x200000 /* Announce device on network */
#define VIRTIO_NET_F_MQ		0x400000 /* Device supports RFS */
#define VIRTIO_NET_F_CTRL_MAC_ADDR 0x800000 /* Set MAC address */
#define VIRTIO_NET_F_MQ			 0x400000 /* Device supports Receive Flow Steering */
#define VIRTIO_NET_F_CTRL_MAC_ADDR	 0x800000 /* Set MAC address */
#define VIRTIO_NET_F_SPEED_DUPLEX	 (1ULL << 63) /* Device set linkspeed and duplex */

#define VIRTIO_NET_S_LINK_UP	1	/* Link is up */
#define VIRTIO_NET_S_ANNOUNCE	2	/* Announcement is needed */

struct virtio_net_config {
	/* The config defining mac address (if VIRTIO_NET_F_MAC) */

	 * Legal values are between 1 and 0x8000.
	 */
	uint16_t	max_virtqueue_pairs;
	/* Default maximum transmit unit advice */
	uint16_t	mtu;
	/*
	 * speed, in units of 1Mb. All values 0 to INT_MAX are legal.
	 * Any other value stands for unknown.
	 */
	uint32_t	speed;
	/*
	 * 0x00 - half duplex
	 * 0x01 - full duplex
	 * Any other value stands for unknown.
	 */
	uint8_t		duplex;
} __packed;

/*
 * This is the first element of the scatter-gather list.  If you don't
 * specify GSO or CSUM features, you can simply ignore the header.
 *
 * This is bitwise-equivalent to the legacy struct virtio_net_hdr_mrg_rxbuf,
 * only flattened.
 */
struct virtio_net_hdr {
#define VIRTIO_NET_HDR_F_NEEDS_CSUM	1	/* Use csum_start,csum_offset*/
#define VIRTIO_NET_HDR_F_DATA_VALID	2	/* Csum is valid */
	uint8_t	flags;
#define VIRTIO_NET_HDR_GSO_NONE		0	/* Not a GSO frame */
#define VIRTIO_NET_HDR_GSO_TCPV4	1	/* GSO frame, IPv4 TCP (TSO) */
#define VIRTIO_NET_HDR_GSO_UDP		3	/* GSO frame, IPv4 UDP (UFO) */

	uint16_t gso_size;	/* Bytes to append to hdr_len per frame */
	uint16_t csum_start;	/* Position to start checksumming from */
	uint16_t csum_offset;	/* Offset after that to place checksum */
	uint16_t num_buffers;	/* Number of merged rx buffers */
};

/*
 * This header comes first in the scatter-gather list.
 * For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated, it must
 * be the first element of the scatter-gather list.  If you don't
 * specify GSO or CSUM features, you can simply ignore the header.
 */
struct virtio_net_hdr {
	/* See VIRTIO_NET_HDR_F_* */
	uint8_t	flags;
	/* See VIRTIO_NET_HDR_GSO_* */
	uint8_t gso_type;
	uint16_t hdr_len;	/* Ethernet + IP + tcp/udp hdrs */
	uint16_t gso_size;	/* Bytes to append to hdr_len per frame */
	uint16_t csum_start;	/* Position to start checksumming from */
	uint16_t csum_offset;	/* Offset after that to place checksum */
};

/*
 * This is the version of the header to use when the MRG_RXBUF
 * feature has been negotiated.
 */

#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET		0
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN		1
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX		0x8000

/*
 * Control network offloads
 *
 * Reconfigures the network offloads that Guest can handle.
 *
 * Available with the VIRTIO_NET_F_CTRL_GUEST_OFFLOADS feature bit.
 *
 * Command data format matches the feature bit mask exactly.
 *
 * See VIRTIO_NET_F_GUEST_* for the list of offloads
 * that can be enabled/disabled.
 */
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS		5
#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET	0

#endif /* _VIRTIO_NET_H */




/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without

#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/sbuf.h>
#include <sys/sysctl.h>
#include <sys/module.h>
#include <sys/malloc.h>


#include <dev/virtio/virtio.h>
#include <dev/virtio/virtqueue.h>
#include <dev/virtio/pci/virtio_pci.h>
#include <dev/virtio/pci/virtio_pci_var.h>

#include "virtio_bus_if.h"
#include "virtio_if.h"

struct vtpci_interrupt {
	struct resource		*vti_irq;
	int			 vti_rid;
	void			*vti_handler;
};

struct vtpci_virtqueue {
	struct virtqueue	*vtv_vq;
	int			 vtv_no_intr;
};

struct vtpci_softc {
	device_t			 vtpci_dev;
	struct resource			*vtpci_res;
	struct resource			*vtpci_msix_res;
	uint64_t			 vtpci_features;
	uint32_t			 vtpci_flags;
#define VTPCI_FLAG_NO_MSI		0x0001
#define VTPCI_FLAG_NO_MSIX		0x0002
#define VTPCI_FLAG_LEGACY		0x1000
#define VTPCI_FLAG_MSI			0x2000
#define VTPCI_FLAG_MSIX			0x4000
#define VTPCI_FLAG_SHARED_MSIX		0x8000
#define VTPCI_FLAG_ITYPE_MASK		0xF000

	/* This "bus" will only ever have one child. */
	device_t			 vtpci_child_dev;
	struct virtio_feature_desc	*vtpci_child_feat_desc;

	int				 vtpci_nvqs;
	struct vtpci_virtqueue		*vtpci_vqs;

	/*
	 * Ideally, each virtqueue that the driver provides a callback for will
	 * receive its own MSIX vector. If there are not sufficient vectors
	 * available, then attempt to have all the VQs share one vector. For
	 * MSIX, the configuration changed notifications must be on their own
	 * vector.
	 *
	 * If MSIX is not available, we will attempt to have the whole device
	 * share one MSI vector, and then, finally, one legacy interrupt.
	 */
	struct vtpci_interrupt		 vtpci_device_interrupt;
	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
	int				 vtpci_nmsix_resources;
};

static int	vtpci_probe(device_t);
static int	vtpci_attach(device_t);
static int	vtpci_detach(device_t);
static int	vtpci_suspend(device_t);
static int	vtpci_resume(device_t);
static int	vtpci_shutdown(device_t);
static void	vtpci_driver_added(device_t, driver_t *);
static void	vtpci_child_detached(device_t, device_t);
static int	vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
static int	vtpci_write_ivar(device_t, device_t, int, uintptr_t);

static uint64_t	vtpci_negotiate_features(device_t, uint64_t);
static int	vtpci_with_feature(device_t, uint64_t);
static int	vtpci_alloc_virtqueues(device_t, int, int,
		    struct vq_alloc_info *);
static int	vtpci_setup_intr(device_t, enum intr_type);
static void	vtpci_stop(device_t);
static int	vtpci_reinit(device_t, uint64_t);
static void	vtpci_reinit_complete(device_t);
static void	vtpci_notify_virtqueue(device_t, uint16_t);
static uint8_t	vtpci_get_status(device_t);
static void	vtpci_set_status(device_t, uint8_t);
static void	vtpci_read_dev_config(device_t, bus_size_t, void *, int);
static void	vtpci_write_dev_config(device_t, bus_size_t, void *, int);

static void	vtpci_describe_features(struct vtpci_softc *, const char *,
		    uint64_t);
static void	vtpci_probe_and_attach_child(struct vtpci_softc *);

static int	vtpci_alloc_msix(struct vtpci_softc *, int);
static int	vtpci_alloc_msi(struct vtpci_softc *);
static int	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
static int	vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
static int	vtpci_alloc_intr_msi(struct vtpci_softc *);
static int	vtpci_alloc_intr_legacy(struct vtpci_softc *);
static int	vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
		    struct vtpci_interrupt *);
static int	vtpci_alloc_intr_resources(struct vtpci_softc *);
		    struct vtpci_interrupt *);

static int	vtpci_setup_legacy_interrupt(struct vtpci_softc *,
static void	vtpci_free_virtqueues(struct vtpci_common *);
static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_common *);
static int	vtpci_alloc_intr_resources(struct vtpci_common *);
static int	vtpci_setup_intx_interrupt(struct vtpci_common *,
		    enum intr_type);
static int	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
		    enum intr_type);
static int	vtpci_setup_msix_interrupts(struct vtpci_softc *,
static int	vtpci_setup_msix_interrupts(struct vtpci_common *,
		    enum intr_type);
static int	vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);

static int	vtpci_register_msix_vector(struct vtpci_softc *, int,
		    struct vtpci_interrupt *);
static int	vtpci_set_host_msix_vectors(struct vtpci_softc *);
static int	vtpci_reinit_virtqueue(struct vtpci_softc *, int);

static void	vtpci_free_interrupt(struct vtpci_softc *,
		    struct vtpci_interrupt *);
static void	vtpci_free_interrupts(struct vtpci_softc *);
static void	vtpci_free_virtqueues(struct vtpci_softc *);
static void	vtpci_release_child_resources(struct vtpci_softc *);
static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
static void	vtpci_reset(struct vtpci_softc *);

static void	vtpci_select_virtqueue(struct vtpci_softc *, int);

static void	vtpci_legacy_intr(void *);
static int	vtpci_vq_shared_intr_filter(void *);
static void	vtpci_vq_shared_intr(void *);
static int	vtpci_vq_intr_filter(void *);
static void	vtpci_vq_intr(void *);
static void	vtpci_config_intr(void *);
static void	vtpci_setup_sysctl(struct vtpci_common *);

#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt

#define VIRTIO_PCI_CONFIG(_sc) \
    VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)

/*
 * I/O port read/write wrappers.
 *   - virtio_pci_legacy (vtpcil) for pre-V1 support
 *   - virtio_pci_modern (vtpcim) for V1 support
 */
#define vtpci_read_config_1(sc, o)	bus_read_1((sc)->vtpci_res, (o))
#define vtpci_read_config_2(sc, o)	bus_read_2((sc)->vtpci_res, (o))
#define vtpci_read_config_4(sc, o)	bus_read_4((sc)->vtpci_res, (o))
#define vtpci_write_config_1(sc, o, v)	bus_write_1((sc)->vtpci_res, (o), (v))
#define vtpci_write_config_2(sc, o, v)	bus_write_2((sc)->vtpci_res, (o), (v))
#define vtpci_write_config_4(sc, o, v)	bus_write_4((sc)->vtpci_res, (o), (v))

/* Tunables. */
static int vtpci_disable_msix = 0;
TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);

static device_method_t vtpci_methods[] = {
	/* Device interface. */
	DEVMETHOD(device_probe,			  vtpci_probe),
	DEVMETHOD(device_attach,		  vtpci_attach),
	DEVMETHOD(device_detach,		  vtpci_detach),
	DEVMETHOD(device_suspend,		  vtpci_suspend),
	DEVMETHOD(device_resume,		  vtpci_resume),
	DEVMETHOD(device_shutdown,		  vtpci_shutdown),

	/* Bus interface. */
	DEVMETHOD(bus_driver_added,		  vtpci_driver_added),
	DEVMETHOD(bus_child_detached,		  vtpci_child_detached),
	DEVMETHOD(bus_read_ivar,		  vtpci_read_ivar),
	DEVMETHOD(bus_write_ivar,		  vtpci_write_ivar),

	/* VirtIO bus interface. */
	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_negotiate_features),
	DEVMETHOD(virtio_bus_with_feature,	  vtpci_with_feature),
	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_alloc_virtqueues),
	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_setup_intr),
	DEVMETHOD(virtio_bus_stop,		  vtpci_stop),
	DEVMETHOD(virtio_bus_reinit,		  vtpci_reinit),
	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_reinit_complete),
	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_notify_virtqueue),
	DEVMETHOD(virtio_bus_read_device_config,  vtpci_read_dev_config),
	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),

	DEVMETHOD_END
};
{
	VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq);
}

static driver_t vtpci_driver = {
	"virtio_pci",
	vtpci_methods,
	sizeof(struct vtpci_softc)
};

devclass_t vtpci_devclass;

DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
MODULE_VERSION(virtio_pci, 1);
MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);

static int
vtpci_probe(device_t dev)
{
	char desc[36];
	const char *name;

	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
		return (ENXIO);

	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
		return (ENXIO);

	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
		return (ENXIO);

	name = virtio_device_name(pci_get_subdevice(dev));
	if (name == NULL)
		name = "Unknown";

	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
	device_set_desc_copy(dev, desc);

	return (BUS_PROBE_DEFAULT);
}

static int
vtpci_attach(device_t dev)
    struct vtpci_interrupt *intr)
{
	struct vtpci_softc *sc;
	device_t child;
	int rid;

	sc = device_get_softc(dev);
	sc->vtpci_dev = dev;
{

	cn->vtpci_dev = dev;

	pci_enable_busmaster(dev);

	rid = PCIR_BAR(0);
	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
	    RF_ACTIVE);
	if (sc->vtpci_res == NULL) {
		device_printf(dev, "cannot map I/O space\n");
		return (ENXIO);
	}

	if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
		sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
	if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)
		cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX;

	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
		rid = PCIR_BAR(1);
		sc->vtpci_msix_res = bus_alloc_resource_any(dev,
		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
	}

	if (sc->vtpci_msix_res == NULL)
		sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
{
	device_t dev, child;

	vtpci_reset(sc);

	/* Tell the host we've noticed this device. */
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);

	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
		device_printf(dev, "cannot create child device\n");
		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
		vtpci_detach(dev);
		return (ENOMEM);
	}

	sc->vtpci_child_dev = child;
	vtpci_probe_and_attach_child(sc);

	return (0);
}

static int
vtpci_detach(device_t dev)
{
	struct vtpci_softc *sc;
	device_t child;
	int error;

	sc = device_get_softc(dev);

	if ((child = sc->vtpci_child_dev) != NULL) {
	if (child != NULL) {
		error = device_delete_child(dev, child);
		if (error)
			return (error);
		sc->vtpci_child_dev = NULL;
	}

	vtpci_reset(sc);

	if (sc->vtpci_msix_res != NULL) {
		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
		    sc->vtpci_msix_res);
		sc->vtpci_msix_res = NULL;
	}

	if (sc->vtpci_res != NULL) {
		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
		    sc->vtpci_res);
		sc->vtpci_res = NULL;
	}

	return (0);
}

static int
vtpci_suspend(device_t dev)
{

	return (bus_generic_suspend(dev));

	cn->vtpci_child_feat_desc = NULL;
	cn->vtpci_host_features = 0;
	cn->vtpci_features = 0;
}

static int
vtpci_resume(device_t dev)
{
	int idx, error;

	return (bus_generic_resume(dev));
}
		if (error)
			return (error);
	}

static int
vtpci_shutdown(device_t dev)
{
			return (error);
	}

	(void) bus_generic_shutdown(dev);
	/* Forcibly stop the host device. */
	vtpci_stop(dev);

	return (0);
}

static void
vtpci_driver_added(device_t dev, driver_t *driver)
    uint64_t features)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);
	child = cn->vtpci_child_dev;

	vtpci_probe_and_attach_child(sc);
		return;

	virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc);
}

static void
vtpci_child_detached(device_t dev, device_t child)
    uint64_t child_features, uint64_t host_features)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);
	vtpci_describe_features(cn, "host", host_features);

	vtpci_reset(sc);
	vtpci_release_child_resources(sc);
	 * host all support.
	 */
	features = host_features & child_features;
	features = virtio_filter_transport_features(features);

	cn->vtpci_features = features;
	vtpci_describe_features(cn, "negotiated", features);

	return (features);
}

static int
vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
{
	struct vtpci_softc *sc;
}

	sc = device_get_softc(dev);
vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result)
{
	device_t dev;
	int error;

	if (sc->vtpci_child_dev != child)
		return (ENOENT);

	switch (index) {
	case VIRTIO_IVAR_DEVTYPE:
	case VIRTIO_IVAR_SUBDEVICE:
		*result = pci_get_subdevice(dev);
		break;

	case VIRTIO_IVAR_SUBVENDOR:
		*result = pci_get_subdevice(dev);
		break;
	case VIRTIO_IVAR_MODERN:
		*result = vtpci_is_modern(cn);
		break;
	default:
		return (ENOENT);
	}

	return (0);
}

static int
vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	if (sc->vtpci_child_dev != child)
		return (ENOENT);

	switch (index) {
	case VIRTIO_IVAR_FEATURE_DESC:
		sc->vtpci_child_feat_desc = (void *) value;
		break;
	default:
		return (ENOENT);
	}

	return (0);
}

static uint64_t
vtpci_negotiate_features(device_t dev, uint64_t child_features)
    struct vq_alloc_info *vq_info)
{
	struct vtpci_softc *sc;
	uint64_t host_features, features;

	sc = device_get_softc(dev);

	host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
	vtpci_describe_features(sc, "host", host_features);

	/*
	 * Limit negotiated features to what the driver, virtqueue, and
	 * host all support.
	 * anyways.
	 */
	features = host_features & child_features;
	features = virtqueue_filter_features(features);
	sc->vtpci_features = features;

	vtpci_describe_features(sc, "negotiated", features);
	vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);

	return (features);
}

static int
vtpci_with_feature(device_t dev, uint64_t feature)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	return ((sc->vtpci_features & feature) != 0);
}

static int
vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
    struct vq_alloc_info *vq_info)
{
	struct vtpci_softc *sc;
	struct virtqueue *vq;
	struct vtpci_virtqueue *vqx;
	struct vq_alloc_info *info;
	int idx, error;
	uint16_t size;

	sc = device_get_softc(dev);

	if (sc->vtpci_nvqs != 0)
		return (EALREADY);
	if (nvqs <= 0)
		return (EINVAL);

	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
	    M_DEVBUF, M_NOWAIT | M_ZERO);
	if (sc->vtpci_vqs == NULL)
		return (ENOMEM);

	for (idx = 0; idx < nvqs; idx++) {
		vqx = &sc->vtpci_vqs[idx];
		struct vq_alloc_info *info;
		struct virtqueue *vq;
		bus_size_t notify_offset;
		uint16_t size;

		vqx = &cn->vtpci_vqs[idx];
		info = &vq_info[idx];

		vtpci_select_virtqueue(sc, idx);
		size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);

		error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
		    0xFFFFFFFFUL, info, &vq);
		if (error) {
			device_printf(dev,

			break;
		}

		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
		    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);

		vqx->vtv_vq = *info->vqai_vq = vq;
		vqx->vtv_no_intr = info->vqai_intr == NULL;

		sc->vtpci_nvqs++;
	}

	if (error)
		vtpci_free_virtqueues(sc);

	return (error);
}

static int
vtpci_setup_intr(device_t dev, enum intr_type type)
{
	struct vtpci_softc *sc;
	int attempt, error;

	sc = device_get_softc(dev);

	for (attempt = 0; attempt < 5; attempt++) {
		/*
		 * Start with the most desirable interrupt configuration and
		 * fallback towards less desirable ones.
		 */
		switch (attempt) {
		case 0:
			error = vtpci_alloc_intr_msix_pervq(sc);
			break;
		case 1:
			error = vtpci_alloc_intr_msix_shared(sc);
			break;
		case 2:
			error = vtpci_alloc_intr_msi(sc);
			break;
		case 3:
			error = vtpci_alloc_intr_legacy(sc);
			break;
		default:
			device_printf(dev,
			    "exhausted all interrupt allocation attempts\n");
			return (ENXIO);
		}

		if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
			break;

		vtpci_cleanup_setup_intr_attempt(sc);
	}

	if (bootverbose) {
		if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
			device_printf(dev, "using legacy interrupt\n");
		else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
			device_printf(dev, "using MSI interrupt\n");
		else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
			device_printf(dev, "using shared MSIX interrupts\n");
		else
			device_printf(dev, "using per VQ MSIX interrupts\n");
	}

	return (0);
}

static void
vtpci_stop(device_t dev)
{

	vtpci_reset(device_get_softc(dev));
}

static int
vtpci_reinit(device_t dev, uint64_t features)
{
	struct vtpci_softc *sc;
	int idx, error;

	sc = device_get_softc(dev);

	/*
	 * Redrive the device initialization. This is a bit of an abuse of
	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
	 * play nice.
	 *
	 * We do not allow the host device to change from what was originally
	 * negotiated beyond what the guest driver changed. MSIX state should
	 * not change, number of virtqueues and their size remain the same, etc.
	 * This will need to be rethought when we want to support migration.
	 */

	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
		vtpci_stop(dev);

	/*
	 * Quickly drive the status through ACK and DRIVER. The device
	 * does not become usable again until vtpci_reinit_complete().
	 */
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);

	vtpci_negotiate_features(dev, features);

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
		error = vtpci_reinit_virtqueue(sc, idx);
		if (error)
			return (error);
	}

	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
		error = vtpci_set_host_msix_vectors(sc);
		if (error)
			return (error);
	}

	return (0);
}

static void
vtpci_reinit_complete(device_t dev)
{

	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
}

static void
vtpci_notify_virtqueue(device_t dev, uint16_t queue)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
}

static uint8_t
vtpci_get_status(device_t dev)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
}

static void
vtpci_set_status(device_t dev, uint8_t status)
{
	struct vtpci_softc *sc;

	sc = device_get_softc(dev);

	if (status != VIRTIO_CONFIG_STATUS_RESET)
		status |= vtpci_get_status(dev);

	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
}

static void
vtpci_read_dev_config(device_t dev, bus_size_t offset,
    void *dst, int length)
{
	struct vtpci_softc *sc;
	bus_size_t off;
	uint8_t *d;
	int size;

	sc = device_get_softc(dev);
	off = VIRTIO_PCI_CONFIG(sc) + offset;

	for (d = dst; length > 0; d += size, off += size, length -= size) {
		if (length >= 4) {
			size = 4;
			*(uint32_t *)d = vtpci_read_config_4(sc, off);
		} else if (length >= 2) {
			size = 2;
			*(uint16_t *)d = vtpci_read_config_2(sc, off);
		} else {
			size = 1;
			*d = vtpci_read_config_1(sc, off);
		}
	}
}

static void
vtpci_write_dev_config(device_t dev, bus_size_t offset,
    void *src, int length)
{
	struct vtpci_softc *sc;
	bus_size_t off;
	uint8_t *s;
	int size;

	sc = device_get_softc(dev);
	off = VIRTIO_PCI_CONFIG(sc) + offset;

	for (s = src; length > 0; s += size, off += size, length -= size) {
		if (length >= 4) {
			size = 4;
			vtpci_write_config_4(sc, off, *(uint32_t *)s);
		} else if (length >= 2) {
			size = 2;
			vtpci_write_config_2(sc, off, *(uint16_t *)s);
		} else {
			size = 1;
			vtpci_write_config_1(sc, off, *s);
		}
	}
}

static void
vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
    uint64_t features)
{
	device_t dev, child;

	dev = sc->vtpci_dev;
	child = sc->vtpci_child_dev;

	if (device_is_attached(child) || bootverbose == 0)
		return;

	virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
}

static void
vtpci_probe_and_attach_child(struct vtpci_softc *sc)
{
	device_t dev, child;

	dev = sc->vtpci_dev;
	child = sc->vtpci_child_dev;

	if (child == NULL)
		return;

	if (device_get_state(child) != DS_NOTPRESENT)
		return;

	if (device_probe(child) != 0)
		return;

	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
	if (device_attach(child) != 0) {
		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
		vtpci_reset(sc);
		vtpci_release_child_resources(sc);
		/* Reset status for future attempt. */
		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
	} else {
		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
		VIRTIO_ATTACH_COMPLETED(child);
	}
}

static int
vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
{
	device_t dev;
	int nmsix, cnt, required;

	dev = sc->vtpci_dev;

	/* Allocate an additional vector for the config changes. */
	required = nvectors + 1;


	cnt = required;
	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
		sc->vtpci_nmsix_resources = required;
		return (0);
	}


}

static int
vtpci_alloc_msi(struct vtpci_softc *sc)
{
	device_t dev;
	int nmsi, cnt, required;

	dev = sc->vtpci_dev;
	required = 1;

	nmsi = pci_msi_count(dev);

}

static int
vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
{
	int i, nvectors, error;

	if (vtpci_disable_msix != 0 ||
	    sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
		return (ENOTSUP);

	for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
		if (sc->vtpci_vqs[i].vtv_no_intr == 0)
			nvectors++;
	}

	error = vtpci_alloc_msix(sc, nvectors);
	if (error)
		return (error);

	sc->vtpci_flags |= VTPCI_FLAG_MSIX;

	return (0);
}

static int
vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
{
	int error;

	if (vtpci_disable_msix != 0 ||
	    sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
		return (ENOTSUP);

	error = vtpci_alloc_msix(sc, 1);
	if (error)
		return (error);

	sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;

	return (0);
}

static int
vtpci_alloc_intr_msi(struct vtpci_softc *sc)
{
	int error;

	/* Only BHyVe supports MSI. */
	if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
		return (ENOTSUP);

	error = vtpci_alloc_msi(sc);
	if (error)
		return (error);

	sc->vtpci_flags |= VTPCI_FLAG_MSI;

	return (0);
}

static int
vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
{

	sc->vtpci_flags |= VTPCI_FLAG_LEGACY;

	return (0);
}

static int
vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
    struct vtpci_interrupt *intr)
{
	struct resource *irq;

	irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
	if (irq == NULL)
		return (ENXIO);


	return (0);
}

static void
vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr)
{
	device_t dev;

	dev = cn->vtpci_dev;

	if (intr->vti_handler != NULL) {
		bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
		intr->vti_handler = NULL;
	}

	if (intr->vti_irq != NULL) {
		bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
		    intr->vti_irq);
		intr->vti_irq = NULL;
		intr->vti_rid = -1;
	}
}

static void
vtpci_free_interrupts(struct vtpci_common *cn)
{
	struct vtpci_interrupt *intr;
	int i, nvq_intrs;

	vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt);

	if (cn->vtpci_nmsix_resources != 0) {
		nvq_intrs = cn->vtpci_nmsix_resources - 1;
		cn->vtpci_nmsix_resources = 0;

		if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) {
			for (i = 0; i < nvq_intrs; i++, intr++)
				vtpci_free_interrupt(cn, intr);

			free(cn->vtpci_msix_vq_interrupts, M_DEVBUF);
			cn->vtpci_msix_vq_interrupts = NULL;
		}
	}

	if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
		pci_release_msi(cn->vtpci_dev);

	cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
}

static void
vtpci_free_virtqueues(struct vtpci_common *cn)
{
	struct vtpci_virtqueue *vqx;
	int idx;

	for (idx = 0; idx < cn->vtpci_nvqs; idx++) {
		vtpci_disable_vq(cn, idx);

		vqx = &cn->vtpci_vqs[idx];
		virtqueue_free(vqx->vtv_vq);
		vqx->vtv_vq = NULL;
	}

	free(cn->vtpci_vqs, M_DEVBUF);
	cn->vtpci_vqs = NULL;
	cn->vtpci_nvqs = 0;
}

void
vtpci_release_child_resources(struct vtpci_common *cn)
{

	vtpci_free_interrupts(cn);
	vtpci_free_virtqueues(cn);
}

static void
vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn)
{
	int idx;

	if (cn->vtpci_flags & VTPCI_FLAG_MSIX) {
		vtpci_register_cfg_msix(cn, NULL);

		for (idx = 0; idx < cn->vtpci_nvqs; idx++)
			vtpci_register_vq_msix(cn, idx, NULL);
	}

	vtpci_free_interrupts(cn);
}

static int
vtpci_alloc_intr_resources(struct vtpci_softc *sc)
{
	struct vtpci_interrupt *intr;
	int i, rid, flags, nvq_intrs, error;

	rid = 0;
	flags = RF_ACTIVE;

	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
		rid = 0;
		flags |= RF_SHAREABLE;
	else
		rid = 1;

	/*
	 * For legacy and MSI interrupts, this single resource handles all
	 * interrupts. For MSIX, this resource is used for the configuration
	 * changed interrupt.
	 */
	intr = &sc->vtpci_device_interrupt;
	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
	if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
	if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI))
		return (error);

	/* Subtract one for the configuration changed interrupt. */
	nvq_intrs = sc->vtpci_nmsix_resources - 1;
	 * for all the virtqueues, or one for each virtqueue. Subtract one
	 * below for because of the configuration changed interrupt.
	 */
	nvq_intrs = cn->vtpci_nmsix_resources - 1;

	intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
	    sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
	if (sc->vtpci_msix_vq_interrupts == NULL)
		return (ENOMEM);

	intr = cn->vtpci_msix_vq_interrupts;

	for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
		error = vtpci_alloc_interrupt(sc, rid, flags, intr);
		if (error)
			return (error);
	}

}

static int
vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
{
	struct vtpci_interrupt *intr;
	int error;

	intr = &sc->vtpci_device_interrupt;
	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
	    vtpci_legacy_intr, sc, &intr->vti_handler);

	error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL,
	    vtpci_intx_intr, cn, &intr->vti_handler);

	return (error);
}

static int
vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
{
	struct vtpci_virtqueue *vqx;
	struct vtpci_interrupt *intr;
	int i, error;

	intr = sc->vtpci_msix_vq_interrupts;

	for (i = 0; i < sc->vtpci_nvqs; i++) {
		vqx = &sc->vtpci_vqs[i];

		if (vqx->vtv_no_intr)
			continue;

		error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
		    vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
		    &intr->vti_handler);
		if (error)

}

static int
vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
{
	device_t dev;
	struct vtpci_interrupt *intr;
	int error;

	dev = sc->vtpci_dev;
	intr = &sc->vtpci_device_interrupt;

	error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
	    vtpci_config_intr, sc, &intr->vti_handler);
	if (error)
		return (error);

	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
		intr = sc->vtpci_msix_vq_interrupts;
		error = bus_setup_intr(dev, intr->vti_irq, type,
		    vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
		    &intr->vti_handler);
	} else
		error = vtpci_setup_pervq_msix_interrupts(sc, type);

	return (error ? error : vtpci_set_host_msix_vectors(sc));
}

static int
vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
{
	int error;

	type |= INTR_MPSAFE;
	KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
	    ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));

	error = vtpci_alloc_intr_resources(sc);
	if (error)
		return (error);

	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
		error = vtpci_setup_legacy_interrupt(sc, type);
	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
		error = vtpci_setup_msi_interrupt(sc, type);
	else
		error = vtpci_setup_msix_interrupts(sc, type);

	return (error);
}

static int
vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
    struct vtpci_interrupt *intr)
{
	device_t dev;
	uint16_t vector;

	dev = sc->vtpci_dev;

	if (intr != NULL) {
		/* Map from guest rid to host vector. */
		vector = intr->vti_rid - 1;
	} else
		vector = VIRTIO_MSI_NO_VECTOR;

	vtpci_write_config_2(sc, offset, vector);

	/* Read vector to determine if the host had sufficient resources. */
	if (vtpci_read_config_2(sc, offset) != vector) {
		device_printf(dev,
		    "insufficient host resources for MSIX interrupts\n");
		return (ENODEV);
	}

	return (0);
}

static int
vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
{
	struct vtpci_interrupt *intr, *tintr;
	int idx, offset, error;

	intr = &sc->vtpci_device_interrupt;
	offset = VIRTIO_MSI_CONFIG_VECTOR;

	error = vtpci_register_msix_vector(sc, offset, intr);
	if (error)
		return (error);

	intr = sc->vtpci_msix_vq_interrupts;
	offset = VIRTIO_MSI_QUEUE_VECTOR;

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
		vtpci_select_virtqueue(sc, idx);

		if (sc->vtpci_vqs[idx].vtv_no_intr)
			tintr = NULL;
		else
			tintr = intr;

		error = vtpci_register_msix_vector(sc, offset, tintr);
		if (error)
			break;


		 * For shared MSIX, all the virtqueues share the first
		 * interrupt.
		 */
		if (!sc->vtpci_vqs[idx].vtv_no_intr &&
		    (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
			intr++;
	}


}

static int
vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
{
	struct vtpci_virtqueue *vqx;
	struct virtqueue *vq;
	int error;
	uint16_t size;

	vqx = &sc->vtpci_vqs[idx];
	vq = vqx->vtv_vq;

	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));

	vtpci_select_virtqueue(sc, idx);
	size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);

	error = virtqueue_reinit(vq, size);
	if (error)
		return (error);

	vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
	    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);

	return (0);
		    vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn,
		    &intr->vti_handler);
	} else
		error = vtpci_setup_pervq_msix_interrupts(cn, type);

	return (error ? error : vtpci_set_host_msix_vectors(cn));
}

static void
vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
{
	device_t dev;

	dev = sc->vtpci_dev;
	KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
	    ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags));

	if (intr->vti_handler != NULL) {
		bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
		intr->vti_handler = NULL;
	}

	if (intr->vti_irq != NULL) {
		bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
		    intr->vti_irq);
		intr->vti_irq = NULL;
		intr->vti_rid = -1;
	}

	return (error);
}

static void
vtpci_free_interrupts(struct vtpci_softc *sc)
{
	struct vtpci_interrupt *intr;
	int i, nvq_intrs;

	vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);

	if (sc->vtpci_nmsix_resources != 0) {
		nvq_intrs = sc->vtpci_nmsix_resources - 1;
		sc->vtpci_nmsix_resources = 0;
		 * fallback towards less desirable ones.
		 */
		switch (attempt) {
		case 0:
			error = vtpci_alloc_intr_msix_pervq(cn);
			break;
		case 1:
			error = vtpci_alloc_intr_msix_shared(cn);
			break;
		case 2:
			error = vtpci_alloc_intr_msi(cn);
			break;
		case 3:
			error = vtpci_alloc_intr_intx(cn);
			break;
		default:
			device_printf(dev,
			    "exhausted all interrupt allocation attempts\n");
			return (ENXIO);
		}

		intr = sc->vtpci_msix_vq_interrupts;
		if (intr != NULL) {
			for (i = 0; i < nvq_intrs; i++, intr++)
				vtpci_free_interrupt(sc, intr);

			free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
			sc->vtpci_msix_vq_interrupts = NULL;
		}
	}

	if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
		pci_release_msi(sc->vtpci_dev);
			device_printf(dev, "using legacy interrupt\n");
		else if (cn->vtpci_flags & VTPCI_FLAG_MSI)
			device_printf(dev, "using MSI interrupt\n");
		else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
			device_printf(dev, "using shared MSIX interrupts\n");
		else
			device_printf(dev, "using per VQ MSIX interrupts\n");
	}

	sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
}

static void
vtpci_free_virtqueues(struct vtpci_softc *sc)
{
	struct vtpci_virtqueue *vqx;
	int idx;
	int error;

	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
		vqx = &sc->vtpci_vqs[idx];

		vtpci_select_virtqueue(sc, idx);
		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);

		virtqueue_free(vqx->vtv_vq);
		vqx->vtv_vq = NULL;
	}

	free(sc->vtpci_vqs, M_DEVBUF);
	sc->vtpci_vqs = NULL;
	sc->vtpci_nvqs = 0;
}

static void
vtpci_release_child_resources(struct vtpci_softc *sc)
{

	vtpci_free_interrupts(sc);
	vtpci_free_virtqueues(sc);
}

static void
vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
{
	int idx;

	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
		vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
		    VIRTIO_MSI_NO_VECTOR);

		for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
			vtpci_select_virtqueue(sc, idx);
			vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
			    VIRTIO_MSI_NO_VECTOR);
		}
	}

	vtpci_free_interrupts(sc);
}

static void
vtpci_reset(struct vtpci_softc *sc)
{

	/*
	 * Setting the status to RESET sets the host device to
	 * the original, uninitialized state.
	 */
	vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
}

static void
vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
{

	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
}

static void
vtpci_legacy_intr(void *xsc)
{
	struct vtpci_softc *sc;
	struct vtpci_virtqueue *vqx;
	int i;
	uint8_t isr;

	sc = xsc;
	vqx = &sc->vtpci_vqs[0];

	/* Reading the ISR also clears it. */
	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);

	if (isr & VIRTIO_PCI_ISR_CONFIG)
		vtpci_config_intr(sc);

	if (isr & VIRTIO_PCI_ISR_INTR) {
		for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
		for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {
			if (vqx->vtv_no_intr == 0)
				virtqueue_intr(vqx->vtv_vq);
		}

}

static int
vtpci_vq_shared_intr_filter(void *xsc)
{
	struct vtpci_softc *sc;
	struct vtpci_virtqueue *vqx;
	int i, rc;

	cn = xcn;
	vqx = &cn->vtpci_vqs[0];
	rc = 0;
	sc = xsc;
	vqx = &sc->vtpci_vqs[0];

	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
		if (vqx->vtv_no_intr == 0)
			rc |= virtqueue_intr_filter(vqx->vtv_vq);
	}

}

static void
vtpci_vq_shared_intr(void *xsc)
{
	struct vtpci_softc *sc;
	struct vtpci_virtqueue *vqx;
	int i;

	sc = xsc;
	vqx = &sc->vtpci_vqs[0];

	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
		if (vqx->vtv_no_intr == 0)
			virtqueue_intr(vqx->vtv_vq);
	}

}

static void
vtpci_config_intr(void *xsc)
{
	struct vtpci_softc *sc;
	device_t child;

	sc = xsc;
	child = sc->vtpci_child_dev;

	if (child != NULL)
		VIRTIO_CONFIG_CHANGE(child);
}

static int
vtpci_feature_sysctl(struct sysctl_req *req, struct vtpci_common *cn,
    uint64_t features)
{
	struct sbuf *sb;
	int error;

	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
	if (sb == NULL)
		return (ENOMEM);

	error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc);
	sbuf_delete(sb);

	return (error);
}

static int
vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS)
{
	struct vtpci_common *cn;

	cn = arg1;

	return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features));
}

static int
vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS)
{
	struct vtpci_common *cn;

	cn = arg1;

	return (vtpci_feature_sysctl(req, cn, cn->vtpci_features));
}

static void
vtpci_setup_sysctl(struct vtpci_common *cn)
{
	device_t dev;
	struct sysctl_ctx_list *ctx;
	struct sysctl_oid *tree;
	struct sysctl_oid_list *child;

	dev = cn->vtpci_dev;
	ctx = device_get_sysctl_ctx(dev);
	tree = device_get_sysctl_tree(dev);
	child = SYSCTL_CHILDREN(tree);

	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs",
	    CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues");

	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features",
	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0,
	    vtpci_host_features_sysctl, "A", "Features supported by the host");
	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features",
	    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0,
	    vtpci_negotiated_features_sysctl, "A", "Features negotiated");
}




/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright IBM Corp. 2007
 * All rights reserved.
 *
 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *
 * This header is BSD licensed so anyone can use the definitions to implement
 * compatible drivers/servers.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of IBM nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.h 326022 2017-11-20 19:36:21Z pfg $
 */

#ifndef _VIRTIO_PCI_H
#define _VIRTIO_PCI_H

/* VirtIO PCI vendor/device ID. */
#define VIRTIO_PCI_VENDORID	0x1AF4
#define VIRTIO_PCI_DEVICEID_MIN	0x1000
#define VIRTIO_PCI_DEVICEID_MAX	0x103F
};

/* VirtIO ABI version, this must match exactly. */
#define VIRTIO_PCI_ABI_VERSION	0
	int			 vtv_no_intr;
	int			 vtv_notify_offset;
};

/*
 * VirtIO Header, located in BAR 0.
 */
#define VIRTIO_PCI_HOST_FEATURES  0  /* host's supported features (32bit, RO)*/
#define VIRTIO_PCI_GUEST_FEATURES 4  /* guest's supported features (32, RW) */
#define VIRTIO_PCI_QUEUE_PFN      8  /* physical address of VQ (32, RW) */
#define VIRTIO_PCI_QUEUE_NUM      12 /* number of ring entries (16, RO) */
#define VIRTIO_PCI_QUEUE_SEL      14 /* current VQ selection (16, RW) */
#define VIRTIO_PCI_QUEUE_NOTIFY	  16 /* notify host regarding VQ (16, RW) */
#define VIRTIO_PCI_STATUS         18 /* device status register (8, RW) */
#define VIRTIO_PCI_ISR            19 /* interrupt status register, reading
				      * also clears the register (8, RO) */
/* Only if MSIX is enabled: */
#define VIRTIO_MSI_CONFIG_VECTOR  20 /* configuration change vector (16, RW) */
#define VIRTIO_MSI_QUEUE_VECTOR   22 /* vector for selected VQ notifications
					(16, RW) */

/* The bit of the ISR which indicates a device has an interrupt. */
#define VIRTIO_PCI_ISR_INTR	0x1
/* The bit of the ISR which indicates a device configuration change. */
#define VIRTIO_PCI_ISR_CONFIG	0x2
/* Vector value used to disable MSI for queue. */
#define VIRTIO_MSI_NO_VECTOR	0xFFFF
#define VTPCI_FLAG_MSIX			0x4000
#define VTPCI_FLAG_SHARED_MSIX		0x8000
#define VTPCI_FLAG_ITYPE_MASK		0xF000

/*
 * The remaining space is defined by each driver as the per-driver
 * configuration space.
 */
#define VIRTIO_PCI_CONFIG_OFF(msix_enabled)     ((msix_enabled) ? 24 : 20)

/*
 * How many bits to shift physical queue address written to QUEUE_PFN.
 * 12 is historical, and due to x86 page size.
 */
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT	12
	 * vector.
	 *
	 * If MSIX is not available, attempt to have the whole device share
	 * one MSI vector, and then, finally, one intx interrupt.
	 */
	struct vtpci_interrupt		 vtpci_device_interrupt;
	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
	int				 vtpci_nmsix_resources;
};

/* The alignment to use between consumer and producer parts of vring. */
#define VIRTIO_PCI_VRING_ALIGN	4096
static inline device_t
vtpci_child_device(struct vtpci_common *cn)
{
	return (cn->vtpci_child_dev);
}

static inline bool
vtpci_is_msix_available(struct vtpci_common *cn)
{
	return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0);
}

static inline bool
vtpci_is_msix_enabled(struct vtpci_common *cn)
{
	return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0);
}

static inline bool
vtpci_is_modern(struct vtpci_common *cn)
{
	return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0);
}

static inline int
vtpci_virtqueue_count(struct vtpci_common *cn)
{
	return (cn->vtpci_nvqs);
}

void	vtpci_init(struct vtpci_common *cn, device_t dev, bool modern);
int	vtpci_add_child(struct vtpci_common *cn);
int	vtpci_delete_child(struct vtpci_common *cn);
void	vtpci_child_detached(struct vtpci_common *cn);
int	vtpci_reinit(struct vtpci_common *cn);

uint64_t vtpci_negotiate_features(struct vtpci_common *cn,
	     uint64_t child_features, uint64_t host_features);
int	 vtpci_with_feature(struct vtpci_common *cn, uint64_t feature);

int	vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result);
int	vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value);

int	vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs,
	    struct vq_alloc_info *vq_info);
int	vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type);
void	vtpci_release_child_resources(struct vtpci_common *cn);

#endif /* _VIRTIO_PCI_H */




#-
# Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
# $FreeBSD$

#include <sys/bus.h>
#include <machine/bus.h>

INTERFACE virtio_pci;

HEADER {
struct virtqueue;
struct vtpci_interrupt;
};

METHOD uint8_t read_isr {
	device_t	dev;
};

METHOD uint16_t get_vq_size {
	device_t	dev;
	int		idx;
};

METHOD bus_size_t get_vq_notify_off {
	device_t	dev;
	int		idx;
};

METHOD void set_vq {
	device_t		dev;
	struct virtqueue	*vq;
};

METHOD void disable_vq {
	device_t		 dev;
	int			 idx;
};

METHOD int register_cfg_msix {
	device_t	dev;
	struct vtpci_interrupt *intr;
};

METHOD int register_vq_msix {
	device_t		dev;
	int			idx;
	struct vtpci_interrupt	*intr;
};




/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/* Driver for the legacy VirtIO PCI interface. */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/module.h>

#include <machine/bus.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>

#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>

#include <dev/virtio/virtio.h>
#include <dev/virtio/virtqueue.h>
#include <dev/virtio/pci/virtio_pci.h>
#include <dev/virtio/pci/virtio_pci_legacy_var.h>

#include "virtio_bus_if.h"
#include "virtio_pci_if.h"
#include "virtio_if.h"

struct vtpci_legacy_softc {
	device_t			 vtpci_dev;
	struct vtpci_common		 vtpci_common;
	struct resource			*vtpci_res;
	struct resource			*vtpci_msix_res;
};

static int	vtpci_legacy_probe(device_t);
static int	vtpci_legacy_attach(device_t);
static int	vtpci_legacy_detach(device_t);
static int	vtpci_legacy_suspend(device_t);
static int	vtpci_legacy_resume(device_t);
static int	vtpci_legacy_shutdown(device_t);

static void	vtpci_legacy_driver_added(device_t, driver_t *);
static void	vtpci_legacy_child_detached(device_t, device_t);
static int	vtpci_legacy_read_ivar(device_t, device_t, int, uintptr_t *);
static int	vtpci_legacy_write_ivar(device_t, device_t, int, uintptr_t);

static uint8_t	vtpci_legacy_read_isr(device_t);
static uint16_t	vtpci_legacy_get_vq_size(device_t, int);
static bus_size_t vtpci_legacy_get_vq_notify_off(device_t, int);
static void	vtpci_legacy_set_vq(device_t, struct virtqueue *);
static void	vtpci_legacy_disable_vq(device_t, int);
static int	vtpci_legacy_register_cfg_msix(device_t,
		    struct vtpci_interrupt *);
static int	vtpci_legacy_register_vq_msix(device_t, int idx,
		    struct vtpci_interrupt *);

static uint64_t	vtpci_legacy_negotiate_features(device_t, uint64_t);
static int	vtpci_legacy_with_feature(device_t, uint64_t);
static int	vtpci_legacy_alloc_virtqueues(device_t, int, int,
		    struct vq_alloc_info *);
static int	vtpci_legacy_setup_interrupts(device_t, enum intr_type);
static void	vtpci_legacy_stop(device_t);
static int	vtpci_legacy_reinit(device_t, uint64_t);
static void	vtpci_legacy_reinit_complete(device_t);
static void	vtpci_legacy_notify_vq(device_t, uint16_t, bus_size_t);
static void	vtpci_legacy_read_dev_config(device_t, bus_size_t, void *, int);
static void	vtpci_legacy_write_dev_config(device_t, bus_size_t, void *, int);

static int	vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *);
static void	vtpci_legacy_free_resources(struct vtpci_legacy_softc *);

static void	vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *);

static uint8_t	vtpci_legacy_get_status(struct vtpci_legacy_softc *);
static void	vtpci_legacy_set_status(struct vtpci_legacy_softc *, uint8_t);
static void	vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *, int);
static void	vtpci_legacy_reset(struct vtpci_legacy_softc *);

#define VIRTIO_PCI_LEGACY_CONFIG(_sc) \
    VIRTIO_PCI_CONFIG_OFF(vtpci_is_msix_enabled(&(_sc)->vtpci_common))

/*
 * I/O port read/write wrappers.
 */
#define vtpci_legacy_read_config_1(sc, o)	bus_read_1((sc)->vtpci_res, (o))
#define vtpci_legacy_read_config_2(sc, o)	bus_read_2((sc)->vtpci_res, (o))
#define vtpci_legacy_read_config_4(sc, o)	bus_read_4((sc)->vtpci_res, (o))
#define vtpci_legacy_write_config_1(sc, o, v) \
    bus_write_1((sc)->vtpci_res, (o), (v))
#define vtpci_legacy_write_config_2(sc, o, v) \
    bus_write_2((sc)->vtpci_res, (o), (v))
#define vtpci_legacy_write_config_4(sc, o, v) \
    bus_write_4((sc)->vtpci_res, (o), (v))

static device_method_t vtpci_legacy_methods[] = {
	/* Device interface. */
	DEVMETHOD(device_probe,			  vtpci_legacy_probe),
	DEVMETHOD(device_attach,		  vtpci_legacy_attach),
	DEVMETHOD(device_detach,		  vtpci_legacy_detach),
	DEVMETHOD(device_suspend,		  vtpci_legacy_suspend),
	DEVMETHOD(device_resume,		  vtpci_legacy_resume),
	DEVMETHOD(device_shutdown,		  vtpci_legacy_shutdown),

	/* Bus interface. */
	DEVMETHOD(bus_driver_added,		  vtpci_legacy_driver_added),
	DEVMETHOD(bus_child_detached,		  vtpci_legacy_child_detached),
	DEVMETHOD(bus_read_ivar,		  vtpci_legacy_read_ivar),
	DEVMETHOD(bus_write_ivar,		  vtpci_legacy_write_ivar),

	/* VirtIO PCI interface. */
	DEVMETHOD(virtio_pci_read_isr,		 vtpci_legacy_read_isr),
	DEVMETHOD(virtio_pci_get_vq_size,	 vtpci_legacy_get_vq_size),
	DEVMETHOD(virtio_pci_get_vq_notify_off,	 vtpci_legacy_get_vq_notify_off),
	DEVMETHOD(virtio_pci_set_vq,		 vtpci_legacy_set_vq),
	DEVMETHOD(virtio_pci_disable_vq,	 vtpci_legacy_disable_vq),
	DEVMETHOD(virtio_pci_register_cfg_msix,  vtpci_legacy_register_cfg_msix),
	DEVMETHOD(virtio_pci_register_vq_msix,	 vtpci_legacy_register_vq_msix),

	/* VirtIO bus interface. */
	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_legacy_negotiate_features),
	DEVMETHOD(virtio_bus_with_feature,	  vtpci_legacy_with_feature),
	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_legacy_alloc_virtqueues),
	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_legacy_setup_interrupts),
	DEVMETHOD(virtio_bus_stop,		  vtpci_legacy_stop),
	DEVMETHOD(virtio_bus_reinit,		  vtpci_legacy_reinit),
	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_legacy_reinit_complete),
	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_legacy_notify_vq),
	DEVMETHOD(virtio_bus_read_device_config,  vtpci_legacy_read_dev_config),
	DEVMETHOD(virtio_bus_write_device_config, vtpci_legacy_write_dev_config),

	DEVMETHOD_END
};

static driver_t vtpci_legacy_driver = {
	.name = "vtpcil",
	.methods = vtpci_legacy_methods,
	.size = sizeof(struct vtpci_legacy_softc)
};

devclass_t vtpci_legacy_devclass;

DRIVER_MODULE(vtpcil, pci, vtpci_legacy_driver, vtpci_legacy_devclass, 0, 0);

static int
vtpci_legacy_probe(device_t dev)
{
	char desc[64];
	const char *name;

	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
		return (ENXIO);

	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_LEGACY_MAX)
		return (ENXIO);

	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
		return (ENXIO);

	name = virtio_device_name(pci_get_subdevice(dev));
	if (name == NULL)
		name = "Unknown";

	snprintf(desc, sizeof(desc), "VirtIO PCI (legacy) %s adapter", name);
	device_set_desc_copy(dev, desc);

	/* Prefer transitional modern VirtIO PCI. */
	return (BUS_PROBE_LOW_PRIORITY);
}

static int
vtpci_legacy_attach(device_t dev)
{
	struct vtpci_legacy_softc *sc;
	int error;

	sc = device_get_softc(dev);
	sc->vtpci_dev = dev;
	vtpci_init(&sc->vtpci_common, dev, false);

	error = vtpci_legacy_alloc_resources(sc);
	if (error) {
		device_printf(dev, "cannot map I/O space\n");
		return (error);
	}

	vtpci_legacy_reset(sc);

	/* Tell the host we've noticed this device. */
	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);

	error = vtpci_add_child(&sc->vtpci_common);
	if (error)
		goto fail;

	vtpci_legacy_probe_and_attach_child(sc);

	return (0);

fail:
	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
	vtpci_legacy_detach(dev);

	return (error);
}

static int
vtpci_legacy_detach(device_t dev)
{
	struct vtpci_legacy_softc *sc;
	int error;

	sc = device_get_softc(dev);

	error = vtpci_delete_child(&sc->vtpci_common);
	if (error)
		return (error);

	vtpci_legacy_reset(sc);
	vtpci_legacy_free_resources(sc);

	return (0);
}

static int
vtpci_legacy_suspend(device_t dev)
{
	return (bus_generic_suspend(dev));
}

static int
vtpci_legacy_resume(device_t dev)
{
	return (bus_generic_resume(dev));
}

static int
vtpci_legacy_shutdown(device_t dev)
{
	(void) bus_generic_shutdown(dev);
	/* Forcibly stop the host device. */
	vtpci_legacy_stop(dev);

	return (0);
}

static void
vtpci_legacy_driver_added(device_t dev, driver_t *driver)
{
	vtpci_legacy_probe_and_attach_child(device_get_softc(dev));
}

static void
vtpci_legacy_child_detached(device_t dev, device_t child)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	vtpci_legacy_reset(sc);
	vtpci_child_detached(&sc->vtpci_common);

	/* After the reset, retell the host we've noticed this device. */
	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
}

static int
vtpci_legacy_read_ivar(device_t dev, device_t child, int index,
    uintptr_t *result)
{
	struct vtpci_legacy_softc *sc;
	struct vtpci_common *cn;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	if (vtpci_child_device(cn) != child)
		return (ENOENT);

	switch (index) {
	case VIRTIO_IVAR_DEVTYPE:
		*result = pci_get_subdevice(dev);
		break;
	default:
		return (vtpci_read_ivar(cn, index, result));
	}

	return (0);
}

static int
vtpci_legacy_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
{
	struct vtpci_legacy_softc *sc;
	struct vtpci_common *cn;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	if (vtpci_child_device(cn) != child)
		return (ENOENT);

	switch (index) {
	default:
		return (vtpci_write_ivar(cn, index, value));
	}

	return (0);
}

static uint64_t
vtpci_legacy_negotiate_features(device_t dev, uint64_t child_features)
{
	struct vtpci_legacy_softc *sc;
	uint64_t host_features, features;

	sc = device_get_softc(dev);
	host_features = vtpci_legacy_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);

	features = vtpci_negotiate_features(&sc->vtpci_common,
	    child_features, host_features);
	vtpci_legacy_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);

	return (features);
}

static int
vtpci_legacy_with_feature(device_t dev, uint64_t feature)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	return (vtpci_with_feature(&sc->vtpci_common, feature));
}

static int
vtpci_legacy_alloc_virtqueues(device_t dev, int flags, int nvqs,
    struct vq_alloc_info *vq_info)
{
	struct vtpci_legacy_softc *sc;
	struct vtpci_common *cn;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info));
}

static int
vtpci_legacy_setup_interrupts(device_t dev, enum intr_type type)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	return (vtpci_setup_interrupts(&sc->vtpci_common, type));
}

static void
vtpci_legacy_stop(device_t dev)
{
	vtpci_legacy_reset(device_get_softc(dev));
}

static int
vtpci_legacy_reinit(device_t dev, uint64_t features)
{
	struct vtpci_legacy_softc *sc;
	struct vtpci_common *cn;
	int error;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	/*
	 * Redrive the device initialization. This is a bit of an abuse of
	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
	 * play nice.
	 *
	 * We do not allow the host device to change from what was originally
	 * negotiated beyond what the guest driver changed. MSIX state should
	 * not change, number of virtqueues and their size remain the same, etc.
	 * This will need to be rethought when we want to support migration.
	 */

	if (vtpci_legacy_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
		vtpci_legacy_stop(dev);

	/*
	 * Quickly drive the status through ACK and DRIVER. The device does
	 * not become usable again until DRIVER_OK in reinit complete.
	 */
	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);

	vtpci_legacy_negotiate_features(dev, features);

	error = vtpci_reinit(cn);
	if (error)
		return (error);

	return (0);
}

static void
vtpci_legacy_reinit_complete(device_t dev)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
}

static void
vtpci_legacy_notify_vq(device_t dev, uint16_t queue, bus_size_t offset)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);
	MPASS(offset == VIRTIO_PCI_QUEUE_NOTIFY);

	vtpci_legacy_write_config_2(sc, offset, queue);
}

static uint8_t
vtpci_legacy_get_status(struct vtpci_legacy_softc *sc)
{
	return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_STATUS));
}

static void
vtpci_legacy_set_status(struct vtpci_legacy_softc *sc, uint8_t status)
{
	if (status != VIRTIO_CONFIG_STATUS_RESET)
		status |= vtpci_legacy_get_status(sc);

	vtpci_legacy_write_config_1(sc, VIRTIO_PCI_STATUS, status);
}

static void
vtpci_legacy_read_dev_config(device_t dev, bus_size_t offset,
    void *dst, int length)
{
	struct vtpci_legacy_softc *sc;
	bus_size_t off;
	uint8_t *d;
	int size;

	sc = device_get_softc(dev);
	off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset;

	for (d = dst; length > 0; d += size, off += size, length -= size) {
		if (length >= 4) {
			size = 4;
			*(uint32_t *)d = vtpci_legacy_read_config_4(sc, off);
		} else if (length >= 2) {
			size = 2;
			*(uint16_t *)d = vtpci_legacy_read_config_2(sc, off);
		} else {
			size = 1;
			*d = vtpci_legacy_read_config_1(sc, off);
		}
	}
}

static void
vtpci_legacy_write_dev_config(device_t dev, bus_size_t offset,
    void *src, int length)
{
	struct vtpci_legacy_softc *sc;
	bus_size_t off;
	uint8_t *s;
	int size;

	sc = device_get_softc(dev);
	off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset;

	for (s = src; length > 0; s += size, off += size, length -= size) {
		if (length >= 4) {
			size = 4;
			vtpci_legacy_write_config_4(sc, off, *(uint32_t *)s);
		} else if (length >= 2) {
			size = 2;
			vtpci_legacy_write_config_2(sc, off, *(uint16_t *)s);
		} else {
			size = 1;
			vtpci_legacy_write_config_1(sc, off, *s);
		}
	}
}

static int
vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *sc)
{
	device_t dev;
	int rid;

	dev = sc->vtpci_dev;
	
	rid = PCIR_BAR(0);
	if ((sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
	    &rid, RF_ACTIVE)) == NULL)
		return (ENXIO);

	if (vtpci_is_msix_available(&sc->vtpci_common)) {
		rid = PCIR_BAR(1);
		if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev,
		    SYS_RES_MEMORY, &rid, RF_ACTIVE)) == NULL)
			return (ENXIO);
	}

	return (0);
}

static void
vtpci_legacy_free_resources(struct vtpci_legacy_softc *sc)
{
	device_t dev;

	dev = sc->vtpci_dev;

	if (sc->vtpci_msix_res != NULL) {
		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
		    sc->vtpci_msix_res);
		sc->vtpci_msix_res = NULL;
	}

	if (sc->vtpci_res != NULL) {
		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
		    sc->vtpci_res);
		sc->vtpci_res = NULL;
	}
}

static void
vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *sc)
{
	device_t dev, child;

	dev = sc->vtpci_dev;
	child = vtpci_child_device(&sc->vtpci_common);

	if (child == NULL || device_get_state(child) != DS_NOTPRESENT)
		return;

	if (device_probe(child) != 0)
		return;

	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);

	if (device_attach(child) != 0) {
		vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
		/* Reset status for future attempt. */
		vtpci_legacy_child_detached(dev, child);
	} else {
		vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
		VIRTIO_ATTACH_COMPLETED(child);
	}
}

static int
vtpci_legacy_register_msix(struct vtpci_legacy_softc *sc, int offset,
    struct vtpci_interrupt *intr)
{
	device_t dev;
	uint16_t vector;

	dev = sc->vtpci_dev;

	if (intr != NULL) {
		/* Map from guest rid to host vector. */
		vector = intr->vti_rid - 1;
	} else
		vector = VIRTIO_MSI_NO_VECTOR;

	vtpci_legacy_write_config_2(sc, offset, vector);
	return (vtpci_legacy_read_config_2(sc, offset) == vector ? 0 : ENODEV);
}

static int
vtpci_legacy_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr)
{
	struct vtpci_legacy_softc *sc;
	int error;

	sc = device_get_softc(dev);

	error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_CONFIG_VECTOR, intr);
	if (error) {
		device_printf(dev,
		    "unable to register config MSIX interrupt\n");
		return (error);
	}

	return (0);
}

static int
vtpci_legacy_register_vq_msix(device_t dev, int idx,
    struct vtpci_interrupt *intr)
{
	struct vtpci_legacy_softc *sc;
	int error;

	sc = device_get_softc(dev);

	vtpci_legacy_select_virtqueue(sc, idx);
	error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_QUEUE_VECTOR, intr);
	if (error) {
		device_printf(dev,
		    "unable to register virtqueue MSIX interrupt\n");
		return (error);
	}

	return (0);
}

static void
vtpci_legacy_reset(struct vtpci_legacy_softc *sc)
{
	/*
	 * Setting the status to RESET sets the host device to the
	 * original, uninitialized state.
	 */
	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_RESET);
	(void) vtpci_legacy_get_status(sc);
}

static void
vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *sc, int idx)
{
	vtpci_legacy_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
}

static uint8_t
vtpci_legacy_read_isr(device_t dev)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_ISR));
}

static uint16_t
vtpci_legacy_get_vq_size(device_t dev, int idx)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	vtpci_legacy_select_virtqueue(sc, idx);
	return (vtpci_legacy_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM));
}

static bus_size_t
vtpci_legacy_get_vq_notify_off(device_t dev, int idx)
{
	return (VIRTIO_PCI_QUEUE_NOTIFY);
}

static void
vtpci_legacy_set_vq(device_t dev, struct virtqueue *vq)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	vtpci_legacy_select_virtqueue(sc, virtqueue_index(vq));
	vtpci_legacy_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
	    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
}

static void
vtpci_legacy_disable_vq(device_t dev, int idx)
{
	struct vtpci_legacy_softc *sc;

	sc = device_get_softc(dev);

	vtpci_legacy_select_virtqueue(sc, idx);
	vtpci_legacy_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
}




/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright IBM Corp. 2007
 *
 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *
 * This header is BSD licensed so anyone can use the definitions to implement
 * compatible drivers/servers.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of IBM nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD$
 */

#ifndef _VIRTIO_PCI_LEGACY_VAR_H
#define _VIRTIO_PCI_LEGACY_VAR_H

#include <dev/virtio/pci/virtio_pci_var.h>

/* VirtIO ABI version, this must match exactly. */
#define VIRTIO_PCI_ABI_VERSION	0

/*
 * VirtIO Header, located in BAR 0.
 */
#define VIRTIO_PCI_HOST_FEATURES  0  /* host's supported features (32bit, RO)*/
#define VIRTIO_PCI_GUEST_FEATURES 4  /* guest's supported features (32, RW) */
#define VIRTIO_PCI_QUEUE_PFN      8  /* physical address of VQ (32, RW) */
#define VIRTIO_PCI_QUEUE_NUM      12 /* number of ring entries (16, RO) */
#define VIRTIO_PCI_QUEUE_SEL      14 /* current VQ selection (16, RW) */
#define VIRTIO_PCI_QUEUE_NOTIFY	  16 /* notify host regarding VQ (16, RW) */
#define VIRTIO_PCI_STATUS         18 /* device status register (8, RW) */
#define VIRTIO_PCI_ISR            19 /* interrupt status register, reading
				      * also clears the register (8, RO) */
/* Only if MSIX is enabled: */
#define VIRTIO_MSI_CONFIG_VECTOR  20 /* configuration change vector (16, RW) */
#define VIRTIO_MSI_QUEUE_VECTOR   22 /* vector for selected VQ notifications
					(16, RW) */

/*
 * The remaining space is defined by each driver as the per-driver
 * configuration space.
 */
#define VIRTIO_PCI_CONFIG_OFF(msix_enabled)     ((msix_enabled) ? 24 : 20)

/*
 * How many bits to shift physical queue address written to QUEUE_PFN.
 * 12 is historical, and due to x86 page size.
 */
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT	12

/* The alignment to use between consumer and producer parts of vring. */
#define VIRTIO_PCI_VRING_ALIGN	4096

#endif /* _VIRTIO_PCI_LEGACY_VAR_H */




/*-
 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
 *
 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/* Driver for the modern VirtIO PCI interface. */

#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/module.h>

#include <machine/bus.h>
#include <machine/cpu.h>
#include <machine/resource.h>
#include <sys/bus.h>
#include <sys/rman.h>

#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>

#include <dev/virtio/virtio.h>
#include <dev/virtio/virtqueue.h>
#include <dev/virtio/pci/virtio_pci.h>
#include <dev/virtio/pci/virtio_pci_modern_var.h>

#include "virtio_bus_if.h"
#include "virtio_pci_if.h"
#include "virtio_if.h"

struct vtpci_modern_resource_map {
	struct resource_map	vtrm_map;
	int			vtrm_cap_offset;
	int			vtrm_bar;
	int			vtrm_offset;
	int			vtrm_length;
	int			vtrm_type;	/* SYS_RES_{MEMORY, IOPORT} */
};

struct vtpci_modern_bar_resource {
	struct resource		*vtbr_res;
	int			 vtbr_type;
};

struct vtpci_modern_softc {
	device_t			 vtpci_dev;
	struct vtpci_common		 vtpci_common;
	uint32_t			 vtpci_notify_offset_multiplier;
	uint16_t			 vtpci_devid;
	int				 vtpci_msix_bar;
	struct resource			*vtpci_msix_res;

	struct vtpci_modern_resource_map vtpci_common_res_map;
	struct vtpci_modern_resource_map vtpci_notify_res_map;
	struct vtpci_modern_resource_map vtpci_isr_res_map;
	struct vtpci_modern_resource_map vtpci_device_res_map;

#define VTPCI_MODERN_MAX_BARS		6
	struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS];
};

static int	vtpci_modern_probe(device_t);
static int	vtpci_modern_attach(device_t);
static int	vtpci_modern_detach(device_t);
static int	vtpci_modern_suspend(device_t);
static int	vtpci_modern_resume(device_t);
static int	vtpci_modern_shutdown(device_t);

static void	vtpci_modern_driver_added(device_t, driver_t *);
static void	vtpci_modern_child_detached(device_t, device_t);
static int	vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *);
static int	vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t);

static uint8_t	vtpci_modern_read_isr(device_t);
static uint16_t	vtpci_modern_get_vq_size(device_t, int);
static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int);
static void	vtpci_modern_set_vq(device_t, struct virtqueue *);
static void	vtpci_modern_disable_vq(device_t, int);
static int	vtpci_modern_register_msix(struct vtpci_modern_softc *, int,
		    struct vtpci_interrupt *);
static int	vtpci_modern_register_cfg_msix(device_t,
		    struct vtpci_interrupt *);
static int	vtpci_modern_register_vq_msix(device_t, int idx,
		    struct vtpci_interrupt *);

static uint64_t	vtpci_modern_negotiate_features(device_t, uint64_t);
static int	vtpci_modern_finalize_features(device_t);
static int	vtpci_modern_with_feature(device_t, uint64_t);
static int	vtpci_modern_alloc_virtqueues(device_t, int, int,
		    struct vq_alloc_info *);
static int	vtpci_modern_setup_interrupts(device_t, enum intr_type);
static void	vtpci_modern_stop(device_t);
static int	vtpci_modern_reinit(device_t, uint64_t);
static void	vtpci_modern_reinit_complete(device_t);
static void	vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t);
static int	vtpci_modern_config_generation(device_t);
static void	vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int);
static void	vtpci_modern_write_dev_config(device_t, bus_size_t, void *, int);

static int	vtpci_modern_probe_configs(device_t);
static int	vtpci_modern_find_cap(device_t, uint8_t, int *);
static int	vtpci_modern_map_configs(struct vtpci_modern_softc *);
static void	vtpci_modern_unmap_configs(struct vtpci_modern_softc *);
static int	vtpci_modern_find_cap_resource(struct vtpci_modern_softc *,
		     uint8_t, int, int, struct vtpci_modern_resource_map *);
static int	vtpci_modern_bar_type(struct vtpci_modern_softc *, int);
static struct resource *vtpci_modern_get_bar_resource(
		    struct vtpci_modern_softc *, int, int);
static struct resource *vtpci_modern_alloc_bar_resource(
		    struct vtpci_modern_softc *, int, int);
static void	vtpci_modern_free_bar_resources(struct vtpci_modern_softc *);
static int	vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *,
		    struct vtpci_modern_resource_map *);
static void	vtpci_modern_free_resource_map(struct vtpci_modern_softc *,
		    struct vtpci_modern_resource_map *);
static void	vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *);
static void	vtpci_modern_free_msix_resource(struct vtpci_modern_softc *);

static void	vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *);

static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *);
static void	vtpci_modern_write_features(struct vtpci_modern_softc *,
		    uint64_t);
static void	vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int);
static uint8_t	vtpci_modern_get_status(struct vtpci_modern_softc *);
static void	vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t);
static void	vtpci_modern_reset(struct vtpci_modern_softc *);
static void	vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *);

static uint8_t	vtpci_modern_read_common_1(struct vtpci_modern_softc *,
		    bus_size_t);
static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *,
		    bus_size_t);
static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *,
		    bus_size_t);
static void	vtpci_modern_write_common_1(struct vtpci_modern_softc *,
		     bus_size_t, uint8_t);
static void	vtpci_modern_write_common_2(struct vtpci_modern_softc *,
		     bus_size_t, uint16_t);
static void	vtpci_modern_write_common_4(struct vtpci_modern_softc *,
		    bus_size_t, uint32_t);
static void	vtpci_modern_write_common_8(struct vtpci_modern_softc *,
		    bus_size_t, uint64_t);
static void	vtpci_modern_write_notify_2(struct vtpci_modern_softc *,
		    bus_size_t, uint16_t);
static uint8_t  vtpci_modern_read_isr_1(struct vtpci_modern_softc *,
		    bus_size_t);
static uint8_t	vtpci_modern_read_device_1(struct vtpci_modern_softc *,
		    bus_size_t);
static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *,
		    bus_size_t);
static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *,
		    bus_size_t);
static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *,
		    bus_size_t);
static void	vtpci_modern_write_device_1(struct vtpci_modern_softc *,
		    bus_size_t, uint8_t);
static void	vtpci_modern_write_device_2(struct vtpci_modern_softc *,
		    bus_size_t, uint16_t);
static void	vtpci_modern_write_device_4(struct vtpci_modern_softc *,
		    bus_size_t, uint32_t);
static void	vtpci_modern_write_device_8(struct vtpci_modern_softc *,
		    bus_size_t, uint64_t);

/* Tunables. */
static int vtpci_modern_transitional = 0;
TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional);

static device_method_t vtpci_modern_methods[] = {
	/* Device interface. */
	DEVMETHOD(device_probe,			vtpci_modern_probe),
	DEVMETHOD(device_attach,		vtpci_modern_attach),
	DEVMETHOD(device_detach,		vtpci_modern_detach),
	DEVMETHOD(device_suspend,		vtpci_modern_suspend),
	DEVMETHOD(device_resume,		vtpci_modern_resume),
	DEVMETHOD(device_shutdown,		vtpci_modern_shutdown),

	/* Bus interface. */
	DEVMETHOD(bus_driver_added,		vtpci_modern_driver_added),
	DEVMETHOD(bus_child_detached,		vtpci_modern_child_detached),
	DEVMETHOD(bus_read_ivar,		vtpci_modern_read_ivar),
	DEVMETHOD(bus_write_ivar,		vtpci_modern_write_ivar),

	/* VirtIO PCI interface. */
	DEVMETHOD(virtio_pci_read_isr,		 vtpci_modern_read_isr),
	DEVMETHOD(virtio_pci_get_vq_size,	 vtpci_modern_get_vq_size),
	DEVMETHOD(virtio_pci_get_vq_notify_off,	 vtpci_modern_get_vq_notify_off),
	DEVMETHOD(virtio_pci_set_vq,		 vtpci_modern_set_vq),
	DEVMETHOD(virtio_pci_disable_vq,	 vtpci_modern_disable_vq),
	DEVMETHOD(virtio_pci_register_cfg_msix,	 vtpci_modern_register_cfg_msix),
	DEVMETHOD(virtio_pci_register_vq_msix,	 vtpci_modern_register_vq_msix),

	/* VirtIO bus interface. */
	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_modern_negotiate_features),
	DEVMETHOD(virtio_bus_finalize_features,	  vtpci_modern_finalize_features),
	DEVMETHOD(virtio_bus_with_feature,	  vtpci_modern_with_feature),
	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_modern_alloc_virtqueues),
	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_modern_setup_interrupts),
	DEVMETHOD(virtio_bus_stop,		  vtpci_modern_stop),
	DEVMETHOD(virtio_bus_reinit,		  vtpci_modern_reinit),
	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_modern_reinit_complete),
	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_modern_notify_vq),
	DEVMETHOD(virtio_bus_config_generation,	  vtpci_modern_config_generation),
	DEVMETHOD(virtio_bus_read_device_config,  vtpci_modern_read_dev_config),
	DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config),

	DEVMETHOD_END
};

static driver_t vtpci_modern_driver = {
	.name = "vtpcim",
	.methods = vtpci_modern_methods,
	.size = sizeof(struct vtpci_modern_softc)
};

devclass_t vtpci_modern_devclass;

DRIVER_MODULE(vtpcim, pci, vtpci_modern_driver, vtpci_modern_devclass, 0, 0);

static int
vtpci_modern_probe(device_t dev)
{
	char desc[64];
	const char *name;
	uint16_t devid;

	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
		return (ENXIO);

	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX)
		return (ENXIO);

	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) {
		if (!vtpci_modern_transitional)
			return (ENXIO);
		devid = pci_get_subdevice(dev);
	} else
		devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN;

	if (vtpci_modern_probe_configs(dev) != 0)
		return (ENXIO);

	name = virtio_device_name(devid);
	if (name == NULL)
		name = "Unknown";

	snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name);
	device_set_desc_copy(dev, desc);

	return (BUS_PROBE_DEFAULT);
}

static int
vtpci_modern_attach(device_t dev)
{
	struct vtpci_modern_softc *sc;
	int error;

	sc = device_get_softc(dev);
	sc->vtpci_dev = dev;
	vtpci_init(&sc->vtpci_common, dev, true);

	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN)
		sc->vtpci_devid = pci_get_subdevice(dev);
	else
		sc->vtpci_devid = pci_get_device(dev) -
		    VIRTIO_PCI_DEVICEID_MODERN_MIN;

	error = vtpci_modern_map_configs(sc);
	if (error) {
		device_printf(dev, "cannot map configs\n");
		vtpci_modern_unmap_configs(sc);
		return (error);
	}

	vtpci_modern_reset(sc);

	/* Tell the host we've noticed this device. */
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);

	error = vtpci_add_child(&sc->vtpci_common);
	if (error)
		goto fail;

	vtpci_modern_probe_and_attach_child(sc);

	return (0);

fail:
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
	vtpci_modern_detach(dev);

	return (error);
}

static int
vtpci_modern_detach(device_t dev)
{
	struct vtpci_modern_softc *sc;
	int error;

	sc = device_get_softc(dev);

	error = vtpci_delete_child(&sc->vtpci_common);
	if (error)
		return (error);

	vtpci_modern_reset(sc);
	vtpci_modern_unmap_configs(sc);

	return (0);
}

static int
vtpci_modern_suspend(device_t dev)
{
	return (bus_generic_suspend(dev));
}

static int
vtpci_modern_resume(device_t dev)
{
	return (bus_generic_resume(dev));
}

static int
vtpci_modern_shutdown(device_t dev)
{
	(void) bus_generic_shutdown(dev);
	/* Forcibly stop the host device. */
	vtpci_modern_stop(dev);

	return (0);
}

static void
vtpci_modern_driver_added(device_t dev, driver_t *driver)
{
	vtpci_modern_probe_and_attach_child(device_get_softc(dev));
}

static void
vtpci_modern_child_detached(device_t dev, device_t child)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	vtpci_modern_reset(sc);
	vtpci_child_detached(&sc->vtpci_common);

	/* After the reset, retell the host we've noticed this device. */
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
}

static int
vtpci_modern_read_ivar(device_t dev, device_t child, int index,
    uintptr_t *result)
{
	struct vtpci_modern_softc *sc;
	struct vtpci_common *cn;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	if (vtpci_child_device(cn) != child)
		return (ENOENT);

	switch (index) {
	case VIRTIO_IVAR_DEVTYPE:
		*result = sc->vtpci_devid;
		break;
	default:
		return (vtpci_read_ivar(cn, index, result));
	}

	return (0);
}

static int
vtpci_modern_write_ivar(device_t dev, device_t child, int index,
    uintptr_t value)
{
	struct vtpci_modern_softc *sc;
	struct vtpci_common *cn;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	if (vtpci_child_device(cn) != child)
		return (ENOENT);

	switch (index) {
	default:
		return (vtpci_write_ivar(cn, index, value));
	}

	return (0);
}

static uint64_t
vtpci_modern_negotiate_features(device_t dev, uint64_t child_features)
{
	struct vtpci_modern_softc *sc;
	uint64_t host_features, features;

	sc = device_get_softc(dev);
	host_features = vtpci_modern_read_features(sc);

	/*
	 * Since the driver was added as a child of the modern PCI bus,
	 * always add the V1 flag.
	 */
	child_features |= VIRTIO_F_VERSION_1;

	features = vtpci_negotiate_features(&sc->vtpci_common,
	    child_features, host_features);
	vtpci_modern_write_features(sc, features);

	return (features);
}

static int
vtpci_modern_finalize_features(device_t dev)
{
	struct vtpci_modern_softc *sc;
	uint8_t status;

	sc = device_get_softc(dev);

	/*
	 * Must re-read the status after setting it to verify the negotiated
	 * features were accepted by the device.
	 */
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK);

	status = vtpci_modern_get_status(sc);
	if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) {
		device_printf(dev, "desired features were not accepted\n");
		return (ENOTSUP);
	}

	return (0);
}

static int
vtpci_modern_with_feature(device_t dev, uint64_t feature)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	return (vtpci_with_feature(&sc->vtpci_common, feature));
}

static uint64_t
vtpci_modern_read_features(struct vtpci_modern_softc *sc)
{
	uint32_t features0, features1;

	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0);
	features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1);
	features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);

	return (((uint64_t) features1 << 32) | features0);
}

static void
vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features)
{
	uint32_t features0, features1;

	features0 = features;
	features1 = features >> 32;

	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0);
	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0);
	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1);
	vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1);
}

static int
vtpci_modern_alloc_virtqueues(device_t dev, int flags, int nvqs,
    struct vq_alloc_info *vq_info)
{
	struct vtpci_modern_softc *sc;
	struct vtpci_common *cn;
	uint16_t max_nvqs;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ);
	if (nvqs > max_nvqs) {
		device_printf(sc->vtpci_dev, "requested virtqueue count %d "
		    "exceeds max %d\n", nvqs, max_nvqs);
		return (E2BIG);
	}

	return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info));
}

static int
vtpci_modern_setup_interrupts(device_t dev, enum intr_type type)
{
	struct vtpci_modern_softc *sc;
	int error;

	sc = device_get_softc(dev);

	error = vtpci_setup_interrupts(&sc->vtpci_common, type);
	if (error == 0)
		vtpci_modern_enable_virtqueues(sc);

	return (error);
}

static void
vtpci_modern_stop(device_t dev)
{
	vtpci_modern_reset(device_get_softc(dev));
}

static int
vtpci_modern_reinit(device_t dev, uint64_t features)
{
	struct vtpci_modern_softc *sc;
	struct vtpci_common *cn;
	int error;

	sc = device_get_softc(dev);
	cn = &sc->vtpci_common;

	/*
	 * Redrive the device initialization. This is a bit of an abuse of
	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
	 * play nice.
	 *
	 * We do not allow the host device to change from what was originally
	 * negotiated beyond what the guest driver changed. MSIX state should
	 * not change, number of virtqueues and their size remain the same, etc.
	 * This will need to be rethought when we want to support migration.
	 */

	if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
		vtpci_modern_stop(dev);

	/*
	 * Quickly drive the status through ACK and DRIVER. The device does
	 * not become usable again until DRIVER_OK in reinit complete.
	 */
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);

	/*
	 * TODO: Check that features are not added as to what was
	 * originally negotiated.
	 */
	vtpci_modern_negotiate_features(dev, features);
	error = vtpci_modern_finalize_features(dev);
	if (error) {
		device_printf(dev, "cannot finalize features during reinit\n");
		return (error);
	}

	error = vtpci_reinit(cn);
	if (error)
		return (error);

	return (0);
}

static void
vtpci_modern_reinit_complete(device_t dev)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	vtpci_modern_enable_virtqueues(sc);
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
}

static void
vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	vtpci_modern_write_notify_2(sc, offset, queue);
}

static uint8_t
vtpci_modern_get_status(struct vtpci_modern_softc *sc)
{
	return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS));
}

static void
vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status)
{
	if (status != VIRTIO_CONFIG_STATUS_RESET)
		status |= vtpci_modern_get_status(sc);

	vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status);
}

static int
vtpci_modern_config_generation(device_t dev)
{
	struct vtpci_modern_softc *sc;
	uint8_t gen;

	sc = device_get_softc(dev);
	gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION);

	return (gen);
}

static void
vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst,
    int length)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
		panic("%s: attempt to read dev config but not present",
		    __func__);
	}

	switch (length) {
	case 1:
		*(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset);
		break;
	case 2:
		*(uint16_t *) dst = virtio_htog16(true,
		    vtpci_modern_read_device_2(sc, offset));
		break;
	case 4:
		*(uint32_t *) dst = virtio_htog32(true,
		    vtpci_modern_read_device_4(sc, offset));
		break;
	case 8:
		*(uint64_t *) dst = virtio_htog64(true,
		    vtpci_modern_read_device_8(sc, offset));
		break;
	default:
		panic("%s: device %s invalid device read length %d offset %d",
		    __func__, device_get_nameunit(dev), length, (int) offset);
	}
}

static void
vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, void *src,
    int length)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
		panic("%s: attempt to write dev config but not present",
		    __func__);
	}

	switch (length) {
	case 1:
		vtpci_modern_write_device_1(sc, offset, *(uint8_t *) src);
		break;
	case 2: {
		uint16_t val = virtio_gtoh16(true, *(uint16_t *) src);
		vtpci_modern_write_device_2(sc, offset, val);
		break;
	}
	case 4: {
		uint32_t val = virtio_gtoh32(true, *(uint32_t *) src);
		vtpci_modern_write_device_4(sc, offset, val);
		break;
	}
	case 8: {
		uint64_t val = virtio_gtoh64(true, *(uint64_t *) src);
		vtpci_modern_write_device_8(sc, offset, val);
		break;
	}
	default:
		panic("%s: device %s invalid device write length %d offset %d",
		    __func__, device_get_nameunit(dev), length, (int) offset);
	}
}

static int
vtpci_modern_probe_configs(device_t dev)
{
	int error;

	/*
	 * These config capabilities must be present. The DEVICE_CFG
	 * capability is only present if the device requires it.
	 */

	error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL);
	if (error) {
		device_printf(dev, "cannot find COMMON_CFG capability\n");
		return (error);
	}

	error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL);
	if (error) {
		device_printf(dev, "cannot find NOTIFY_CFG capability\n");
		return (error);
	}

	error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL);
	if (error) {
		device_printf(dev, "cannot find ISR_CFG capability\n");
		return (error);
	}

	return (0);
}

static int
vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset)
{
	uint32_t type, bar;
	int capreg, error;

	for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg);
	     error == 0;
	     error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) {

		type = pci_read_config(dev, capreg +
		    offsetof(struct virtio_pci_cap, cfg_type), 1);
		bar = pci_read_config(dev, capreg +
		    offsetof(struct virtio_pci_cap, bar), 1);

		/* Must ignore reserved BARs. */
		if (bar >= VTPCI_MODERN_MAX_BARS)
			continue;

		if (type == cfg_type) {
			if (cap_offset != NULL)
				*cap_offset = capreg;
			break;
		}
	}

	return (error);
}

static int
vtpci_modern_map_common_config(struct vtpci_modern_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtpci_dev;

	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG,
	    sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map);
	if (error) {
		device_printf(dev, "cannot find cap COMMON_CFG resource\n");
		return (error);
	}

	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map);
	if (error) {
		device_printf(dev, "cannot alloc resource for COMMON_CFG\n");
		return (error);
	}

	return (0);
}

static int
vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc)
{
	device_t dev;
	int cap_offset, error;

	dev = sc->vtpci_dev;

	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG,
	    -1, 2, &sc->vtpci_notify_res_map);
	if (error) {
		device_printf(dev, "cannot find cap NOTIFY_CFG resource\n");
		return (error);
	}

	cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset;

	sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset +
	    offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4);

	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map);
	if (error) {
		device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n");
		return (error);
	}

	return (0);
}

static int
vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtpci_dev;

	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG,
	    sizeof(uint8_t), 1, &sc->vtpci_isr_res_map);
	if (error) {
		device_printf(dev, "cannot find cap ISR_CFG resource\n");
		return (error);
	}

	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map);
	if (error) {
		device_printf(dev, "cannot alloc resource for ISR_CFG\n");
		return (error);
	}

	return (0);
}

static int
vtpci_modern_map_device_config(struct vtpci_modern_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtpci_dev;

	error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG,
	    -1, 4, &sc->vtpci_device_res_map);
	if (error == ENOENT) {
		/* Device configuration is optional depending on device. */
		return (0);
	} else if (error) {
		device_printf(dev, "cannot find cap DEVICE_CFG resource\n");
		return (error);
	}

	error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map);
	if (error) {
		device_printf(dev, "cannot alloc resource for DEVICE_CFG\n");
		return (error);
	}

	return (error);
}

static int
vtpci_modern_map_configs(struct vtpci_modern_softc *sc)
{
	int error;

	error = vtpci_modern_map_common_config(sc);
	if (error)
		return (error);

	error = vtpci_modern_map_notify_config(sc);
	if (error)
		return (error);

	error = vtpci_modern_map_isr_config(sc);
	if (error)
		return (error);

	error = vtpci_modern_map_device_config(sc);
	if (error)
		return (error);

	vtpci_modern_alloc_msix_resource(sc);

	return (0);
}

static void
vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc)
{

	vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map);
	vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map);
	vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map);
	vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map);

	vtpci_modern_free_bar_resources(sc);
	vtpci_modern_free_msix_resource(sc);

	sc->vtpci_notify_offset_multiplier = 0;
}

static int
vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type,
    int min_size, int alignment, struct vtpci_modern_resource_map *res)
{
	device_t dev;
	int cap_offset, offset, length, error;
	uint8_t bar, cap_length;

	dev = sc->vtpci_dev;

	error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset);
	if (error)
		return (error);

	cap_length = pci_read_config(dev,
	    cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1);

	if (cap_length < sizeof(struct virtio_pci_cap)) {
		device_printf(dev, "cap %u length %d less than expected\n",
		    cfg_type, cap_length);
		return (ENXIO);
	}

	bar = pci_read_config(dev,
	    cap_offset + offsetof(struct virtio_pci_cap, bar), 1);
	offset = pci_read_config(dev,
	    cap_offset + offsetof(struct virtio_pci_cap, offset), 4);
	length = pci_read_config(dev,
	    cap_offset + offsetof(struct virtio_pci_cap, length), 4);

	if (min_size != -1 && length < min_size) {
		device_printf(dev, "cap %u struct length %d less than min %d\n",
		    cfg_type, length, min_size);
		return (ENXIO);
	}

	if (offset % alignment) {
		device_printf(dev, "cap %u struct offset %d not aligned to %d\n",
		    cfg_type, offset, alignment);
		return (ENXIO);
	}

	/* BMV: TODO Can we determine the size of the BAR here? */

	res->vtrm_cap_offset = cap_offset;
	res->vtrm_bar = bar;
	res->vtrm_offset = offset;
	res->vtrm_length = length;
	res->vtrm_type = vtpci_modern_bar_type(sc, bar);

	return (0);
}

static int
vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar)
{
	uint32_t val;

	/*
	 * The BAR described by a config capability may be either an IOPORT or
	 * MEM, but we must know the type when calling bus_alloc_resource().
	 */
	val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4);
	if (PCI_BAR_IO(val))
		return (SYS_RES_IOPORT);
	else
		return (SYS_RES_MEMORY);
}

static struct resource *
vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type)
{
	struct resource *res;

	MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
	res = sc->vtpci_bar_res[bar].vtbr_res;
	MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type);

	return (res);
}

static struct resource *
vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar,
    int type)
{
	struct resource *res;
	int rid;

	MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
	MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT);

	res = sc->vtpci_bar_res[bar].vtbr_res;
	if (res != NULL) {
		MPASS(sc->vtpci_bar_res[bar].vtbr_type == type);
		return (res);
	}

	rid = PCIR_BAR(bar);
	res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid,
	    RF_ACTIVE | RF_UNMAPPED);
	if (res != NULL) {
		sc->vtpci_bar_res[bar].vtbr_res = res;
		sc->vtpci_bar_res[bar].vtbr_type = type;
	}

	return (res);
}

static void
vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc)
{
	device_t dev;
	struct resource *res;
	int bar, rid, type;

	dev = sc->vtpci_dev;

	for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) {
		res = sc->vtpci_bar_res[bar].vtbr_res;
		type = sc->vtpci_bar_res[bar].vtbr_type;

		if (res != NULL) {
			rid = PCIR_BAR(bar);
			bus_release_resource(dev, type, rid, res);
			sc->vtpci_bar_res[bar].vtbr_res = NULL;
			sc->vtpci_bar_res[bar].vtbr_type = 0;
		}
	}
}

static int
vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc,
    struct vtpci_modern_resource_map *map)
{
	struct resource_map_request req;
	struct resource *res;
	int type;

	type = map->vtrm_type;

	res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type);
	if (res == NULL)
		return (ENXIO);

	resource_init_map_request(&req);
	req.offset = map->vtrm_offset;
	req.length = map->vtrm_length;

	return (bus_map_resource(sc->vtpci_dev, type, res, &req,
	    &map->vtrm_map));
}

static void
vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc,
    struct vtpci_modern_resource_map *map)
{
	struct resource *res;
	int type;

	type = map->vtrm_type;
	res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type);

	if (res != NULL && map->vtrm_map.r_size != 0) {
		bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map);
		bzero(map, sizeof(struct vtpci_modern_resource_map));
	}
}

static void
vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc)
{
	device_t dev;
	int bar;

	dev = sc->vtpci_dev;

	if (!vtpci_is_msix_available(&sc->vtpci_common) ||
	    (bar = pci_msix_table_bar(dev)) == -1)
		return;

	sc->vtpci_msix_bar = bar;
	if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
	    &bar, RF_ACTIVE)) == NULL)
		device_printf(dev, "Unable to map MSIX table\n");
}

static void
vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc)
{
	device_t dev;

	dev = sc->vtpci_dev;

	if (sc->vtpci_msix_res != NULL) {
		bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar,
		    sc->vtpci_msix_res);
		sc->vtpci_msix_bar = 0;
		sc->vtpci_msix_res = NULL;
	}
}

static void
vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc)
{
	device_t dev, child;

	dev = sc->vtpci_dev;
	child = vtpci_child_device(&sc->vtpci_common);

	if (child == NULL || device_get_state(child) != DS_NOTPRESENT)
		return;

	if (device_probe(child) != 0)
		return;

	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);

	if (device_attach(child) != 0) {
		vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
		/* Reset state for later attempt. */
		vtpci_modern_child_detached(dev, child);
	} else {
		vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
		VIRTIO_ATTACH_COMPLETED(child);
	}
}

static int
vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset,
    struct vtpci_interrupt *intr)
{
	uint16_t vector;

	if (intr != NULL) {
		/* Map from guest rid to host vector. */
		vector = intr->vti_rid - 1;
	} else
		vector = VIRTIO_MSI_NO_VECTOR;

	vtpci_modern_write_common_2(sc, offset, vector);
	return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV);
}

static int
vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr)
{
	struct vtpci_modern_softc *sc;
	int error;

	sc = device_get_softc(dev);

	error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr);
	if (error) {
		device_printf(dev,
		    "unable to register config MSIX interrupt\n");
		return (error);
	}

	return (0);
}

static int
vtpci_modern_register_vq_msix(device_t dev, int idx,
    struct vtpci_interrupt *intr)
{
	struct vtpci_modern_softc *sc;
	int error;

	sc = device_get_softc(dev);

	vtpci_modern_select_virtqueue(sc, idx);
	error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr);
	if (error) {
		device_printf(dev,
		    "unable to register virtqueue MSIX interrupt\n");
		return (error);
	}

	return (0);
}

static void
vtpci_modern_reset(struct vtpci_modern_softc *sc)
{
	/*
	 * Setting the status to RESET sets the host device to the
	 * original, uninitialized state. Must poll the status until
	 * the reset is complete.
	 */
	vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET);

	while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
		cpu_spinwait();
}

static void
vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx)
{
	vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx);
}

static uint8_t
vtpci_modern_read_isr(device_t dev)
{
	return (vtpci_modern_read_isr_1(device_get_softc(dev), 0));
}

static uint16_t
vtpci_modern_get_vq_size(device_t dev, int idx)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	vtpci_modern_select_virtqueue(sc, idx);
	return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE));
}

static bus_size_t
vtpci_modern_get_vq_notify_off(device_t dev, int idx)
{
	struct vtpci_modern_softc *sc;
	uint16_t q_notify_off;

	sc = device_get_softc(dev);

	vtpci_modern_select_virtqueue(sc, idx);
	q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF);

	return (q_notify_off * sc->vtpci_notify_offset_multiplier);
}

static void
vtpci_modern_set_vq(device_t dev, struct virtqueue *vq)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	vtpci_modern_select_virtqueue(sc, virtqueue_index(vq));

	/* BMV: Currently we never adjust the device's proposed VQ size. */
	vtpci_modern_write_common_2(sc,
	    VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq));

	vtpci_modern_write_common_8(sc,
	    VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq));
	vtpci_modern_write_common_8(sc,
	    VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq));
        vtpci_modern_write_common_8(sc,
	    VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq));
}

static void
vtpci_modern_disable_vq(device_t dev, int idx)
{
	struct vtpci_modern_softc *sc;

	sc = device_get_softc(dev);

	vtpci_modern_select_virtqueue(sc, idx);
	vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 0);
	vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL);
	vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL);
        vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL);
}

static void
vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc)
{
	int idx;

	for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) {
		vtpci_modern_select_virtqueue(sc, idx);
		vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1);
	}
}

static uint8_t
vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off));
}

static uint16_t
vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off));
}

static uint32_t
vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off));
}

static void
vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off,
    uint8_t val)
{
	bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val);
}

static void
vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off,
    uint16_t val)
{
	bus_write_2(&sc->vtpci_common_res_map.vtrm_map, off, val);
}

static void
vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off,
    uint32_t val)
{
	bus_write_4(&sc->vtpci_common_res_map.vtrm_map, off, val);
}

static void
vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off,
    uint64_t val)
{
	uint32_t val0, val1;

	val0 = (uint32_t) val;
	val1 = val >> 32;

	vtpci_modern_write_common_4(sc, off, val0);
	vtpci_modern_write_common_4(sc, off + 4, val1);
}

static void
vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off,
    uint16_t val)
{
	bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val);
}

static uint8_t
vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off));
}

static uint8_t
vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off));
}

static uint16_t
vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off));
}

static uint32_t
vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off)
{
	return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off));
}

static uint64_t
vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off)
{
	device_t dev;
	int gen;
	uint32_t val0, val1;

	dev = sc->vtpci_dev;

	/*
	 * Treat the 64-bit field as two 32-bit fields. Use the generation
	 * to ensure a consistent read.
	 */
	do {
		gen = vtpci_modern_config_generation(dev);
		val0 = vtpci_modern_read_device_4(sc, off);
		val1 = vtpci_modern_read_device_4(sc, off + 4);
	} while (gen != vtpci_modern_config_generation(dev));

	return (((uint64_t) val1 << 32) | val0);
}

static void
vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off,
    uint8_t val)
{
	bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val);
}

static void
vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off,
    uint16_t val)
{
	bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val);
}

static void
vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off,
    uint32_t val)
{
	bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val);
}

static void
vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off,
    uint64_t val)
{
	uint32_t val0, val1;

	val0 = (uint32_t) val;
	val1 = val >> 32;

	vtpci_modern_write_device_4(sc, off, val0);
	vtpci_modern_write_device_4(sc, off + 4, val1);
}




/*
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright IBM Corp. 2007
 *
 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *
 * This header is BSD licensed so anyone can use the definitions to implement
 * compatible drivers/servers.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of IBM nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD$
 */

#ifndef _VIRTIO_PCI_MODERN_VAR_H
#define _VIRTIO_PCI_MODERN_VAR_H

#include <dev/virtio/pci/virtio_pci_var.h>

/* IDs for different capabilities.  Must all exist. */
/* Common configuration */
#define VIRTIO_PCI_CAP_COMMON_CFG	1
/* Notifications */
#define VIRTIO_PCI_CAP_NOTIFY_CFG	2
/* ISR access */
#define VIRTIO_PCI_CAP_ISR_CFG		3
/* Device specific configuration */
#define VIRTIO_PCI_CAP_DEVICE_CFG	4
/* PCI configuration access */
#define VIRTIO_PCI_CAP_PCI_CFG		5

/* This is the PCI capability header: */
struct virtio_pci_cap {
	uint8_t cap_vndr;		/* Generic PCI field: PCI_CAP_ID_VNDR */
	uint8_t cap_next;		/* Generic PCI field: next ptr. */
	uint8_t cap_len;		/* Generic PCI field: capability length */
	uint8_t cfg_type;		/* Identifies the structure. */
	uint8_t bar;			/* Where to find it. */
	uint8_t padding[3];		/* Pad to full dword. */
	uint32_t offset;		/* Offset within bar. */
	uint32_t length;		/* Length of the structure, in bytes. */
};

struct virtio_pci_notify_cap {
	struct virtio_pci_cap cap;
	uint32_t notify_off_multiplier;	/* Multiplier for queue_notify_off. */
};

/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */
struct virtio_pci_common_cfg {
	/* About the whole device. */
	uint32_t device_feature_select;	/* read-write */
	uint32_t device_feature;	/* read-only */
	uint32_t guest_feature_select;	/* read-write */
	uint32_t guest_feature;		/* read-write */
	uint16_t msix_config;		/* read-write */
	uint16_t num_queues;		/* read-only */
	uint8_t device_status;		/* read-write */
	uint8_t config_generation;	/* read-only */

	/* About a specific virtqueue. */
	uint16_t queue_select;		/* read-write */
	uint16_t queue_size;		/* read-write, power of 2. */
	uint16_t queue_msix_vector;	/* read-write */
	uint16_t queue_enable;		/* read-write */
	uint16_t queue_notify_off;	/* read-only */
	uint32_t queue_desc_lo;		/* read-write */
	uint32_t queue_desc_hi;		/* read-write */
	uint32_t queue_avail_lo;	/* read-write */
	uint32_t queue_avail_hi;	/* read-write */
	uint32_t queue_used_lo;		/* read-write */
	uint32_t queue_used_hi;		/* read-write */
};

/* Fields in VIRTIO_PCI_CAP_PCI_CFG: */
struct virtio_pci_cfg_cap {
	struct virtio_pci_cap cap;
	uint8_t pci_cfg_data[4]; /* Data for BAR access. */
};

/* Macro versions of offsets for the Old Timers! */
#define VIRTIO_PCI_CAP_VNDR		0
#define VIRTIO_PCI_CAP_NEXT		1
#define VIRTIO_PCI_CAP_LEN		2
#define VIRTIO_PCI_CAP_CFG_TYPE		3
#define VIRTIO_PCI_CAP_BAR		4
#define VIRTIO_PCI_CAP_OFFSET		8
#define VIRTIO_PCI_CAP_LENGTH		12

#define VIRTIO_PCI_NOTIFY_CAP_MULT	16

#define VIRTIO_PCI_COMMON_DFSELECT	0
#define VIRTIO_PCI_COMMON_DF		4
#define VIRTIO_PCI_COMMON_GFSELECT	8
#define VIRTIO_PCI_COMMON_GF		12
#define VIRTIO_PCI_COMMON_MSIX		16
#define VIRTIO_PCI_COMMON_NUMQ		18
#define VIRTIO_PCI_COMMON_STATUS	20
#define VIRTIO_PCI_COMMON_CFGGENERATION	21
#define VIRTIO_PCI_COMMON_Q_SELECT	22
#define VIRTIO_PCI_COMMON_Q_SIZE	24
#define VIRTIO_PCI_COMMON_Q_MSIX	26
#define VIRTIO_PCI_COMMON_Q_ENABLE	28
#define VIRTIO_PCI_COMMON_Q_NOFF	30
#define VIRTIO_PCI_COMMON_Q_DESCLO	32
#define VIRTIO_PCI_COMMON_Q_DESCHI	36
#define VIRTIO_PCI_COMMON_Q_AVAILLO	40
#define VIRTIO_PCI_COMMON_Q_AVAILHI	44
#define VIRTIO_PCI_COMMON_Q_USEDLO	48
#define VIRTIO_PCI_COMMON_Q_USEDHI	52

#endif /* _VIRTIO_PCI_MODERN_VAR_H */




/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright IBM Corp. 2007
 *
 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *
 * This header is BSD licensed so anyone can use the definitions to implement
 * compatible drivers/servers.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of IBM nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * $FreeBSD$
 */

#ifndef _VIRTIO_PCI_VAR_H
#define _VIRTIO_PCI_VAR_H

/* VirtIO PCI vendor/device ID. */
#define VIRTIO_PCI_VENDORID	0x1AF4
#define VIRTIO_PCI_DEVICEID_MIN	0x1000
#define VIRTIO_PCI_DEVICEID_LEGACY_MAX	0x103F
#define VIRTIO_PCI_DEVICEID_MODERN_MIN	0x1040
#define VIRTIO_PCI_DEVICEID_MODERN_MAX	0x107F

/* The bit of the ISR which indicates a device has an interrupt. */
#define VIRTIO_PCI_ISR_INTR	0x1
/* The bit of the ISR which indicates a device configuration change. */
#define VIRTIO_PCI_ISR_CONFIG	0x2
/* Vector value used to disable MSI for queue. */
#define VIRTIO_MSI_NO_VECTOR	0xFFFF

#endif /* _VIRTIO_PCI_VAR_H */




static int	vtrnd_attach(device_t);
static int	vtrnd_detach(device_t);

static void	vtrnd_negotiate_features(struct vtrnd_softc *);
static int	vtrnd_setup_features(struct vtrnd_softc *);
static int	vtrnd_alloc_virtqueue(struct vtrnd_softc *);
static void	vtrnd_harvest(struct vtrnd_softc *);
static void	vtrnd_timer(void *);

};
static devclass_t vtrnd_devclass;

DRIVER_MODULE(virtio_random, virtio_pci, vtrnd_driver, vtrnd_devclass,
    vtrnd_modevent, 0);
DRIVER_MODULE(virtio_random, vtpcim, vtrnd_driver, vtrnd_devclass,
    vtrnd_modevent, 0);
MODULE_VERSION(virtio_random, 1);
MODULE_DEPEND(virtio_random, virtio, 1, 1, 1);



	sc = device_get_softc(dev);
	sc->vtrnd_dev = dev;

	callout_init(&sc->vtrnd_callout, 1);

	virtio_set_feature_desc(dev, vtrnd_feature_desc);
	vtrnd_negotiate_features(sc);

		goto fail;
	}
	error = vtrnd_alloc_virtqueue(sc);
	if (error) {
		device_printf(dev, "cannot allocate virtqueue\n");

	return (0);
}

static void
vtrnd_negotiate_features(struct vtrnd_softc *sc)
{
	device_t dev;

	features = VTRND_FEATURES;

	sc->vtrnd_features = virtio_negotiate_features(dev, features);
	return (virtio_finalize_features(dev));
}

static int
vtrnd_setup_features(struct vtrnd_softc *sc)
{
	int error;

	error = vtrnd_negotiate_features(sc);
	if (error)
		return (error);

	return (0);
}

static int
vtrnd_alloc_virtqueue(struct vtrnd_softc *sc)
{
	device_t dev;

	virtqueue_notify(vq);
	virtqueue_poll(vq, NULL);

//	random_harvest_queue(&value, sizeof(value), sizeof(value) * NBBY / 2,
//	    RANDOM_PURE_VIRTIO);
	random_harvest_queue(&value, sizeof(value), RANDOM_PURE_VIRTIO);
}





static int	vtscsi_suspend(device_t);
static int	vtscsi_resume(device_t);

static void	vtscsi_negotiate_features(struct vtscsi_softc *);
static int	vtscsi_setup_features(struct vtscsi_softc *);
static void	vtscsi_read_config(struct vtscsi_softc *,
		    struct virtio_scsi_config *);
static int	vtscsi_maximum_segments(struct vtscsi_softc *, int);


static void	vtscsi_get_request_lun(uint8_t [], target_id_t *, lun_id_t *);
static void	vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);
static void	vtscsi_init_scsi_cmd_req(struct ccb_scsiio *,
		    struct virtio_scsi_cmd_req *);
static void	vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t,
		    uintptr_t, struct virtio_scsi_ctrl_tmf_req *);

static void	vtscsi_freeze_simq(struct vtscsi_softc *, int);
static int	vtscsi_thaw_simq(struct vtscsi_softc *, int);

static void	vtscsi_enable_vqs_intr(struct vtscsi_softc *);

static void	vtscsi_get_tunables(struct vtscsi_softc *);
static void	vtscsi_add_sysctl(struct vtscsi_softc *);

static void	vtscsi_printf_req(struct vtscsi_request *, const char *,
		    const char *, ...);

#define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0)
#define vtscsi_htog16(_sc, _val)	virtio_htog16(vtscsi_modern(_sc), _val)
#define vtscsi_htog32(_sc, _val)	virtio_htog32(vtscsi_modern(_sc), _val)
#define vtscsi_htog64(_sc, _val)	virtio_htog64(vtscsi_modern(_sc), _val)
#define vtscsi_gtoh16(_sc, _val)	virtio_gtoh16(vtscsi_modern(_sc), _val)
#define vtscsi_gtoh32(_sc, _val)	virtio_gtoh32(vtscsi_modern(_sc), _val)
#define vtscsi_gtoh64(_sc, _val)	virtio_gtoh64(vtscsi_modern(_sc), _val)

/* Global tunables. */
/*
 * The current QEMU VirtIO SCSI implementation does not cancel in-flight

static struct virtio_feature_desc vtscsi_feature_desc[] = {
	{ VIRTIO_SCSI_F_INOUT,		"InOut"		},
	{ VIRTIO_SCSI_F_HOTPLUG,	"Hotplug"	},
	{ VIRTIO_SCSI_F_CHANGE,		"ChangeEvent"	},
	{ VIRTIO_SCSI_F_T10_PI, 	"T10PI"		},

	{ 0, NULL }
};

};
static devclass_t vtscsi_devclass;

DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass,
    vtscsi_modevent, 0);
DRIVER_MODULE(virtio_scsi, vtpcim, vtscsi_driver, vtscsi_devclass,
    vtscsi_modevent, 0);
MODULE_VERSION(virtio_scsi, 1);
MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1);
MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1);


	sc = device_get_softc(dev);
	sc->vtscsi_dev = dev;
	virtio_set_feature_desc(dev, vtscsi_feature_desc);

	VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev));
	TAILQ_INIT(&sc->vtscsi_req_free);

	vtscsi_get_tunables(sc);
	vtscsi_add_sysctl(sc);

	virtio_set_feature_desc(dev, vtscsi_feature_desc);
	vtscsi_negotiate_features(sc);
		device_printf(dev, "cannot setup features\n");
		goto fail;
	}

	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT;
	if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
		sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL;
	if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
		sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG;

	vtscsi_read_config(sc, &scsicfg);

	sc->vtscsi_max_channel = scsicfg.max_channel;

	return (0);
}

static void
vtscsi_negotiate_features(struct vtscsi_softc *sc)
{
	device_t dev;
	uint64_t features;

	dev = sc->vtscsi_dev;
	features = virtio_negotiate_features(dev, VTSCSI_FEATURES);
	sc->vtscsi_features = features;
	sc->vtscsi_features = virtio_negotiate_features(dev, features);
	return (virtio_finalize_features(dev));
}

static int
vtscsi_setup_features(struct vtscsi_softc *sc)
{
	device_t dev;
	int error;

	dev = sc->vtscsi_dev;

	error = vtscsi_negotiate_features(sc);
	if (error)
		return (error);

	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		sc->vtscsi_flags |= VTSCSI_FLAG_INDIRECT;
	if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
		sc->vtscsi_flags |= VTSCSI_FLAG_BIDIRECTIONAL;
	if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
		sc->vtscsi_flags |= VTSCSI_FLAG_HOTPLUG;

	return (0);
}

#define VTSCSI_GET_CONFIG(_dev, _field, _cfg)			\
	virtio_read_device_config(_dev,				\
	    offsetof(struct virtio_scsi_config, _field),	\

	error = virtio_reinit(dev, sc->vtscsi_features);
	if (error == 0) {
		vtscsi_write_device_config(sc);
		virtio_reinit_complete(dev);
		vtscsi_reinit_event_vq(sc);
		virtio_reinit_complete(dev);

		vtscsi_enable_vqs_intr(sc);
	}


	cpi->max_target = sc->vtscsi_max_target;
	cpi->max_lun = sc->vtscsi_max_lun;
	cpi->initiator_id = VTSCSI_INITIATOR_ID;

	strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
	strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);

	cmd_req = &req->vsr_cmd_req;
	cmd_resp = &req->vsr_cmd_resp;

	vtscsi_init_scsi_cmd_req(csio, cmd_req);

	error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable);
	if (error)

	tmf_req = &req->vsr_tmf_req;
	tmf_resp = &req->vsr_tmf_resp;

	vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
	    (uintptr_t) to_ccbh, tmf_req);

	sglist_reset(sg);

vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc,
    struct ccb_scsiio *csio, struct virtio_scsi_cmd_resp *cmd_resp)
{
	uint32_t resp_sense_length;
	cam_status status;

	csio->scsi_status = cmd_resp->status;
	csio->resid = cmd_resp->resid;

	if (csio->scsi_status == SCSI_STATUS_OK)
		status = CAM_REQ_CMP;
	else
		status = CAM_SCSI_STATUS_ERROR;

	if (cmd_resp->sense_len > 0) {

	if (resp_sense_length > 0) {
		status |= CAM_AUTOSNS_VALID;

		if (cmd_resp->sense_len < csio->sense_len)
			csio->sense_resid = csio->sense_len -
			    cmd_resp->sense_len;
		else
			csio->sense_resid = 0;

		memcpy(&csio->sense_data, cmd_resp->sense,
		memcpy(cmd_resp->sense, &csio->sense_data,
		    csio->sense_len - csio->sense_resid);
	}


	if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
		callout_stop(&abort_req->vsr_callout);

	vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
	    (uintptr_t) abort_ccbh, tmf_req);

	sglist_reset(sg);

	else
		subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;

	vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req);

	sglist_reset(sg);
	sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));

}

static void
vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio,
    struct virtio_scsi_cmd_req *cmd_req)
{
	uint8_t attr;

	}

	vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun);
	cmd_req->tag = (uintptr_t) csio;
	cmd_req->task_attr = attr;

	memcpy(cmd_req->cdb,

}

static void
vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype,
    uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)
{

	vtscsi_set_request_lun(ccbh, tmf_req->lun);

	tmf_req->type = VIRTIO_SCSI_T_TMF;
	tmf_req->subtype = subtype;
	tmf_req->tag = tag;
}

static void

}

static void
vtscsi_add_sysctl(struct vtscsi_softc *sc)
{
	device_t dev;
	struct vtscsi_statistics *stats;




#ifndef _VIRTIO_SCSI_H
#define _VIRTIO_SCSI_H

/* Feature bits */
#define VIRTIO_SCSI_F_INOUT	0x0001	/* Single request can contain both
					 * read and write buffers */
#define VIRTIO_SCSI_F_HOTPLUG	0x0002	/* Host should enable hot plug/unplug
					 * of new LUNs and targets.
					 */

#define VIRTIO_SCSI_CDB_SIZE	32
#define VIRTIO_SCSI_SENSE_SIZE	96


	uint8_t lun[8];		/* Logical Unit Number */
	uint64_t tag;		/* Command identifier */
	uint8_t task_attr;	/* Task attribute */
	uint8_t prio;
	uint8_t crn;
	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
} __packed;

/* SCSI command request, followed by protection information */
struct virtio_scsi_cmd_req_pi {
	uint8_t lun[8];		/* Logical Unit Number */
	uint64_t tag;		/* Command identifier */
	uint8_t task_attr;	/* Task attribute */
	uint8_t prio;		/* SAM command priority field */
	uint8_t crn;
	uint32_t pi_bytesout;	/* DataOUT PI Number of bytes */
	uint32_t pi_bytesin;	/* DataIN PI Number of bytes */
	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
} __packed;

/* Response, followed by sense data and data-in */
struct virtio_scsi_cmd_resp {
	uint32_t sense_len;		/* Sense data length */

	uint32_t max_lun;
} __packed;

/* Feature bits */
#define VIRTIO_SCSI_F_INOUT	0x0001	/* Single request can contain both
					 * read and write buffers.
					 */
#define VIRTIO_SCSI_F_HOTPLUG	0x0002	/* Host should enable hot plug/unplug
					 * of new LUNs and targets.
					 */
#define VIRTIO_SCSI_F_CHANGE	0x0004	/* Host will report changes to LUN
					 * parameters via a
					 * VIRTIO_SCSI_T_PARAM_CHANGE event.
					 */
#define VIRTIO_SCSI_F_T10_PI 	0x0008	/* Extended fields for T10 protection
					 * information (DIF/DIX) are included
					 * in the SCSI request header.
					 */

/* Response codes */
#define VIRTIO_SCSI_S_OK                       0
#define VIRTIO_SCSI_S_FUNCTION_COMPLETE        0

#define VIRTIO_SCSI_T_NO_EVENT                 0
#define VIRTIO_SCSI_T_TRANSPORT_RESET          1
#define VIRTIO_SCSI_T_ASYNC_NOTIFY             2
#define VIRTIO_SCSI_T_PARAM_CHANGE             3

/* Reasons of transport reset event */
#define VIRTIO_SCSI_EVT_RESET_HARD             0

Lines 205-210 Link Here

(-)sys/dev/virtio.ori/scsi/virtio_scsivar.h (+5 lines)
205	#define VTSCSI_RESERVED_REQUESTS 10	205	#define VTSCSI_RESERVED_REQUESTS 10
206		206
207	/*	207	/*
		208	* Specification doesn't say, use traditional SCSI default.
		209	*/
		210	#define VTSCSI_INITIATOR_ID 7
		211
		212	/*
208	* How to wait (or not) for request completion.	213	* How to wait (or not) for request completion.
209	*/	214	*/
210	#define VTSCSI_EXECUTE_ASYNC 0	215	#define VTSCSI_EXECUTE_ASYNC 0





/* Device independent features. */
static struct virtio_feature_desc virtio_common_feature_desc[] = {
	{ VIRTIO_F_NOTIFY_ON_EMPTY,	"NotifyOnEmpty"	},
	{ VIRTIO_RING_F_INDIRECT_DESC,	"RingIndirect"	},
	{ VIRTIO_RING_F_EVENT_IDX,	"EventIdx"	},
	{ VIRTIO_F_BAD_FEATURE,		"BadFeature"	},
	{ VIRTIO_F_BAD_FEATURE,		"BadFeature"		}, /* Legacy */
	{ VIRTIO_F_VERSION_1,		"Version1"		},
	{ VIRTIO_F_IOMMU_PLATFORM,	"IOMMUPlatform"		},

	{ 0, NULL }
};

	return (NULL);
}

void
virtio_describe(device_t dev, const char *msg,
    uint64_t features, struct virtio_feature_desc *desc)
{
	struct sbuf sb;
	uint64_t val;
	char *buf;
	const char *name;
	int n;

	if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) {
		device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features);
		return;
	}

	sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN);
	sbuf_printf(&sb, "%s features: %#jx", msg, (uintmax_t) features);

	for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) {
		/*
		 * BAD_FEATURE is used to detect broken Linux clients

			continue;

		if (n++ == 0)
			sbuf_cat(&sb, " <");
		else
			sbuf_cat(&sb, ",");

		name = virtio_feature_name(val, desc);
		if (name == NULL)
			sbuf_printf(&sb, "%#jx", (uintmax_t) val);
		else
			sbuf_cat(&sb, name);
	}

	if (n > 0)
		sbuf_cat(&sb, ">");

#if __FreeBSD_version < 900020
	sbuf_finish(&sb);
	if (sbuf_overflowed(&sb) == 0)
#else
	if (sbuf_finish(&sb) == 0)
#endif
{
	struct sbuf sb;
	char *buf;
	int error;

	if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) {
		error = ENOMEM;
		goto out;
	}

	sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN);
	sbuf_printf(&sb, "%s features: ", msg);

	error = virtio_describe_sbuf(&sb, features, desc);
	if (error == 0)
		device_printf(dev, "%s\n", sbuf_data(&sb));

	sbuf_delete(&sb);
	free(buf, M_TEMP);

out:
	if (error != 0) {
		device_printf(dev, "%s features: %#jx\n", msg,
		    (uintmax_t) features);
	}
}

uint64_t
virtio_filter_transport_features(uint64_t features)
{
	uint64_t transport, mask;

	transport = (1ULL <<
	    (VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1;
	transport <<= VIRTIO_TRANSPORT_F_START;

	mask = -1ULL & ~transport;
	mask |= VIRTIO_RING_F_INDIRECT_DESC;
	mask |= VIRTIO_RING_F_EVENT_IDX;
	mask |= VIRTIO_F_VERSION_1;

	return (features & mask);
}

int
virtio_bus_is_modern(device_t dev)
{
	uintptr_t modern;

	virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern);
	return (modern != 0);
}

void
virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst,
    int size, int count)
{
	int i, gen;

	do {
		gen = virtio_config_generation(dev);

		for (i = 0; i < count; i++) {
			virtio_read_device_config(dev, offset + i * size,
			    (uint8_t *) dst + i * size, size);
		}
	} while (gen != virtio_config_generation(dev));
}

/*
 * VirtIO bus method wrappers.
 */


	return (VIRTIO_BUS_NEGOTIATE_FEATURES(device_get_parent(dev),
	    child_features));
}

int
virtio_finalize_features(device_t dev)
{

	return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev)));
}

int




#ifndef _VIRTIO_H_
#define _VIRTIO_H_

#include <dev/virtio/virtio_endian.h>
#include <dev/virtio/virtio_ids.h>
#include <dev/virtio/virtio_config.h>

struct sbuf;
struct vq_alloc_info;

/*

#define VIRTIO_IVAR_DEVICE		4
#define VIRTIO_IVAR_SUBVENDOR		5
#define VIRTIO_IVAR_SUBDEVICE		6
#define VIRTIO_IVAR_MODERN		7

struct virtio_feature_desc {
	uint64_t	 vfd_val;


const char *virtio_device_name(uint16_t devid);
void	 virtio_describe(device_t dev, const char *msg,
	     uint64_t features, struct virtio_feature_desc *feature_desc);
int	 virtio_describe_sbuf(struct sbuf *sb, uint64_t features,
	     struct virtio_feature_desc *desc);
uint64_t virtio_filter_transport_features(uint64_t features);
int	 virtio_bus_is_modern(device_t dev);
void	 virtio_read_device_config_array(device_t dev, bus_size_t offset,
	     void *dst, int size, int count);

/*
 * VirtIO Bus Methods.

void	 virtio_read_ivar(device_t dev, int ivar, uintptr_t *val);
void	 virtio_write_ivar(device_t dev, int ivar, uintptr_t val);
uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features);
int	 virtio_finalize_features(device_t dev);
int	 virtio_alloc_virtqueues(device_t dev, int flags, int nvqs,
	     struct vq_alloc_info *info);
int	 virtio_setup_intr(device_t dev, enum intr_type type);

VIRTIO_READ_IVAR(device,	VIRTIO_IVAR_DEVICE);
VIRTIO_READ_IVAR(subvendor,	VIRTIO_IVAR_SUBVENDOR);
VIRTIO_READ_IVAR(subdevice,	VIRTIO_IVAR_SUBDEVICE);
VIRTIO_READ_IVAR(modern,	VIRTIO_IVAR_MODERN);

#undef VIRTIO_READ_IVAR






CODE {
	static int
	virtio_bus_default_finalize_features(device_t dev)
	{
		return (0);
	}

	static int
	virtio_bus_default_config_generation(device_t dev)
	{
		return (0);

	uint64_t	child_features;
};

METHOD int finalize_features {
	device_t	dev;
} DEFAULT virtio_bus_default_finalize_features;

METHOD int with_feature {
	device_t	dev;
	uint64_t	feature;

METHOD void notify_vq {
	device_t	dev;
	uint16_t	queue;
	bus_size_t	offset;
};

METHOD int config_generation {




/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice unmodified, this list of conditions, and the following
 *    disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD$
 */

#ifndef _VIRTIO_ENDIAN_H_
#define _VIRTIO_ENDIAN_H_

#include <sys/endian.h>

/*
 * VirtIO V1 (modern) uses little endian, while legacy VirtIO uses the guest's
 * native endian. These functions convert to and from the Guest's (driver's)
 * and the Host's (device's) endianness when needed.
 */

static inline bool
virtio_swap_endian(bool modern)
{
#if _BYTE_ORDER == _LITTLE_ENDIAN
	return (false);
#else
	return (modern);
#endif
}

static inline uint16_t
virtio_htog16(bool modern, uint16_t val)
{
	if (virtio_swap_endian(modern))
		return (le16toh(val));
	else
		return (val);
}

static inline uint16_t
virtio_gtoh16(bool modern, uint16_t val)
{
	if (virtio_swap_endian(modern))
		return (htole16(val));
	else
		return (val);
}

static inline uint32_t
virtio_htog32(bool modern, uint32_t val)
{
	if (virtio_swap_endian(modern))
		return (le32toh(val));
	else
		return (val);
}

static inline uint32_t
virtio_gtoh32(bool modern, uint32_t val)
{
	if (virtio_swap_endian(modern))
		return (htole32(val));
	else
		return (val);
}

static inline uint64_t
virtio_htog64(bool modern, uint64_t val)
{
	if (virtio_swap_endian(modern))
		return (le64toh(val));
	else
		return (val);
}

static inline uint64_t
virtio_gtoh64(bool modern, uint64_t val)
{
	if (virtio_swap_endian(modern))
		return (htole64(val));
	else
		return (val);
}

#endif /* _VIRTIO_ENDIAN_H_ */





struct virtqueue {
	device_t		 vq_dev;
	char			 vq_name[VIRTQUEUE_MAX_NAME_SZ];
	uint16_t		 vq_queue_index;
	uint16_t		 vq_nentries;
	uint32_t		 vq_flags;
#define	VIRTQUEUE_FLAG_INDIRECT	 0x0001
#define	VIRTQUEUE_FLAG_EVENT_IDX 0x0002
#define	VIRTQUEUE_FLAG_EVENT_IDX 0x0004

	int			 vq_alignment;
	int			 vq_ring_size;
	void			*vq_ring_mem;
	int			 vq_max_indirect_size;
	int			 vq_indirect_mem_size;
	virtqueue_intr_t	*vq_intrhand;
	void			*vq_intrhand_arg;


	 */
	uint16_t		 vq_used_cons_idx;

	void			*vq_ring_mem;
	int			 vq_indirect_mem_size;
	int			 vq_alignment;
	int			 vq_ring_size;
	char			 vq_name[VIRTQUEUE_MAX_NAME_SZ];

	struct vq_desc_extra {
		void		  *cookie;
		struct vring_desc *indirect;

static void	vq_ring_notify_host(struct virtqueue *);
static void	vq_ring_free_chain(struct virtqueue *, uint16_t);

#define vq_modern(_vq) 		(((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
#define vq_htog16(_vq, _val) 	virtio_htog16(vq_modern(_vq), _val)
#define vq_htog32(_vq, _val) 	virtio_htog32(vq_modern(_vq), _val)
#define vq_htog64(_vq, _val) 	virtio_htog64(vq_modern(_vq), _val)
#define vq_gtoh16(_vq, _val) 	virtio_gtoh16(vq_modern(_vq), _val)
#define vq_gtoh32(_vq, _val) 	virtio_gtoh32(vq_modern(_vq), _val)
#define vq_gtoh64(_vq, _val) 	virtio_gtoh64(vq_modern(_vq), _val)

uint64_t
virtqueue_filter_features(uint64_t features)
{

}

int
virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
    vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp)
    struct vq_alloc_info *info, struct virtqueue **vqp)
{
	struct virtqueue *vq;
	int error;

	vq->vq_dev = dev;
	strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
	vq->vq_queue_index = queue;
	vq->vq_notify_offset = notify_offset;
	vq->vq_alignment = align;
	vq->vq_nentries = size;
	vq->vq_free_cnt = size;
	vq->vq_intrhand = info->vqai_intr;
	vq->vq_intrhand_arg = info->vqai_intr_arg;

	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
		vq->vq_flags |= VIRTQUEUE_FLAG_MODERN;
	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
		vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;


	bzero(indirect, vq->vq_indirect_mem_size);

	for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
		indirect[i].next = i + 1;
	indirect[i].next = VQ_RING_DESC_CHAIN_END;
}

int

{
	uint16_t used_idx, nused;

	used_idx = vq->vq_ring.used->idx;

	nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
	VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");

virtqueue_intr_filter(struct virtqueue *vq)
{

	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
		return (0);

	virtqueue_disable_intr(vq);

{
	uint16_t ndesc, avail_idx;

	avail_idx = vq->vq_ring.avail->idx;
	ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);

	switch (hint) {

{

	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
		vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
		    vq->vq_nentries - 1;
	} else
		vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;

	vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
}

int

	void *cookie;
	uint16_t used_idx, desc_idx;

	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
		return (NULL);

	used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
	uep = &vq->vq_ring.used->ring[used_idx];

	rmb();
	desc_idx = (uint16_t) uep->id;
	if (len != NULL)
		*len = uep->len;

	vq_ring_free_chain(vq, desc_idx);


	printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
	    "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
	    "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
	    vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
	    virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
	    vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
	    vq->vq_ring.used->idx,
		vring_used_event(&vq->vq_ring),
	    vq->vq_ring.avail->flags,
	    vq->vq_ring.used->flags);
}

static void

	vring_init(vr, size, ring_mem, vq->vq_alignment);

	for (i = 0; i < size - 1; i++)
		vr->desc[i].next = i + 1;
	vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
}

static void
vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
{
	uint16_t avail_idx;

	/*
	 * Place the head of the descriptor chain into the next slot and make

	 * currently running on another CPU, we can keep it processing the new
	 * descriptor.
	 */
	avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
	vq->vq_ring.avail->ring[avail_idx] = desc_idx;

	wmb();
	vq->vq_ring.avail->idx++;

	/* Keep pending count until virtqueue_notify(). */
	vq->vq_queued_cnt++;


	for (i = 0, idx = head_idx, seg = sg->sg_segs;
	     i < needed;
	     i++, idx = dp->next, seg++) {
		VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
		    "premature end of free desc chain");

		dp = &desc[idx];
		dp->addr = seg->ss_paddr;
		dp->len = seg->ss_len;
		dp->flags = 0;

		if (i < needed - 1)
			dp->flags |= VRING_DESC_F_NEXT;
		if (i >= readable)
			dp->flags |= VRING_DESC_F_WRITE;
	}

	return (idx);

	dxp->cookie = cookie;
	dxp->ndescs = 1;

	dp->addr = dxp->indirect_paddr;
	dp->len = needed * sizeof(struct vring_desc);
	dp->flags = VRING_DESC_F_INDIRECT;

	vq_ring_enqueue_segments(vq, dxp->indirect, 0,
	    sg, readable, writable);

	vq->vq_desc_head_idx = dp->next;
	vq->vq_free_cnt--;
	if (vq->vq_free_cnt == 0)
		VQ_RING_ASSERT_CHAIN_TERM(vq);

	 * Enable interrupts, making sure we get the latest index of
	 * what's already been consumed.
	 */
	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
		vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
	else
		vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;

		    vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
	}
	mb();

	/*

static int
vq_ring_must_notify_host(struct virtqueue *vq)
{
	uint16_t new_idx, prev_idx, event_idx;

	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
		new_idx = vq->vq_ring.avail->idx;
		prev_idx = new_idx - vq->vq_queued_cnt;
		event_idx = vring_avail_event(&vq->vq_ring);

		return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
	}

	return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
	return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
}

static void
vq_ring_notify_host(struct virtqueue *vq)
{

	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
	    vq->vq_notify_offset);
}

static void

	vq->vq_free_cnt += dxp->ndescs;
	dxp->ndescs--;

	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
		while (dp->flags & VRING_DESC_F_NEXT) {
			VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
			dp = &vq->vq_ring.desc[dp->next];
			dp = &vq->vq_ring.desc[next_idx];
			dxp->ndescs--;
		}
	}

	 * newly freed chain. If the virtqueue was completely used, then
	 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
	 */
	dp->next = vq->vq_desc_head_idx;
	vq->vq_desc_head_idx = desc_idx;
}

Lines 70-77 Link Here

(-)sys/dev/virtio.ori/virtqueue.h (-2 / +2 lines)
70	uint64_t virtqueue_filter_features(uint64_t features);	70	uint64_t virtqueue_filter_features(uint64_t features);
71		71
72	int virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,	72	int virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
73	bus_size_t notify_offset, int align, vm_paddr_t highaddr,	73	int align, vm_paddr_t highaddr, struct vq_alloc_info *info,
74	struct vq_alloc_info info, struct virtqueue *vqp);	74	struct virtqueue **vqp);
75	void virtqueue_drain(struct virtqueue vq, int *last);	75	void virtqueue_drain(struct virtqueue vq, int *last);
76	void virtqueue_free(struct virtqueue *vq);	76	void virtqueue_free(struct virtqueue *vq);
77	int virtqueue_reinit(struct virtqueue *vq, uint16_t size);	77	int virtqueue_reinit(struct virtqueue *vq, uint16_t size);

Return to bug 236922

Lines 80-86 Link Here

(-)sys/dev/virtio.ori/balloon/virtio_balloon.c (-49 / +15 lines)
80	static struct virtio_feature_desc vtballoon_feature_desc[] = {	80	static struct virtio_feature_desc vtballoon_feature_desc[] = {
81	{ VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },	81	{ VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" },
82	{ VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },	82	{ VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" },
83	{ VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" },
84		83
85	{ 0, NULL }	84	{ 0, NULL }
86	};	85	};
Lines 90-97 Link Here
90	static int vtballoon_detach(device_t);	89	static int vtballoon_detach(device_t);
91	static int vtballoon_config_change(device_t);	90	static int vtballoon_config_change(device_t);
92		91
93	static int vtballoon_negotiate_features(struct vtballoon_softc *);	92	static void vtballoon_negotiate_features(struct vtballoon_softc *);
94	static int vtballoon_setup_features(struct vtballoon_softc *);
95	static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);	93	static int vtballoon_alloc_virtqueues(struct vtballoon_softc *);
96		94
97	static void vtballoon_vq_intr(void *);	95	static void vtballoon_vq_intr(void *);
Lines 111-123 Link Here
111		109
112	static int vtballoon_sleep(struct vtballoon_softc *);	110	static int vtballoon_sleep(struct vtballoon_softc *);
113	static void vtballoon_thread(void *);	111	static void vtballoon_thread(void *);
114	static void vtballoon_setup_sysctl(struct vtballoon_softc *);	112	static void vtballoon_add_sysctl(struct vtballoon_softc *);
115		113
116	#define vtballoon_modern(_sc) \
117	(((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
118
119	/* Features desired/implemented by this driver. */	114	/* Features desired/implemented by this driver. */
120	#define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST	115	#define VTBALLOON_FEATURES 0
121		116
122	/* Timeout between retries when the balloon needs inflating. */	117	/* Timeout between retries when the balloon needs inflating. */
123	#define VTBALLOON_LOWMEM_TIMEOUT hz	118	#define VTBALLOON_LOWMEM_TIMEOUT hz
Lines 158-167 Link Here
158	};	153	};
159	static devclass_t vtballoon_devclass;	154	static devclass_t vtballoon_devclass;
160		155
161	DRIVER_MODULE(virtio_balloon, vtpcil, vtballoon_driver,	156	DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
162	vtballoon_devclass, 0, 0);	157	vtballoon_devclass, 0, 0);
163	DRIVER_MODULE(virtio_balloon, vtpcim, vtballoon_driver,
164	vtballoon_devclass, 0, 0);
165	MODULE_VERSION(virtio_balloon, 1);	158	MODULE_VERSION(virtio_balloon, 1);
166	MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);	159	MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
167		160
Lines 185-202 Link Here
185		178
186	sc = device_get_softc(dev);	179	sc = device_get_softc(dev);
187	sc->vtballoon_dev = dev;	180	sc->vtballoon_dev = dev;
188	virtio_set_feature_desc(dev, vtballoon_feature_desc);
189		181
190	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));	182	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
191	TAILQ_INIT(&sc->vtballoon_pages);	183	TAILQ_INIT(&sc->vtballoon_pages);
192		184
193	vtballoon_setup_sysctl(sc);	185	vtballoon_add_sysctl(sc);
194		186
195	error = vtballoon_setup_features(sc);	187	virtio_set_feature_desc(dev, vtballoon_feature_desc);
196	if (error) {	188	vtballoon_negotiate_features(sc);
197	device_printf(dev, "cannot setup features\n");
198	goto fail;
199	}
200		189
201	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *	190	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
202	sizeof(uint32_t), M_DEVBUF, M_NOWAIT \| M_ZERO);	191	sizeof(uint32_t), M_DEVBUF, M_NOWAIT \| M_ZERO);
Lines 282-313 Link Here
282	return (1);	271	return (1);
283	}	272	}
284		273
285	static int	274	static void
286	vtballoon_negotiate_features(struct vtballoon_softc *sc)	275	vtballoon_negotiate_features(struct vtballoon_softc *sc)
287	{	276	{
288	device_t dev;	277	device_t dev;
289	uint64_t features;	278	uint64_t features;
290		279
291	dev = sc->vtballoon_dev;	280	dev = sc->vtballoon_dev;
292	features = VTBALLOON_FEATURES;	281	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
293		282	sc->vtballoon_features = features;
294	sc->vtballoon_features = virtio_negotiate_features(dev, features);
295	return (virtio_finalize_features(dev));
296	}	283	}
297		284
298	static int	285	static int
299	vtballoon_setup_features(struct vtballoon_softc *sc)
300	{
301	int error;
302
303	error = vtballoon_negotiate_features(sc);
304	if (error)
305	return (error);
306
307	return (0);
308	}
309
310	static int
311	vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)	286	vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
312	{	287	{
313	device_t dev;	288	device_t dev;
Lines 465-472 Link Here
465	{	440	{
466	vm_page_t m;	441	vm_page_t m;
467		442
468	m = vm_page_alloc(NULL, 0,	443	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ);
469	VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ \| VM_ALLOC_NODUMP);
470	if (m != NULL)	444	if (m != NULL)
471	sc->vtballoon_current_npages++;	445	sc->vtballoon_current_npages++;
472		446
Lines 489-512 Link Here
489	desired = virtio_read_dev_config_4(sc->vtballoon_dev,	463	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
490	offsetof(struct virtio_balloon_config, num_pages));	464	offsetof(struct virtio_balloon_config, num_pages));
491		465
492	if (vtballoon_modern(sc))	466	return (le32toh(desired));
493	return (desired);
494	else
495	return (le32toh(desired));
496	}	467	}
497		468
498	static void	469	static void
499	vtballoon_update_size(struct vtballoon_softc *sc)	470	vtballoon_update_size(struct vtballoon_softc *sc)
500	{	471	{
501	uint32_t npages;
502		472
503	npages = sc->vtballoon_current_npages;
504	if (!vtballoon_modern(sc))
505	npages = htole32(npages);
506
507	virtio_write_dev_config_4(sc->vtballoon_dev,	473	virtio_write_dev_config_4(sc->vtballoon_dev,
508	offsetof(struct virtio_balloon_config, actual), npages);	474	offsetof(struct virtio_balloon_config, actual),
509		475	htole32(sc->vtballoon_current_npages));
510	}	476	}
511		477
512	static int	478	static int
Lines 578-584 Link Here
578	}	544	}
579		545
580	static void	546	static void
581	vtballoon_setup_sysctl(struct vtballoon_softc *sc)	547	vtballoon_add_sysctl(struct vtballoon_softc *sc)
582	{	548	{
583	device_t dev;	549	device_t dev;
584	struct sysctl_ctx_list *ctx;	550	struct sysctl_ctx_list *ctx;

Lines 158-165 Link Here

(-)sys/dev/virtio.ori/console/virtio_console.c (-39 / +16 lines)
158	static int vtcon_detach(device_t);	158	static int vtcon_detach(device_t);
159	static int vtcon_config_change(device_t);	159	static int vtcon_config_change(device_t);
160		160
161	static int vtcon_setup_features(struct vtcon_softc *);	161	static void vtcon_setup_features(struct vtcon_softc *);
162	static int vtcon_negotiate_features(struct vtcon_softc *);	162	static void vtcon_negotiate_features(struct vtcon_softc *);
163	static int vtcon_alloc_scports(struct vtcon_softc *);	163	static int vtcon_alloc_scports(struct vtcon_softc *);
164	static int vtcon_alloc_virtqueues(struct vtcon_softc *);	164	static int vtcon_alloc_virtqueues(struct vtcon_softc *);
165	static void vtcon_read_config(struct vtcon_softc *,	165	static void vtcon_read_config(struct vtcon_softc *,
Lines 227-240 Link Here
227	static void vtcon_enable_interrupts(struct vtcon_softc *);	227	static void vtcon_enable_interrupts(struct vtcon_softc *);
228	static void vtcon_disable_interrupts(struct vtcon_softc *);	228	static void vtcon_disable_interrupts(struct vtcon_softc *);
229		229
230	#define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0)
231	#define vtcon_htog16(_sc, _val) virtio_htog16(vtcon_modern(_sc), _val)
232	#define vtcon_htog32(_sc, _val) virtio_htog32(vtcon_modern(_sc), _val)
233	#define vtcon_htog64(_sc, _val) virtio_htog64(vtcon_modern(_sc), _val)
234	#define vtcon_gtoh16(_sc, _val) virtio_gtoh16(vtcon_modern(_sc), _val)
235	#define vtcon_gtoh32(_sc, _val) virtio_gtoh32(vtcon_modern(_sc), _val)
236	#define vtcon_gtoh64(_sc, _val) virtio_gtoh64(vtcon_modern(_sc), _val)
237
238	static int vtcon_pending_free;	230	static int vtcon_pending_free;
239		231
240	static struct ttydevsw vtcon_tty_class = {	232	static struct ttydevsw vtcon_tty_class = {
Lines 264-273 Link Here
264	};	256	};
265	static devclass_t vtcon_devclass;	257	static devclass_t vtcon_devclass;
266		258
267	DRIVER_MODULE(virtio_console, vtpcil, vtcon_driver, vtcon_devclass,	259	DRIVER_MODULE(virtio_console, virtio_pci, vtcon_driver, vtcon_devclass,
268	vtcon_modevent, 0);	260	vtcon_modevent, 0);
269	DRIVER_MODULE(virtio_console, vtpcim, vtcon_driver, vtcon_devclass,
270	vtcon_modevent, 0);
271	MODULE_VERSION(virtio_console, 1);	261	MODULE_VERSION(virtio_console, 1);
272	MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);	262	MODULE_DEPEND(virtio_console, virtio, 1, 1, 1);
273		263
Lines 333-348 Link Here
333		323
334	sc = device_get_softc(dev);	324	sc = device_get_softc(dev);
335	sc->vtcon_dev = dev;	325	sc->vtcon_dev = dev;
336	virtio_set_feature_desc(dev, vtcon_feature_desc);
337		326
338	mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);	327	mtx_init(&sc->vtcon_mtx, "vtconmtx", NULL, MTX_DEF);
339	mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);	328	mtx_init(&sc->vtcon_ctrl_tx_mtx, "vtconctrlmtx", NULL, MTX_DEF);
340		329
341	error = vtcon_setup_features(sc);	330	virtio_set_feature_desc(dev, vtcon_feature_desc);
342	if (error) {	331	vtcon_setup_features(sc);
343	device_printf(dev, "cannot setup features\n");
344	goto fail;
345	}
346		332
347	vtcon_read_config(sc, &concfg);	333	vtcon_read_config(sc, &concfg);
348	vtcon_determine_max_ports(sc, &concfg);	334	vtcon_determine_max_ports(sc, &concfg);
Lines 434-440 Link Here
434	return (0);	420	return (0);
435	}	421	}
436		422
437	static int	423	static void
438	vtcon_negotiate_features(struct vtcon_softc *sc)	424	vtcon_negotiate_features(struct vtcon_softc *sc)
439	{	425	{
440	device_t dev;	426	device_t dev;
Lines 444-470 Link Here
444	features = VTCON_FEATURES;	430	features = VTCON_FEATURES;
445		431
446	sc->vtcon_features = virtio_negotiate_features(dev, features);	432	sc->vtcon_features = virtio_negotiate_features(dev, features);
447	return (virtio_finalize_features(dev));
448	}	433	}
449		434
450	static int	435	static void
451	vtcon_setup_features(struct vtcon_softc *sc)	436	vtcon_setup_features(struct vtcon_softc *sc)
452	{	437	{
453	device_t dev;	438	device_t dev;
454	int error;
455		439
456	dev = sc->vtcon_dev;	440	dev = sc->vtcon_dev;
457		441
458	error = vtcon_negotiate_features(sc);	442	vtcon_negotiate_features(sc);
459	if (error)
460	return (error);
461		443
462	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))	444	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_SIZE))
463	sc->vtcon_flags \|= VTCON_FLAG_SIZE;	445	sc->vtcon_flags \|= VTCON_FLAG_SIZE;
464	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))	446	if (virtio_with_feature(dev, VIRTIO_CONSOLE_F_MULTIPORT))
465	sc->vtcon_flags \|= VTCON_FLAG_MULTIPORT;	447	sc->vtcon_flags \|= VTCON_FLAG_MULTIPORT;
466
467	return (0);
468	}	448	}
469		449
470	#define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg) \	450	#define VTCON_GET_CONFIG(_dev, _feature, _field, _cfg) \
Lines 867-886 Link Here
867	struct virtio_console_control control, void data, size_t data_len)	847	struct virtio_console_control control, void data, size_t data_len)
868	{	848	{
869	device_t dev;	849	device_t dev;
870	uint32_t id;	850	int id;
871	uint16_t event;
872		851
873	dev = sc->vtcon_dev;	852	dev = sc->vtcon_dev;
874	id = vtcon_htog32(sc, control->id);	853	id = control->id;
875	event = vtcon_htog16(sc, control->event);
876		854
877	if (id >= sc->vtcon_max_ports) {	855	if (id < 0 \|\| id >= sc->vtcon_max_ports) {
878	device_printf(dev, "%s: event %d invalid port ID %d\n",	856	device_printf(dev, "%s: invalid port ID %d\n", __func__, id);
879	__func__, event, id);
880	return;	857	return;
881	}	858	}
882		859
883	switch (event) {	860	switch (control->event) {
884	case VIRTIO_CONSOLE_PORT_ADD:	861	case VIRTIO_CONSOLE_PORT_ADD:
885	vtcon_ctrl_port_add_event(sc, id);	862	vtcon_ctrl_port_add_event(sc, id);
886	break;	863	break;
Lines 1008-1016 Link Here
1008	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)	985	if ((sc->vtcon_flags & VTCON_FLAG_MULTIPORT) == 0)
1009	return;	986	return;
1010		987
1011	control.id = vtcon_gtoh32(sc, portid);	988	control.id = portid;
1012	control.event = vtcon_gtoh16(sc, event);	989	control.event = event;
1013	control.value = vtcon_gtoh16(sc, value);	990	control.value = value;
1014		991
1015	vtcon_ctrl_poll(sc, &control);	992	vtcon_ctrl_poll(sc, &control);
1016	}	993	}

Lines 58-65 Link Here

(-)sys/dev/virtio.ori/random/virtio_random.c (-27 / +7 lines)
58	static int vtrnd_attach(device_t);	58	static int vtrnd_attach(device_t);
59	static int vtrnd_detach(device_t);	59	static int vtrnd_detach(device_t);
60		60
61	static int vtrnd_negotiate_features(struct vtrnd_softc *);	61	static void vtrnd_negotiate_features(struct vtrnd_softc *);
62	static int vtrnd_setup_features(struct vtrnd_softc *);
63	static int vtrnd_alloc_virtqueue(struct vtrnd_softc *);	62	static int vtrnd_alloc_virtqueue(struct vtrnd_softc *);
64	static void vtrnd_harvest(struct vtrnd_softc *);	63	static void vtrnd_harvest(struct vtrnd_softc *);
65	static void vtrnd_timer(void *);	64	static void vtrnd_timer(void *);
Lines 86-95 Link Here
86	};	85	};
87	static devclass_t vtrnd_devclass;	86	static devclass_t vtrnd_devclass;
88		87
89	DRIVER_MODULE(virtio_random, vtpcil, vtrnd_driver, vtrnd_devclass,	88	DRIVER_MODULE(virtio_random, virtio_pci, vtrnd_driver, vtrnd_devclass,
90	vtrnd_modevent, 0);	89	vtrnd_modevent, 0);
91	DRIVER_MODULE(virtio_random, vtpcim, vtrnd_driver, vtrnd_devclass,
92	vtrnd_modevent, 0);
93	MODULE_VERSION(virtio_random, 1);	90	MODULE_VERSION(virtio_random, 1);
94	MODULE_DEPEND(virtio_random, virtio, 1, 1, 1);	91	MODULE_DEPEND(virtio_random, virtio, 1, 1, 1);
95		92
Lines 133-146 Link Here
133		130
134	sc = device_get_softc(dev);	131	sc = device_get_softc(dev);
135	sc->vtrnd_dev = dev;	132	sc->vtrnd_dev = dev;
136	virtio_set_feature_desc(dev, vtrnd_feature_desc);	133
137	callout_init(&sc->vtrnd_callout, 1);	134	callout_init(&sc->vtrnd_callout, 1);
138		135
139	error = vtrnd_setup_features(sc);	136	virtio_set_feature_desc(dev, vtrnd_feature_desc);
140	if (error) {	137	vtrnd_negotiate_features(sc);
141	device_printf(dev, "cannot setup features\n");	138
142	goto fail;
143	}
144	error = vtrnd_alloc_virtqueue(sc);	139	error = vtrnd_alloc_virtqueue(sc);
145	if (error) {	140	if (error) {
146	device_printf(dev, "cannot allocate virtqueue\n");	141	device_printf(dev, "cannot allocate virtqueue\n");
Lines 168-174 Link Here
168	return (0);	163	return (0);
169	}	164	}
170		165
171	static int	166	static void
172	vtrnd_negotiate_features(struct vtrnd_softc *sc)	167	vtrnd_negotiate_features(struct vtrnd_softc *sc)
173	{	168	{
174	device_t dev;	169	device_t dev;
Lines 178-199 Link Here
178	features = VTRND_FEATURES;	173	features = VTRND_FEATURES;
179		174
180	sc->vtrnd_features = virtio_negotiate_features(dev, features);	175	sc->vtrnd_features = virtio_negotiate_features(dev, features);
181	return (virtio_finalize_features(dev));
182	}	176	}
183		177
184	static int	178	static int
185	vtrnd_setup_features(struct vtrnd_softc *sc)
186	{
187	int error;
188
189	error = vtrnd_negotiate_features(sc);
190	if (error)
191	return (error);
192
193	return (0);
194	}
195
196	static int
197	vtrnd_alloc_virtqueue(struct vtrnd_softc *sc)	179	vtrnd_alloc_virtqueue(struct vtrnd_softc *sc)
198	{	180	{
199	device_t dev;	181	device_t dev;
Lines 235-242 Link Here
235	virtqueue_notify(vq);	217	virtqueue_notify(vq);
236	virtqueue_poll(vq, NULL);	218	virtqueue_poll(vq, NULL);
237		219
238	// random_harvest_queue(&value, sizeof(value), sizeof(value) * NBBY / 2,
239	// RANDOM_PURE_VIRTIO);
240	random_harvest_queue(&value, sizeof(value), RANDOM_PURE_VIRTIO);	220	random_harvest_queue(&value, sizeof(value), RANDOM_PURE_VIRTIO);
241	}	221	}
242		222

Lines 75-87 Link Here

(-)sys/dev/virtio.ori/virtio.c (-92 / +27 lines)
75		75
76	/* Device independent features. */	76	/* Device independent features. */
77	static struct virtio_feature_desc virtio_common_feature_desc[] = {	77	static struct virtio_feature_desc virtio_common_feature_desc[] = {
78	{ VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, /* Legacy */	78	{ VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" },
79	{ VIRTIO_F_ANY_LAYOUT, "AnyLayout" }, /* Legacy */	79	{ VIRTIO_RING_F_INDIRECT_DESC, "RingIndirect" },
80	{ VIRTIO_RING_F_INDIRECT_DESC, "RingIndirectDesc" },	80	{ VIRTIO_RING_F_EVENT_IDX, "EventIdx" },
81	{ VIRTIO_RING_F_EVENT_IDX, "RingEventIdx" },	81	{ VIRTIO_F_BAD_FEATURE, "BadFeature" },
82	{ VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */
83	{ VIRTIO_F_VERSION_1, "Version1" },
84	{ VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" },
85		82
86	{ 0, NULL }	83	{ 0, NULL }
87	};	84	};
Lines 119-134 Link Here
119	return (NULL);	116	return (NULL);
120	}	117	}
121		118
122	int	119	void
123	virtio_describe_sbuf(struct sbuf *sb, uint64_t features,	120	virtio_describe(device_t dev, const char *msg,
124	struct virtio_feature_desc *desc)	121	uint64_t features, struct virtio_feature_desc *desc)
125	{	122	{
		123	struct sbuf sb;
126	uint64_t val;	124	uint64_t val;
		125	char *buf;
127	const char *name;	126	const char *name;
128	int n;	127	int n;
129		128
130	sbuf_printf(sb, "%#jx", (uintmax_t) features);	129	if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) {
		130	device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features);
		131	return;
		132	}
131		133
		134	sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN);
		135	sbuf_printf(&sb, "%s features: %#jx", msg, (uintmax_t) features);
		136
132	for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) {	137	for (n = 0, val = 1ULL << 63; val != 0; val >>= 1) {
133	/*	138	/*
134	* BAD_FEATURE is used to detect broken Linux clients	139	* BAD_FEATURE is used to detect broken Linux clients
Lines 138-232 Link Here
138	continue;	143	continue;
139		144
140	if (n++ == 0)	145	if (n++ == 0)
141	sbuf_cat(sb, " <");	146	sbuf_cat(&sb, " <");
142	else	147	else
143	sbuf_cat(sb, ",");	148	sbuf_cat(&sb, ",");
144		149
145	name = virtio_feature_name(val, desc);	150	name = virtio_feature_name(val, desc);
146	if (name == NULL)	151	if (name == NULL)
147	sbuf_printf(sb, "%#jx", (uintmax_t) val);	152	sbuf_printf(&sb, "%#jx", (uintmax_t) val);
148	else	153	else
149	sbuf_cat(sb, name);	154	sbuf_cat(&sb, name);
150	}	155	}
151		156
152	if (n > 0)	157	if (n > 0)
153	sbuf_cat(sb, ">");	158	sbuf_cat(&sb, ">");
154		159
155	return (sbuf_finish(sb));	160	#if __FreeBSD_version < 900020
156	}	161	sbuf_finish(&sb);
157		162	if (sbuf_overflowed(&sb) == 0)
158	void	163	#else
159	virtio_describe(device_t dev, const char *msg, uint64_t features,	164	if (sbuf_finish(&sb) == 0)
160	struct virtio_feature_desc *desc)	165	#endif
161	{
162	struct sbuf sb;
163	char *buf;
164	int error;
165
166	if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) {
167	error = ENOMEM;
168	goto out;
169	}
170
171	sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN);
172	sbuf_printf(&sb, "%s features: ", msg);
173
174	error = virtio_describe_sbuf(&sb, features, desc);
175	if (error == 0)
176	device_printf(dev, "%s\n", sbuf_data(&sb));	166	device_printf(dev, "%s\n", sbuf_data(&sb));
177		167
178	sbuf_delete(&sb);	168	sbuf_delete(&sb);
179	free(buf, M_TEMP);	169	free(buf, M_TEMP);
180
181	out:
182	if (error != 0) {
183	device_printf(dev, "%s features: %#jx\n", msg,
184	(uintmax_t) features);
185	}
186	}	170	}
187		171
188	uint64_t
189	virtio_filter_transport_features(uint64_t features)
190	{
191	uint64_t transport, mask;
192
193	transport = (1ULL <<
194	(VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1;
195	transport <<= VIRTIO_TRANSPORT_F_START;
196
197	mask = -1ULL & ~transport;
198	mask \|= VIRTIO_RING_F_INDIRECT_DESC;
199	mask \|= VIRTIO_RING_F_EVENT_IDX;
200	mask \|= VIRTIO_F_VERSION_1;
201
202	return (features & mask);
203	}
204
205	int
206	virtio_bus_is_modern(device_t dev)
207	{
208	uintptr_t modern;
209
210	virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern);
211	return (modern != 0);
212	}
213
214	void
215	virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst,
216	int size, int count)
217	{
218	int i, gen;
219
220	do {
221	gen = virtio_config_generation(dev);
222
223	for (i = 0; i < count; i++) {
224	virtio_read_device_config(dev, offset + i * size,
225	(uint8_t ) dst + i size, size);
226	}
227	} while (gen != virtio_config_generation(dev));
228	}
229
230	/*	172	/*
231	* VirtIO bus method wrappers.	173	* VirtIO bus method wrappers.
232	*/	174	*/
Lines 252-264 Link Here
252		194
253	return (VIRTIO_BUS_NEGOTIATE_FEATURES(device_get_parent(dev),	195	return (VIRTIO_BUS_NEGOTIATE_FEATURES(device_get_parent(dev),
254	child_features));	196	child_features));
255	}
256
257	int
258	virtio_finalize_features(device_t dev)
259	{
260
261	return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev)));
262	}	197	}
263		198
264	int	199	int

Lines 36-42 Link Here

(-)sys/dev/virtio.ori/balloon/virtio_balloon.h (-27 / +1 lines)
36	/* Feature bits. */	36	/* Feature bits. */
37	#define VIRTIO_BALLOON_F_MUST_TELL_HOST 0x1 /* Tell before reclaiming pages */	37	#define VIRTIO_BALLOON_F_MUST_TELL_HOST 0x1 /* Tell before reclaiming pages */
38	#define VIRTIO_BALLOON_F_STATS_VQ 0x2 /* Memory stats virtqueue */	38	#define VIRTIO_BALLOON_F_STATS_VQ 0x2 /* Memory stats virtqueue */
39	#define VIRTIO_BALLOON_F_DEFLATE_ON_OOM 0x4 /* Deflate balloon on OOM */
40		39
41	/* Size of a PFN in the balloon interface. */	40	/* Size of a PFN in the balloon interface. */
42	#define VIRTIO_BALLOON_PFN_SHIFT 12	41	#define VIRTIO_BALLOON_PFN_SHIFT 12
Lines 55-87 Link Here
55	#define VIRTIO_BALLOON_S_MINFLT 3 /* Number of minor faults */	54	#define VIRTIO_BALLOON_S_MINFLT 3 /* Number of minor faults */
56	#define VIRTIO_BALLOON_S_MEMFREE 4 /* Total amount of free memory */	55	#define VIRTIO_BALLOON_S_MEMFREE 4 /* Total amount of free memory */
57	#define VIRTIO_BALLOON_S_MEMTOT 5 /* Total amount of memory */	56	#define VIRTIO_BALLOON_S_MEMTOT 5 /* Total amount of memory */
58	#define VIRTIO_BALLOON_S_AVAIL 6 /* Available memory as in /proc */	57	#define VIRTIO_BALLOON_S_NR 6
59	#define VIRTIO_BALLOON_S_CACHES 7 /* Disk caches */
60	#define VIRTIO_BALLOON_S_NR 8
61		58
62	/*
63	* Memory statistics structure.
64	* Driver fills an array of these structures and passes to device.
65	*
66	* NOTE: fields are laid out in a way that would make compiler add padding
67	* between and after fields, so we have to use compiler-specific attributes to
68	* pack it, to disable this padding. This also often causes compiler to
69	* generate suboptimal code.
70	*
71	* We maintain this statistics structure format for backwards compatibility,
72	* but don't follow this example.
73	*
74	* If implementing a similar structure, do something like the below instead:
75	* struct virtio_balloon_stat {
76	* __virtio16 tag;
77	* __u8 reserved[6];
78	* __virtio64 val;
79	* };
80	*
81	* In other words, add explicit reserved fields to align field and
82	* structure boundaries at field size, avoiding compiler padding
83	* without the packed attribute.
84	*/
85	struct virtio_balloon_stat {	59	struct virtio_balloon_stat {
86	uint16_t tag;	60	uint16_t tag;
87	uint64_t val;	61	uint64_t val;

Lines 76-85 Link Here

(-)sys/dev/virtio.ori/block/virtio_blk.c (-95 / +64 lines)
76	uint64_t vtblk_features;	76	uint64_t vtblk_features;
77	uint32_t vtblk_flags;	77	uint32_t vtblk_flags;
78	#define VTBLK_FLAG_INDIRECT 0x0001	78	#define VTBLK_FLAG_INDIRECT 0x0001
79	#define VTBLK_FLAG_DETACH 0x0002	79	#define VTBLK_FLAG_READONLY 0x0002
80	#define VTBLK_FLAG_SUSPEND 0x0004	80	#define VTBLK_FLAG_DETACH 0x0004
81	#define VTBLK_FLAG_BARRIER 0x0008	81	#define VTBLK_FLAG_SUSPEND 0x0008
82	#define VTBLK_FLAG_WCE_CONFIG 0x0010	82	#define VTBLK_FLAG_BARRIER 0x0010
		83	#define VTBLK_FLAG_WC_CONFIG 0x0020
83		84
84	struct virtqueue *vtblk_vq;	85	struct virtqueue *vtblk_vq;
85	struct sglist *vtblk_sglist;	86	struct sglist *vtblk_sglist;
Lines 108-117 Link Here
108	{ VIRTIO_BLK_F_RO, "ReadOnly" },	109	{ VIRTIO_BLK_F_RO, "ReadOnly" },
109	{ VIRTIO_BLK_F_BLK_SIZE, "BlockSize" },	110	{ VIRTIO_BLK_F_BLK_SIZE, "BlockSize" },
110	{ VIRTIO_BLK_F_SCSI, "SCSICmds" },	111	{ VIRTIO_BLK_F_SCSI, "SCSICmds" },
111	{ VIRTIO_BLK_F_FLUSH, "FlushCmd" },	112	{ VIRTIO_BLK_F_WCE, "WriteCache" },
112	{ VIRTIO_BLK_F_TOPOLOGY, "Topology" },	113	{ VIRTIO_BLK_F_TOPOLOGY, "Topology" },
113	{ VIRTIO_BLK_F_CONFIG_WCE, "ConfigWCE" },	114	{ VIRTIO_BLK_F_CONFIG_WCE, "ConfigWCE" },
114	{ VIRTIO_BLK_F_MQ, "Multiqueue" },
115		115
116	{ 0, NULL }	116	{ 0, NULL }
117	};	117	};
Lines 133-140 Link Here
133	static int vtblk_dump(void , void , vm_offset_t, off_t, size_t);	133	static int vtblk_dump(void , void , vm_offset_t, off_t, size_t);
134	static void vtblk_strategy(struct bio *);	134	static void vtblk_strategy(struct bio *);
135		135
136	static int vtblk_negotiate_features(struct vtblk_softc *);	136	static void vtblk_negotiate_features(struct vtblk_softc *);
137	static int vtblk_setup_features(struct vtblk_softc *);	137	static void vtblk_setup_features(struct vtblk_softc *);
138	static int vtblk_maximum_segments(struct vtblk_softc *,	138	static int vtblk_maximum_segments(struct vtblk_softc *,
139	struct virtio_blk_config *);	139	struct virtio_blk_config *);
140	static int vtblk_alloc_virtqueue(struct vtblk_softc *);	140	static int vtblk_alloc_virtqueue(struct vtblk_softc *);
Lines 193-206 Link Here
193	static void vtblk_setup_sysctl(struct vtblk_softc *);	193	static void vtblk_setup_sysctl(struct vtblk_softc *);
194	static int vtblk_tunable_int(struct vtblk_softc , const char , int);	194	static int vtblk_tunable_int(struct vtblk_softc , const char , int);
195		195
196	#define vtblk_modern(_sc) (((_sc)->vtblk_features & VIRTIO_F_VERSION_1) != 0)
197	#define vtblk_htog16(_sc, _val) virtio_htog16(vtblk_modern(_sc), _val)
198	#define vtblk_htog32(_sc, _val) virtio_htog32(vtblk_modern(_sc), _val)
199	#define vtblk_htog64(_sc, _val) virtio_htog64(vtblk_modern(_sc), _val)
200	#define vtblk_gtoh16(_sc, _val) virtio_gtoh16(vtblk_modern(_sc), _val)
201	#define vtblk_gtoh32(_sc, _val) virtio_gtoh32(vtblk_modern(_sc), _val)
202	#define vtblk_gtoh64(_sc, _val) virtio_gtoh64(vtblk_modern(_sc), _val)
203
204	/* Tunables. */	196	/* Tunables. */
205	static int vtblk_no_ident = 0;	197	static int vtblk_no_ident = 0;
206	TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);	198	TUNABLE_INT("hw.vtblk.no_ident", &vtblk_no_ident);
Lines 208-227 Link Here
208	TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);	200	TUNABLE_INT("hw.vtblk.writecache_mode", &vtblk_writecache_mode);
209		201
210	/* Features desired/implemented by this driver. */	202	/* Features desired/implemented by this driver. */
211	#define VTBLK_COMMON_FEATURES \	203	#define VTBLK_FEATURES \
212	(VIRTIO_BLK_F_SIZE_MAX \| \	204	(VIRTIO_BLK_F_BARRIER \| \
		205	VIRTIO_BLK_F_SIZE_MAX \| \
213	VIRTIO_BLK_F_SEG_MAX \| \	206	VIRTIO_BLK_F_SEG_MAX \| \
214	VIRTIO_BLK_F_GEOMETRY \| \	207	VIRTIO_BLK_F_GEOMETRY \| \
215	VIRTIO_BLK_F_RO \| \	208	VIRTIO_BLK_F_RO \| \
216	VIRTIO_BLK_F_BLK_SIZE \| \	209	VIRTIO_BLK_F_BLK_SIZE \| \
217	VIRTIO_BLK_F_FLUSH \| \	210	VIRTIO_BLK_F_WCE \| \
218	VIRTIO_BLK_F_TOPOLOGY \| \	211	VIRTIO_BLK_F_TOPOLOGY \| \
219	VIRTIO_BLK_F_CONFIG_WCE \| \	212	VIRTIO_BLK_F_CONFIG_WCE \| \
220	VIRTIO_RING_F_INDIRECT_DESC)	213	VIRTIO_RING_F_INDIRECT_DESC)
221		214
222	#define VTBLK_MODERN_FEATURES (VTBLK_COMMON_FEATURES)
223	#define VTBLK_LEGACY_FEATURES (VIRTIO_BLK_F_BARRIER \| VTBLK_COMMON_FEATURES)
224
225	#define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx	215	#define VTBLK_MTX(_sc) &(_sc)->vtblk_mtx
226	#define VTBLK_LOCK_INIT(_sc, _name) \	216	#define VTBLK_LOCK_INIT(_sc, _name) \
227	mtx_init(VTBLK_MTX((_sc)), (_name), \	217	mtx_init(VTBLK_MTX((_sc)), (_name), \
Lines 266-275 Link Here
266		256
267	DRIVER_MODULE(virtio_blk, virtio_mmio, vtblk_driver, vtblk_devclass,	257	DRIVER_MODULE(virtio_blk, virtio_mmio, vtblk_driver, vtblk_devclass,
268	vtblk_modevent, 0);	258	vtblk_modevent, 0);
269	DRIVER_MODULE(virtio_blk, vtpcil, vtblk_driver, vtblk_devclass,	259	DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
270	vtblk_modevent, 0);	260	vtblk_modevent, 0);
271	DRIVER_MODULE(virtio_blk, vtpcim, vtblk_driver, vtblk_devclass,
272	vtblk_modevent, 0);
273	MODULE_VERSION(virtio_blk, 1);	261	MODULE_VERSION(virtio_blk, 1);
274	MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);	262	MODULE_DEPEND(virtio_blk, virtio, 1, 1, 1);
275		263
Lines 313-322 Link Here
313	struct virtio_blk_config blkcfg;	301	struct virtio_blk_config blkcfg;
314	int error;	302	int error;
315		303
316	sc = device_get_softc(dev);
317	sc->vtblk_dev = dev;
318	virtio_set_feature_desc(dev, vtblk_feature_desc);	304	virtio_set_feature_desc(dev, vtblk_feature_desc);
319		305
		306	sc = device_get_softc(dev);
		307	sc->vtblk_dev = dev;
320	VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));	308	VTBLK_LOCK_INIT(sc, device_get_nameunit(dev));
321	bioq_init(&sc->vtblk_bioq);	309	bioq_init(&sc->vtblk_bioq);
322	TAILQ_INIT(&sc->vtblk_dump_queue);	310	TAILQ_INIT(&sc->vtblk_dump_queue);
Lines 324-336 Link Here
324	TAILQ_INIT(&sc->vtblk_req_ready);	312	TAILQ_INIT(&sc->vtblk_req_ready);
325		313
326	vtblk_setup_sysctl(sc);	314	vtblk_setup_sysctl(sc);
		315	vtblk_setup_features(sc);
327		316
328	error = vtblk_setup_features(sc);
329	if (error) {
330	device_printf(dev, "cannot setup features\n");
331	goto fail;
332	}
333
334	vtblk_read_config(sc, &blkcfg);	317	vtblk_read_config(sc, &blkcfg);
335		318
336	/*	319	/*
Lines 558-563 Link Here
558	return;	541	return;
559	}	542	}
560		543
		544	/*
		545	* Fail any write if RO. Unfortunately, there does not seem to
		546	* be a better way to report our readonly'ness to GEOM above.
		547	*/
		548	if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
		549	(bp->bio_cmd == BIO_WRITE \|\| bp->bio_cmd == BIO_FLUSH)) {
		550	vtblk_bio_done(sc, bp, EROFS);
		551	return;
		552	}
		553
561	VTBLK_LOCK(sc);	554	VTBLK_LOCK(sc);
562		555
563	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {	556	if (sc->vtblk_flags & VTBLK_FLAG_DETACH) {
Lines 572-611 Link Here
572	VTBLK_UNLOCK(sc);	565	VTBLK_UNLOCK(sc);
573	}	566	}
574		567
575	static int	568	static void
576	vtblk_negotiate_features(struct vtblk_softc *sc)	569	vtblk_negotiate_features(struct vtblk_softc *sc)
577	{	570	{
578	device_t dev;	571	device_t dev;
579	uint64_t features;	572	uint64_t features;
580		573
581	dev = sc->vtblk_dev;	574	dev = sc->vtblk_dev;
582	features = virtio_bus_is_modern(dev) ? VTBLK_MODERN_FEATURES :	575	features = VTBLK_FEATURES;
583	VTBLK_LEGACY_FEATURES;
584		576
585	sc->vtblk_features = virtio_negotiate_features(dev, features);	577	sc->vtblk_features = virtio_negotiate_features(dev, features);
586	return (virtio_finalize_features(dev));
587	}	578	}
588		579
589	static int	580	static void
590	vtblk_setup_features(struct vtblk_softc *sc)	581	vtblk_setup_features(struct vtblk_softc *sc)
591	{	582	{
592	device_t dev;	583	device_t dev;
593	int error;	584
594
595	dev = sc->vtblk_dev;	585	dev = sc->vtblk_dev;
596		586
597	error = vtblk_negotiate_features(sc);	587	vtblk_negotiate_features(sc);
598	if (error)
599	return (error);
600		588
601	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))	589	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
602	sc->vtblk_flags \|= VTBLK_FLAG_INDIRECT;	590	sc->vtblk_flags \|= VTBLK_FLAG_INDIRECT;
603	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))	591	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
604	sc->vtblk_flags \|= VTBLK_FLAG_WCE_CONFIG;	592	sc->vtblk_flags \|= VTBLK_FLAG_READONLY;
605	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))	593	if (virtio_with_feature(dev, VIRTIO_BLK_F_BARRIER))
606	sc->vtblk_flags \|= VTBLK_FLAG_BARRIER; /* Legacy. */	594	sc->vtblk_flags \|= VTBLK_FLAG_BARRIER;
607		595	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
608	return (0);	596	sc->vtblk_flags \|= VTBLK_FLAG_WC_CONFIG;
609	}	597	}
610		598
611	static int	599	static int
Lines 684-702 Link Here
684	dp->d_name = VTBLK_DISK_NAME;	672	dp->d_name = VTBLK_DISK_NAME;
685	dp->d_unit = device_get_unit(dev);	673	dp->d_unit = device_get_unit(dev);
686	dp->d_drv1 = sc;	674	dp->d_drv1 = sc;
687	dp->d_flags = DISKFLAG_UNMAPPED_BIO \| DISKFLAG_DIRECT_COMPLETION;	675	dp->d_flags = DISKFLAG_CANFLUSHCACHE \| DISKFLAG_UNMAPPED_BIO \|
		676	DISKFLAG_DIRECT_COMPLETION;
688	dp->d_hba_vendor = virtio_get_vendor(dev);	677	dp->d_hba_vendor = virtio_get_vendor(dev);
689	dp->d_hba_device = virtio_get_device(dev);	678	dp->d_hba_device = virtio_get_device(dev);
690	dp->d_hba_subvendor = virtio_get_subvendor(dev);	679	dp->d_hba_subvendor = virtio_get_subvendor(dev);
691	dp->d_hba_subdevice = virtio_get_subdevice(dev);	680	dp->d_hba_subdevice = virtio_get_subdevice(dev);
692		681
693	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))	682	if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0)
694	dp->d_flags \|= DISKFLAG_WRITE_PROTECT;
695	else {
696	if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH))
697	dp->d_flags \|= DISKFLAG_CANFLUSHCACHE;
698	dp->d_dump = vtblk_dump;	683	dp->d_dump = vtblk_dump;
699	}
700		684
701	/* Capacity is always in 512-byte units. */	685	/* Capacity is always in 512-byte units. */
702	dp->d_mediasize = blkcfg->capacity * 512;	686	dp->d_mediasize = blkcfg->capacity * 512;
Lines 880-906 Link Here
880	bp = bioq_takefirst(bioq);	864	bp = bioq_takefirst(bioq);
881	req->vbr_bp = bp;	865	req->vbr_bp = bp;
882	req->vbr_ack = -1;	866	req->vbr_ack = -1;
883	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	867	req->vbr_hdr.ioprio = 1;
884		868
885	switch (bp->bio_cmd) {	869	switch (bp->bio_cmd) {
886	case BIO_FLUSH:	870	case BIO_FLUSH:
887	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);	871	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
888	req->vbr_hdr.sector = 0;
889	break;	872	break;
890	case BIO_READ:	873	case BIO_READ:
891	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN);	874	req->vbr_hdr.type = VIRTIO_BLK_T_IN;
892	req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / 512);	875	req->vbr_hdr.sector = bp->bio_offset / 512;
893	break;	876	break;
894	case BIO_WRITE:	877	case BIO_WRITE:
895	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);	878	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
896	req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / 512);	879	req->vbr_hdr.sector = bp->bio_offset / 512;
897	break;	880	break;
898	default:	881	default:
899	panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);	882	panic("%s: bio with unhandled cmd: %d", __func__, bp->bio_cmd);
900	}	883	}
901		884
902	if (bp->bio_flags & BIO_ORDERED)	885	if (bp->bio_flags & BIO_ORDERED)
903	req->vbr_hdr.type \|= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER);	886	req->vbr_hdr.type \|= VIRTIO_BLK_T_BARRIER;
904		887
905	return (req);	888	return (req);
906	}	889	}
Lines 931-938 Link Here
931	if (!virtqueue_empty(vq))	914	if (!virtqueue_empty(vq))
932	return (EBUSY);	915	return (EBUSY);
933	ordered = 1;	916	ordered = 1;
934	req->vbr_hdr.type &= vtblk_gtoh32(sc,	917	req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER;
935	~VIRTIO_BLK_T_BARRIER);
936	}	918	}
937	}	919	}
938		920
Lines 1036-1051 Link Here
1036	static void	1018	static void
1037	vtblk_drain(struct vtblk_softc *sc)	1019	vtblk_drain(struct vtblk_softc *sc)
1038	{	1020	{
		1021	struct bio_queue queue;
1039	struct bio_queue_head *bioq;	1022	struct bio_queue_head *bioq;
1040	struct vtblk_request *req;	1023	struct vtblk_request *req;
1041	struct bio *bp;	1024	struct bio *bp;
1042		1025
1043	bioq = &sc->vtblk_bioq;	1026	bioq = &sc->vtblk_bioq;
		1027	TAILQ_INIT(&queue);
1044		1028
1045	if (sc->vtblk_vq != NULL) {	1029	if (sc->vtblk_vq != NULL) {
1046	struct bio_queue queue;
1047
1048	TAILQ_INIT(&queue);
1049	vtblk_queue_completed(sc, &queue);	1030	vtblk_queue_completed(sc, &queue);
1050	vtblk_done_completed(sc, &queue);	1031	vtblk_done_completed(sc, &queue);
1051		1032
Lines 1136-1157 Link Here
1136	/* Read the configuration if the feature was negotiated. */	1117	/* Read the configuration if the feature was negotiated. */
1137	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg);	1118	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SIZE_MAX, size_max, blkcfg);
1138	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg);	1119	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_SEG_MAX, seg_max, blkcfg);
1139	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,	1120	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg);
1140	geometry.cylinders, blkcfg);
1141	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1142	geometry.heads, blkcfg);
1143	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1144	geometry.sectors, blkcfg);
1145	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg);	1121	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_BLK_SIZE, blk_size, blkcfg);
1146	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,	1122	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg);
1147	topology.physical_block_exp, blkcfg);	1123	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg);
1148	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1149	topology.alignment_offset, blkcfg);
1150	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1151	topology.min_io_size, blkcfg);
1152	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1153	topology.opt_io_size, blkcfg);
1154	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, wce, blkcfg);
1155	}	1124	}
1156		1125
1157	#undef VTBLK_GET_CONFIG	1126	#undef VTBLK_GET_CONFIG
Lines 1175-1182 Link Here
1175	return;	1144	return;
1176		1145
1177	req->vbr_ack = -1;	1146	req->vbr_ack = -1;
1178	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID);	1147	req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID;
1179	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	1148	req->vbr_hdr.ioprio = 1;
1180	req->vbr_hdr.sector = 0;	1149	req->vbr_hdr.sector = 0;
1181		1150
1182	req->vbr_bp = &buf;	1151	req->vbr_bp = &buf;
Lines 1307-1315 Link Here
1307		1276
1308	req = &sc->vtblk_dump_request;	1277	req = &sc->vtblk_dump_request;
1309	req->vbr_ack = -1;	1278	req->vbr_ack = -1;
1310	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);	1279	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
1311	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	1280	req->vbr_hdr.ioprio = 1;
1312	req->vbr_hdr.sector = vtblk_gtoh64(sc, offset / 512);	1281	req->vbr_hdr.sector = offset / 512;
1313		1282
1314	req->vbr_bp = &buf;	1283	req->vbr_bp = &buf;
1315	g_reset_bio(&buf);	1284	g_reset_bio(&buf);
Lines 1329-1336 Link Here
1329		1298
1330	req = &sc->vtblk_dump_request;	1299	req = &sc->vtblk_dump_request;
1331	req->vbr_ack = -1;	1300	req->vbr_ack = -1;
1332	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);	1301	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
1333	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	1302	req->vbr_hdr.ioprio = 1;
1334	req->vbr_hdr.sector = 0;	1303	req->vbr_hdr.sector = 0;
1335		1304
1336	req->vbr_bp = &buf;	1305	req->vbr_bp = &buf;
Lines 1358-1364 Link Here
1358		1327
1359	/* Set either writeback (1) or writethrough (0) mode. */	1328	/* Set either writeback (1) or writethrough (0) mode. */
1360	virtio_write_dev_config_1(sc->vtblk_dev,	1329	virtio_write_dev_config_1(sc->vtblk_dev,
1361	offsetof(struct virtio_blk_config, wce), wc);	1330	offsetof(struct virtio_blk_config, writeback), wc);
1362	}	1331	}
1363		1332
1364	static int	1333	static int
Lines 1367-1381 Link Here
1367	{	1336	{
1368	int wc;	1337	int wc;
1369		1338
1370	if (sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) {	1339	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
1371	wc = vtblk_tunable_int(sc, "writecache_mode",	1340	wc = vtblk_tunable_int(sc, "writecache_mode",
1372	vtblk_writecache_mode);	1341	vtblk_writecache_mode);
1373	if (wc >= 0 && wc < VTBLK_CACHE_MAX)	1342	if (wc >= 0 && wc < VTBLK_CACHE_MAX)
1374	vtblk_set_write_cache(sc, wc);	1343	vtblk_set_write_cache(sc, wc);
1375	else	1344	else
1376	wc = blkcfg->wce;	1345	wc = blkcfg->writeback;
1377	} else	1346	} else
1378	wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_FLUSH);	1347	wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
1379		1348
1380	return (wc);	1349	return (wc);
1381	}	1350	}
Lines 1392-1398 Link Here
1392	error = sysctl_handle_int(oidp, &wc, 0, req);	1361	error = sysctl_handle_int(oidp, &wc, 0, req);
1393	if (error \|\| req->newptr == NULL)	1362	if (error \|\| req->newptr == NULL)
1394	return (error);	1363	return (error);
1395	if ((sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) == 0)	1364	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
1396	return (EPERM);	1365	return (EPERM);
1397	if (wc < 0 \|\| wc >= VTBLK_CACHE_MAX)	1366	if (wc < 0 \|\| wc >= VTBLK_CACHE_MAX)
1398	return (EINVAL);	1367	return (EINVAL);

Lines 34-55 Link Here

(-)sys/dev/virtio.ori/block/virtio_blk.h (-22 / +5 lines)
34	#define _VIRTIO_BLK_H	34	#define _VIRTIO_BLK_H
35		35
36	/* Feature bits */	36	/* Feature bits */
		37	#define VIRTIO_BLK_F_BARRIER 0x0001 /* Does host support barriers? */
37	#define VIRTIO_BLK_F_SIZE_MAX 0x0002 /* Indicates maximum segment size */	38	#define VIRTIO_BLK_F_SIZE_MAX 0x0002 /* Indicates maximum segment size */
38	#define VIRTIO_BLK_F_SEG_MAX 0x0004 /* Indicates maximum # of segments */	39	#define VIRTIO_BLK_F_SEG_MAX 0x0004 /* Indicates maximum # of segments */
39	#define VIRTIO_BLK_F_GEOMETRY 0x0010 /* Legacy geometry available */	40	#define VIRTIO_BLK_F_GEOMETRY 0x0010 /* Legacy geometry available */
40	#define VIRTIO_BLK_F_RO 0x0020 /* Disk is read-only */	41	#define VIRTIO_BLK_F_RO 0x0020 /* Disk is read-only */
41	#define VIRTIO_BLK_F_BLK_SIZE 0x0040 /* Block size of disk is available*/	42	#define VIRTIO_BLK_F_BLK_SIZE 0x0040 /* Block size of disk is available*/
42	#define VIRTIO_BLK_F_FLUSH 0x0200 /* Flush command supported */	43	#define VIRTIO_BLK_F_SCSI 0x0080 /* Supports scsi command passthru */
		44	#define VIRTIO_BLK_F_WCE 0x0200 /* Writeback mode enabled after reset */
43	#define VIRTIO_BLK_F_TOPOLOGY 0x0400 /* Topology information is available */	45	#define VIRTIO_BLK_F_TOPOLOGY 0x0400 /* Topology information is available */
44	#define VIRTIO_BLK_F_CONFIG_WCE 0x0800 /* Writeback mode available in config */	46	#define VIRTIO_BLK_F_CONFIG_WCE 0x0800 /* Writeback mode available in config */
45	#define VIRTIO_BLK_F_MQ 0x1000 /* Support more than one vq */
46		47
47	/* Legacy feature bits */
48	#define VIRTIO_BLK_F_BARRIER 0x0001 /* Does host support barriers? */
49	#define VIRTIO_BLK_F_SCSI 0x0080 /* Supports scsi command passthru */
50
51	/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
52	#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
53	#define VIRTIO_BLK_ID_BYTES 20 /* ID string length */	48	#define VIRTIO_BLK_ID_BYTES 20 /* ID string length */
54		49
55	struct virtio_blk_config {	50	struct virtio_blk_config {
Lines 71-93 Link Here
71		66
72	/* Topology of the device (if VIRTIO_BLK_F_TOPOLOGY) */	67	/* Topology of the device (if VIRTIO_BLK_F_TOPOLOGY) */
73	struct virtio_blk_topology {	68	struct virtio_blk_topology {
74	/* exponent for physical block per logical block. */
75	uint8_t physical_block_exp;	69	uint8_t physical_block_exp;
76	/* alignment offset in logical blocks. */
77	uint8_t alignment_offset;	70	uint8_t alignment_offset;
78	/* minimum I/O size without performance penalty in logical
79	* blocks. */
80	uint16_t min_io_size;	71	uint16_t min_io_size;
81	/* optimal sustained I/O size in logical blocks. */
82	uint32_t opt_io_size;	72	uint32_t opt_io_size;
83	} topology;	73	} topology;
84		74
85	/* Writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */	75	/* Writeback mode (if VIRTIO_BLK_F_CONFIG_WCE) */
86	uint8_t wce;	76	uint8_t writeback;
87	uint8_t unused;
88		77
89	/* Number of vqs, only available when VIRTIO_BLK_F_MQ is set */
90	uint16_t num_queues;
91	} __packed;	78	} __packed;
92		79
93	/*	80	/*
Lines 120-130 Link Here
120	/* ID string length */	107	/* ID string length */
121	#define VIRTIO_BLK_ID_BYTES 20	108	#define VIRTIO_BLK_ID_BYTES 20
122		109
123	/*	110	/* This is the first element of the read scatter-gather list. */
124	* This comes first in the read scatter-gather list.
125	* For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated,
126	* this is the first element of the read scatter-gather list.
127	*/
128	struct virtio_blk_outhdr {	111	struct virtio_blk_outhdr {
129	/* VIRTIO_BLK_T* */	112	/* VIRTIO_BLK_T* */
130	uint32_t type;	113	uint32_t type;

Lines 43-52 Link Here

(-)sys/dev/virtio.ori/network/if_vtnetvar.h (-73 / +51 lines)
43	uint64_t rx_csum_bad_ipproto;	43	uint64_t rx_csum_bad_ipproto;
44	uint64_t rx_csum_bad_offset;	44	uint64_t rx_csum_bad_offset;
45	uint64_t rx_csum_bad_proto;	45	uint64_t rx_csum_bad_proto;
46	uint64_t tx_csum_unknown_ethtype;	46	uint64_t tx_csum_bad_ethtype;
47	uint64_t tx_csum_proto_mismatch;	47	uint64_t tx_tso_bad_ethtype;
48	uint64_t tx_tso_not_tcp;	48	uint64_t tx_tso_not_tcp;
49	uint64_t tx_tso_without_csum;
50	uint64_t tx_defragged;	49	uint64_t tx_defragged;
51	uint64_t tx_defrag_failed;	50	uint64_t tx_defrag_failed;
52		51
Lines 68-74 Link Here
68	uint64_t vrxs_ierrors; /* if_ierrors */	67	uint64_t vrxs_ierrors; /* if_ierrors */
69	uint64_t vrxs_csum;	68	uint64_t vrxs_csum;
70	uint64_t vrxs_csum_failed;	69	uint64_t vrxs_csum_failed;
71	uint64_t vrxs_host_lro;
72	uint64_t vrxs_rescheduled;	70	uint64_t vrxs_rescheduled;
73	};	71	};
74		72
Lines 81-87 Link Here
81	struct vtnet_rxq_stats vtnrx_stats;	79	struct vtnet_rxq_stats vtnrx_stats;
82	struct taskqueue *vtnrx_tq;	80	struct taskqueue *vtnrx_tq;
83	struct task vtnrx_intrtask;	81	struct task vtnrx_intrtask;
84	struct lro_ctrl vtnrx_lro;
85	#ifdef DEV_NETMAP	82	#ifdef DEV_NETMAP
86	struct virtio_net_hdr_mrg_rxbuf vtnrx_shrhdr;	83	struct virtio_net_hdr_mrg_rxbuf vtnrx_shrhdr;
87	#endif /* DEV_NETMAP */	84	#endif /* DEV_NETMAP */
Lines 114-120 Link Here
114	#endif	111	#endif
115	int vtntx_id;	112	int vtntx_id;
116	int vtntx_watchdog;	113	int vtntx_watchdog;
117	int vtntx_intr_threshold;
118	struct vtnet_txq_stats vtntx_stats;	114	struct vtnet_txq_stats vtntx_stats;
119	struct taskqueue *vtntx_tq;	115	struct taskqueue *vtntx_tq;
120	struct task vtntx_intrtask;	116	struct task vtntx_intrtask;
Lines 140-149 Link Here
140	struct ifnet *vtnet_ifp;	136	struct ifnet *vtnet_ifp;
141	struct vtnet_rxq *vtnet_rxqs;	137	struct vtnet_rxq *vtnet_rxqs;
142	struct vtnet_txq *vtnet_txqs;	138	struct vtnet_txq *vtnet_txqs;
143	uint64_t vtnet_features;
144		139
145	uint32_t vtnet_flags;	140	uint32_t vtnet_flags;
146	#define VTNET_FLAG_MODERN 0x0001	141	#define VTNET_FLAG_SUSPENDED 0x0001
147	#define VTNET_FLAG_MAC 0x0002	142	#define VTNET_FLAG_MAC 0x0002
148	#define VTNET_FLAG_CTRL_VQ 0x0004	143	#define VTNET_FLAG_CTRL_VQ 0x0004
149	#define VTNET_FLAG_CTRL_RX 0x0008	144	#define VTNET_FLAG_CTRL_RX 0x0008
Lines 152-184 Link Here
152	#define VTNET_FLAG_TSO_ECN 0x0040	147	#define VTNET_FLAG_TSO_ECN 0x0040
153	#define VTNET_FLAG_MRG_RXBUFS 0x0080	148	#define VTNET_FLAG_MRG_RXBUFS 0x0080
154	#define VTNET_FLAG_LRO_NOMRG 0x0100	149	#define VTNET_FLAG_LRO_NOMRG 0x0100
155	#define VTNET_FLAG_MQ 0x0200	150	#define VTNET_FLAG_MULTIQ 0x0200
156	#define VTNET_FLAG_INDIRECT 0x0400	151	#define VTNET_FLAG_INDIRECT 0x0400
157	#define VTNET_FLAG_EVENT_IDX 0x0800	152	#define VTNET_FLAG_EVENT_IDX 0x0800
158	#define VTNET_FLAG_SUSPENDED 0x1000
159	#define VTNET_FLAG_FIXUP_NEEDS_CSUM 0x2000
160	#define VTNET_FLAG_SW_LRO 0x4000
161		153
		154	int vtnet_link_active;
162	int vtnet_hdr_size;	155	int vtnet_hdr_size;
163	int vtnet_rx_nmbufs;
164	int vtnet_rx_clustersz;
165	int vtnet_rx_nsegs;
166	int vtnet_rx_process_limit;	156	int vtnet_rx_process_limit;
167	int vtnet_link_active;	157	int vtnet_rx_nsegs;
168	int vtnet_act_vq_pairs;	158	int vtnet_rx_nmbufs;
169	int vtnet_req_vq_pairs;	159	int vtnet_rx_clsize;
170	int vtnet_max_vq_pairs;	160	int vtnet_rx_new_clsize;
		161	int vtnet_tx_intr_thresh;
171	int vtnet_tx_nsegs;	162	int vtnet_tx_nsegs;
172	int vtnet_if_flags;	163	int vtnet_if_flags;
173	int vtnet_max_mtu;	164	int vtnet_act_vq_pairs;
174	int vtnet_lro_entry_count;	165	int vtnet_max_vq_pairs;
175	int vtnet_lro_mbufq_depth;	166	int vtnet_requested_vq_pairs;
176		167
177	struct virtqueue *vtnet_ctrl_vq;	168	struct virtqueue *vtnet_ctrl_vq;
178	struct vtnet_mac_filter *vtnet_mac_filter;	169	struct vtnet_mac_filter *vtnet_mac_filter;
179	uint32_t *vtnet_vlan_filter;	170	uint32_t *vtnet_vlan_filter;
180		171
181	uint64_t vtnet_negotiated_features;	172	uint64_t vtnet_features;
182	struct vtnet_statistics vtnet_stats;	173	struct vtnet_statistics vtnet_stats;
183	struct callout vtnet_tick_ch;	174	struct callout vtnet_tick_ch;
184	struct ifmedia vtnet_media;	175	struct ifmedia vtnet_media;
Lines 190-211 Link Here
190	char vtnet_hwaddr[ETHER_ADDR_LEN];	181	char vtnet_hwaddr[ETHER_ADDR_LEN];
191	};	182	};
192		183
193	static bool
194	vtnet_modern(struct vtnet_softc *sc)
195	{
196	return ((sc->vtnet_flags & VTNET_FLAG_MODERN) != 0);
197	}
198
199	static bool
200	vtnet_software_lro(struct vtnet_softc *sc)
201	{
202	return ((sc->vtnet_flags & VTNET_FLAG_SW_LRO) != 0);
203	}
204
205	/*	184	/*
206	* Maximum number of queue pairs we will autoconfigure to.	185	* Maximum number of queue pairs we will autoconfigure to.
207	*/	186	*/
208	#define VTNET_MAX_QUEUE_PAIRS 32	187	#define VTNET_MAX_QUEUE_PAIRS 8
209		188
210	/*	189	/*
211	* Additional completed entries can appear in a virtqueue before we can	190	* Additional completed entries can appear in a virtqueue before we can
Lines 223-246 Link Here
223	#define VTNET_NOTIFY_RETRIES 4	202	#define VTNET_NOTIFY_RETRIES 4
224		203
225	/*	204	/*
		205	* Fake the media type. The host does not provide us with any real media
		206	* information.
		207	*/
		208	#define VTNET_MEDIATYPE (IFM_ETHER \| IFM_10G_T \| IFM_FDX)
		209
		210	/*
226	* Number of words to allocate for the VLAN shadow table. There is one	211	* Number of words to allocate for the VLAN shadow table. There is one
227	* bit for each VLAN.	212	* bit for each VLAN.
228	*/	213	*/
229	#define VTNET_VLAN_FILTER_NWORDS (4096 / 32)	214	#define VTNET_VLAN_FILTER_NWORDS (4096 / 32)
230		215
231	/* We depend on these being the same size (and same layout). */
232	CTASSERT(sizeof(struct virtio_net_hdr_mrg_rxbuf) ==
233	sizeof(struct virtio_net_hdr_v1));
234
235	/*	216	/*
236	* In legacy VirtIO when mergeable buffers are not negotiated, this structure	217	* When mergeable buffers are not negotiated, the vtnet_rx_header structure
237	* is placed at the beginning of the mbuf data. Use 4 bytes of pad to keep	218	* below is placed at the beginning of the mbuf data. Use 4 bytes of pad to
238	* both the VirtIO header and the data non-contiguous and the frame's payload	219	* both keep the VirtIO header and the data non-contiguous and to keep the
239	* 4 byte aligned. Note this padding would not be necessary if the	220	* frame's payload 4 byte aligned.
240	* VIRTIO_F_ANY_LAYOUT feature was negotiated (but we don't support that yet).
241	*	221	*
242	* In modern VirtIO or when mergeable buffers are negotiated, the host puts	222	* When mergeable buffers are negotiated, the host puts the VirtIO header in
243	* the VirtIO header in the beginning of the first mbuf's data.	223	* the beginning of the first mbuf's data.
244	*/	224	*/
245	#define VTNET_RX_HEADER_PAD 4	225	#define VTNET_RX_HEADER_PAD 4
246	struct vtnet_rx_header {	226	struct vtnet_rx_header {
Lines 256-262 Link Here
256	union {	236	union {
257	struct virtio_net_hdr hdr;	237	struct virtio_net_hdr hdr;
258	struct virtio_net_hdr_mrg_rxbuf mhdr;	238	struct virtio_net_hdr_mrg_rxbuf mhdr;
259	struct virtio_net_hdr_v1 v1hdr;
260	} vth_uhdr;	239	} vth_uhdr;
261		240
262	struct mbuf *vth_mbuf;	241	struct mbuf *vth_mbuf;
Lines 271-281 Link Here
271	*/	250	*/
272	#define VTNET_MAX_MAC_ENTRIES 128	251	#define VTNET_MAX_MAC_ENTRIES 128
273		252
274	/*
275	* The driver version of struct virtio_net_ctrl_mac but with our predefined
276	* number of MAC addresses allocated. This structure is shared with the host,
277	* so nentries field is in the correct VirtIO endianness.
278	*/
279	struct vtnet_mac_table {	253	struct vtnet_mac_table {
280	uint32_t nentries;	254	uint32_t nentries;
281	uint8_t macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];	255	uint8_t macs[VTNET_MAX_MAC_ENTRIES][ETHER_ADDR_LEN];
Lines 301-316 Link Here
301	(VTNET_CSUM_OFFLOAD \| VTNET_CSUM_OFFLOAD_IPV6 \| CSUM_TSO)	275	(VTNET_CSUM_OFFLOAD \| VTNET_CSUM_OFFLOAD_IPV6 \| CSUM_TSO)
302		276
303	/* Features desired/implemented by this driver. */	277	/* Features desired/implemented by this driver. */
304	#define VTNET_COMMON_FEATURES \	278	#define VTNET_FEATURES \
305	(VIRTIO_NET_F_MAC \| \	279	(VIRTIO_NET_F_MAC \| \
306	VIRTIO_NET_F_STATUS \| \	280	VIRTIO_NET_F_STATUS \| \
307	VIRTIO_NET_F_CTRL_GUEST_OFFLOADS \| \
308	VIRTIO_NET_F_MTU \| \
309	VIRTIO_NET_F_CTRL_VQ \| \	281	VIRTIO_NET_F_CTRL_VQ \| \
310	VIRTIO_NET_F_CTRL_RX \| \	282	VIRTIO_NET_F_CTRL_RX \| \
311	VIRTIO_NET_F_CTRL_MAC_ADDR \| \	283	VIRTIO_NET_F_CTRL_MAC_ADDR \| \
312	VIRTIO_NET_F_CTRL_VLAN \| \	284	VIRTIO_NET_F_CTRL_VLAN \| \
313	VIRTIO_NET_F_CSUM \| \	285	VIRTIO_NET_F_CSUM \| \
		286	VIRTIO_NET_F_GSO \| \
314	VIRTIO_NET_F_HOST_TSO4 \| \	287	VIRTIO_NET_F_HOST_TSO4 \| \
315	VIRTIO_NET_F_HOST_TSO6 \| \	288	VIRTIO_NET_F_HOST_TSO6 \| \
316	VIRTIO_NET_F_HOST_ECN \| \	289	VIRTIO_NET_F_HOST_ECN \| \
Lines 320-332 Link Here
320	VIRTIO_NET_F_GUEST_ECN \| \	293	VIRTIO_NET_F_GUEST_ECN \| \
321	VIRTIO_NET_F_MRG_RXBUF \| \	294	VIRTIO_NET_F_MRG_RXBUF \| \
322	VIRTIO_NET_F_MQ \| \	295	VIRTIO_NET_F_MQ \| \
323	VIRTIO_NET_F_SPEED_DUPLEX \| \
324	VIRTIO_RING_F_EVENT_IDX \| \	296	VIRTIO_RING_F_EVENT_IDX \| \
325	VIRTIO_RING_F_INDIRECT_DESC)	297	VIRTIO_RING_F_INDIRECT_DESC)
326		298
327	#define VTNET_MODERN_FEATURES (VTNET_COMMON_FEATURES)
328	#define VTNET_LEGACY_FEATURES (VTNET_COMMON_FEATURES \| VIRTIO_NET_F_GSO)
329
330	/*	299	/*
331	* The VIRTIO_NET_F_HOST_TSO[46] features permit us to send the host	300	* The VIRTIO_NET_F_HOST_TSO[46] features permit us to send the host
332	* frames larger than 1514 bytes.	301	* frames larger than 1514 bytes.
Lines 336-374 Link Here
336		305
337	/*	306	/*
338	* The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us	307	* The VIRTIO_NET_F_GUEST_TSO[46] features permit the host to send us
339	* frames larger than 1514 bytes.	308	* frames larger than 1514 bytes. We do not yet support software LRO
340		309	* via tcp_lro_rx().
341	*/	310	*/
342	#define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 \| \	311	#define VTNET_LRO_FEATURES (VIRTIO_NET_F_GUEST_TSO4 \| \
343	VIRTIO_NET_F_GUEST_TSO6 \| VIRTIO_NET_F_GUEST_ECN)	312	VIRTIO_NET_F_GUEST_TSO6 \| VIRTIO_NET_F_GUEST_ECN)
344		313
345	#define VTNET_MIN_MTU 68
346	#define VTNET_MAX_MTU 65536	314	#define VTNET_MAX_MTU 65536
347	#define VTNET_MAX_RX_SIZE 65550	315	#define VTNET_MAX_RX_SIZE 65550
348		316
349	/*	317	/*
350	* Used to preallocate the VQ indirect descriptors. Modern and mergeable	318	* Used to preallocate the Vq indirect descriptors. The first segment
351	* buffers do not required one segment for the VirtIO header since it is	319	* is reserved for the header, except for mergeable buffers since the
352	* placed inline at the beginning of the receive buffer.	320	* header is placed inline with the data.
353	*/	321	*/
354	#define VTNET_RX_SEGS_HDR_INLINE 1	322	#define VTNET_MRG_RX_SEGS 1
355	#define VTNET_RX_SEGS_HDR_SEPARATE 2	323	#define VTNET_MIN_RX_SEGS 2
356	#define VTNET_RX_SEGS_LRO_NOMRG 34	324	#define VTNET_MAX_RX_SEGS 34
357	#define VTNET_TX_SEGS_MIN 32	325	#define VTNET_MIN_TX_SEGS 32
358	#define VTNET_TX_SEGS_MAX 64	326	#define VTNET_MAX_TX_SEGS 64
359		327
360	/*	328	/*
361	* Assert we can receive and transmit the maximum with regular	329	* Assert we can receive and transmit the maximum with regular
362	* size clusters.	330	* size clusters.
363	*/	331	*/
364	CTASSERT(((VTNET_RX_SEGS_LRO_NOMRG - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);	332	CTASSERT(((VTNET_MAX_RX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_RX_SIZE);
365	CTASSERT(((VTNET_TX_SEGS_MAX - 1) * MCLBYTES) >= VTNET_MAX_MTU);	333	CTASSERT(((VTNET_MAX_TX_SEGS - 1) * MCLBYTES) >= VTNET_MAX_MTU);
366		334
367	/*	335	/*
368	* Number of slots in the Tx bufrings. This value matches most other	336	* Number of slots in the Tx bufrings. This value matches most other
369	* multiqueue drivers.	337	* multiqueue drivers.
370	*/	338	*/
371	#define VTNET_DEFAULT_BUFRING_SIZE 4096	339	#define VTNET_DEFAULT_BUFRING_SIZE 4096
		340
		341	/*
		342	* Determine how many mbufs are in each receive buffer. For LRO without
		343	* mergeable buffers, we must allocate an mbuf chain large enough to
		344	* hold both the vtnet_rx_header and the maximum receivable data.
		345	*/
		346	#define VTNET_NEEDED_RX_MBUFS(_sc, _clsize) \
		347	((_sc)->vtnet_flags & VTNET_FLAG_LRO_NOMRG) == 0 ? 1 : \
		348	howmany(sizeof(struct vtnet_rx_header) + VTNET_MAX_RX_SIZE, \
		349	(_clsize))
372		350
373	#define VTNET_CORE_MTX(_sc) &(_sc)->vtnet_mtx	351	#define VTNET_CORE_MTX(_sc) &(_sc)->vtnet_mtx
374	#define VTNET_CORE_LOCK(_sc) mtx_lock(VTNET_CORE_MTX((_sc)))	352	#define VTNET_CORE_LOCK(_sc) mtx_lock(VTNET_CORE_MTX((_sc)))

Lines 34-66 Link Here

(-)sys/dev/virtio.ori/network/virtio_net.h (-78 / +25 lines)
34	#define _VIRTIO_NET_H	34	#define _VIRTIO_NET_H
35		35
36	/* The feature bitmap for virtio net */	36	/* The feature bitmap for virtio net */
37	#define VIRTIO_NET_F_CSUM 0x000001 /* Host handles pkts w/ partial csum */	37	#define VIRTIO_NET_F_CSUM 0x00001 /* Host handles pkts w/ partial csum */
38	#define VIRTIO_NET_F_GUEST_CSUM 0x000002 /* Guest handles pkts w/ partial csum*/	38	#define VIRTIO_NET_F_GUEST_CSUM 0x00002 /* Guest handles pkts w/ partial csum*/
39	#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS 0x000004 /* Dynamic offload configuration. */	39	#define VIRTIO_NET_F_MAC 0x00020 /* Host has given MAC address. */
40	#define VIRTIO_NET_F_MTU 0x000008 /* Initial MTU advice */	40	#define VIRTIO_NET_F_GSO 0x00040 /* Host handles pkts w/ any GSO type */
41	#define VIRTIO_NET_F_MAC 0x000020 /* Host has given MAC address. */	41	#define VIRTIO_NET_F_GUEST_TSO4 0x00080 /* Guest can handle TSOv4 in. */
42	#define VIRTIO_NET_F_GSO 0x000040 /* Host handles pkts w/ any GSO type */	42	#define VIRTIO_NET_F_GUEST_TSO6 0x00100 /* Guest can handle TSOv6 in. */
43	#define VIRTIO_NET_F_GUEST_TSO4 0x000080 /* Guest can handle TSOv4 in. */	43	#define VIRTIO_NET_F_GUEST_ECN 0x00200 /* Guest can handle TSO[6] w/ ECN in.*/
44	#define VIRTIO_NET_F_GUEST_TSO6 0x000100 /* Guest can handle TSOv6 in. */	44	#define VIRTIO_NET_F_GUEST_UFO 0x00400 /* Guest can handle UFO in. */
45	#define VIRTIO_NET_F_GUEST_ECN 0x000200 /* Guest can handle TSO[6] w/ ECN in. */	45	#define VIRTIO_NET_F_HOST_TSO4 0x00800 /* Host can handle TSOv4 in. */
46	#define VIRTIO_NET_F_GUEST_UFO 0x000400 /* Guest can handle UFO in. */	46	#define VIRTIO_NET_F_HOST_TSO6 0x01000 /* Host can handle TSOv6 in. */
47	#define VIRTIO_NET_F_HOST_TSO4 0x000800 /* Host can handle TSOv4 in. */	47	#define VIRTIO_NET_F_HOST_ECN 0x02000 /* Host can handle TSO[6] w/ ECN in. */
48	#define VIRTIO_NET_F_HOST_TSO6 0x001000 /* Host can handle TSOv6 in. */	48	#define VIRTIO_NET_F_HOST_UFO 0x04000 /* Host can handle UFO in. */
49	#define VIRTIO_NET_F_HOST_ECN 0x002000 /* Host can handle TSO[6] w/ ECN in. */	49	#define VIRTIO_NET_F_MRG_RXBUF 0x08000 /* Host can merge receive buffers. */
50	#define VIRTIO_NET_F_HOST_UFO 0x004000 /* Host can handle UFO in. */	50	#define VIRTIO_NET_F_STATUS 0x10000 /* virtio_net_config.status available*/
51	#define VIRTIO_NET_F_MRG_RXBUF 0x008000 /* Host can merge receive buffers. */	51	#define VIRTIO_NET_F_CTRL_VQ 0x20000 /* Control channel available */
52	#define VIRTIO_NET_F_STATUS 0x010000 /* virtio_net_config.status available*/	52	#define VIRTIO_NET_F_CTRL_RX 0x40000 /* Control channel RX mode support */
53	#define VIRTIO_NET_F_CTRL_VQ 0x020000 /* Control channel available */	53	#define VIRTIO_NET_F_CTRL_VLAN 0x80000 /* Control channel VLAN filtering */
54	#define VIRTIO_NET_F_CTRL_RX 0x040000 /* Control channel RX mode support */	54	#define VIRTIO_NET_F_CTRL_RX_EXTRA 0x100000 /* Extra RX mode control support */
55	#define VIRTIO_NET_F_CTRL_VLAN 0x080000 /* Control channel VLAN filtering */	55	#define VIRTIO_NET_F_GUEST_ANNOUNCE 0x200000 /* Announce device on network */
56	#define VIRTIO_NET_F_CTRL_RX_EXTRA 0x100000 /* Extra RX mode control support */	56	#define VIRTIO_NET_F_MQ 0x400000 /* Device supports RFS */
57	#define VIRTIO_NET_F_GUEST_ANNOUNCE 0x200000 /* Announce device on network */	57	#define VIRTIO_NET_F_CTRL_MAC_ADDR 0x800000 /* Set MAC address */
58	#define VIRTIO_NET_F_MQ 0x400000 /* Device supports Receive Flow Steering */
59	#define VIRTIO_NET_F_CTRL_MAC_ADDR 0x800000 /* Set MAC address */
60	#define VIRTIO_NET_F_SPEED_DUPLEX (1ULL << 63) /* Device set linkspeed and duplex */
61		58
62	#define VIRTIO_NET_S_LINK_UP 1 /* Link is up */	59	#define VIRTIO_NET_S_LINK_UP 1 /* Link is up */
63	#define VIRTIO_NET_S_ANNOUNCE 2 /* Announcement is needed */
64		60
65	struct virtio_net_config {	61	struct virtio_net_config {
66	/* The config defining mac address (if VIRTIO_NET_F_MAC) */	62	/* The config defining mac address (if VIRTIO_NET_F_MAC) */
Lines 72-103 Link Here
72	* Legal values are between 1 and 0x8000.	68	* Legal values are between 1 and 0x8000.
73	*/	69	*/
74	uint16_t max_virtqueue_pairs;	70	uint16_t max_virtqueue_pairs;
75	/* Default maximum transmit unit advice */
76	uint16_t mtu;
77	/*
78	* speed, in units of 1Mb. All values 0 to INT_MAX are legal.
79	* Any other value stands for unknown.
80	*/
81	uint32_t speed;
82	/*
83	* 0x00 - half duplex
84	* 0x01 - full duplex
85	* Any other value stands for unknown.
86	*/
87	uint8_t duplex;
88	} __packed;	71	} __packed;
89		72
90	/*	73	/*
91	* This header comes first in the scatter-gather list. If you don't	74	* This is the first element of the scatter-gather list. If you don't
92	* specify GSO or CSUM features, you can simply ignore the header.	75	* specify GSO or CSUM features, you can simply ignore the header.
93	*
94	* This is bitwise-equivalent to the legacy struct virtio_net_hdr_mrg_rxbuf,
95	* only flattened.
96	*/	76	*/
97	struct virtio_net_hdr_v1 {	77	struct virtio_net_hdr {
98	#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start, csum_offset */	78	#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* Use csum_start,csum_offset*/
99	#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */	79	#define VIRTIO_NET_HDR_F_DATA_VALID 2 /* Csum is valid */
100	uint8_t flags;	80	uint8_t flags;
101	#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */	81	#define VIRTIO_NET_HDR_GSO_NONE 0 /* Not a GSO frame */
102	#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */	82	#define VIRTIO_NET_HDR_GSO_TCPV4 1 /* GSO frame, IPv4 TCP (TSO) */
103	#define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */	83	#define VIRTIO_NET_HDR_GSO_UDP 3 /* GSO frame, IPv4 UDP (UFO) */
Lines 108-134 Link Here
108	uint16_t gso_size; /* Bytes to append to hdr_len per frame */	88	uint16_t gso_size; /* Bytes to append to hdr_len per frame */
109	uint16_t csum_start; /* Position to start checksumming from */	89	uint16_t csum_start; /* Position to start checksumming from */
110	uint16_t csum_offset; /* Offset after that to place checksum */	90	uint16_t csum_offset; /* Offset after that to place checksum */
111	uint16_t num_buffers; /* Number of merged rx buffers */
112	};	91	};
113		92
114	/*	93	/*
115	* This header comes first in the scatter-gather list.
116	* For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated, it must
117	* be the first element of the scatter-gather list. If you don't
118	* specify GSO or CSUM features, you can simply ignore the header.
119	*/
120	struct virtio_net_hdr {
121	/* See VIRTIO_NET_HDR_F_* */
122	uint8_t flags;
123	/* See VIRTIO_NET_HDR_GSO_* */
124	uint8_t gso_type;
125	uint16_t hdr_len; /* Ethernet + IP + tcp/udp hdrs */
126	uint16_t gso_size; /* Bytes to append to hdr_len per frame */
127	uint16_t csum_start; /* Position to start checksumming from */
128	uint16_t csum_offset; /* Offset after that to place checksum */
129	};
130
131	/*
132	* This is the version of the header to use when the MRG_RXBUF	94	* This is the version of the header to use when the MRG_RXBUF
133	* feature has been negotiated.	95	* feature has been negotiated.
134	*/	96	*/
Lines 238-257 Link Here
238	#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0	200	#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
239	#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1	201	#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
240	#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000	202	#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
241
242	/*
243	* Control network offloads
244	*
245	* Reconfigures the network offloads that Guest can handle.
246	*
247	* Available with the VIRTIO_NET_F_CTRL_GUEST_OFFLOADS feature bit.
248	*
249	* Command data format matches the feature bit mask exactly.
250	*
251	* See VIRTIO_NET_F_GUEST_* for the list of offloads
252	* that can be enabled/disabled.
253	*/
254	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS 5
255	#define VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET 0
256		203
257	#endif /* _VIRTIO_NET_H */	204	#endif /* _VIRTIO_NET_H */

Lines 1-132 Link Here

(-)sys/dev/virtio.ori/pci/virtio_pci.h (-107 / +63 lines)
1	/*-	1	/*-
2	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD	2	* SPDX-License-Identifier: BSD-3-Clause
3	*	3	*
4	* Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>	4	* Copyright IBM Corp. 2007
5	* All rights reserved.
6	*	5	*
		6	* Authors:
		7	* Anthony Liguori <aliguori@us.ibm.com>
		8	*
		9	* This header is BSD licensed so anyone can use the definitions to implement
		10	* compatible drivers/servers.
		11	*
7	* Redistribution and use in source and binary forms, with or without	12	* Redistribution and use in source and binary forms, with or without
8	* modification, are permitted provided that the following conditions	13	* modification, are permitted provided that the following conditions
9	* are met:	14	* are met:
10	* 1. Redistributions of source code must retain the above copyright	15	* 1. Redistributions of source code must retain the above copyright
11	* notice unmodified, this list of conditions, and the following	16	* notice, this list of conditions and the following disclaimer.
12	* disclaimer.
13	* 2. Redistributions in binary form must reproduce the above copyright	17	* 2. Redistributions in binary form must reproduce the above copyright
14	* notice, this list of conditions and the following disclaimer in the	18	* notice, this list of conditions and the following disclaimer in the
15	* documentation and/or other materials provided with the distribution.	19	* documentation and/or other materials provided with the distribution.
		20	* 3. Neither the name of IBM nor the names of its contributors
		21	* may be used to endorse or promote products derived from this software
		22	* without specific prior written permission.
		23	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
		24	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
		25	* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
		26	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
		27	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
		28	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
		29	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
		30	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
		31	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
		32	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
		33	* SUCH DAMAGE.
16	*	34	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*
28	* $FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.h 326022 2017-11-20 19:36:21Z pfg $	35	* $FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.h 326022 2017-11-20 19:36:21Z pfg $
29	*/	36	*/
30		37
31	#ifndef _VIRTIO_PCI_H	38	#ifndef _VIRTIO_PCI_H
32	#define _VIRTIO_PCI_H	39	#define _VIRTIO_PCI_H
33		40
34	struct vtpci_interrupt {	41	/* VirtIO PCI vendor/device ID. */
35	struct resource *vti_irq;	42	#define VIRTIO_PCI_VENDORID 0x1AF4
36	int vti_rid;	43	#define VIRTIO_PCI_DEVICEID_MIN 0x1000
37	void *vti_handler;	44	#define VIRTIO_PCI_DEVICEID_MAX 0x103F
38	};
39		45
40	struct vtpci_virtqueue {	46	/* VirtIO ABI version, this must match exactly. */
41	struct virtqueue *vtv_vq;	47	#define VIRTIO_PCI_ABI_VERSION 0
42	int vtv_no_intr;
43	int vtv_notify_offset;
44	};
45		48
46	struct vtpci_common {	49	/*
47	device_t vtpci_dev;	50	* VirtIO Header, located in BAR 0.
48	uint64_t vtpci_host_features;	51	*/
49	uint64_t vtpci_features;	52	#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/
50	struct vtpci_virtqueue *vtpci_vqs;	53	#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */
51	int vtpci_nvqs;	54	#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */
		55	#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */
		56	#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */
		57	#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */
		58	#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */
		59	#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading
		60	* also clears the register (8, RO) */
		61	/* Only if MSIX is enabled: */
		62	#define VIRTIO_MSI_CONFIG_VECTOR 20 /* configuration change vector (16, RW) */
		63	#define VIRTIO_MSI_QUEUE_VECTOR 22 /* vector for selected VQ notifications
		64	(16, RW) */
52		65
53	uint32_t vtpci_flags;	66	/* The bit of the ISR which indicates a device has an interrupt. */
54	#define VTPCI_FLAG_NO_MSI 0x0001	67	#define VIRTIO_PCI_ISR_INTR 0x1
55	#define VTPCI_FLAG_NO_MSIX 0x0002	68	/* The bit of the ISR which indicates a device configuration change. */
56	#define VTPCI_FLAG_MODERN 0x0004	69	#define VIRTIO_PCI_ISR_CONFIG 0x2
57	#define VTPCI_FLAG_INTX 0x1000	70	/* Vector value used to disable MSI for queue. */
58	#define VTPCI_FLAG_MSI 0x2000	71	#define VIRTIO_MSI_NO_VECTOR 0xFFFF
59	#define VTPCI_FLAG_MSIX 0x4000
60	#define VTPCI_FLAG_SHARED_MSIX 0x8000
61	#define VTPCI_FLAG_ITYPE_MASK 0xF000
62		72
63	/* The VirtIO PCI "bus" will only ever have one child. */	73	/*
64	device_t vtpci_child_dev;	74	* The remaining space is defined by each driver as the per-driver
65	struct virtio_feature_desc *vtpci_child_feat_desc;	75	* configuration space.
		76	*/
		77	#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20)
66		78
67	/*	79	/*
68	* Ideally, each virtqueue that the driver provides a callback for will	80	* How many bits to shift physical queue address written to QUEUE_PFN.
69	* receive its own MSIX vector. If there are not sufficient vectors	81	* 12 is historical, and due to x86 page size.
70	* available, then attempt to have all the VQs share one vector. For	82	*/
71	* MSIX, the configuration changed notifications must be on their own	83	#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12
72	* vector.
73	*
74	* If MSIX is not available, attempt to have the whole device share
75	* one MSI vector, and then, finally, one intx interrupt.
76	*/
77	struct vtpci_interrupt vtpci_device_interrupt;
78	struct vtpci_interrupt *vtpci_msix_vq_interrupts;
79	int vtpci_nmsix_resources;
80	};
81		84
82	extern int vtpci_disable_msix;	85	/* The alignment to use between consumer and producer parts of vring. */
83		86	#define VIRTIO_PCI_VRING_ALIGN 4096
84	static inline device_t
85	vtpci_child_device(struct vtpci_common *cn)
86	{
87	return (cn->vtpci_child_dev);
88	}
89
90	static inline bool
91	vtpci_is_msix_available(struct vtpci_common *cn)
92	{
93	return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0);
94	}
95
96	static inline bool
97	vtpci_is_msix_enabled(struct vtpci_common *cn)
98	{
99	return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0);
100	}
101
102	static inline bool
103	vtpci_is_modern(struct vtpci_common *cn)
104	{
105	return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0);
106	}
107
108	static inline int
109	vtpci_virtqueue_count(struct vtpci_common *cn)
110	{
111	return (cn->vtpci_nvqs);
112	}
113
114	void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern);
115	int vtpci_add_child(struct vtpci_common *cn);
116	int vtpci_delete_child(struct vtpci_common *cn);
117	void vtpci_child_detached(struct vtpci_common *cn);
118	int vtpci_reinit(struct vtpci_common *cn);
119
120	uint64_t vtpci_negotiate_features(struct vtpci_common *cn,
121	uint64_t child_features, uint64_t host_features);
122	int vtpci_with_feature(struct vtpci_common *cn, uint64_t feature);
123
124	int vtpci_read_ivar(struct vtpci_common cn, int index, uintptr_t result);
125	int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value);
126
127	int vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs,
128	struct vq_alloc_info *vq_info);
129	int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type);
130	void vtpci_release_child_resources(struct vtpci_common *cn);
131		87
132	#endif /* _VIRTIO_PCI_H */	88	#endif /* _VIRTIO_PCI_H */

Lines 1-71 Link Here

(-)sys/dev/virtio.ori/pci/virtio_pci_if.m (-71 lines)
1	#-
2	# Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
3	# All rights reserved.
4	#
5	# Redistribution and use in source and binary forms, with or without
6	# modification, are permitted provided that the following conditions
7	# are met:
8	# 1. Redistributions of source code must retain the above copyright
9	# notice, this list of conditions and the following disclaimer.
10	# 2. Redistributions in binary form must reproduce the above copyright
11	# notice, this list of conditions and the following disclaimer in the
12	# documentation and/or other materials provided with the distribution.
13	#
14	# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15	# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16	# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17	# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18	# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19	# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20	# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21	# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22	# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23	# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24	# SUCH DAMAGE.
25	#
26	# $FreeBSD$
27
28	#include <sys/bus.h>
29	#include <machine/bus.h>
30
31	INTERFACE virtio_pci;
32
33	HEADER {
34	struct virtqueue;
35	struct vtpci_interrupt;
36	};
37
38	METHOD uint8_t read_isr {
39	device_t dev;
40	};
41
42	METHOD uint16_t get_vq_size {
43	device_t dev;
44	int idx;
45	};
46
47	METHOD bus_size_t get_vq_notify_off {
48	device_t dev;
49	int idx;
50	};
51
52	METHOD void set_vq {
53	device_t dev;
54	struct virtqueue *vq;
55	};
56
57	METHOD void disable_vq {
58	device_t dev;
59	int idx;
60	};
61
62	METHOD int register_cfg_msix {
63	device_t dev;
64	struct vtpci_interrupt *intr;
65	};
66
67	METHOD int register_vq_msix {
68	device_t dev;
69	int idx;
70	struct vtpci_interrupt *intr;
71	};

Lines 1-714 Link Here

(-)sys/dev/virtio.ori/pci/virtio_pci_legacy.c (-714 lines)
1	/*-
2	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3	*
4	* Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5	* All rights reserved.
6	*
7	* Redistribution and use in source and binary forms, with or without
8	* modification, are permitted provided that the following conditions
9	* are met:
10	* 1. Redistributions of source code must retain the above copyright
11	* notice unmodified, this list of conditions, and the following
12	* disclaimer.
13	* 2. Redistributions in binary form must reproduce the above copyright
14	* notice, this list of conditions and the following disclaimer in the
15	* documentation and/or other materials provided with the distribution.
16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/
28
29	/* Driver for the legacy VirtIO PCI interface. */
30
31	#include <sys/cdefs.h>
32	__FBSDID("$FreeBSD$");
33
34	#include <sys/param.h>
35	#include <sys/systm.h>
36	#include <sys/bus.h>
37	#include <sys/kernel.h>
38	#include <sys/module.h>
39
40	#include <machine/bus.h>
41	#include <machine/resource.h>
42	#include <sys/bus.h>
43	#include <sys/rman.h>
44
45	#include <dev/pci/pcivar.h>
46	#include <dev/pci/pcireg.h>
47
48	#include <dev/virtio/virtio.h>
49	#include <dev/virtio/virtqueue.h>
50	#include <dev/virtio/pci/virtio_pci.h>
51	#include <dev/virtio/pci/virtio_pci_legacy_var.h>
52
53	#include "virtio_bus_if.h"
54	#include "virtio_pci_if.h"
55	#include "virtio_if.h"
56
57	struct vtpci_legacy_softc {
58	device_t vtpci_dev;
59	struct vtpci_common vtpci_common;
60	struct resource *vtpci_res;
61	struct resource *vtpci_msix_res;
62	};
63
64	static int vtpci_legacy_probe(device_t);
65	static int vtpci_legacy_attach(device_t);
66	static int vtpci_legacy_detach(device_t);
67	static int vtpci_legacy_suspend(device_t);
68	static int vtpci_legacy_resume(device_t);
69	static int vtpci_legacy_shutdown(device_t);
70
71	static void vtpci_legacy_driver_added(device_t, driver_t *);
72	static void vtpci_legacy_child_detached(device_t, device_t);
73	static int vtpci_legacy_read_ivar(device_t, device_t, int, uintptr_t *);
74	static int vtpci_legacy_write_ivar(device_t, device_t, int, uintptr_t);
75
76	static uint8_t vtpci_legacy_read_isr(device_t);
77	static uint16_t vtpci_legacy_get_vq_size(device_t, int);
78	static bus_size_t vtpci_legacy_get_vq_notify_off(device_t, int);
79	static void vtpci_legacy_set_vq(device_t, struct virtqueue *);
80	static void vtpci_legacy_disable_vq(device_t, int);
81	static int vtpci_legacy_register_cfg_msix(device_t,
82	struct vtpci_interrupt *);
83	static int vtpci_legacy_register_vq_msix(device_t, int idx,
84	struct vtpci_interrupt *);
85
86	static uint64_t vtpci_legacy_negotiate_features(device_t, uint64_t);
87	static int vtpci_legacy_with_feature(device_t, uint64_t);
88	static int vtpci_legacy_alloc_virtqueues(device_t, int, int,
89	struct vq_alloc_info *);
90	static int vtpci_legacy_setup_interrupts(device_t, enum intr_type);
91	static void vtpci_legacy_stop(device_t);
92	static int vtpci_legacy_reinit(device_t, uint64_t);
93	static void vtpci_legacy_reinit_complete(device_t);
94	static void vtpci_legacy_notify_vq(device_t, uint16_t, bus_size_t);
95	static void vtpci_legacy_read_dev_config(device_t, bus_size_t, void *, int);
96	static void vtpci_legacy_write_dev_config(device_t, bus_size_t, void *, int);
97
98	static int vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *);
99	static void vtpci_legacy_free_resources(struct vtpci_legacy_softc *);
100
101	static void vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *);
102
103	static uint8_t vtpci_legacy_get_status(struct vtpci_legacy_softc *);
104	static void vtpci_legacy_set_status(struct vtpci_legacy_softc *, uint8_t);
105	static void vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *, int);
106	static void vtpci_legacy_reset(struct vtpci_legacy_softc *);
107
108	#define VIRTIO_PCI_LEGACY_CONFIG(_sc) \
109	VIRTIO_PCI_CONFIG_OFF(vtpci_is_msix_enabled(&(_sc)->vtpci_common))
110
111	/*
112	* I/O port read/write wrappers.
113	*/
114	#define vtpci_legacy_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
115	#define vtpci_legacy_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
116	#define vtpci_legacy_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
117	#define vtpci_legacy_write_config_1(sc, o, v) \
118	bus_write_1((sc)->vtpci_res, (o), (v))
119	#define vtpci_legacy_write_config_2(sc, o, v) \
120	bus_write_2((sc)->vtpci_res, (o), (v))
121	#define vtpci_legacy_write_config_4(sc, o, v) \
122	bus_write_4((sc)->vtpci_res, (o), (v))
123
124	static device_method_t vtpci_legacy_methods[] = {
125	/* Device interface. */
126	DEVMETHOD(device_probe, vtpci_legacy_probe),
127	DEVMETHOD(device_attach, vtpci_legacy_attach),
128	DEVMETHOD(device_detach, vtpci_legacy_detach),
129	DEVMETHOD(device_suspend, vtpci_legacy_suspend),
130	DEVMETHOD(device_resume, vtpci_legacy_resume),
131	DEVMETHOD(device_shutdown, vtpci_legacy_shutdown),
132
133	/* Bus interface. */
134	DEVMETHOD(bus_driver_added, vtpci_legacy_driver_added),
135	DEVMETHOD(bus_child_detached, vtpci_legacy_child_detached),
136	DEVMETHOD(bus_read_ivar, vtpci_legacy_read_ivar),
137	DEVMETHOD(bus_write_ivar, vtpci_legacy_write_ivar),
138
139	/* VirtIO PCI interface. */
140	DEVMETHOD(virtio_pci_read_isr, vtpci_legacy_read_isr),
141	DEVMETHOD(virtio_pci_get_vq_size, vtpci_legacy_get_vq_size),
142	DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_legacy_get_vq_notify_off),
143	DEVMETHOD(virtio_pci_set_vq, vtpci_legacy_set_vq),
144	DEVMETHOD(virtio_pci_disable_vq, vtpci_legacy_disable_vq),
145	DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_legacy_register_cfg_msix),
146	DEVMETHOD(virtio_pci_register_vq_msix, vtpci_legacy_register_vq_msix),
147
148	/* VirtIO bus interface. */
149	DEVMETHOD(virtio_bus_negotiate_features, vtpci_legacy_negotiate_features),
150	DEVMETHOD(virtio_bus_with_feature, vtpci_legacy_with_feature),
151	DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_legacy_alloc_virtqueues),
152	DEVMETHOD(virtio_bus_setup_intr, vtpci_legacy_setup_interrupts),
153	DEVMETHOD(virtio_bus_stop, vtpci_legacy_stop),
154	DEVMETHOD(virtio_bus_reinit, vtpci_legacy_reinit),
155	DEVMETHOD(virtio_bus_reinit_complete, vtpci_legacy_reinit_complete),
156	DEVMETHOD(virtio_bus_notify_vq, vtpci_legacy_notify_vq),
157	DEVMETHOD(virtio_bus_read_device_config, vtpci_legacy_read_dev_config),
158	DEVMETHOD(virtio_bus_write_device_config, vtpci_legacy_write_dev_config),
159
160	DEVMETHOD_END
161	};
162
163	static driver_t vtpci_legacy_driver = {
164	.name = "vtpcil",
165	.methods = vtpci_legacy_methods,
166	.size = sizeof(struct vtpci_legacy_softc)
167	};
168
169	devclass_t vtpci_legacy_devclass;
170
171	DRIVER_MODULE(vtpcil, pci, vtpci_legacy_driver, vtpci_legacy_devclass, 0, 0);
172
173	static int
174	vtpci_legacy_probe(device_t dev)
175	{
176	char desc[64];
177	const char *name;
178
179	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
180	return (ENXIO);
181
182	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN \|\|
183	pci_get_device(dev) > VIRTIO_PCI_DEVICEID_LEGACY_MAX)
184	return (ENXIO);
185
186	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
187	return (ENXIO);
188
189	name = virtio_device_name(pci_get_subdevice(dev));
190	if (name == NULL)
191	name = "Unknown";
192
193	snprintf(desc, sizeof(desc), "VirtIO PCI (legacy) %s adapter", name);
194	device_set_desc_copy(dev, desc);
195
196	/* Prefer transitional modern VirtIO PCI. */
197	return (BUS_PROBE_LOW_PRIORITY);
198	}
199
200	static int
201	vtpci_legacy_attach(device_t dev)
202	{
203	struct vtpci_legacy_softc *sc;
204	int error;
205
206	sc = device_get_softc(dev);
207	sc->vtpci_dev = dev;
208	vtpci_init(&sc->vtpci_common, dev, false);
209
210	error = vtpci_legacy_alloc_resources(sc);
211	if (error) {
212	device_printf(dev, "cannot map I/O space\n");
213	return (error);
214	}
215
216	vtpci_legacy_reset(sc);
217
218	/* Tell the host we've noticed this device. */
219	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
220
221	error = vtpci_add_child(&sc->vtpci_common);
222	if (error)
223	goto fail;
224
225	vtpci_legacy_probe_and_attach_child(sc);
226
227	return (0);
228
229	fail:
230	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
231	vtpci_legacy_detach(dev);
232
233	return (error);
234	}
235
236	static int
237	vtpci_legacy_detach(device_t dev)
238	{
239	struct vtpci_legacy_softc *sc;
240	int error;
241
242	sc = device_get_softc(dev);
243
244	error = vtpci_delete_child(&sc->vtpci_common);
245	if (error)
246	return (error);
247
248	vtpci_legacy_reset(sc);
249	vtpci_legacy_free_resources(sc);
250
251	return (0);
252	}
253
254	static int
255	vtpci_legacy_suspend(device_t dev)
256	{
257	return (bus_generic_suspend(dev));
258	}
259
260	static int
261	vtpci_legacy_resume(device_t dev)
262	{
263	return (bus_generic_resume(dev));
264	}
265
266	static int
267	vtpci_legacy_shutdown(device_t dev)
268	{
269	(void) bus_generic_shutdown(dev);
270	/* Forcibly stop the host device. */
271	vtpci_legacy_stop(dev);
272
273	return (0);
274	}
275
276	static void
277	vtpci_legacy_driver_added(device_t dev, driver_t *driver)
278	{
279	vtpci_legacy_probe_and_attach_child(device_get_softc(dev));
280	}
281
282	static void
283	vtpci_legacy_child_detached(device_t dev, device_t child)
284	{
285	struct vtpci_legacy_softc *sc;
286
287	sc = device_get_softc(dev);
288
289	vtpci_legacy_reset(sc);
290	vtpci_child_detached(&sc->vtpci_common);
291
292	/* After the reset, retell the host we've noticed this device. */
293	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
294	}
295
296	static int
297	vtpci_legacy_read_ivar(device_t dev, device_t child, int index,
298	uintptr_t *result)
299	{
300	struct vtpci_legacy_softc *sc;
301	struct vtpci_common *cn;
302
303	sc = device_get_softc(dev);
304	cn = &sc->vtpci_common;
305
306	if (vtpci_child_device(cn) != child)
307	return (ENOENT);
308
309	switch (index) {
310	case VIRTIO_IVAR_DEVTYPE:
311	*result = pci_get_subdevice(dev);
312	break;
313	default:
314	return (vtpci_read_ivar(cn, index, result));
315	}
316
317	return (0);
318	}
319
320	static int
321	vtpci_legacy_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
322	{
323	struct vtpci_legacy_softc *sc;
324	struct vtpci_common *cn;
325
326	sc = device_get_softc(dev);
327	cn = &sc->vtpci_common;
328
329	if (vtpci_child_device(cn) != child)
330	return (ENOENT);
331
332	switch (index) {
333	default:
334	return (vtpci_write_ivar(cn, index, value));
335	}
336
337	return (0);
338	}
339
340	static uint64_t
341	vtpci_legacy_negotiate_features(device_t dev, uint64_t child_features)
342	{
343	struct vtpci_legacy_softc *sc;
344	uint64_t host_features, features;
345
346	sc = device_get_softc(dev);
347	host_features = vtpci_legacy_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
348
349	features = vtpci_negotiate_features(&sc->vtpci_common,
350	child_features, host_features);
351	vtpci_legacy_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
352
353	return (features);
354	}
355
356	static int
357	vtpci_legacy_with_feature(device_t dev, uint64_t feature)
358	{
359	struct vtpci_legacy_softc *sc;
360
361	sc = device_get_softc(dev);
362
363	return (vtpci_with_feature(&sc->vtpci_common, feature));
364	}
365
366	static int
367	vtpci_legacy_alloc_virtqueues(device_t dev, int flags, int nvqs,
368	struct vq_alloc_info *vq_info)
369	{
370	struct vtpci_legacy_softc *sc;
371	struct vtpci_common *cn;
372
373	sc = device_get_softc(dev);
374	cn = &sc->vtpci_common;
375
376	return (vtpci_alloc_virtqueues(cn, flags, nvqs, vq_info));
377	}
378
379	static int
380	vtpci_legacy_setup_interrupts(device_t dev, enum intr_type type)
381	{
382	struct vtpci_legacy_softc *sc;
383
384	sc = device_get_softc(dev);
385
386	return (vtpci_setup_interrupts(&sc->vtpci_common, type));
387	}
388
389	static void
390	vtpci_legacy_stop(device_t dev)
391	{
392	vtpci_legacy_reset(device_get_softc(dev));
393	}
394
395	static int
396	vtpci_legacy_reinit(device_t dev, uint64_t features)
397	{
398	struct vtpci_legacy_softc *sc;
399	struct vtpci_common *cn;
400	int error;
401
402	sc = device_get_softc(dev);
403	cn = &sc->vtpci_common;
404
405	/*
406	* Redrive the device initialization. This is a bit of an abuse of
407	* the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
408	* play nice.
409	*
410	* We do not allow the host device to change from what was originally
411	* negotiated beyond what the guest driver changed. MSIX state should
412	* not change, number of virtqueues and their size remain the same, etc.
413	* This will need to be rethought when we want to support migration.
414	*/
415
416	if (vtpci_legacy_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
417	vtpci_legacy_stop(dev);
418
419	/*
420	* Quickly drive the status through ACK and DRIVER. The device does
421	* not become usable again until DRIVER_OK in reinit complete.
422	*/
423	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
424	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
425
426	vtpci_legacy_negotiate_features(dev, features);
427
428	error = vtpci_reinit(cn);
429	if (error)
430	return (error);
431
432	return (0);
433	}
434
435	static void
436	vtpci_legacy_reinit_complete(device_t dev)
437	{
438	struct vtpci_legacy_softc *sc;
439
440	sc = device_get_softc(dev);
441
442	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
443	}
444
445	static void
446	vtpci_legacy_notify_vq(device_t dev, uint16_t queue, bus_size_t offset)
447	{
448	struct vtpci_legacy_softc *sc;
449
450	sc = device_get_softc(dev);
451	MPASS(offset == VIRTIO_PCI_QUEUE_NOTIFY);
452
453	vtpci_legacy_write_config_2(sc, offset, queue);
454	}
455
456	static uint8_t
457	vtpci_legacy_get_status(struct vtpci_legacy_softc *sc)
458	{
459	return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_STATUS));
460	}
461
462	static void
463	vtpci_legacy_set_status(struct vtpci_legacy_softc *sc, uint8_t status)
464	{
465	if (status != VIRTIO_CONFIG_STATUS_RESET)
466	status \|= vtpci_legacy_get_status(sc);
467
468	vtpci_legacy_write_config_1(sc, VIRTIO_PCI_STATUS, status);
469	}
470
471	static void
472	vtpci_legacy_read_dev_config(device_t dev, bus_size_t offset,
473	void *dst, int length)
474	{
475	struct vtpci_legacy_softc *sc;
476	bus_size_t off;
477	uint8_t *d;
478	int size;
479
480	sc = device_get_softc(dev);
481	off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset;
482
483	for (d = dst; length > 0; d += size, off += size, length -= size) {
484	if (length >= 4) {
485	size = 4;
486	(uint32_t )d = vtpci_legacy_read_config_4(sc, off);
487	} else if (length >= 2) {
488	size = 2;
489	(uint16_t )d = vtpci_legacy_read_config_2(sc, off);
490	} else {
491	size = 1;
492	*d = vtpci_legacy_read_config_1(sc, off);
493	}
494	}
495	}
496
497	static void
498	vtpci_legacy_write_dev_config(device_t dev, bus_size_t offset,
499	void *src, int length)
500	{
501	struct vtpci_legacy_softc *sc;
502	bus_size_t off;
503	uint8_t *s;
504	int size;
505
506	sc = device_get_softc(dev);
507	off = VIRTIO_PCI_LEGACY_CONFIG(sc) + offset;
508
509	for (s = src; length > 0; s += size, off += size, length -= size) {
510	if (length >= 4) {
511	size = 4;
512	vtpci_legacy_write_config_4(sc, off, (uint32_t )s);
513	} else if (length >= 2) {
514	size = 2;
515	vtpci_legacy_write_config_2(sc, off, (uint16_t )s);
516	} else {
517	size = 1;
518	vtpci_legacy_write_config_1(sc, off, *s);
519	}
520	}
521	}
522
523	static int
524	vtpci_legacy_alloc_resources(struct vtpci_legacy_softc *sc)
525	{
526	device_t dev;
527	int rid;
528
529	dev = sc->vtpci_dev;
530
531	rid = PCIR_BAR(0);
532	if ((sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT,
533	&rid, RF_ACTIVE)) == NULL)
534	return (ENXIO);
535
536	if (vtpci_is_msix_available(&sc->vtpci_common)) {
537	rid = PCIR_BAR(1);
538	if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev,
539	SYS_RES_MEMORY, &rid, RF_ACTIVE)) == NULL)
540	return (ENXIO);
541	}
542
543	return (0);
544	}
545
546	static void
547	vtpci_legacy_free_resources(struct vtpci_legacy_softc *sc)
548	{
549	device_t dev;
550
551	dev = sc->vtpci_dev;
552
553	if (sc->vtpci_msix_res != NULL) {
554	bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
555	sc->vtpci_msix_res);
556	sc->vtpci_msix_res = NULL;
557	}
558
559	if (sc->vtpci_res != NULL) {
560	bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
561	sc->vtpci_res);
562	sc->vtpci_res = NULL;
563	}
564	}
565
566	static void
567	vtpci_legacy_probe_and_attach_child(struct vtpci_legacy_softc *sc)
568	{
569	device_t dev, child;
570
571	dev = sc->vtpci_dev;
572	child = vtpci_child_device(&sc->vtpci_common);
573
574	if (child == NULL \|\| device_get_state(child) != DS_NOTPRESENT)
575	return;
576
577	if (device_probe(child) != 0)
578	return;
579
580	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
581
582	if (device_attach(child) != 0) {
583	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
584	/* Reset status for future attempt. */
585	vtpci_legacy_child_detached(dev, child);
586	} else {
587	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
588	VIRTIO_ATTACH_COMPLETED(child);
589	}
590	}
591
592	static int
593	vtpci_legacy_register_msix(struct vtpci_legacy_softc *sc, int offset,
594	struct vtpci_interrupt *intr)
595	{
596	device_t dev;
597	uint16_t vector;
598
599	dev = sc->vtpci_dev;
600
601	if (intr != NULL) {
602	/* Map from guest rid to host vector. */
603	vector = intr->vti_rid - 1;
604	} else
605	vector = VIRTIO_MSI_NO_VECTOR;
606
607	vtpci_legacy_write_config_2(sc, offset, vector);
608	return (vtpci_legacy_read_config_2(sc, offset) == vector ? 0 : ENODEV);
609	}
610
611	static int
612	vtpci_legacy_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr)
613	{
614	struct vtpci_legacy_softc *sc;
615	int error;
616
617	sc = device_get_softc(dev);
618
619	error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_CONFIG_VECTOR, intr);
620	if (error) {
621	device_printf(dev,
622	"unable to register config MSIX interrupt\n");
623	return (error);
624	}
625
626	return (0);
627	}
628
629	static int
630	vtpci_legacy_register_vq_msix(device_t dev, int idx,
631	struct vtpci_interrupt *intr)
632	{
633	struct vtpci_legacy_softc *sc;
634	int error;
635
636	sc = device_get_softc(dev);
637
638	vtpci_legacy_select_virtqueue(sc, idx);
639	error = vtpci_legacy_register_msix(sc, VIRTIO_MSI_QUEUE_VECTOR, intr);
640	if (error) {
641	device_printf(dev,
642	"unable to register virtqueue MSIX interrupt\n");
643	return (error);
644	}
645
646	return (0);
647	}
648
649	static void
650	vtpci_legacy_reset(struct vtpci_legacy_softc *sc)
651	{
652	/*
653	* Setting the status to RESET sets the host device to the
654	* original, uninitialized state.
655	*/
656	vtpci_legacy_set_status(sc, VIRTIO_CONFIG_STATUS_RESET);
657	(void) vtpci_legacy_get_status(sc);
658	}
659
660	static void
661	vtpci_legacy_select_virtqueue(struct vtpci_legacy_softc *sc, int idx)
662	{
663	vtpci_legacy_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
664	}
665
666	static uint8_t
667	vtpci_legacy_read_isr(device_t dev)
668	{
669	struct vtpci_legacy_softc *sc;
670
671	sc = device_get_softc(dev);
672
673	return (vtpci_legacy_read_config_1(sc, VIRTIO_PCI_ISR));
674	}
675
676	static uint16_t
677	vtpci_legacy_get_vq_size(device_t dev, int idx)
678	{
679	struct vtpci_legacy_softc *sc;
680
681	sc = device_get_softc(dev);
682
683	vtpci_legacy_select_virtqueue(sc, idx);
684	return (vtpci_legacy_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM));
685	}
686
687	static bus_size_t
688	vtpci_legacy_get_vq_notify_off(device_t dev, int idx)
689	{
690	return (VIRTIO_PCI_QUEUE_NOTIFY);
691	}
692
693	static void
694	vtpci_legacy_set_vq(device_t dev, struct virtqueue *vq)
695	{
696	struct vtpci_legacy_softc *sc;
697
698	sc = device_get_softc(dev);
699
700	vtpci_legacy_select_virtqueue(sc, virtqueue_index(vq));
701	vtpci_legacy_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
702	virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
703	}
704
705	static void
706	vtpci_legacy_disable_vq(device_t dev, int idx)
707	{
708	struct vtpci_legacy_softc *sc;
709
710	sc = device_get_softc(dev);
711
712	vtpci_legacy_select_virtqueue(sc, idx);
713	vtpci_legacy_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
714	}

Lines 1-135 Link Here

(-)sys/dev/virtio.ori/pci/virtio_pci_modern_var.h (-135 lines)
1	/*
2	* SPDX-License-Identifier: BSD-3-Clause
3	*
4	* Copyright IBM Corp. 2007
5	*
6	* Authors:
7	* Anthony Liguori <aliguori@us.ibm.com>
8	*
9	* This header is BSD licensed so anyone can use the definitions to implement
10	* compatible drivers/servers.
11	*
12	* Redistribution and use in source and binary forms, with or without
13	* modification, are permitted provided that the following conditions
14	* are met:
15	* 1. Redistributions of source code must retain the above copyright
16	* notice, this list of conditions and the following disclaimer.
17	* 2. Redistributions in binary form must reproduce the above copyright
18	* notice, this list of conditions and the following disclaimer in the
19	* documentation and/or other materials provided with the distribution.
20	* 3. Neither the name of IBM nor the names of its contributors
21	* may be used to endorse or promote products derived from this software
22	* without specific prior written permission.
23	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25	* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
27	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33	* SUCH DAMAGE.
34	*
35	* $FreeBSD$
36	*/
37
38	#ifndef _VIRTIO_PCI_MODERN_VAR_H
39	#define _VIRTIO_PCI_MODERN_VAR_H
40
41	#include <dev/virtio/pci/virtio_pci_var.h>
42
43	/* IDs for different capabilities. Must all exist. */
44	/* Common configuration */
45	#define VIRTIO_PCI_CAP_COMMON_CFG 1
46	/* Notifications */
47	#define VIRTIO_PCI_CAP_NOTIFY_CFG 2
48	/* ISR access */
49	#define VIRTIO_PCI_CAP_ISR_CFG 3
50	/* Device specific configuration */
51	#define VIRTIO_PCI_CAP_DEVICE_CFG 4
52	/* PCI configuration access */
53	#define VIRTIO_PCI_CAP_PCI_CFG 5
54
55	/* This is the PCI capability header: */
56	struct virtio_pci_cap {
57	uint8_t cap_vndr; /* Generic PCI field: PCI_CAP_ID_VNDR */
58	uint8_t cap_next; /* Generic PCI field: next ptr. */
59	uint8_t cap_len; /* Generic PCI field: capability length */
60	uint8_t cfg_type; /* Identifies the structure. */
61	uint8_t bar; /* Where to find it. */
62	uint8_t padding[3]; /* Pad to full dword. */
63	uint32_t offset; /* Offset within bar. */
64	uint32_t length; /* Length of the structure, in bytes. */
65	};
66
67	struct virtio_pci_notify_cap {
68	struct virtio_pci_cap cap;
69	uint32_t notify_off_multiplier; /* Multiplier for queue_notify_off. */
70	};
71
72	/* Fields in VIRTIO_PCI_CAP_COMMON_CFG: */
73	struct virtio_pci_common_cfg {
74	/* About the whole device. */
75	uint32_t device_feature_select; /* read-write */
76	uint32_t device_feature; /* read-only */
77	uint32_t guest_feature_select; /* read-write */
78	uint32_t guest_feature; /* read-write */
79	uint16_t msix_config; /* read-write */
80	uint16_t num_queues; /* read-only */
81	uint8_t device_status; /* read-write */
82	uint8_t config_generation; /* read-only */
83
84	/* About a specific virtqueue. */
85	uint16_t queue_select; /* read-write */
86	uint16_t queue_size; /* read-write, power of 2. */
87	uint16_t queue_msix_vector; /* read-write */
88	uint16_t queue_enable; /* read-write */
89	uint16_t queue_notify_off; /* read-only */
90	uint32_t queue_desc_lo; /* read-write */
91	uint32_t queue_desc_hi; /* read-write */
92	uint32_t queue_avail_lo; /* read-write */
93	uint32_t queue_avail_hi; /* read-write */
94	uint32_t queue_used_lo; /* read-write */
95	uint32_t queue_used_hi; /* read-write */
96	};
97
98	/* Fields in VIRTIO_PCI_CAP_PCI_CFG: */
99	struct virtio_pci_cfg_cap {
100	struct virtio_pci_cap cap;
101	uint8_t pci_cfg_data[4]; /* Data for BAR access. */
102	};
103
104	/* Macro versions of offsets for the Old Timers! */
105	#define VIRTIO_PCI_CAP_VNDR 0
106	#define VIRTIO_PCI_CAP_NEXT 1
107	#define VIRTIO_PCI_CAP_LEN 2
108	#define VIRTIO_PCI_CAP_CFG_TYPE 3
109	#define VIRTIO_PCI_CAP_BAR 4
110	#define VIRTIO_PCI_CAP_OFFSET 8
111	#define VIRTIO_PCI_CAP_LENGTH 12
112
113	#define VIRTIO_PCI_NOTIFY_CAP_MULT 16
114
115	#define VIRTIO_PCI_COMMON_DFSELECT 0
116	#define VIRTIO_PCI_COMMON_DF 4
117	#define VIRTIO_PCI_COMMON_GFSELECT 8
118	#define VIRTIO_PCI_COMMON_GF 12
119	#define VIRTIO_PCI_COMMON_MSIX 16
120	#define VIRTIO_PCI_COMMON_NUMQ 18
121	#define VIRTIO_PCI_COMMON_STATUS 20
122	#define VIRTIO_PCI_COMMON_CFGGENERATION 21
123	#define VIRTIO_PCI_COMMON_Q_SELECT 22
124	#define VIRTIO_PCI_COMMON_Q_SIZE 24
125	#define VIRTIO_PCI_COMMON_Q_MSIX 26
126	#define VIRTIO_PCI_COMMON_Q_ENABLE 28
127	#define VIRTIO_PCI_COMMON_Q_NOFF 30
128	#define VIRTIO_PCI_COMMON_Q_DESCLO 32
129	#define VIRTIO_PCI_COMMON_Q_DESCHI 36
130	#define VIRTIO_PCI_COMMON_Q_AVAILLO 40
131	#define VIRTIO_PCI_COMMON_Q_AVAILHI 44
132	#define VIRTIO_PCI_COMMON_Q_USEDLO 48
133	#define VIRTIO_PCI_COMMON_Q_USEDHI 52
134
135	#endif /* _VIRTIO_PCI_MODERN_VAR_H */

Lines 1-55 Link Here

(-)sys/dev/virtio.ori/pci/virtio_pci_var.h (-55 lines)
1	/*-
2	* SPDX-License-Identifier: BSD-3-Clause
3	*
4	* Copyright IBM Corp. 2007
5	*
6	* Authors:
7	* Anthony Liguori <aliguori@us.ibm.com>
8	*
9	* This header is BSD licensed so anyone can use the definitions to implement
10	* compatible drivers/servers.
11	*
12	* Redistribution and use in source and binary forms, with or without
13	* modification, are permitted provided that the following conditions
14	* are met:
15	* 1. Redistributions of source code must retain the above copyright
16	* notice, this list of conditions and the following disclaimer.
17	* 2. Redistributions in binary form must reproduce the above copyright
18	* notice, this list of conditions and the following disclaimer in the
19	* documentation and/or other materials provided with the distribution.
20	* 3. Neither the name of IBM nor the names of its contributors
21	* may be used to endorse or promote products derived from this software
22	* without specific prior written permission.
23	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
25	* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
26	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
27	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33	* SUCH DAMAGE.
34	*
35	* $FreeBSD$
36	*/
37
38	#ifndef _VIRTIO_PCI_VAR_H
39	#define _VIRTIO_PCI_VAR_H
40
41	/* VirtIO PCI vendor/device ID. */
42	#define VIRTIO_PCI_VENDORID 0x1AF4
43	#define VIRTIO_PCI_DEVICEID_MIN 0x1000
44	#define VIRTIO_PCI_DEVICEID_LEGACY_MAX 0x103F
45	#define VIRTIO_PCI_DEVICEID_MODERN_MIN 0x1040
46	#define VIRTIO_PCI_DEVICEID_MODERN_MAX 0x107F
47
48	/* The bit of the ISR which indicates a device has an interrupt. */
49	#define VIRTIO_PCI_ISR_INTR 0x1
50	/* The bit of the ISR which indicates a device configuration change. */
51	#define VIRTIO_PCI_ISR_CONFIG 0x2
52	/* Vector value used to disable MSI for queue. */
53	#define VIRTIO_MSI_NO_VECTOR 0xFFFF
54
55	#endif /* _VIRTIO_PCI_VAR_H */

Lines 76-83 Link Here

(-)sys/dev/virtio.ori/scsi/virtio_scsi.c (-77 / +40 lines)
76	static int vtscsi_suspend(device_t);	76	static int vtscsi_suspend(device_t);
77	static int vtscsi_resume(device_t);	77	static int vtscsi_resume(device_t);
78		78
79	static int vtscsi_negotiate_features(struct vtscsi_softc *);	79	static void vtscsi_negotiate_features(struct vtscsi_softc *);
80	static int vtscsi_setup_features(struct vtscsi_softc *);
81	static void vtscsi_read_config(struct vtscsi_softc *,	80	static void vtscsi_read_config(struct vtscsi_softc *,
82	struct virtio_scsi_config *);	81	struct virtio_scsi_config *);
83	static int vtscsi_maximum_segments(struct vtscsi_softc *, int);	82	static int vtscsi_maximum_segments(struct vtscsi_softc *, int);
Lines 136-145 Link Here
136		135
137	static void vtscsi_get_request_lun(uint8_t [], target_id_t , lun_id_t );	136	static void vtscsi_get_request_lun(uint8_t [], target_id_t , lun_id_t );
138	static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);	137	static void vtscsi_set_request_lun(struct ccb_hdr *, uint8_t []);
139	static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *,	138	static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *,
140	struct ccb_scsiio , struct virtio_scsi_cmd_req );	139	struct virtio_scsi_cmd_req *);
141	static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc , struct ccb_hdr ,	140	static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t,
142	uint32_t, uintptr_t, struct virtio_scsi_ctrl_tmf_req *);	141	uintptr_t, struct virtio_scsi_ctrl_tmf_req *);
143		142
144	static void vtscsi_freeze_simq(struct vtscsi_softc *, int);	143	static void vtscsi_freeze_simq(struct vtscsi_softc *, int);
145	static int vtscsi_thaw_simq(struct vtscsi_softc *, int);	144	static int vtscsi_thaw_simq(struct vtscsi_softc *, int);
Lines 185-203 Link Here
185	static void vtscsi_enable_vqs_intr(struct vtscsi_softc *);	184	static void vtscsi_enable_vqs_intr(struct vtscsi_softc *);
186		185
187	static void vtscsi_get_tunables(struct vtscsi_softc *);	186	static void vtscsi_get_tunables(struct vtscsi_softc *);
188	static void vtscsi_setup_sysctl(struct vtscsi_softc *);	187	static void vtscsi_add_sysctl(struct vtscsi_softc *);
189		188
190	static void vtscsi_printf_req(struct vtscsi_request , const char ,	189	static void vtscsi_printf_req(struct vtscsi_request , const char ,
191	const char *, ...);	190	const char *, ...);
192		191
193	#define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0)
194	#define vtscsi_htog16(_sc, _val) virtio_htog16(vtscsi_modern(_sc), _val)
195	#define vtscsi_htog32(_sc, _val) virtio_htog32(vtscsi_modern(_sc), _val)
196	#define vtscsi_htog64(_sc, _val) virtio_htog64(vtscsi_modern(_sc), _val)
197	#define vtscsi_gtoh16(_sc, _val) virtio_gtoh16(vtscsi_modern(_sc), _val)
198	#define vtscsi_gtoh32(_sc, _val) virtio_gtoh32(vtscsi_modern(_sc), _val)
199	#define vtscsi_gtoh64(_sc, _val) virtio_gtoh64(vtscsi_modern(_sc), _val)
200
201	/* Global tunables. */	192	/* Global tunables. */
202	/*	193	/*
203	* The current QEMU VirtIO SCSI implementation does not cancel in-flight	194	* The current QEMU VirtIO SCSI implementation does not cancel in-flight
Lines 215-222 Link Here
215	static struct virtio_feature_desc vtscsi_feature_desc[] = {	206	static struct virtio_feature_desc vtscsi_feature_desc[] = {
216	{ VIRTIO_SCSI_F_INOUT, "InOut" },	207	{ VIRTIO_SCSI_F_INOUT, "InOut" },
217	{ VIRTIO_SCSI_F_HOTPLUG, "Hotplug" },	208	{ VIRTIO_SCSI_F_HOTPLUG, "Hotplug" },
218	{ VIRTIO_SCSI_F_CHANGE, "ChangeEvent" },
219	{ VIRTIO_SCSI_F_T10_PI, "T10PI" },
220		209
221	{ 0, NULL }	210	{ 0, NULL }
222	};	211	};
Lines 239-248 Link Here
239	};	228	};
240	static devclass_t vtscsi_devclass;	229	static devclass_t vtscsi_devclass;
241		230
242	DRIVER_MODULE(virtio_scsi, vtpcil, vtscsi_driver, vtscsi_devclass,	231	DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass,
243	vtscsi_modevent, 0);	232	vtscsi_modevent, 0);
244	DRIVER_MODULE(virtio_scsi, vtpcim, vtscsi_driver, vtscsi_devclass,
245	vtscsi_modevent, 0);
246	MODULE_VERSION(virtio_scsi, 1);	233	MODULE_VERSION(virtio_scsi, 1);
247	MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1);	234	MODULE_DEPEND(virtio_scsi, virtio, 1, 1, 1);
248	MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1);	235	MODULE_DEPEND(virtio_scsi, cam, 1, 1, 1);
Lines 288-307 Link Here
288		275
289	sc = device_get_softc(dev);	276	sc = device_get_softc(dev);
290	sc->vtscsi_dev = dev;	277	sc->vtscsi_dev = dev;
291	virtio_set_feature_desc(dev, vtscsi_feature_desc);
292		278
293	VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev));	279	VTSCSI_LOCK_INIT(sc, device_get_nameunit(dev));
294	TAILQ_INIT(&sc->vtscsi_req_free);	280	TAILQ_INIT(&sc->vtscsi_req_free);
295		281
296	vtscsi_get_tunables(sc);	282	vtscsi_get_tunables(sc);
297	vtscsi_setup_sysctl(sc);	283	vtscsi_add_sysctl(sc);
298		284
299	error = vtscsi_setup_features(sc);	285	virtio_set_feature_desc(dev, vtscsi_feature_desc);
300	if (error) {	286	vtscsi_negotiate_features(sc);
301	device_printf(dev, "cannot setup features\n");
302	goto fail;
303	}
304		287
		288	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		289	sc->vtscsi_flags \|= VTSCSI_FLAG_INDIRECT;
		290	if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
		291	sc->vtscsi_flags \|= VTSCSI_FLAG_BIDIRECTIONAL;
		292	if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
		293	sc->vtscsi_flags \|= VTSCSI_FLAG_HOTPLUG;
		294
305	vtscsi_read_config(sc, &scsicfg);	295	vtscsi_read_config(sc, &scsicfg);
306		296
307	sc->vtscsi_max_channel = scsicfg.max_channel;	297	sc->vtscsi_max_channel = scsicfg.max_channel;
Lines 413-453 Link Here
413	return (0);	403	return (0);
414	}	404	}
415		405
416	static int	406	static void
417	vtscsi_negotiate_features(struct vtscsi_softc *sc)	407	vtscsi_negotiate_features(struct vtscsi_softc *sc)
418	{	408	{
419	device_t dev;	409	device_t dev;
420	uint64_t features;	410	uint64_t features;
421		411
422	dev = sc->vtscsi_dev;	412	dev = sc->vtscsi_dev;
423	features = VTSCSI_FEATURES;	413	features = virtio_negotiate_features(dev, VTSCSI_FEATURES);
424		414	sc->vtscsi_features = features;
425	sc->vtscsi_features = virtio_negotiate_features(dev, features);
426	return (virtio_finalize_features(dev));
427	}	415	}
428		416
429	static int
430	vtscsi_setup_features(struct vtscsi_softc *sc)
431	{
432	device_t dev;
433	int error;
434
435	dev = sc->vtscsi_dev;
436
437	error = vtscsi_negotiate_features(sc);
438	if (error)
439	return (error);
440
441	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
442	sc->vtscsi_flags \|= VTSCSI_FLAG_INDIRECT;
443	if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
444	sc->vtscsi_flags \|= VTSCSI_FLAG_BIDIRECTIONAL;
445	if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
446	sc->vtscsi_flags \|= VTSCSI_FLAG_HOTPLUG;
447
448	return (0);
449	}
450
451	#define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \	417	#define VTSCSI_GET_CONFIG(_dev, _field, _cfg) \
452	virtio_read_device_config(_dev, \	418	virtio_read_device_config(_dev, \
453	offsetof(struct virtio_scsi_config, _field), \	419	offsetof(struct virtio_scsi_config, _field), \
Lines 565-572 Link Here
565	error = virtio_reinit(dev, sc->vtscsi_features);	531	error = virtio_reinit(dev, sc->vtscsi_features);
566	if (error == 0) {	532	if (error == 0) {
567	vtscsi_write_device_config(sc);	533	vtscsi_write_device_config(sc);
568	virtio_reinit_complete(dev);
569	vtscsi_reinit_event_vq(sc);	534	vtscsi_reinit_event_vq(sc);
		535	virtio_reinit_complete(dev);
570		536
571	vtscsi_enable_vqs_intr(sc);	537	vtscsi_enable_vqs_intr(sc);
572	}	538	}
Lines 974-980 Link Here
974		940
975	cpi->max_target = sc->vtscsi_max_target;	941	cpi->max_target = sc->vtscsi_max_target;
976	cpi->max_lun = sc->vtscsi_max_lun;	942	cpi->max_lun = sc->vtscsi_max_lun;
977	cpi->initiator_id = cpi->max_target + 1;	943	cpi->initiator_id = VTSCSI_INITIATOR_ID;
978		944
979	strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);	945	strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
980	strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);	946	strlcpy(cpi->hba_vid, "VirtIO", HBA_IDLEN);
Lines 1120-1126 Link Here
1120	cmd_req = &req->vsr_cmd_req;	1086	cmd_req = &req->vsr_cmd_req;
1121	cmd_resp = &req->vsr_cmd_resp;	1087	cmd_resp = &req->vsr_cmd_resp;
1122		1088
1123	vtscsi_init_scsi_cmd_req(sc, csio, cmd_req);	1089	vtscsi_init_scsi_cmd_req(csio, cmd_req);
1124		1090
1125	error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable);	1091	error = vtscsi_fill_scsi_cmd_sglist(sc, req, &readable, &writable);
1126	if (error)	1092	if (error)
Lines 1240-1246 Link Here
1240	tmf_req = &req->vsr_tmf_req;	1206	tmf_req = &req->vsr_tmf_req;
1241	tmf_resp = &req->vsr_tmf_resp;	1207	tmf_resp = &req->vsr_tmf_resp;
1242		1208
1243	vtscsi_init_ctrl_tmf_req(sc, to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,	1209	vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1244	(uintptr_t) to_ccbh, tmf_req);	1210	(uintptr_t) to_ccbh, tmf_req);
1245		1211
1246	sglist_reset(sg);	1212	sglist_reset(sg);
Lines 1348-1376 Link Here
1348	vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc,	1314	vtscsi_complete_scsi_cmd_response(struct vtscsi_softc *sc,
1349	struct ccb_scsiio csio, struct virtio_scsi_cmd_resp cmd_resp)	1315	struct ccb_scsiio csio, struct virtio_scsi_cmd_resp cmd_resp)
1350	{	1316	{
1351	uint32_t resp_sense_length;
1352	cam_status status;	1317	cam_status status;
1353		1318
1354	csio->scsi_status = cmd_resp->status;	1319	csio->scsi_status = cmd_resp->status;
1355	csio->resid = vtscsi_htog32(sc, cmd_resp->resid);	1320	csio->resid = cmd_resp->resid;
1356		1321
1357	if (csio->scsi_status == SCSI_STATUS_OK)	1322	if (csio->scsi_status == SCSI_STATUS_OK)
1358	status = CAM_REQ_CMP;	1323	status = CAM_REQ_CMP;
1359	else	1324	else
1360	status = CAM_SCSI_STATUS_ERROR;	1325	status = CAM_SCSI_STATUS_ERROR;
1361		1326
1362	resp_sense_length = vtscsi_htog32(sc, cmd_resp->sense_len);	1327	if (cmd_resp->sense_len > 0) {
1363
1364	if (resp_sense_length > 0) {
1365	status \|= CAM_AUTOSNS_VALID;	1328	status \|= CAM_AUTOSNS_VALID;
1366		1329
1367	if (resp_sense_length < csio->sense_len)	1330	if (cmd_resp->sense_len < csio->sense_len)
1368	csio->sense_resid = csio->sense_len - resp_sense_length;	1331	csio->sense_resid = csio->sense_len -
		1332	cmd_resp->sense_len;
1369	else	1333	else
1370	csio->sense_resid = 0;	1334	csio->sense_resid = 0;
1371		1335
1372	bzero(&csio->sense_data, sizeof(csio->sense_data));	1336	memcpy(&csio->sense_data, cmd_resp->sense,
1373	memcpy(cmd_resp->sense, &csio->sense_data,
1374	csio->sense_len - csio->sense_resid);	1337	csio->sense_len - csio->sense_resid);
1375	}	1338	}
1376		1339
Lines 1531-1537 Link Here
1531	if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)	1494	if (abort_req->vsr_flags & VTSCSI_REQ_FLAG_TIMEOUT_SET)
1532	callout_stop(&abort_req->vsr_callout);	1495	callout_stop(&abort_req->vsr_callout);
1533		1496
1534	vtscsi_init_ctrl_tmf_req(sc, ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,	1497	vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1535	(uintptr_t) abort_ccbh, tmf_req);	1498	(uintptr_t) abort_ccbh, tmf_req);
1536		1499
1537	sglist_reset(sg);	1500	sglist_reset(sg);
Lines 1600-1606 Link Here
1600	else	1563	else
1601	subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;	1564	subtype = VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET;
1602		1565
1603	vtscsi_init_ctrl_tmf_req(sc, ccbh, subtype, 0, tmf_req);	1566	vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req);
1604		1567
1605	sglist_reset(sg);	1568	sglist_reset(sg);
1606	sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));	1569	sglist_append(sg, tmf_req, sizeof(struct virtio_scsi_ctrl_tmf_req));
Lines 1637-1643 Link Here
1637	}	1600	}
1638		1601
1639	static void	1602	static void
1640	vtscsi_init_scsi_cmd_req(struct vtscsi_softc sc, struct ccb_scsiio csio,	1603	vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio,
1641	struct virtio_scsi_cmd_req *cmd_req)	1604	struct virtio_scsi_cmd_req *cmd_req)
1642	{	1605	{
1643	uint8_t attr;	1606	uint8_t attr;
Lines 1658-1664 Link Here
1658	}	1621	}
1659		1622
1660	vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun);	1623	vtscsi_set_request_lun(&csio->ccb_h, cmd_req->lun);
1661	cmd_req->tag = vtscsi_gtoh64(sc, (uintptr_t) csio);	1624	cmd_req->tag = (uintptr_t) csio;
1662	cmd_req->task_attr = attr;	1625	cmd_req->task_attr = attr;
1663		1626
1664	memcpy(cmd_req->cdb,	1627	memcpy(cmd_req->cdb,
Lines 1668-1682 Link Here
1668	}	1631	}
1669		1632
1670	static void	1633	static void
1671	vtscsi_init_ctrl_tmf_req(struct vtscsi_softc sc, struct ccb_hdr ccbh,	1634	vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype,
1672	uint32_t subtype, uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)	1635	uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)
1673	{	1636	{
1674		1637
1675	vtscsi_set_request_lun(ccbh, tmf_req->lun);	1638	vtscsi_set_request_lun(ccbh, tmf_req->lun);
1676		1639
1677	tmf_req->type = vtscsi_gtoh32(sc, VIRTIO_SCSI_T_TMF);	1640	tmf_req->type = VIRTIO_SCSI_T_TMF;
1678	tmf_req->subtype = vtscsi_gtoh32(sc, subtype);	1641	tmf_req->subtype = subtype;
1679	tmf_req->tag = vtscsi_gtoh64(sc, tag);	1642	tmf_req->tag = tag;
1680	}	1643	}
1681		1644
1682	static void	1645	static void
Lines 2310-2316 Link Here
2310	}	2273	}
2311		2274
2312	static void	2275	static void
2313	vtscsi_setup_sysctl(struct vtscsi_softc *sc)	2276	vtscsi_add_sysctl(struct vtscsi_softc *sc)
2314	{	2277	{
2315	device_t dev;	2278	device_t dev;
2316	struct vtscsi_statistics *stats;	2279	struct vtscsi_statistics *stats;

Lines 31-37 Link Here

(-)sys/dev/virtio.ori/scsi/virtio_scsi.h (-31 / +8 lines)
31	#ifndef _VIRTIO_SCSI_H	31	#ifndef _VIRTIO_SCSI_H
32	#define _VIRTIO_SCSI_H	32	#define _VIRTIO_SCSI_H
33		33
34	/* Default values of the CDB and sense data size configuration fields */	34	/* Feature bits */
		35	#define VIRTIO_SCSI_F_INOUT 0x0001 /* Single request can contain both
		36	* read and write buffers */
		37	#define VIRTIO_SCSI_F_HOTPLUG 0x0002 /* Host should enable hot plug/unplug
		38	* of new LUNs and targets.
		39	*/
		40
35	#define VIRTIO_SCSI_CDB_SIZE 32	41	#define VIRTIO_SCSI_CDB_SIZE 32
36	#define VIRTIO_SCSI_SENSE_SIZE 96	42	#define VIRTIO_SCSI_SENSE_SIZE 96
37		43
Lines 40-62 Link Here
40	uint8_t lun[8]; /* Logical Unit Number */	46	uint8_t lun[8]; /* Logical Unit Number */
41	uint64_t tag; /* Command identifier */	47	uint64_t tag; /* Command identifier */
42	uint8_t task_attr; /* Task attribute */	48	uint8_t task_attr; /* Task attribute */
43	uint8_t prio; /* SAM command priority field */	49	uint8_t prio;
44	uint8_t crn;	50	uint8_t crn;
45	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];	51	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
46	} __packed;	52	} __packed;
47		53
48	/* SCSI command request, followed by protection information */
49	struct virtio_scsi_cmd_req_pi {
50	uint8_t lun[8]; /* Logical Unit Number */
51	uint64_t tag; /* Command identifier */
52	uint8_t task_attr; /* Task attribute */
53	uint8_t prio; /* SAM command priority field */
54	uint8_t crn;
55	uint32_t pi_bytesout; /* DataOUT PI Number of bytes */
56	uint32_t pi_bytesin; /* DataIN PI Number of bytes */
57	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
58	} __packed;
59
60	/* Response, followed by sense data and data-in */	54	/* Response, followed by sense data and data-in */
61	struct virtio_scsi_cmd_resp {	55	struct virtio_scsi_cmd_resp {
62	uint32_t sense_len; /* Sense data length */	56	uint32_t sense_len; /* Sense data length */
Lines 110-131 Link Here
110	uint32_t max_lun;	104	uint32_t max_lun;
111	} __packed;	105	} __packed;
112		106
113	/* Feature bits */
114	#define VIRTIO_SCSI_F_INOUT 0x0001 /* Single request can contain both
115	* read and write buffers.
116	*/
117	#define VIRTIO_SCSI_F_HOTPLUG 0x0002 /* Host should enable hot plug/unplug
118	* of new LUNs and targets.
119	*/
120	#define VIRTIO_SCSI_F_CHANGE 0x0004 /* Host will report changes to LUN
121	* parameters via a
122	* VIRTIO_SCSI_T_PARAM_CHANGE event.
123	*/
124	#define VIRTIO_SCSI_F_T10_PI 0x0008 /* Extended fields for T10 protection
125	* information (DIF/DIX) are included
126	* in the SCSI request header.
127	*/
128
129	/* Response codes */	107	/* Response codes */
130	#define VIRTIO_SCSI_S_OK 0	108	#define VIRTIO_SCSI_S_OK 0
131	#define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0	109	#define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0
Lines 162-168 Link Here
162	#define VIRTIO_SCSI_T_NO_EVENT 0	140	#define VIRTIO_SCSI_T_NO_EVENT 0
163	#define VIRTIO_SCSI_T_TRANSPORT_RESET 1	141	#define VIRTIO_SCSI_T_TRANSPORT_RESET 1
164	#define VIRTIO_SCSI_T_ASYNC_NOTIFY 2	142	#define VIRTIO_SCSI_T_ASYNC_NOTIFY 2
165	#define VIRTIO_SCSI_T_PARAM_CHANGE 3
166		143
167	/* Reasons of transport reset event */	144	/* Reasons of transport reset event */
168	#define VIRTIO_SCSI_EVT_RESET_HARD 0	145	#define VIRTIO_SCSI_EVT_RESET_HARD 0

Lines 31-41 Link Here

(-)sys/dev/virtio.ori/virtio.h (-12 / +1 lines)
31	#ifndef _VIRTIO_H_	31	#ifndef _VIRTIO_H_
32	#define _VIRTIO_H_	32	#define _VIRTIO_H_
33		33
34	#include <dev/virtio/virtio_endian.h>
35	#include <dev/virtio/virtio_ids.h>	34	#include <dev/virtio/virtio_ids.h>
36	#include <dev/virtio/virtio_config.h>	35	#include <dev/virtio/virtio_config.h>
37		36
38	struct sbuf;
39	struct vq_alloc_info;	37	struct vq_alloc_info;
40		38
41	/*	39	/*
Lines 59-65 Link Here
59	#define VIRTIO_IVAR_DEVICE 4	57	#define VIRTIO_IVAR_DEVICE 4
60	#define VIRTIO_IVAR_SUBVENDOR 5	58	#define VIRTIO_IVAR_SUBVENDOR 5
61	#define VIRTIO_IVAR_SUBDEVICE 6	59	#define VIRTIO_IVAR_SUBDEVICE 6
62	#define VIRTIO_IVAR_MODERN 7
63		60
64	struct virtio_feature_desc {	61	struct virtio_feature_desc {
65	uint64_t vfd_val;	62	uint64_t vfd_val;
Lines 68-80 Link Here
68		65
69	const char *virtio_device_name(uint16_t devid);	66	const char *virtio_device_name(uint16_t devid);
70	void virtio_describe(device_t dev, const char *msg,	67	void virtio_describe(device_t dev, const char *msg,
71	uint64_t features, struct virtio_feature_desc *desc);	68	uint64_t features, struct virtio_feature_desc *feature_desc);
72	int virtio_describe_sbuf(struct sbuf *sb, uint64_t features,
73	struct virtio_feature_desc *desc);
74	uint64_t virtio_filter_transport_features(uint64_t features);
75	int virtio_bus_is_modern(device_t dev);
76	void virtio_read_device_config_array(device_t dev, bus_size_t offset,
77	void *dst, int size, int count);
78		69
79	/*	70	/*
80	* VirtIO Bus Methods.	71	* VirtIO Bus Methods.
Lines 82-88 Link Here
82	void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val);	73	void virtio_read_ivar(device_t dev, int ivar, uintptr_t *val);
83	void virtio_write_ivar(device_t dev, int ivar, uintptr_t val);	74	void virtio_write_ivar(device_t dev, int ivar, uintptr_t val);
84	uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features);	75	uint64_t virtio_negotiate_features(device_t dev, uint64_t child_features);
85	int virtio_finalize_features(device_t dev);
86	int virtio_alloc_virtqueues(device_t dev, int flags, int nvqs,	76	int virtio_alloc_virtqueues(device_t dev, int flags, int nvqs,
87	struct vq_alloc_info *info);	77	struct vq_alloc_info *info);
88	int virtio_setup_intr(device_t dev, enum intr_type type);	78	int virtio_setup_intr(device_t dev, enum intr_type type);
Lines 140-146 Link Here
140	VIRTIO_READ_IVAR(device, VIRTIO_IVAR_DEVICE);	130	VIRTIO_READ_IVAR(device, VIRTIO_IVAR_DEVICE);
141	VIRTIO_READ_IVAR(subvendor, VIRTIO_IVAR_SUBVENDOR);	131	VIRTIO_READ_IVAR(subvendor, VIRTIO_IVAR_SUBVENDOR);
142	VIRTIO_READ_IVAR(subdevice, VIRTIO_IVAR_SUBDEVICE);	132	VIRTIO_READ_IVAR(subdevice, VIRTIO_IVAR_SUBDEVICE);
143	VIRTIO_READ_IVAR(modern, VIRTIO_IVAR_MODERN);
144		133
145	#undef VIRTIO_READ_IVAR	134	#undef VIRTIO_READ_IVAR
146		135

Lines 36-47 Link Here

(-)sys/dev/virtio.ori/virtio_bus_if.m (-11 lines)
36		36
37	CODE {	37	CODE {
38	static int	38	static int
39	virtio_bus_default_finalize_features(device_t dev)
40	{
41	return (0);
42	}
43
44	static int
45	virtio_bus_default_config_generation(device_t dev)	39	virtio_bus_default_config_generation(device_t dev)
46	{	40	{
47	return (0);	41	return (0);
Lines 53-62 Link Here
53	uint64_t child_features;	47	uint64_t child_features;
54	};	48	};
55		49
56	METHOD int finalize_features {
57	device_t dev;
58	} DEFAULT virtio_bus_default_finalize_features;
59
60	METHOD int with_feature {	50	METHOD int with_feature {
61	device_t dev;	51	device_t dev;
62	uint64_t feature;	52	uint64_t feature;
Lines 90-96 Link Here
90	METHOD void notify_vq {	80	METHOD void notify_vq {
91	device_t dev;	81	device_t dev;
92	uint16_t queue;	82	uint16_t queue;
93	bus_size_t offset;
94	};	83	};
95		84
96	METHOD int config_generation {	85	METHOD int config_generation {

Lines 1-106 Link Here

(-)sys/dev/virtio.ori/virtio_endian.h (-106 lines)
1	/*-
2	* SPDX-License-Identifier: BSD-2-Clause
3	*
4	* Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
5	* All rights reserved.
6	*
7	* Redistribution and use in source and binary forms, with or without
8	* modification, are permitted provided that the following conditions
9	* are met:
10	* 1. Redistributions of source code must retain the above copyright
11	* notice unmodified, this list of conditions, and the following
12	* disclaimer.
13	* 2. Redistributions in binary form must reproduce the above copyright
14	* notice, this list of conditions and the following disclaimer in the
15	* documentation and/or other materials provided with the distribution.
16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*
28	* $FreeBSD$
29	*/
30
31	#ifndef _VIRTIO_ENDIAN_H_
32	#define _VIRTIO_ENDIAN_H_
33
34	#include <sys/endian.h>
35
36	/*
37	* VirtIO V1 (modern) uses little endian, while legacy VirtIO uses the guest's
38	* native endian. These functions convert to and from the Guest's (driver's)
39	* and the Host's (device's) endianness when needed.
40	*/
41
42	static inline bool
43	virtio_swap_endian(bool modern)
44	{
45	#if _BYTE_ORDER == _LITTLE_ENDIAN
46	return (false);
47	#else
48	return (modern);
49	#endif
50	}
51
52	static inline uint16_t
53	virtio_htog16(bool modern, uint16_t val)
54	{
55	if (virtio_swap_endian(modern))
56	return (le16toh(val));
57	else
58	return (val);
59	}
60
61	static inline uint16_t
62	virtio_gtoh16(bool modern, uint16_t val)
63	{
64	if (virtio_swap_endian(modern))
65	return (htole16(val));
66	else
67	return (val);
68	}
69
70	static inline uint32_t
71	virtio_htog32(bool modern, uint32_t val)
72	{
73	if (virtio_swap_endian(modern))
74	return (le32toh(val));
75	else
76	return (val);
77	}
78
79	static inline uint32_t
80	virtio_gtoh32(bool modern, uint32_t val)
81	{
82	if (virtio_swap_endian(modern))
83	return (htole32(val));
84	else
85	return (val);
86	}
87
88	static inline uint64_t
89	virtio_htog64(bool modern, uint64_t val)
90	{
91	if (virtio_swap_endian(modern))
92	return (le64toh(val));
93	else
94	return (val);
95	}
96
97	static inline uint64_t
98	virtio_gtoh64(bool modern, uint64_t val)
99	{
100	if (virtio_swap_endian(modern))
101	return (htole64(val));
102	else
103	return (val);
104	}
105
106	#endif /* _VIRTIO_ENDIAN_H_ */

Lines 57-71 Link Here

(-)sys/dev/virtio.ori/virtqueue.c (-81 / +59 lines)
57		57
58	struct virtqueue {	58	struct virtqueue {
59	device_t vq_dev;	59	device_t vq_dev;
		60	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
60	uint16_t vq_queue_index;	61	uint16_t vq_queue_index;
61	uint16_t vq_nentries;	62	uint16_t vq_nentries;
62	uint32_t vq_flags;	63	uint32_t vq_flags;
63	#define VIRTQUEUE_FLAG_MODERN 0x0001	64	#define VIRTQUEUE_FLAG_INDIRECT 0x0001
64	#define VIRTQUEUE_FLAG_INDIRECT 0x0002	65	#define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
65	#define VIRTQUEUE_FLAG_EVENT_IDX 0x0004
66		66
		67	int vq_alignment;
		68	int vq_ring_size;
		69	void *vq_ring_mem;
67	int vq_max_indirect_size;	70	int vq_max_indirect_size;
68	bus_size_t vq_notify_offset;	71	int vq_indirect_mem_size;
69	virtqueue_intr_t *vq_intrhand;	72	virtqueue_intr_t *vq_intrhand;
70	void *vq_intrhand_arg;	73	void *vq_intrhand_arg;
71		74
Lines 84-95 Link Here
84	*/	87	*/
85	uint16_t vq_used_cons_idx;	88	uint16_t vq_used_cons_idx;
86		89
87	void *vq_ring_mem;
88	int vq_indirect_mem_size;
89	int vq_alignment;
90	int vq_ring_size;
91	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
92
93	struct vq_desc_extra {	90	struct vq_desc_extra {
94	void *cookie;	91	void *cookie;
95	struct vring_desc *indirect;	92	struct vring_desc *indirect;
Lines 137-150 Link Here
137	static void vq_ring_notify_host(struct virtqueue *);	134	static void vq_ring_notify_host(struct virtqueue *);
138	static void vq_ring_free_chain(struct virtqueue *, uint16_t);	135	static void vq_ring_free_chain(struct virtqueue *, uint16_t);
139		136
140	#define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
141	#define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val)
142	#define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val)
143	#define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val)
144	#define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val)
145	#define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val)
146	#define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val)
147
148	uint64_t	137	uint64_t
149	virtqueue_filter_features(uint64_t features)	138	virtqueue_filter_features(uint64_t features)
150	{	139	{
Lines 158-166 Link Here
158	}	147	}
159		148
160	int	149	int
161	virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,	150	virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
162	bus_size_t notify_offset, int align, vm_paddr_t highaddr,	151	vm_paddr_t highaddr, struct vq_alloc_info info, struct virtqueue *vqp)
163	struct vq_alloc_info info, struct virtqueue *vqp)
164	{	152	{
165	struct virtqueue *vq;	153	struct virtqueue *vq;
166	int error;	154	int error;
Lines 196-210 Link Here
196	vq->vq_dev = dev;	184	vq->vq_dev = dev;
197	strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));	185	strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
198	vq->vq_queue_index = queue;	186	vq->vq_queue_index = queue;
199	vq->vq_notify_offset = notify_offset;
200	vq->vq_alignment = align;	187	vq->vq_alignment = align;
201	vq->vq_nentries = size;	188	vq->vq_nentries = size;
202	vq->vq_free_cnt = size;	189	vq->vq_free_cnt = size;
203	vq->vq_intrhand = info->vqai_intr;	190	vq->vq_intrhand = info->vqai_intr;
204	vq->vq_intrhand_arg = info->vqai_intr_arg;	191	vq->vq_intrhand_arg = info->vqai_intr_arg;
205		192
206	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
207	vq->vq_flags \|= VIRTQUEUE_FLAG_MODERN;
208	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)	193	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
209	vq->vq_flags \|= VIRTQUEUE_FLAG_EVENT_IDX;	194	vq->vq_flags \|= VIRTQUEUE_FLAG_EVENT_IDX;
210		195
Lines 309-316 Link Here
309	bzero(indirect, vq->vq_indirect_mem_size);	294	bzero(indirect, vq->vq_indirect_mem_size);
310		295
311	for (i = 0; i < vq->vq_max_indirect_size - 1; i++)	296	for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
312	indirect[i].next = vq_gtoh16(vq, i + 1);	297	indirect[i].next = i + 1;
313	indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);	298	indirect[i].next = VQ_RING_DESC_CHAIN_END;
314	}	299	}
315		300
316	int	301	int
Lines 456-462 Link Here
456	{	441	{
457	uint16_t used_idx, nused;	442	uint16_t used_idx, nused;
458		443
459	used_idx = vq_htog16(vq, vq->vq_ring.used->idx);	444	used_idx = vq->vq_ring.used->idx;
460		445
461	nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);	446	nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
462	VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");	447	VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
Lines 468-474 Link Here
468	virtqueue_intr_filter(struct virtqueue *vq)	453	virtqueue_intr_filter(struct virtqueue *vq)
469	{	454	{
470		455
471	if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))	456	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
472	return (0);	457	return (0);
473		458
474	virtqueue_disable_intr(vq);	459	virtqueue_disable_intr(vq);
Lines 495-501 Link Here
495	{	480	{
496	uint16_t ndesc, avail_idx;	481	uint16_t ndesc, avail_idx;
497		482
498	avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);	483	avail_idx = vq->vq_ring.avail->idx;
499	ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);	484	ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
500		485
501	switch (hint) {	486	switch (hint) {
Lines 520-531 Link Here
520	{	505	{
521		506
522	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {	507	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
523	vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,	508	vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
524	vq->vq_used_cons_idx - vq->vq_nentries - 1);	509	vq->vq_nentries - 1;
525	return;	510	} else
526	}	511	vq->vq_ring.avail->flags \|= VRING_AVAIL_F_NO_INTERRUPT;
527
528	vq->vq_ring.avail->flags \|= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
529	}	512	}
530		513
531	int	514	int
Lines 588-603 Link Here
588	void *cookie;	571	void *cookie;
589	uint16_t used_idx, desc_idx;	572	uint16_t used_idx, desc_idx;
590		573
591	if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))	574	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
592	return (NULL);	575	return (NULL);
593		576
594	used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);	577	used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
595	uep = &vq->vq_ring.used->ring[used_idx];	578	uep = &vq->vq_ring.used->ring[used_idx];
596		579
597	rmb();	580	rmb();
598	desc_idx = (uint16_t) vq_htog32(vq, uep->id);	581	desc_idx = (uint16_t) uep->id;
599	if (len != NULL)	582	if (len != NULL)
600	*len = vq_htog32(vq, uep->len);	583	*len = uep->len;
601		584
602	vq_ring_free_chain(vq, desc_idx);	585	vq_ring_free_chain(vq, desc_idx);
603		586
Lines 655-667 Link Here
655	printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "	638	printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
656	"desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "	639	"desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
657	"used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",	640	"used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
658	vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),	641	vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
659	vq->vq_queued_cnt, vq->vq_desc_head_idx,	642	virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
660	vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,	643	vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
661	vq_htog16(vq, vq->vq_ring.used->idx),	644	vq->vq_ring.used->idx,
662	vq_htog16(vq, vring_used_event(&vq->vq_ring)),	645	vring_used_event(&vq->vq_ring),
663	vq_htog16(vq, vq->vq_ring.avail->flags),	646	vq->vq_ring.avail->flags,
664	vq_htog16(vq, vq->vq_ring.used->flags));	647	vq->vq_ring.used->flags);
665	}	648	}
666		649
667	static void	650	static void
Lines 678-691 Link Here
678	vring_init(vr, size, ring_mem, vq->vq_alignment);	661	vring_init(vr, size, ring_mem, vq->vq_alignment);
679		662
680	for (i = 0; i < size - 1; i++)	663	for (i = 0; i < size - 1; i++)
681	vr->desc[i].next = vq_gtoh16(vq, i + 1);	664	vr->desc[i].next = i + 1;
682	vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);	665	vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
683	}	666	}
684		667
685	static void	668	static void
686	vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)	669	vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
687	{	670	{
688	uint16_t avail_idx, avail_ring_idx;	671	uint16_t avail_idx;
689		672
690	/*	673	/*
691	* Place the head of the descriptor chain into the next slot and make	674	* Place the head of the descriptor chain into the next slot and make
Lines 694-704 Link Here
694	* currently running on another CPU, we can keep it processing the new	677	* currently running on another CPU, we can keep it processing the new
695	* descriptor.	678	* descriptor.
696	*/	679	*/
697	avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);	680	avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
698	avail_ring_idx = avail_idx & (vq->vq_nentries - 1);	681	vq->vq_ring.avail->ring[avail_idx] = desc_idx;
699	vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);	682
700	wmb();	683	wmb();
701	vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);	684	vq->vq_ring.avail->idx++;
702		685
703	/* Keep pending count until virtqueue_notify(). */	686	/* Keep pending count until virtqueue_notify(). */
704	vq->vq_queued_cnt++;	687	vq->vq_queued_cnt++;
Lines 717-735 Link Here
717		700
718	for (i = 0, idx = head_idx, seg = sg->sg_segs;	701	for (i = 0, idx = head_idx, seg = sg->sg_segs;
719	i < needed;	702	i < needed;
720	i++, idx = vq_htog16(vq, dp->next), seg++) {	703	i++, idx = dp->next, seg++) {
721	VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,	704	VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
722	"premature end of free desc chain");	705	"premature end of free desc chain");
723		706
724	dp = &desc[idx];	707	dp = &desc[idx];
725	dp->addr = vq_gtoh64(vq, seg->ss_paddr);	708	dp->addr = seg->ss_paddr;
726	dp->len = vq_gtoh32(vq, seg->ss_len);	709	dp->len = seg->ss_len;
727	dp->flags = 0;	710	dp->flags = 0;
728		711
729	if (i < needed - 1)	712	if (i < needed - 1)
730	dp->flags \|= vq_gtoh16(vq, VRING_DESC_F_NEXT);	713	dp->flags \|= VRING_DESC_F_NEXT;
731	if (i >= readable)	714	if (i >= readable)
732	dp->flags \|= vq_gtoh16(vq, VRING_DESC_F_WRITE);	715	dp->flags \|= VRING_DESC_F_WRITE;
733	}	716	}
734		717
735	return (idx);	718	return (idx);
Lines 774-787 Link Here
774	dxp->cookie = cookie;	757	dxp->cookie = cookie;
775	dxp->ndescs = 1;	758	dxp->ndescs = 1;
776		759
777	dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);	760	dp->addr = dxp->indirect_paddr;
778	dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));	761	dp->len = needed * sizeof(struct vring_desc);
779	dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);	762	dp->flags = VRING_DESC_F_INDIRECT;
780		763
781	vq_ring_enqueue_segments(vq, dxp->indirect, 0,	764	vq_ring_enqueue_segments(vq, dxp->indirect, 0,
782	sg, readable, writable);	765	sg, readable, writable);
783		766
784	vq->vq_desc_head_idx = vq_htog16(vq, dp->next);	767	vq->vq_desc_head_idx = dp->next;
785	vq->vq_free_cnt--;	768	vq->vq_free_cnt--;
786	if (vq->vq_free_cnt == 0)	769	if (vq->vq_free_cnt == 0)
787	VQ_RING_ASSERT_CHAIN_TERM(vq);	770	VQ_RING_ASSERT_CHAIN_TERM(vq);
Lines 799-811 Link Here
799	* Enable interrupts, making sure we get the latest index of	782	* Enable interrupts, making sure we get the latest index of
800	* what's already been consumed.	783	* what's already been consumed.
801	*/	784	*/
802	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {	785	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
803	vring_used_event(&vq->vq_ring) =	786	vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
804	vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);	787	else
805	} else {	788	vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
806	vq->vq_ring.avail->flags &=	789
807	vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
808	}
809	mb();	790	mb();
810		791
811	/*	792	/*
Lines 822-847 Link Here
822	static int	803	static int
823	vq_ring_must_notify_host(struct virtqueue *vq)	804	vq_ring_must_notify_host(struct virtqueue *vq)
824	{	805	{
825	uint16_t new_idx, prev_idx, event_idx, flags;	806	uint16_t new_idx, prev_idx, event_idx;
826		807
827	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {	808	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
828	new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);	809	new_idx = vq->vq_ring.avail->idx;
829	prev_idx = new_idx - vq->vq_queued_cnt;	810	prev_idx = new_idx - vq->vq_queued_cnt;
830	event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));	811	event_idx = vring_avail_event(&vq->vq_ring);
831		812
832	return (vring_need_event(event_idx, new_idx, prev_idx) != 0);	813	return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
833	}	814	}
834		815
835	flags = vq->vq_ring.used->flags;	816	return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
836	return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
837	}	817	}
838		818
839	static void	819	static void
840	vq_ring_notify_host(struct virtqueue *vq)	820	vq_ring_notify_host(struct virtqueue *vq)
841	{	821	{
842		822
843	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,	823	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
844	vq->vq_notify_offset);
845	}	824	}
846		825
847	static void	826	static void
Lines 860-870 Link Here
860	vq->vq_free_cnt += dxp->ndescs;	839	vq->vq_free_cnt += dxp->ndescs;
861	dxp->ndescs--;	840	dxp->ndescs--;
862		841
863	if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {	842	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
864	while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {	843	while (dp->flags & VRING_DESC_F_NEXT) {
865	uint16_t next_idx = vq_htog16(vq, dp->next);	844	VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
866	VQ_RING_ASSERT_VALID_IDX(vq, next_idx);	845	dp = &vq->vq_ring.desc[dp->next];
867	dp = &vq->vq_ring.desc[next_idx];
868	dxp->ndescs--;	846	dxp->ndescs--;
869	}	847	}
870	}	848	}
Lines 877-882 Link Here
877	* newly freed chain. If the virtqueue was completely used, then	855	* newly freed chain. If the virtqueue was completely used, then
878	* head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).	856	* head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
879	*/	857	*/
880	dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);	858	dp->next = vq->vq_desc_head_idx;
881	vq->vq_desc_head_idx = desc_idx;	859	vq->vq_desc_head_idx = desc_idx;
882	}	860	}