Attachment #210808 for bug #236922




static int pmap_allow_2m_x_ept;
SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN | CTLFLAG_NOFETCH,
    &pmap_allow_2m_x_ept, 0,
    "Allow executable superpage mappings in EPT");

void
pmap_allow_2m_x_ept_recalculate(void)
{
	/*
	 * SKL002, SKL012S.  Since the EPT format is only used by
	 * Intel CPUs, the vendor check is merely a formality.
	 */
	if (!(cpu_vendor_id != CPU_VENDOR_INTEL ||
	    (cpu_ia32_arch_caps & IA32_ARCH_CAP_IF_PSCHANGE_MC_NO) != 0 ||
	    (CPUID_TO_FAMILY(cpu_id) == 0x6 &&
	    (CPUID_TO_MODEL(cpu_id) == 0x26 ||	/* Atoms */
	    CPUID_TO_MODEL(cpu_id) == 0x27 ||
	    CPUID_TO_MODEL(cpu_id) == 0x35 ||
	    CPUID_TO_MODEL(cpu_id) == 0x36 ||
	    CPUID_TO_MODEL(cpu_id) == 0x37 ||
	    CPUID_TO_MODEL(cpu_id) == 0x86 ||
	    CPUID_TO_MODEL(cpu_id) == 0x1c ||
	    CPUID_TO_MODEL(cpu_id) == 0x4a ||
	    CPUID_TO_MODEL(cpu_id) == 0x4c ||
	    CPUID_TO_MODEL(cpu_id) == 0x4d ||
	    CPUID_TO_MODEL(cpu_id) == 0x5a ||
	    CPUID_TO_MODEL(cpu_id) == 0x5c ||
	    CPUID_TO_MODEL(cpu_id) == 0x5d ||
	    CPUID_TO_MODEL(cpu_id) == 0x5f ||
	    CPUID_TO_MODEL(cpu_id) == 0x6e ||
	    CPUID_TO_MODEL(cpu_id) == 0x7a ||
	    CPUID_TO_MODEL(cpu_id) == 0x57 ||	/* Knights */
	    CPUID_TO_MODEL(cpu_id) == 0x85))))
		pmap_allow_2m_x_ept = 1;
	TUNABLE_INT_FETCH("hw.allow_2m_x_ept", &pmap_allow_2m_x_ept);
}

static bool
pmap_allow_2m_x_page(pmap_t pmap, bool executable)
{

	return (pmap->pm_type != PT_EPT || !executable ||
	    !pmap_allow_2m_x_ept);
}


	/* IFU */
	pmap_allow_2m_x_ept_recalculate();


static bool
pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde)
{

	if (pmap->pm_type != PT_EPT)
		return (false);
	return ((pde & EPT_PG_EXECUTE) != 0);
}


	if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
	    !pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
	    newpde))) {
--

	if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
	    newpde))) {
		CTR2(KTR_PMAP, "pmap_enter_pde: 2m x blocked for va %#lx"
		    " in pmap %p", va, pmap);
		return (KERN_FAILURE);
	}

		    pmap_allow_2m_x_page(pmap, (prot & VM_PROT_EXECUTE) != 0) &&

Line 427 Link Here

(-)/usr/src/sys/amd64/include/pmap.h (-1 lines)
427	void pmap_allow_2m_x_ept_recalculate(void);

Lines 3481-3483 Link Here

(-)/usr/src/sys/conf/files (-3 lines)
3481	dev/virtio/pci/virtio_pci_if.m optional virtio_pci
3482	dev/virtio/pci/virtio_pci_legacy.c optional virtio_pci
3483	dev/virtio/pci/virtio_pci_modern.c optional virtio_pci

Line 49 Link Here

(-)/usr/src/sys/conf/newvers.sh (-2 / +1 lines)
49	BRANCH="RELEASE-p1"	49	BRANCH="RELEASE"
50	--

Lines 53-56 Link Here

(-)/usr/src/sys/dev/cpuctl/cpuctl.c (-5 lines)
53	#include <vm/vm.h>
54	#include <vm/vm_param.h>
55	#include <vm/pmap.h>
56
Line 546 Link Here
546	pmap_allow_2m_x_ept_recalculate();

Lines 90-91 Link Here

(-)/usr/src/sys/dev/netmap/if_ptnet.c (-3 lines)
90	#ifdef WITH_PTNETMAP
91
Line 2296 Link Here
2296	#endif /* WITH_PTNETMAP */

Line 32 Link Here

(-)/usr/src/sys/dev/virtio/balloon/virtio_balloon.c (-62 / +16 lines)
32	__FBSDID("$FreeBSD$");	32	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/balloon/virtio_balloon.c 326255 2017-11-27 14:52:40Z pfg $");
33	--
Line 83 Link Here
83	{ VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOOM" },
Lines 93-94 Link Here
93	static int vtballoon_negotiate_features(struct vtballoon_softc *);	92	static void vtballoon_negotiate_features(struct vtballoon_softc *);
94	static int vtballoon_setup_features(struct vtballoon_softc *);
95	--
Line 114 Link Here
114	static void vtballoon_setup_sysctl(struct vtballoon_softc *);	112	static void vtballoon_add_sysctl(struct vtballoon_softc *);
115	--
Lines 116-118 Link Here
116	#define vtballoon_modern(_sc) \
117	(((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
118
Line 120 Link Here
120	#define VTBALLOON_FEATURES VIRTIO_BALLOON_F_MUST_TELL_HOST	115	#define VTBALLOON_FEATURES 0
121	--
Line 161 Link Here
161	DRIVER_MODULE(virtio_balloon, vtpcil, vtballoon_driver,	156	DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
162	--
Lines 163-164 Link Here
163	DRIVER_MODULE(virtio_balloon, vtpcim, vtballoon_driver,
164	vtballoon_devclass, 0, 0);
Line 188 Link Here
188	virtio_set_feature_desc(dev, vtballoon_feature_desc);
Line 193 Link Here
193	vtballoon_setup_sysctl(sc);	185	vtballoon_add_sysctl(sc);
194	--
Lines 195-199 Link Here
195	error = vtballoon_setup_features(sc);	187	virtio_set_feature_desc(dev, vtballoon_feature_desc);
196	if (error) {	188	vtballoon_negotiate_features(sc);
197	device_printf(dev, "cannot setup features\n");
198	goto fail;
199	}
200	--
Line 285 Link Here
285	static int	274	static void
286	--
Lines 292-295 Link Here
292	features = VTBALLOON_FEATURES;	281	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
293		282	sc->vtballoon_features = features;
294	sc->vtballoon_features = virtio_negotiate_features(dev, features);
295	return (virtio_finalize_features(dev));
296	--
Lines 299-310 Link Here
299	vtballoon_setup_features(struct vtballoon_softc *sc)
300	{
301	int error;
302
303	error = vtballoon_negotiate_features(sc);
304	if (error)
305	return (error);
306
307	return (0);
308	}
309
310	static int
Lines 468-469 Link Here
468	m = vm_page_alloc(NULL, 0,	443	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ);
469	VM_ALLOC_NORMAL \| VM_ALLOC_NOOBJ \| VM_ALLOC_NODUMP);
470	--
Lines 492-495 Link Here
492	if (vtballoon_modern(sc))	466	return (le32toh(desired));
493	return (desired);
494	else
495	return (le32toh(desired));
496	--
Line 501 Link Here
501	uint32_t npages;
Lines 503-506 Link Here
503	npages = sc->vtballoon_current_npages;
504	if (!vtballoon_modern(sc))
505	npages = htole32(npages);
506
Line 508 Link Here
508	offsetof(struct virtio_balloon_config, actual), npages);	474	offsetof(struct virtio_balloon_config, actual),
509	--	475	htole32(sc->vtballoon_current_npages));
Line 580 Link Here
580	vtballoon_setup_sysctl(struct vtballoon_softc *sc)	547	vtballoon_add_sysctl(struct vtballoon_softc *sc)
581	--




 * $FreeBSD: releng/12.1/sys/dev/virtio/balloon/virtio_balloon.h 326022 2017-11-20 19:36:21Z pfg $
--

#define VIRTIO_BALLOON_F_DEFLATE_ON_OOM	0x4 /* Deflate balloon on OOM */

#define VIRTIO_BALLOON_S_NR       6
#define VIRTIO_BALLOON_S_CACHES   7   /* Disk caches */
#define VIRTIO_BALLOON_S_NR       8
--

/*
 * Memory statistics structure.
 * Driver fills an array of these structures and passes to device.
 *
 * NOTE: fields are laid out in a way that would make compiler add padding
 * between and after fields, so we have to use compiler-specific attributes to
 * pack it, to disable this padding. This also often causes compiler to
 * generate suboptimal code.
 *
 * We maintain this statistics structure format for backwards compatibility,
 * but don't follow this example.
 *
 * If implementing a similar structure, do something like the below instead:
 *     struct virtio_balloon_stat {
 *         __virtio16 tag;
 *         __u8 reserved[6];
 *         __virtio64 val;
 *     };
 *
 * In other words, add explicit reserved fields to align field and
 * structure boundaries at field size, avoiding compiler padding
 * without the packed attribute.
 */

Line 32 Link Here

(-)/usr/src/sys/dev/virtio/block/virtio_blk.c (-126 / +65 lines)
32	__FBSDID("$FreeBSD$");	32	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/block/virtio_blk.c 326255 2017-11-27 14:52:40Z pfg $");
33	--
Lines 79-82 Link Here
79	#define VTBLK_FLAG_DETACH 0x0002	79	#define VTBLK_FLAG_READONLY 0x0002
80	#define VTBLK_FLAG_SUSPEND 0x0004	80	#define VTBLK_FLAG_DETACH 0x0004
81	#define VTBLK_FLAG_BARRIER 0x0008	81	#define VTBLK_FLAG_SUSPEND 0x0008
82	#define VTBLK_FLAG_WCE_CONFIG 0x0010	82	#define VTBLK_FLAG_BARRIER 0x0010
83	--	83	#define VTBLK_FLAG_WC_CONFIG 0x0020
Line 111 Link Here
111	{ VIRTIO_BLK_F_FLUSH, "FlushCmd" },	112	{ VIRTIO_BLK_F_WCE, "WriteCache" },
112	--
Line 114 Link Here
114	{ VIRTIO_BLK_F_MQ, "Multiqueue" },
Lines 136-137 Link Here
136	static int vtblk_negotiate_features(struct vtblk_softc *);	136	static void vtblk_negotiate_features(struct vtblk_softc *);
137	static int vtblk_setup_features(struct vtblk_softc *);	137	static void vtblk_setup_features(struct vtblk_softc *);
138	--
Lines 196-203 Link Here
196	#define vtblk_modern(_sc) (((_sc)->vtblk_features & VIRTIO_F_VERSION_1) != 0)
197	#define vtblk_htog16(_sc, _val) virtio_htog16(vtblk_modern(_sc), _val)
198	#define vtblk_htog32(_sc, _val) virtio_htog32(vtblk_modern(_sc), _val)
199	#define vtblk_htog64(_sc, _val) virtio_htog64(vtblk_modern(_sc), _val)
200	#define vtblk_gtoh16(_sc, _val) virtio_gtoh16(vtblk_modern(_sc), _val)
201	#define vtblk_gtoh32(_sc, _val) virtio_gtoh32(vtblk_modern(_sc), _val)
202	#define vtblk_gtoh64(_sc, _val) virtio_gtoh64(vtblk_modern(_sc), _val)
203
Lines 210-211 Link Here
210	#define VTBLK_COMMON_FEATURES \	202	/* Features desired/implemented by this driver. */
211	(VIRTIO_BLK_F_SIZE_MAX \| \	203	#define VTBLK_FEATURES \
212	--	204	(VIRTIO_BLK_F_BARRIER \| \
		205	VIRTIO_BLK_F_SIZE_MAX \| \
Line 216 Link Here
216	VIRTIO_BLK_F_FLUSH \| \	210	VIRTIO_BLK_F_WCE \| \
217	--
Lines 221-223 Link Here
221	#define VTBLK_MODERN_FEATURES (VTBLK_COMMON_FEATURES)
222	#define VTBLK_LEGACY_FEATURES (VIRTIO_BLK_F_BARRIER \| VTBLK_COMMON_FEATURES)
223
Line 268 Link Here
268	DRIVER_MODULE(virtio_blk, vtpcil, vtblk_driver, vtblk_devclass,	259	DRIVER_MODULE(virtio_blk, virtio_pci, vtblk_driver, vtblk_devclass,
269	--
Lines 270-271 Link Here
270	DRIVER_MODULE(virtio_blk, vtpcim, vtblk_driver, vtblk_devclass,
271	vtblk_modevent, 0);
Lines 315-316 Link Here
315	sc = device_get_softc(dev);
316	sc->vtblk_dev = dev;
Line 319 Link Here
		306	sc = device_get_softc(dev);
		307	sc->vtblk_dev = dev;
Line 326 Link Here
		315	vtblk_setup_features(sc);
Lines 327-332 Link Here
327	error = vtblk_setup_features(sc);
328	if (error) {
329	device_printf(dev, "cannot setup features\n");
330	goto fail;
331	}
332
Line 560 Link Here
		544	/*
		545	* Fail any write if RO. Unfortunately, there does not seem to
		546	* be a better way to report our readonly'ness to GEOM above.
		547	*/
		548	if (sc->vtblk_flags & VTBLK_FLAG_READONLY &&
		549	(bp->bio_cmd == BIO_WRITE \|\| bp->bio_cmd == BIO_FLUSH)) {
		550	vtblk_bio_done(sc, bp, EROFS);
		551	return;
		552	}
		553
Line 574 Link Here
574	static int	568	static void
575	--
Lines 581-582 Link Here
581	features = virtio_bus_is_modern(dev) ? VTBLK_MODERN_FEATURES :	575	features = VTBLK_FEATURES;
582	VTBLK_LEGACY_FEATURES;
583	--
Line 585 Link Here
585	return (virtio_finalize_features(dev));
Line 588 Link Here
588	static int	580	static void
589	--
Line 592 Link Here
592	int error;
Lines 596-598 Link Here
596	error = vtblk_negotiate_features(sc);	587	vtblk_negotiate_features(sc);
597	if (error)
598	return (error);
599	--
Lines 602-603 Link Here
602	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))	591	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))
603	sc->vtblk_flags \|= VTBLK_FLAG_WCE_CONFIG;	592	sc->vtblk_flags \|= VTBLK_FLAG_READONLY;
604	--
Lines 605-607 Link Here
605	sc->vtblk_flags \|= VTBLK_FLAG_BARRIER; /* Legacy. */	594	sc->vtblk_flags \|= VTBLK_FLAG_BARRIER;
606		595	if (virtio_with_feature(dev, VIRTIO_BLK_F_CONFIG_WCE))
607	return (0);	596	sc->vtblk_flags \|= VTBLK_FLAG_WC_CONFIG;
608	--
Line 686 Link Here
686	dp->d_flags = DISKFLAG_UNMAPPED_BIO \| DISKFLAG_DIRECT_COMPLETION;	675	dp->d_flags = DISKFLAG_CANFLUSHCACHE \| DISKFLAG_UNMAPPED_BIO \|
687	--	676	DISKFLAG_DIRECT_COMPLETION;
Lines 692-696 Link Here
692	if (virtio_with_feature(dev, VIRTIO_BLK_F_RO))	682	if ((sc->vtblk_flags & VTBLK_FLAG_READONLY) == 0)
693	dp->d_flags \|= DISKFLAG_WRITE_PROTECT;
694	else {
695	if (virtio_with_feature(dev, VIRTIO_BLK_F_FLUSH))
696	dp->d_flags \|= DISKFLAG_CANFLUSHCACHE;
697	--
Line 698 Link Here
698	}
Line 882 Link Here
882	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	867	req->vbr_hdr.ioprio = 1;
883	--
Lines 886-887 Link Here
886	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);	871	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
887	req->vbr_hdr.sector = 0;
888	--
Lines 890-891 Link Here
890	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_IN);	874	req->vbr_hdr.type = VIRTIO_BLK_T_IN;
891	req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / 512);	875	req->vbr_hdr.sector = bp->bio_offset / 512;
892	--
Lines 894-895 Link Here
894	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);	878	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
895	req->vbr_hdr.sector = vtblk_gtoh64(sc, bp->bio_offset / 512);	879	req->vbr_hdr.sector = bp->bio_offset / 512;
896	--
Line 902 Link Here
902	req->vbr_hdr.type \|= vtblk_gtoh32(sc, VIRTIO_BLK_T_BARRIER);	886	req->vbr_hdr.type \|= VIRTIO_BLK_T_BARRIER;
903	--
Lines 933-934 Link Here
933	req->vbr_hdr.type &= vtblk_gtoh32(sc,	917	req->vbr_hdr.type &= ~VIRTIO_BLK_T_BARRIER;
934	~VIRTIO_BLK_T_BARRIER);
935	--
Line 1038 Link Here
		1021	struct bio_queue queue;
Line 1043 Link Here
		1027	TAILQ_INIT(&queue);
Lines 1045-1047 Link Here
1045	struct bio_queue queue;
1046
1047	TAILQ_INIT(&queue);
Lines 1138-1143 Link Here
1138	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,	1120	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY, geometry, blkcfg);
1139	geometry.cylinders, blkcfg);
1140	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1141	geometry.heads, blkcfg);
1142	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_GEOMETRY,
1143	geometry.sectors, blkcfg);
1144	--
Lines 1145-1153 Link Here
1145	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,	1122	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY, topology, blkcfg);
1146	topology.physical_block_exp, blkcfg);	1123	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, writeback, blkcfg);
1147	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1148	topology.alignment_offset, blkcfg);
1149	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1150	topology.min_io_size, blkcfg);
1151	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_TOPOLOGY,
1152	topology.opt_io_size, blkcfg);
1153	VTBLK_GET_CONFIG(dev, VIRTIO_BLK_F_CONFIG_WCE, wce, blkcfg);
1154	--
Lines 1177-1178 Link Here
1177	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_GET_ID);	1147	req->vbr_hdr.type = VIRTIO_BLK_T_GET_ID;
1178	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	1148	req->vbr_hdr.ioprio = 1;
1179	--
Lines 1309-1311 Link Here
1309	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_OUT);	1279	req->vbr_hdr.type = VIRTIO_BLK_T_OUT;
1310	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	1280	req->vbr_hdr.ioprio = 1;
1311	req->vbr_hdr.sector = vtblk_gtoh64(sc, offset / 512);	1281	req->vbr_hdr.sector = offset / 512;
1312	--
Lines 1331-1332 Link Here
1331	req->vbr_hdr.type = vtblk_gtoh32(sc, VIRTIO_BLK_T_FLUSH);	1301	req->vbr_hdr.type = VIRTIO_BLK_T_FLUSH;
1332	req->vbr_hdr.ioprio = vtblk_gtoh32(sc, 1);	1302	req->vbr_hdr.ioprio = 1;
1333	--
Line 1360 Link Here
1360	offsetof(struct virtio_blk_config, wce), wc);	1330	offsetof(struct virtio_blk_config, writeback), wc);
1361	--
Line 1369 Link Here
1369	if (sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) {	1339	if (sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) {
1370	--
Line 1375 Link Here
1375	wc = blkcfg->wce;	1345	wc = blkcfg->writeback;
1376	--
Line 1377 Link Here
1377	wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_FLUSH);	1347	wc = virtio_with_feature(sc->vtblk_dev, VIRTIO_BLK_F_WCE);
1378	--
Line 1394 Link Here
1394	if ((sc->vtblk_flags & VTBLK_FLAG_WCE_CONFIG) == 0)	1364	if ((sc->vtblk_flags & VTBLK_FLAG_WC_CONFIG) == 0)
1395	--

Line 30 Link Here

(-)/usr/src/sys/dev/virtio/block/virtio_blk.h (-28 / +6 lines)
30	* $FreeBSD$	30	* $FreeBSD: releng/12.1/sys/dev/virtio/block/virtio_blk.h 326022 2017-11-20 19:36:21Z pfg $
31	--
Line 37 Link Here
		37	#define VIRTIO_BLK_F_BARRIER 0x0001 /* Does host support barriers? */
Line 42 Link Here
42	#define VIRTIO_BLK_F_FLUSH 0x0200 /* Flush command supported */	43	#define VIRTIO_BLK_F_SCSI 0x0080 /* Supports scsi command passthru */
43	--	44	#define VIRTIO_BLK_F_WCE 0x0200 /* Writeback mode enabled after reset */
Line 45 Link Here
45	#define VIRTIO_BLK_F_MQ 0x1000 /* Support more than one vq */
Lines 47-53 Link Here
47	/* Legacy feature bits */
48	#define VIRTIO_BLK_F_BARRIER 0x0001 /* Does host support barriers? */
49	#define VIRTIO_BLK_F_SCSI 0x0080 /* Supports scsi command passthru */
50
51	/* Old (deprecated) name for VIRTIO_BLK_F_FLUSH. */
52	#define VIRTIO_BLK_F_WCE VIRTIO_BLK_F_FLUSH
53
Line 75 Link Here
75	/* exponent for physical block per logical block. */
Line 77 Link Here
77	/* alignment offset in logical blocks. */
Lines 79-80 Link Here
79	/* minimum I/O size without performance penalty in logical
80	* blocks. */
Line 82 Link Here
82	/* optimal sustained I/O size in logical blocks. */
Lines 87-88 Link Here
87	uint8_t wce;	76	uint8_t writeback;
88	uint8_t unused;
89	--
Lines 90-91 Link Here
90	/* Number of vqs, only available when VIRTIO_BLK_F_MQ is set */
91	uint16_t num_queues;
Lines 124-128 Link Here
124	/*	110	/* This is the first element of the read scatter-gather list. */
125	* This comes first in the read scatter-gather list.
126	* For legacy virtio, if VIRTIO_F_ANY_LAYOUT is not negotiated,
127	* this is the first element of the read scatter-gather list.
128	*/
129	--

Line 30 Link Here

(-)/usr/src/sys/dev/virtio/console/virtio_console.c (-52 / +17 lines)
30	__FBSDID("$FreeBSD$");	30	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/console/virtio_console.c 328218 2018-01-21 15:42:36Z pfg $");
31	--
Lines 161-162 Link Here
161	static int vtcon_setup_features(struct vtcon_softc *);	161	static void vtcon_setup_features(struct vtcon_softc *);
162	static int vtcon_negotiate_features(struct vtcon_softc *);	162	static void vtcon_negotiate_features(struct vtcon_softc *);
163	--
Lines 230-237 Link Here
230	#define vtcon_modern(_sc) (((_sc)->vtcon_features & VIRTIO_F_VERSION_1) != 0)
231	#define vtcon_htog16(_sc, _val) virtio_htog16(vtcon_modern(_sc), _val)
232	#define vtcon_htog32(_sc, _val) virtio_htog32(vtcon_modern(_sc), _val)
233	#define vtcon_htog64(_sc, _val) virtio_htog64(vtcon_modern(_sc), _val)
234	#define vtcon_gtoh16(_sc, _val) virtio_gtoh16(vtcon_modern(_sc), _val)
235	#define vtcon_gtoh32(_sc, _val) virtio_gtoh32(vtcon_modern(_sc), _val)
236	#define vtcon_gtoh64(_sc, _val) virtio_gtoh64(vtcon_modern(_sc), _val)
237
Line 267 Link Here
267	DRIVER_MODULE(virtio_console, vtpcil, vtcon_driver, vtcon_devclass,	259	DRIVER_MODULE(virtio_console, virtio_pci, vtcon_driver, vtcon_devclass,
268	--
Lines 269-270 Link Here
269	DRIVER_MODULE(virtio_console, vtpcim, vtcon_driver, vtcon_devclass,
270	vtcon_modevent, 0);
Line 336 Link Here
336	virtio_set_feature_desc(dev, vtcon_feature_desc);
Lines 341-345 Link Here
341	error = vtcon_setup_features(sc);	330	virtio_set_feature_desc(dev, vtcon_feature_desc);
342	if (error) {	331	vtcon_setup_features(sc);
343	device_printf(dev, "cannot setup features\n");
344	goto fail;
345	}
346	--
Line 437 Link Here
437	static int	423	static void
438	--
Line 447 Link Here
447	return (virtio_finalize_features(dev));
Line 450 Link Here
450	static int	435	static void
451	--
Line 454 Link Here
454	int error;
Lines 458-460 Link Here
458	error = vtcon_negotiate_features(sc);	442	vtcon_negotiate_features(sc);
459	if (error)
460	return (error);
461	--
Lines 466-467 Link Here
466
467	return (0);
Lines 870-871 Link Here
870	uint32_t id;	850	int id;
871	uint16_t event;
872	--
Lines 874-875 Link Here
874	id = vtcon_htog32(sc, control->id);	853	id = control->id;
875	event = vtcon_htog16(sc, control->event);
876	--
Lines 877-879 Link Here
877	if (id >= sc->vtcon_max_ports) {	855	if (id < 0 \|\| id >= sc->vtcon_max_ports) {
878	device_printf(dev, "%s: event %d invalid port ID %d\n",	856	device_printf(dev, "%s: invalid port ID %d\n", __func__, id);
879	__func__, event, id);
880	--
Line 883 Link Here
883	switch (event) {	860	switch (control->event) {
884	--
Lines 1011-1013 Link Here
1011	control.id = vtcon_gtoh32(sc, portid);	988	control.id = portid;
1012	control.event = vtcon_gtoh16(sc, event);	989	control.event = event;
1013	control.value = vtcon_gtoh16(sc, value);	990	control.value = value;
1014	--

Line 31 Link Here

(-)/usr/src/sys/dev/virtio/console/virtio_console.h (-2 / +1 lines)
31	* $FreeBSD$	31	* $FreeBSD: releng/12.1/sys/dev/virtio/console/virtio_console.h 273515 2014-10-23 04:47:32Z bryanv $
32	--

Line 51 Link Here

(-)/usr/src/sys/dev/virtio/mmio/virtio_mmio.c (-6 / +1 lines)
51	__FBSDID("$FreeBSD$");	51	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/mmio/virtio_mmio.c 335211 2018-06-15 16:19:10Z br $");
52	--
Lines 428-431 Link Here
428	break;
429	case VIRTIO_IVAR_SUBVENDOR:
430	case VIRTIO_IVAR_MODERN:
431	*result = 0;

Line 30 Link Here

(-)/usr/src/sys/dev/virtio/mmio/virtio_mmio.h (-2 / +1 lines)
30	* $FreeBSD$	30	* $FreeBSD: releng/12.1/sys/dev/virtio/mmio/virtio_mmio.h 284544 2015-06-18 10:33:04Z br $
31	--

Line 30 Link Here

(-)/usr/src/sys/dev/virtio/mmio/virtio_mmio_if.m (-2 / +1 lines)
30	# $FreeBSD$	30	# $FreeBSD: releng/12.1/sys/dev/virtio/mmio/virtio_mmio_if.m 285091 2015-07-03 14:13:16Z br $
31	--

Line 4 Link Here

(-)/usr/src/sys/dev/virtio/pci/virtio_pci.c (-668 / +840 lines)
4	* Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>	4	* Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5	--
Line 29 Link Here
		29	/* Driver for the VirtIO PCI interface. */
		30
Line 30 Link Here
30	__FBSDID("$FreeBSD$");	32	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.c 328218 2018-01-21 15:42:36Z pfg $");
31	--
Lines 36-37 Link Here
36	#include <sys/sbuf.h>
37	#include <sys/sysctl.h>
Line 52 Link Here
52	#include <dev/virtio/pci/virtio_pci_var.h>
Line 54 Link Here
54	#include "virtio_pci_if.h"	53	#include "virtio_bus_if.h"
55	--
Line 57 Link Here
57	static void vtpci_describe_features(struct vtpci_common , const char ,	56	struct vtpci_interrupt {
58	--	57	struct resource *vti_irq;
		58	int vti_rid;
		59	void *vti_handler;
		60	};
		61
		62	struct vtpci_virtqueue {
		63	struct virtqueue *vtv_vq;
		64	int vtv_no_intr;
		65	};
		66
		67	struct vtpci_softc {
		68	device_t vtpci_dev;
		69	struct resource *vtpci_res;
		70	struct resource *vtpci_msix_res;
		71	uint64_t vtpci_features;
		72	uint32_t vtpci_flags;
		73	#define VTPCI_FLAG_NO_MSI 0x0001
		74	#define VTPCI_FLAG_NO_MSIX 0x0002
		75	#define VTPCI_FLAG_LEGACY 0x1000
		76	#define VTPCI_FLAG_MSI 0x2000
		77	#define VTPCI_FLAG_MSIX 0x4000
		78	#define VTPCI_FLAG_SHARED_MSIX 0x8000
		79	#define VTPCI_FLAG_ITYPE_MASK 0xF000
		80
		81	/* This "bus" will only ever have one child. */
		82	device_t vtpci_child_dev;
		83	struct virtio_feature_desc *vtpci_child_feat_desc;
		84
		85	int vtpci_nvqs;
		86	struct vtpci_virtqueue *vtpci_vqs;
		87
		88	/*
		89	* Ideally, each virtqueue that the driver provides a callback for will
		90	* receive its own MSIX vector. If there are not sufficient vectors
		91	* available, then attempt to have all the VQs share one vector. For
		92	* MSIX, the configuration changed notifications must be on their own
		93	* vector.
		94	*
		95	* If MSIX is not available, we will attempt to have the whole device
		96	* share one MSI vector, and then, finally, one legacy interrupt.
		97	*/
		98	struct vtpci_interrupt vtpci_device_interrupt;
		99	struct vtpci_interrupt *vtpci_msix_vq_interrupts;
		100	int vtpci_nmsix_resources;
		101	};
		102
		103	static int vtpci_probe(device_t);
		104	static int vtpci_attach(device_t);
		105	static int vtpci_detach(device_t);
		106	static int vtpci_suspend(device_t);
		107	static int vtpci_resume(device_t);
		108	static int vtpci_shutdown(device_t);
		109	static void vtpci_driver_added(device_t, driver_t *);
		110	static void vtpci_child_detached(device_t, device_t);
		111	static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
		112	static int vtpci_write_ivar(device_t, device_t, int, uintptr_t);
		113
		114	static uint64_t vtpci_negotiate_features(device_t, uint64_t);
		115	static int vtpci_with_feature(device_t, uint64_t);
		116	static int vtpci_alloc_virtqueues(device_t, int, int,
		117	struct vq_alloc_info *);
		118	static int vtpci_setup_intr(device_t, enum intr_type);
		119	static void vtpci_stop(device_t);
		120	static int vtpci_reinit(device_t, uint64_t);
		121	static void vtpci_reinit_complete(device_t);
122	static void vtpci_notify_virtqueue(device_t, uint16_t);
123	static uint8_t vtpci_get_status(device_t);
124	static void vtpci_set_status(device_t, uint8_t);
125	static void vtpci_read_dev_config(device_t, bus_size_t, void *, int);
126	static void vtpci_write_dev_config(device_t, bus_size_t, void *, int);
127
128	static void vtpci_describe_features(struct vtpci_softc , const char ,
Lines 59-65 Link Here
59	static int vtpci_alloc_msix(struct vtpci_common *, int);	130	static void vtpci_probe_and_attach_child(struct vtpci_softc *);
60	static int vtpci_alloc_msi(struct vtpci_common *);	131
61	static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *);	132	static int vtpci_alloc_msix(struct vtpci_softc *, int);
62	static int vtpci_alloc_intr_msix_shared(struct vtpci_common *);	133	static int vtpci_alloc_msi(struct vtpci_softc *);
63	static int vtpci_alloc_intr_msi(struct vtpci_common *);	134	static int vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
64	static int vtpci_alloc_intr_intx(struct vtpci_common *);	135	static int vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
65	static int vtpci_alloc_interrupt(struct vtpci_common *, int, int,	136	static int vtpci_alloc_intr_msi(struct vtpci_softc *);
66	--	137	static int vtpci_alloc_intr_legacy(struct vtpci_softc *);
		138	static int vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
Lines 67-68 Link Here
67	static void vtpci_free_interrupt(struct vtpci_common *,	140	static int vtpci_alloc_intr_resources(struct vtpci_softc *);
68	struct vtpci_interrupt *);
69	--
Lines 70-74 Link Here
70	static void vtpci_free_interrupts(struct vtpci_common *);	142	static int vtpci_setup_legacy_interrupt(struct vtpci_softc *,
71	static void vtpci_free_virtqueues(struct vtpci_common *);
72	static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *);
73	static int vtpci_alloc_intr_resources(struct vtpci_common *);
74	static int vtpci_setup_intx_interrupt(struct vtpci_common *,
75	--
Line 76 Link Here
76	static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *,	144	static int vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
77	--
Lines 78-79 Link Here
78	static int vtpci_set_host_msix_vectors(struct vtpci_common *);	146	static int vtpci_setup_msix_interrupts(struct vtpci_softc *,
79	static int vtpci_setup_msix_interrupts(struct vtpci_common *,
80	--
Lines 81-83 Link Here
81	static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type);	148	static int vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
82	static int vtpci_reinit_virtqueue(struct vtpci_common *, int);	149
83	static void vtpci_intx_intr(void *);	150	static int vtpci_register_msix_vector(struct vtpci_softc *, int,
84	--	151	struct vtpci_interrupt *);
		152	static int vtpci_set_host_msix_vectors(struct vtpci_softc *);
		153	static int vtpci_reinit_virtqueue(struct vtpci_softc *, int);
		154
		155	static void vtpci_free_interrupt(struct vtpci_softc *,
		156	struct vtpci_interrupt *);
		157	static void vtpci_free_interrupts(struct vtpci_softc *);
		158	static void vtpci_free_virtqueues(struct vtpci_softc *);
		159	static void vtpci_release_child_resources(struct vtpci_softc *);
		160	static void vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
		161	static void vtpci_reset(struct vtpci_softc *);
		162
		163	static void vtpci_select_virtqueue(struct vtpci_softc *, int);
		164
		165	static void vtpci_legacy_intr(void *);
Line 90 Link Here
90	static void vtpci_setup_sysctl(struct vtpci_common *);	172	#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
91	--
Line 92 Link Here
92	#define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt	174	#define VIRTIO_PCI_CONFIG(_sc) \
93	--	175	VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
Lines 95-97 Link Here
95	* This module contains two drivers:	178	* I/O port read/write wrappers.
96	* - virtio_pci_legacy (vtpcil) for pre-V1 support
97	* - virtio_pci_modern (vtpcim) for V1 support
98	--
Lines 99-101 Link Here
99	MODULE_VERSION(virtio_pci, 1);	180	#define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o))
100	MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);	181	#define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o))
101	MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);	182	#define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o))
102	--	183	#define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v))
		184	#define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v))
		185	#define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v))
Line 103 Link Here
103	int vtpci_disable_msix = 0;	187	/* Tunables. */
104	--	188	static int vtpci_disable_msix = 0;
Lines 106-110 Link Here
106	static uint8_t	191	static device_method_t vtpci_methods[] = {
107	vtpci_read_isr(struct vtpci_common *cn)	192	/* Device interface. */
108	{	193	DEVMETHOD(device_probe, vtpci_probe),
109	return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev));	194	DEVMETHOD(device_attach, vtpci_attach),
110	}	195	DEVMETHOD(device_detach, vtpci_detach),
111	--	196	DEVMETHOD(device_suspend, vtpci_suspend),
		197	DEVMETHOD(device_resume, vtpci_resume),
		198	DEVMETHOD(device_shutdown, vtpci_shutdown),
Lines 112-116 Link Here
112	static uint16_t	200	/* Bus interface. */
113	vtpci_get_vq_size(struct vtpci_common *cn, int idx)	201	DEVMETHOD(bus_driver_added, vtpci_driver_added),
114	{	202	DEVMETHOD(bus_child_detached, vtpci_child_detached),
115	return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx));	203	DEVMETHOD(bus_read_ivar, vtpci_read_ivar),
116	}	204	DEVMETHOD(bus_write_ivar, vtpci_write_ivar),
117	--
Lines 118-122 Link Here
118	static bus_size_t	206	/* VirtIO bus interface. */
119	vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx)	207	DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features),
120	{	208	DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature),
121	return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx));	209	DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues),
122	}	210	DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr),
123	--	211	DEVMETHOD(virtio_bus_stop, vtpci_stop),
		212	DEVMETHOD(virtio_bus_reinit, vtpci_reinit),
		213	DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete),
		214	DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue),
		215	DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config),
		216	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
Lines 124-128 Link Here
124	static void	218	DEVMETHOD_END
125	vtpci_set_vq(struct vtpci_common cn, struct virtqueue vq)	219	};
126	{
127	VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq);
128	}
129	--
Lines 130-134 Link Here
130	static void	221	static driver_t vtpci_driver = {
131	vtpci_disable_vq(struct vtpci_common *cn, int idx)	222	"virtio_pci",
132	{	223	vtpci_methods,
133	VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx);	224	sizeof(struct vtpci_softc)
134	}	225	};
135	--
Line 136 Link Here
		227	devclass_t vtpci_devclass;
		228
		229	DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
		230	MODULE_VERSION(virtio_pci, 1);
		231	MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
		232	MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
		233
Line 137 Link Here
137	vtpci_register_cfg_msix(struct vtpci_common cn, struct vtpci_interrupt intr)	235	vtpci_probe(device_t dev)
138	--
Line 139 Link Here
139	return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr));	237	char desc[36];
140	--	238	const char *name;
		239
		240	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
		241	return (ENXIO);
		242
		243	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN \|\|
		244	pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
		245	return (ENXIO);
		246
		247	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
		248	return (ENXIO);
		249
		250	name = virtio_device_name(pci_get_subdevice(dev));
		251	if (name == NULL)
		252	name = "Unknown";
		253
		254	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
		255	device_set_desc_copy(dev, desc);
		256
		257	return (BUS_PROBE_DEFAULT);
Lines 143-144 Link Here
143	vtpci_register_vq_msix(struct vtpci_common *cn, int idx,	261	vtpci_attach(device_t dev)
144	struct vtpci_interrupt *intr)
145	--
Lines 146-147 Link Here
146	return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr));	263	struct vtpci_softc *sc;
147	}	264	device_t child;
148	--	265	int rid;
Lines 149-151 Link Here
149	void	267	sc = device_get_softc(dev);
150	vtpci_init(struct vtpci_common *cn, device_t dev, bool modern)	268	sc->vtpci_dev = dev;
151	{
152	--
Lines 153-154 Link Here
153	cn->vtpci_dev = dev;
154
Lines 157-158 Link Here
157	if (modern)	272	rid = PCIR_BAR(0);
158	cn->vtpci_flags \|= VTPCI_FLAG_MODERN;	273	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
159	--	274	RF_ACTIVE);
		275	if (sc->vtpci_res == NULL) {
		276	device_printf(dev, "cannot map I/O space\n");
		277	return (ENXIO);
		278	}
		279
Lines 160-162 Link Here
160	cn->vtpci_flags \|= VTPCI_FLAG_NO_MSI;	281	sc->vtpci_flags \|= VTPCI_FLAG_NO_MSI;
161	if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)
162	cn->vtpci_flags \|= VTPCI_FLAG_NO_MSIX;
163	--
Lines 164-165 Link Here
164	vtpci_setup_sysctl(cn);	283	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
165	}	284	rid = PCIR_BAR(1);
166	--	285	sc->vtpci_msix_res = bus_alloc_resource_any(dev,
		286	SYS_RES_MEMORY, &rid, RF_ACTIVE);
		287	}
Lines 167-170 Link Here
167	int	289	if (sc->vtpci_msix_res == NULL)
168	vtpci_add_child(struct vtpci_common *cn)	290	sc->vtpci_flags \|= VTPCI_FLAG_NO_MSIX;
169	{
170	device_t dev, child;
171	--
Line 172 Link Here
172	dev = cn->vtpci_dev;	292	vtpci_reset(sc);
173	--
Lines 174-175 Link Here
174	child = device_add_child(dev, NULL, -1);	294	/* Tell the host we've noticed this device. */
175	if (child == NULL) {	295	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
176	--	296
		297	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
Line 177 Link Here
		299	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
		300	vtpci_detach(dev);
Line 180 Link Here
180	cn->vtpci_child_dev = child;	304	sc->vtpci_child_dev = child;
181	--	305	vtpci_probe_and_attach_child(sc);
Lines 185-186 Link Here
185	int	310	static int
186	vtpci_delete_child(struct vtpci_common *cn)	311	vtpci_detach(device_t dev)
187	--
Line 188 Link Here
188	device_t dev, child;	313	struct vtpci_softc *sc;
189	--	314	device_t child;
Line 191 Link Here
191	dev = cn->vtpci_dev;	317	sc = device_get_softc(dev);
192	--
Lines 193-194 Link Here
193	child = cn->vtpci_child_dev;	319	if ((child = sc->vtpci_child_dev) != NULL) {
194	if (child != NULL) {
195	--
Line 198 Link Here
198	cn->vtpci_child_dev = NULL;	323	sc->vtpci_child_dev = NULL;
199	--
Line 201 Link Here
		326	vtpci_reset(sc);
		327
		328	if (sc->vtpci_msix_res != NULL) {
		329	bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
		330	sc->vtpci_msix_res);
		331	sc->vtpci_msix_res = NULL;
		332	}
		333
		334	if (sc->vtpci_res != NULL) {
		335	bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
		336	sc->vtpci_res);
		337	sc->vtpci_res = NULL;
		338	}
		339
Lines 204-205 Link Here
204	void	343	static int
205	vtpci_child_detached(struct vtpci_common *cn)	344	vtpci_suspend(device_t dev)
206	--
Lines 208-212 Link Here
208	vtpci_release_child_resources(cn);	347	return (bus_generic_suspend(dev));
209
210	cn->vtpci_child_feat_desc = NULL;
211	cn->vtpci_host_features = 0;
212	cn->vtpci_features = 0;
213	--
Lines 215-216 Link Here
215	int	350	static int
216	vtpci_reinit(struct vtpci_common *cn)	351	vtpci_resume(device_t dev)
217	--
Line 218 Link Here
218	int idx, error;
Lines 220-224 Link Here
220	for (idx = 0; idx < cn->vtpci_nvqs; idx++) {	354	return (bus_generic_resume(dev));
221	error = vtpci_reinit_virtqueue(cn, idx);	355	}
222	if (error)
223	return (error);
224	}
225	--
Lines 226-230 Link Here
226	if (vtpci_is_msix_enabled(cn)) {	357	static int
227	error = vtpci_set_host_msix_vectors(cn);	358	vtpci_shutdown(device_t dev)
228	if (error)	359	{
229	return (error);
230	}
231	--
Line 232 Link Here
		361	(void) bus_generic_shutdown(dev);
		362	/* Forcibly stop the host device. */
		363	vtpci_stop(dev);
		364
Lines 236-237 Link Here
236	vtpci_describe_features(struct vtpci_common cn, const char msg,	369	vtpci_driver_added(device_t dev, driver_t *driver)
237	uint64_t features)
238	--
Line 239 Link Here
239	device_t dev, child;	371	struct vtpci_softc *sc;
240	--
Lines 241-242 Link Here
241	dev = cn->vtpci_dev;	373	sc = device_get_softc(dev);
242	child = cn->vtpci_child_dev;
243	--
Lines 244-247 Link Here
244	if (device_is_attached(child) \|\| bootverbose == 0)	375	vtpci_probe_and_attach_child(sc);
245	return;
246
247	virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc);
248	--
Lines 250-252 Link Here
250	uint64_t	378	static void
251	vtpci_negotiate_features(struct vtpci_common *cn,	379	vtpci_child_detached(device_t dev, device_t child)
252	uint64_t child_features, uint64_t host_features)
253	--
Line 254 Link Here
254	uint64_t features;	381	struct vtpci_softc *sc;
255	--
Lines 256-257 Link Here
256	cn->vtpci_host_features = host_features;	383	sc = device_get_softc(dev);
257	vtpci_describe_features(cn, "host", host_features);
258	--
Lines 259-269 Link Here
259	/*	385	vtpci_reset(sc);
260	* Limit negotiated features to what the driver, virtqueue, and	386	vtpci_release_child_resources(sc);
261	* host all support.
262	*/
263	features = host_features & child_features;
264	features = virtio_filter_transport_features(features);
265
266	cn->vtpci_features = features;
267	vtpci_describe_features(cn, "negotiated", features);
268
269	return (features);
270	--
Lines 272-273 Link Here
272	int	389	static int
273	vtpci_with_feature(struct vtpci_common *cn, uint64_t feature)	390	vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
274	--
Lines 275-276 Link Here
275	return ((cn->vtpci_features & feature) != 0);	392	struct vtpci_softc *sc;
276	}
277	--
Lines 278-282 Link Here
278	int	394	sc = device_get_softc(dev);
279	vtpci_read_ivar(struct vtpci_common cn, int index, uintptr_t result)
280	{
281	device_t dev;
282	int error;
283	--
Lines 284-285 Link Here
284	dev = cn->vtpci_dev;	396	if (sc->vtpci_child_dev != child)
285	error = 0;	397	return (ENOENT);
286	--
Line 288 Link Here
		400	case VIRTIO_IVAR_DEVTYPE:
Lines 300-302 Link Here
300	case VIRTIO_IVAR_MODERN:
301	*result = vtpci_is_modern(cn);
302	break;
Line 304 Link Here
304	error = ENOENT;	414	return (ENOENT);
305	--
Line 307 Link Here
307	return (error);	417	return (0);
308	--
Lines 310-311 Link Here
310	int	420	static int
311	vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value)	421	vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
312	--
Line 313 Link Here
313	int error;	423	struct vtpci_softc *sc;
314	--
Line 315 Link Here
315	error = 0;	425	sc = device_get_softc(dev);
316	--
Line 317 Link Here
		427	if (sc->vtpci_child_dev != child)
		428	return (ENOENT);
		429
Line 319 Link Here
319	cn->vtpci_child_feat_desc = (void *) value;	432	sc->vtpci_child_feat_desc = (void *) value;
320	--
Line 322 Link Here
322	error = ENOENT;	435	return (ENOENT);
323	--
Line 325 Link Here
325	return (error);	438	return (0);
326	--
Lines 328-330 Link Here
328	int	441	static uint64_t
329	vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs,	442	vtpci_negotiate_features(device_t dev, uint64_t child_features)
330	struct vq_alloc_info *vq_info)
331	--
Lines 332-333 Link Here
332	device_t dev;	444	struct vtpci_softc *sc;
333	int idx, align, error;	445	uint64_t host_features, features;
334	--
Line 335 Link Here
335	dev = cn->vtpci_dev;	447	sc = device_get_softc(dev);
336	--
Line 337 Link Here
		449	host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
		450	vtpci_describe_features(sc, "host", host_features);
		451
Lines 338-340 Link Here
338	* This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO,	453	* Limit negotiated features to what the driver, virtqueue, and
339	* the tables do not have to be allocated contiguously, but we do so	454	* host all support.
340	* anyways.
341	--
Line 342 Link Here
342	align = 4096;	456	features = host_features & child_features;
343	--	457	features = virtqueue_filter_features(features);
		458	sc->vtpci_features = features;
Line 344 Link Here
344	if (cn->vtpci_nvqs != 0)	460	vtpci_describe_features(sc, "negotiated", features);
345	--	461	vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
		462
		463	return (features);
		464	}
		465
		466	static int
		467	vtpci_with_feature(device_t dev, uint64_t feature)
		468	{
		469	struct vtpci_softc *sc;
		470
		471	sc = device_get_softc(dev);
		472
		473	return ((sc->vtpci_features & feature) != 0);
		474	}
		475
		476	static int
		477	vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
		478	struct vq_alloc_info *vq_info)
		479	{
		480	struct vtpci_softc *sc;
		481	struct virtqueue *vq;
		482	struct vtpci_virtqueue *vqx;
		483	struct vq_alloc_info *info;
		484	int idx, error;
		485	uint16_t size;
		486
		487	sc = device_get_softc(dev);
		488
		489	if (sc->vtpci_nvqs != 0)
Line 349 Link Here
349	cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),	494	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
350	--
Line 351 Link Here
351	if (cn->vtpci_vqs == NULL)	496	if (sc->vtpci_vqs == NULL)
352	--
Lines 355-361 Link Here
355	struct vtpci_virtqueue *vqx;	500	vqx = &sc->vtpci_vqs[idx];
356	struct vq_alloc_info *info;
357	struct virtqueue *vq;
358	bus_size_t notify_offset;
359	uint16_t size;
360
361	vqx = &cn->vtpci_vqs[idx];
362	--
Lines 364-365 Link Here
364	size = vtpci_get_vq_size(cn, idx);	503	vtpci_select_virtqueue(sc, idx);
365	notify_offset = vtpci_get_vq_notify_off(cn, idx);	504	size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
366	--
Line 367 Link Here
367	error = virtqueue_alloc(dev, idx, size, notify_offset, align,	506	error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
368	--
Line 375 Link Here
375	vtpci_set_vq(cn, vq);	514	vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
376	--	515	virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
Line 380 Link Here
380	cn->vtpci_nvqs++;	520	sc->vtpci_nvqs++;
381	--
Line 384 Link Here
384	vtpci_free_virtqueues(cn);	524	vtpci_free_virtqueues(sc);
385	--
Line 390 Link Here
390	vtpci_alloc_msix(struct vtpci_common *cn, int nvectors)	530	vtpci_setup_intr(device_t dev, enum intr_type type)
391	--
Line 392 Link Here
		532	struct vtpci_softc *sc;
		533	int attempt, error;
		534
		535	sc = device_get_softc(dev);
		536
		537	for (attempt = 0; attempt < 5; attempt++) {
		538	/*
		539	* Start with the most desirable interrupt configuration and
		540	* fallback towards less desirable ones.
		541	*/
		542	switch (attempt) {
		543	case 0:
		544	error = vtpci_alloc_intr_msix_pervq(sc);
		545	break;
		546	case 1:
		547	error = vtpci_alloc_intr_msix_shared(sc);
		548	break;
		549	case 2:
		550	error = vtpci_alloc_intr_msi(sc);
		551	break;
		552	case 3:
		553	error = vtpci_alloc_intr_legacy(sc);
		554	break;
		555	default:
		556	device_printf(dev,
		557	"exhausted all interrupt allocation attempts\n");
		558	return (ENXIO);
		559	}
		560
		561	if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
		562	break;
		563
		564	vtpci_cleanup_setup_intr_attempt(sc);
		565	}
		566
		567	if (bootverbose) {
		568	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
		569	device_printf(dev, "using legacy interrupt\n");
		570	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
		571	device_printf(dev, "using MSI interrupt\n");
		572	else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
		573	device_printf(dev, "using shared MSIX interrupts\n");
		574	else
		575	device_printf(dev, "using per VQ MSIX interrupts\n");
		576	}
		577
		578	return (0);
		579	}
		580
		581	static void
		582	vtpci_stop(device_t dev)
		583	{
		584
		585	vtpci_reset(device_get_softc(dev));
		586	}
		587
		588	static int
		589	vtpci_reinit(device_t dev, uint64_t features)
		590	{
		591	struct vtpci_softc *sc;
		592	int idx, error;
		593
		594	sc = device_get_softc(dev);
		595
596	/*
597	* Redrive the device initialization. This is a bit of an abuse of
598	* the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
599	* play nice.
600	*
601	* We do not allow the host device to change from what was originally
602	* negotiated beyond what the guest driver changed. MSIX state should
603	* not change, number of virtqueues and their size remain the same, etc.
604	* This will need to be rethought when we want to support migration.
605	*/
606
607	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
608	vtpci_stop(dev);
609
610	/*
611	* Quickly drive the status through ACK and DRIVER. The device
612	* does not become usable again until vtpci_reinit_complete().
613	*/
614	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
615	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
616
617	vtpci_negotiate_features(dev, features);
618
619	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
620	error = vtpci_reinit_virtqueue(sc, idx);
621	if (error)
622	return (error);
623	}
624
625	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
626	error = vtpci_set_host_msix_vectors(sc);
627	if (error)
628	return (error);
629	}
630
631	return (0);
632	}
633
634	static void
635	vtpci_reinit_complete(device_t dev)
636	{
637
638	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
639	}
640
641	static void
642	vtpci_notify_virtqueue(device_t dev, uint16_t queue)
643	{
644	struct vtpci_softc *sc;
645
646	sc = device_get_softc(dev);
647
648	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
649	}
650
651	static uint8_t
652	vtpci_get_status(device_t dev)
653	{
654	struct vtpci_softc *sc;
655
656	sc = device_get_softc(dev);
657
658	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
659	}
660
661	static void
662	vtpci_set_status(device_t dev, uint8_t status)
663	{
664	struct vtpci_softc *sc;
665
666	sc = device_get_softc(dev);
667
668	if (status != VIRTIO_CONFIG_STATUS_RESET)
669	status \|= vtpci_get_status(dev);
670
671	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
672	}
673
674	static void
675	vtpci_read_dev_config(device_t dev, bus_size_t offset,
676	void *dst, int length)
677	{
678	struct vtpci_softc *sc;
679	bus_size_t off;
680	uint8_t *d;
681	int size;
682
683	sc = device_get_softc(dev);
684	off = VIRTIO_PCI_CONFIG(sc) + offset;
685
686	for (d = dst; length > 0; d += size, off += size, length -= size) {
687	if (length >= 4) {
688	size = 4;
689	(uint32_t )d = vtpci_read_config_4(sc, off);
690	} else if (length >= 2) {
691	size = 2;
692	(uint16_t )d = vtpci_read_config_2(sc, off);
693	} else {
694	size = 1;
695	*d = vtpci_read_config_1(sc, off);
696	}
697	}
698	}
699
700	static void
701	vtpci_write_dev_config(device_t dev, bus_size_t offset,
702	void *src, int length)
703	{
704	struct vtpci_softc *sc;
705	bus_size_t off;
706	uint8_t *s;
707	int size;
708
709	sc = device_get_softc(dev);
710	off = VIRTIO_PCI_CONFIG(sc) + offset;
711
712	for (s = src; length > 0; s += size, off += size, length -= size) {
713	if (length >= 4) {
714	size = 4;
715	vtpci_write_config_4(sc, off, (uint32_t )s);
716	} else if (length >= 2) {
717	size = 2;
718	vtpci_write_config_2(sc, off, (uint16_t )s);
719	} else {
720	size = 1;
721	vtpci_write_config_1(sc, off, *s);
722	}
723	}
724	}
725
726	static void
727	vtpci_describe_features(struct vtpci_softc sc, const char msg,
728	uint64_t features)
729	{
730	device_t dev, child;
731
732	dev = sc->vtpci_dev;
733	child = sc->vtpci_child_dev;
734
735	if (device_is_attached(child) \|\| bootverbose == 0)
736	return;
737
738	virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
739	}
740
741	static void
742	vtpci_probe_and_attach_child(struct vtpci_softc *sc)
743	{
744	device_t dev, child;
745
746	dev = sc->vtpci_dev;
747	child = sc->vtpci_child_dev;
748
749	if (child == NULL)
750	return;
751
752	if (device_get_state(child) != DS_NOTPRESENT)
753	return;
754
755	if (device_probe(child) != 0)
756	return;
757
758	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
759	if (device_attach(child) != 0) {
760	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
761	vtpci_reset(sc);
762	vtpci_release_child_resources(sc);
763	/* Reset status for future attempt. */
764	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
765	} else {
766	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
767	VIRTIO_ATTACH_COMPLETED(child);
768	}
769	}
770
771	static int
772	vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
773	{
Line 395 Link Here
395	dev = cn->vtpci_dev;	777	dev = sc->vtpci_dev;
396	--
Line 406 Link Here
406	cn->vtpci_nmsix_resources = required;	788	sc->vtpci_nmsix_resources = required;
407	--
Line 416 Link Here
416	vtpci_alloc_msi(struct vtpci_common *cn)	798	vtpci_alloc_msi(struct vtpci_softc *sc)
417	--
Line 421 Link Here
421	dev = cn->vtpci_dev;	803	dev = sc->vtpci_dev;
422	--
Line 438 Link Here
438	vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn)	820	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
439	--
Line 442 Link Here
442	if (vtpci_disable_msix != 0 \|\| cn->vtpci_flags & VTPCI_FLAG_NO_MSIX)	824	if (vtpci_disable_msix != 0 \|\|
443	--	825	sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
Lines 445-446 Link Here
445	for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) {	828	for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
446	if (cn->vtpci_vqs[i].vtv_no_intr == 0)	829	if (sc->vtpci_vqs[i].vtv_no_intr == 0)
447	--
Line 450 Link Here
450	error = vtpci_alloc_msix(cn, nvectors);	833	error = vtpci_alloc_msix(sc, nvectors);
451	--
Line 454 Link Here
454	cn->vtpci_flags \|= VTPCI_FLAG_MSIX;	837	sc->vtpci_flags \|= VTPCI_FLAG_MSIX;
455	--
Line 460 Link Here
460	vtpci_alloc_intr_msix_shared(struct vtpci_common *cn)	843	vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
461	--
Line 464 Link Here
464	if (vtpci_disable_msix != 0 \|\| cn->vtpci_flags & VTPCI_FLAG_NO_MSIX)	847	if (vtpci_disable_msix != 0 \|\|
465	--	848	sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
Line 467 Link Here
467	error = vtpci_alloc_msix(cn, 1);	851	error = vtpci_alloc_msix(sc, 1);
468	--
Line 471 Link Here
471	cn->vtpci_flags \|= VTPCI_FLAG_MSIX \| VTPCI_FLAG_SHARED_MSIX;	855	sc->vtpci_flags \|= VTPCI_FLAG_MSIX \| VTPCI_FLAG_SHARED_MSIX;
472	--
Line 477 Link Here
477	vtpci_alloc_intr_msi(struct vtpci_common *cn)	861	vtpci_alloc_intr_msi(struct vtpci_softc *sc)
478	--
Line 482 Link Here
482	if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI)	866	if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
483	--
Line 485 Link Here
485	error = vtpci_alloc_msi(cn);	869	error = vtpci_alloc_msi(sc);
486	--
Line 489 Link Here
489	cn->vtpci_flags \|= VTPCI_FLAG_MSI;	873	sc->vtpci_flags \|= VTPCI_FLAG_MSI;
490	--
Line 495 Link Here
495	vtpci_alloc_intr_intx(struct vtpci_common *cn)	879	vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
496	--
Line 498 Link Here
498	cn->vtpci_flags \|= VTPCI_FLAG_INTX;	882	sc->vtpci_flags \|= VTPCI_FLAG_LEGACY;
499	--
Line 504 Link Here
504	vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags,	888	vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
505	--
Line 509 Link Here
509	irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags);	893	irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
510	--
Lines 519-607 Link Here
519	static void
520	vtpci_free_interrupt(struct vtpci_common cn, struct vtpci_interrupt intr)
521	{
522	device_t dev;
523
524	dev = cn->vtpci_dev;
525
526	if (intr->vti_handler != NULL) {
527	bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
528	intr->vti_handler = NULL;
529	}
530
531	if (intr->vti_irq != NULL) {
532	bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
533	intr->vti_irq);
534	intr->vti_irq = NULL;
535	intr->vti_rid = -1;
536	}
537	}
538
539	static void
540	vtpci_free_interrupts(struct vtpci_common *cn)
541	{
542	struct vtpci_interrupt *intr;
543	int i, nvq_intrs;
544
545	vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt);
546
547	if (cn->vtpci_nmsix_resources != 0) {
548	nvq_intrs = cn->vtpci_nmsix_resources - 1;
549	cn->vtpci_nmsix_resources = 0;
550
551	if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) {
552	for (i = 0; i < nvq_intrs; i++, intr++)
553	vtpci_free_interrupt(cn, intr);
554
555	free(cn->vtpci_msix_vq_interrupts, M_DEVBUF);
556	cn->vtpci_msix_vq_interrupts = NULL;
557	}
558	}
559
560	if (cn->vtpci_flags & (VTPCI_FLAG_MSI \| VTPCI_FLAG_MSIX))
561	pci_release_msi(cn->vtpci_dev);
562
563	cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
564	}
565
566	static void
567	vtpci_free_virtqueues(struct vtpci_common *cn)
568	{
569	struct vtpci_virtqueue *vqx;
570	int idx;
571
572	for (idx = 0; idx < cn->vtpci_nvqs; idx++) {
573	vtpci_disable_vq(cn, idx);
574
575	vqx = &cn->vtpci_vqs[idx];
576	virtqueue_free(vqx->vtv_vq);
577	vqx->vtv_vq = NULL;
578	}
579
580	free(cn->vtpci_vqs, M_DEVBUF);
581	cn->vtpci_vqs = NULL;
582	cn->vtpci_nvqs = 0;
583	}
584
585	void
586	vtpci_release_child_resources(struct vtpci_common *cn)
587	{
588
589	vtpci_free_interrupts(cn);
590	vtpci_free_virtqueues(cn);
591	}
592
593	static void
594	vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn)
595	{
596	int idx;
597
598	if (cn->vtpci_flags & VTPCI_FLAG_MSIX) {
599	vtpci_register_cfg_msix(cn, NULL);
600
601	for (idx = 0; idx < cn->vtpci_nvqs; idx++)
602	vtpci_register_vq_msix(cn, idx, NULL);
603	}
604
605	vtpci_free_interrupts(cn);
606	}
607
Line 609 Link Here
609	vtpci_alloc_intr_resources(struct vtpci_common *cn)	904	vtpci_alloc_intr_resources(struct vtpci_softc *sc)
610	--
Line 614 Link Here
		909	rid = 0;
Lines 616-617 Link Here
616	if (cn->vtpci_flags & VTPCI_FLAG_INTX) {	912	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
617	rid = 0;
618	--
Line 619 Link Here
619	} else	914	else
620	--
Lines 623-625 Link Here
623	* When using INTX or MSI interrupts, this resource handles all	918	* For legacy and MSI interrupts, this single resource handles all
624	* interrupts. When using MSIX, this resource handles just the	919	* interrupts. For MSIX, this resource is used for the configuration
625	* configuration changed interrupt.	920	* changed interrupt.
626	--
Lines 627-630 Link Here
627	intr = &cn->vtpci_device_interrupt;	922	intr = &sc->vtpci_device_interrupt;
628		923	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
629	error = vtpci_alloc_interrupt(cn, rid, flags, intr);	924	if (error \|\| sc->vtpci_flags & (VTPCI_FLAG_LEGACY \| VTPCI_FLAG_MSI))
630	if (error \|\| cn->vtpci_flags & (VTPCI_FLAG_INTX \| VTPCI_FLAG_MSI))
631	--
Lines 633-638 Link Here
633	/*	927	/* Subtract one for the configuration changed interrupt. */
634	* Now allocate the interrupts for the virtqueues. This may be one	928	nvq_intrs = sc->vtpci_nmsix_resources - 1;
635	* for all the virtqueues, or one for each virtqueue. Subtract one
636	* below for because of the configuration changed interrupt.
637	*/
638	nvq_intrs = cn->vtpci_nmsix_resources - 1;
639	--
Line 640 Link Here
640	cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs *	930	intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
641	--
Line 642 Link Here
642	if (cn->vtpci_msix_vq_interrupts == NULL)	932	if (sc->vtpci_msix_vq_interrupts == NULL)
643	--
Lines 645-646 Link Here
645	intr = cn->vtpci_msix_vq_interrupts;
646
Line 648 Link Here
648	error = vtpci_alloc_interrupt(cn, rid, flags, intr);	936	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
649	--
Line 657 Link Here
657	vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type)	945	vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
658	--
Line 662 Link Here
662	intr = &cn->vtpci_device_interrupt;	950	intr = &sc->vtpci_device_interrupt;
663	--	951	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
		952	vtpci_legacy_intr, sc, &intr->vti_handler);
Lines 664-666 Link Here
664	error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL,
665	vtpci_intx_intr, cn, &intr->vti_handler);
666
Line 671 Link Here
671	vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type)	958	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
672	--
Line 677 Link Here
677	intr = cn->vtpci_msix_vq_interrupts;	964	intr = sc->vtpci_msix_vq_interrupts;
678	--
Lines 679-680 Link Here
679	for (i = 0; i < cn->vtpci_nvqs; i++) {	966	for (i = 0; i < sc->vtpci_nvqs; i++) {
680	vqx = &cn->vtpci_vqs[i];	967	vqx = &sc->vtpci_vqs[i];
681	--
Line 685 Link Here
685	error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type,	972	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
686	--
Line 698 Link Here
698	vtpci_set_host_msix_vectors(struct vtpci_common *cn)	985	vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
699	--
Line 700 Link Here
		987	device_t dev;
		988	struct vtpci_interrupt *intr;
		989	int error;
		990
		991	dev = sc->vtpci_dev;
		992	intr = &sc->vtpci_device_interrupt;
		993
		994	error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
		995	vtpci_config_intr, sc, &intr->vti_handler);
		996	if (error)
		997	return (error);
		998
		999	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
		1000	intr = sc->vtpci_msix_vq_interrupts;
		1001	error = bus_setup_intr(dev, intr->vti_irq, type,
		1002	vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
		1003	&intr->vti_handler);
		1004	} else
		1005	error = vtpci_setup_pervq_msix_interrupts(sc, type);
		1006
		1007	return (error ? error : vtpci_set_host_msix_vectors(sc));
		1008	}
		1009
		1010	static int
		1011	vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
		1012	{
		1013	int error;
		1014
		1015	type \|= INTR_MPSAFE;
		1016	KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
		1017	("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
		1018
		1019	error = vtpci_alloc_intr_resources(sc);
		1020	if (error)
		1021	return (error);
		1022
		1023	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
		1024	error = vtpci_setup_legacy_interrupt(sc, type);
		1025	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
		1026	error = vtpci_setup_msi_interrupt(sc, type);
		1027	else
		1028	error = vtpci_setup_msix_interrupts(sc, type);
		1029
		1030	return (error);
		1031	}
		1032
		1033	static int
		1034	vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
		1035	struct vtpci_interrupt *intr)
		1036	{
		1037	device_t dev;
		1038	uint16_t vector;
		1039
		1040	dev = sc->vtpci_dev;
		1041
		1042	if (intr != NULL) {
		1043	/* Map from guest rid to host vector. */
		1044	vector = intr->vti_rid - 1;
		1045	} else
		1046	vector = VIRTIO_MSI_NO_VECTOR;
		1047
		1048	vtpci_write_config_2(sc, offset, vector);
		1049
		1050	/* Read vector to determine if the host had sufficient resources. */
1051	if (vtpci_read_config_2(sc, offset) != vector) {
1052	device_printf(dev,
1053	"insufficient host resources for MSIX interrupts\n");
1054	return (ENODEV);
1055	}
1056
1057	return (0);
1058	}
1059
1060	static int
1061	vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1062	{
Line 701 Link Here
701	int idx, error;	1064	int idx, offset, error;
702	--
Lines 703-704 Link Here
703	intr = &cn->vtpci_device_interrupt;	1066	intr = &sc->vtpci_device_interrupt;
704	error = vtpci_register_cfg_msix(cn, intr);	1067	offset = VIRTIO_MSI_CONFIG_VECTOR;
705	--	1068
		1069	error = vtpci_register_msix_vector(sc, offset, intr);
Lines 708-710 Link Here
708	intr = cn->vtpci_msix_vq_interrupts;	1073	intr = sc->vtpci_msix_vq_interrupts;
709	for (idx = 0; idx < cn->vtpci_nvqs; idx++) {	1074	offset = VIRTIO_MSI_QUEUE_VECTOR;
710	if (cn->vtpci_vqs[idx].vtv_no_intr)	1075
711	--	1076	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
		1077	vtpci_select_virtqueue(sc, idx);
		1078
		1079	if (sc->vtpci_vqs[idx].vtv_no_intr)
Line 715 Link Here
715	error = vtpci_register_vq_msix(cn, idx, tintr);	1084	error = vtpci_register_msix_vector(sc, offset, tintr);
716	--
Lines 723-724 Link Here
723	if (!cn->vtpci_vqs[idx].vtv_no_intr &&	1092	if (!sc->vtpci_vqs[idx].vtv_no_intr &&
724	(cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)	1093	(sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
725	--
Line 732 Link Here
732	vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type)	1101	vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
733	--
Line 734 Link Here
734	struct vtpci_interrupt *intr;	1103	struct vtpci_virtqueue *vqx;
735	--	1104	struct virtqueue *vq;
Line 736 Link Here
		1106	uint16_t size;
Line 737 Link Here
737	intr = &cn->vtpci_device_interrupt;	1108	vqx = &sc->vtpci_vqs[idx];
738	--	1109	vq = vqx->vtv_vq;
Lines 739-740 Link Here
739	error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL,	1111	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
740	vtpci_config_intr, cn, &intr->vti_handler);	1112
741	--	1113	vtpci_select_virtqueue(sc, idx);
		1114	size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
		1115
		1116	error = virtqueue_reinit(vq, size);
Lines 744-745 Link Here
744	if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {	1120	vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
745	intr = &cn->vtpci_msix_vq_interrupts[0];	1121	virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
746	--
Lines 747-753 Link Here
747	error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type,	1123	return (0);
748	vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn,
749	&intr->vti_handler);
750	} else
751	error = vtpci_setup_pervq_msix_interrupts(cn, type);
752
753	return (error ? error : vtpci_set_host_msix_vectors(cn));
754	--
Lines 756-757 Link Here
756	static int	1126	static void
757	vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type)	1127	vtpci_free_interrupt(struct vtpci_softc sc, struct vtpci_interrupt intr)
758	--
Line 759 Link Here
759	int error;	1129	device_t dev;
760	--
Lines 761-763 Link Here
761	type \|= INTR_MPSAFE;	1131	dev = sc->vtpci_dev;
762	KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
763	("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags));
764	--
Lines 765-767 Link Here
765	error = vtpci_alloc_intr_resources(cn);	1133	if (intr->vti_handler != NULL) {
766	if (error)	1134	bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
767	return (error);	1135	intr->vti_handler = NULL;
768	--	1136	}
Lines 769-776 Link Here
769	if (cn->vtpci_flags & VTPCI_FLAG_INTX)	1138	if (intr->vti_irq != NULL) {
770	error = vtpci_setup_intx_interrupt(cn, type);	1139	bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
771	else if (cn->vtpci_flags & VTPCI_FLAG_MSI)	1140	intr->vti_irq);
772	error = vtpci_setup_msi_interrupt(cn, type);	1141	intr->vti_irq = NULL;
773	else	1142	intr->vti_rid = -1;
774	error = vtpci_setup_msix_interrupts(cn, type);	1143	}
775
776	return (error);
777	--
Lines 779-780 Link Here
779	int	1146	static void
780	vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type)	1147	vtpci_free_interrupts(struct vtpci_softc *sc)
781	--
Lines 782-783 Link Here
782	device_t dev;	1149	struct vtpci_interrupt *intr;
783	int attempt, error;	1150	int i, nvq_intrs;
784	--
Line 785 Link Here
785	dev = cn->vtpci_dev;	1152	vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
786	--
Lines 787-809 Link Here
787	for (attempt = 0; attempt < 5; attempt++) {	1154	if (sc->vtpci_nmsix_resources != 0) {
788	/*	1155	nvq_intrs = sc->vtpci_nmsix_resources - 1;
789	* Start with the most desirable interrupt configuration and	1156	sc->vtpci_nmsix_resources = 0;
790	* fallback towards less desirable ones.
791	*/
792	switch (attempt) {
793	case 0:
794	error = vtpci_alloc_intr_msix_pervq(cn);
795	break;
796	case 1:
797	error = vtpci_alloc_intr_msix_shared(cn);
798	break;
799	case 2:
800	error = vtpci_alloc_intr_msi(cn);
801	break;
802	case 3:
803	error = vtpci_alloc_intr_intx(cn);
804	break;
805	default:
806	device_printf(dev,
807	"exhausted all interrupt allocation attempts\n");
808	return (ENXIO);
809	}
810	--
Lines 811-812 Link Here
811	if (error == 0 && vtpci_setup_intrs(cn, type) == 0)	1158	intr = sc->vtpci_msix_vq_interrupts;
812	break;	1159	if (intr != NULL) {
813	--	1160	for (i = 0; i < nvq_intrs; i++, intr++)
		1161	vtpci_free_interrupt(sc, intr);
Line 814 Link Here
814	vtpci_cleanup_setup_intr_attempt(cn);	1163	free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
815	--	1164	sc->vtpci_msix_vq_interrupts = NULL;
		1165	}
Lines 817-826 Link Here
817	if (bootverbose) {	1168	if (sc->vtpci_flags & (VTPCI_FLAG_MSI \| VTPCI_FLAG_MSIX))
818	if (cn->vtpci_flags & VTPCI_FLAG_INTX)	1169	pci_release_msi(sc->vtpci_dev);
819	device_printf(dev, "using legacy interrupt\n");
820	else if (cn->vtpci_flags & VTPCI_FLAG_MSI)
821	device_printf(dev, "using MSI interrupt\n");
822	else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
823	device_printf(dev, "using shared MSIX interrupts\n");
824	else
825	device_printf(dev, "using per VQ MSIX interrupts\n");
826	}
827	--
Line 828 Link Here
828	return (0);	1171	sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
829	--
Lines 831-832 Link Here
831	static int	1174	static void
832	vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx)	1175	vtpci_free_virtqueues(struct vtpci_softc *sc)
833	--
Lines 835-836 Link Here
835	struct virtqueue *vq;	1178	int idx;
836	int error;
837	--
Lines 838-839 Link Here
838	vqx = &cn->vtpci_vqs[idx];	1180	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
839	vq = vqx->vtv_vq;	1181	vqx = &sc->vtpci_vqs[idx];
840	--
Line 841 Link Here
841	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));	1183	vtpci_select_virtqueue(sc, idx);
842	--	1184	vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
Lines 843-845 Link Here
843	error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx));	1186	virtqueue_free(vqx->vtv_vq);
844	if (error == 0)	1187	vqx->vtv_vq = NULL;
845	vtpci_set_vq(cn, vq);	1188	}
846	--
Line 847 Link Here
847	return (error);	1190	free(sc->vtpci_vqs, M_DEVBUF);
848	--	1191	sc->vtpci_vqs = NULL;
		1192	sc->vtpci_nvqs = 0;
Line 851 Link Here
851	vtpci_intx_intr(void *xcn)	1196	vtpci_release_child_resources(struct vtpci_softc *sc)
852	--
Line 853 Link Here
853	struct vtpci_common *cn;	1198
854	--	1199	vtpci_free_interrupts(sc);
		1200	vtpci_free_virtqueues(sc);
		1201	}
		1202
		1203	static void
		1204	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
		1205	{
		1206	int idx;
		1207
		1208	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
		1209	vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
		1210	VIRTIO_MSI_NO_VECTOR);
		1211
		1212	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
		1213	vtpci_select_virtqueue(sc, idx);
		1214	vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
		1215	VIRTIO_MSI_NO_VECTOR);
		1216	}
		1217	}
		1218
		1219	vtpci_free_interrupts(sc);
		1220	}
		1221
		1222	static void
		1223	vtpci_reset(struct vtpci_softc *sc)
		1224	{
		1225
		1226	/*
		1227	* Setting the status to RESET sets the host device to
		1228	* the original, uninitialized state.
		1229	*/
		1230	vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
		1231	}
		1232
		1233	static void
		1234	vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
		1235	{
		1236
		1237	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
		1238	}
		1239
		1240	static void
		1241	vtpci_legacy_intr(void *xsc)
		1242	{
		1243	struct vtpci_softc *sc;
Lines 858-859 Link Here
858	cn = xcn;	1248	sc = xsc;
859	isr = vtpci_read_isr(cn);	1249	vqx = &sc->vtpci_vqs[0];
860	--
Line 861 Link Here
		1251	/* Reading the ISR also clears it. */
		1252	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
		1253
Line 862 Link Here
862	vtpci_config_intr(cn);	1255	vtpci_config_intr(sc);
863	--
Lines 865-866 Link Here
865	vqx = &cn->vtpci_vqs[0];	1258	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
866	for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {
867	--
Line 874 Link Here
874	vtpci_vq_shared_intr_filter(void *xcn)	1266	vtpci_vq_shared_intr_filter(void *xsc)
875	--
Line 876 Link Here
876	struct vtpci_common *cn;	1268	struct vtpci_softc *sc;
877	--
Lines 880-881 Link Here
880	cn = xcn;
881	vqx = &cn->vtpci_vqs[0];
Line 883 Link Here
		1273	sc = xsc;
		1274	vqx = &sc->vtpci_vqs[0];
Line 884 Link Here
884	for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {	1276	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
885	--
Line 893 Link Here
893	vtpci_vq_shared_intr(void *xcn)	1285	vtpci_vq_shared_intr(void *xsc)
894	--
Line 895 Link Here
895	struct vtpci_common *cn;	1287	struct vtpci_softc *sc;
896	--
Lines 899-900 Link Here
899	cn = xcn;	1291	sc = xsc;
900	vqx = &cn->vtpci_vqs[0];	1292	vqx = &sc->vtpci_vqs[0];
901	--
Line 902 Link Here
902	for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {	1294	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
903	--
Line 930 Link Here
930	vtpci_config_intr(void *xcn)	1322	vtpci_config_intr(void *xsc)
931	--
Line 932 Link Here
932	struct vtpci_common *cn;	1324	struct vtpci_softc *sc;
933	--
Lines 935-936 Link Here
935	cn = xcn;	1327	sc = xsc;
936	child = cn->vtpci_child_dev;	1328	child = sc->vtpci_child_dev;
937	--
Lines 940-1000 Link Here
940	}
941
942	static int
943	vtpci_feature_sysctl(struct sysctl_req req, struct vtpci_common cn,
944	uint64_t features)
945	{
946	struct sbuf *sb;
947	int error;
948
949	sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
950	if (sb == NULL)
951	return (ENOMEM);
952
953	error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc);
954	sbuf_delete(sb);
955
956	return (error);
957	}
958
959	static int
960	vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS)
961	{
962	struct vtpci_common *cn;
963
964	cn = arg1;
965
966	return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features));
967	}
968
969	static int
970	vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS)
971	{
972	struct vtpci_common *cn;
973
974	cn = arg1;
975
976	return (vtpci_feature_sysctl(req, cn, cn->vtpci_features));
977	}
978
979	static void
980	vtpci_setup_sysctl(struct vtpci_common *cn)
981	{
982	device_t dev;
983	struct sysctl_ctx_list *ctx;
984	struct sysctl_oid *tree;
985	struct sysctl_oid_list *child;
986
987	dev = cn->vtpci_dev;
988	ctx = device_get_sysctl_ctx(dev);
989	tree = device_get_sysctl_tree(dev);
990	child = SYSCTL_CHILDREN(tree);
991
992	SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs",
993	CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues");
994
995	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features",
996	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, cn, 0,
997	vtpci_host_features_sysctl, "A", "Features supported by the host");
998	SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features",
999	CTLTYPE_STRING \| CTLFLAG_RD \| CTLFLAG_MPSAFE, cn, 0,
1000	vtpci_negotiated_features_sysctl, "A", "Features negotiated");




 * SPDX-License-Identifier: BSD-3-Clause
--

 * Copyright IBM Corp. 2007
 * All rights reserved.
--

 * Authors:
 *  Anthony Liguori  <aliguori@us.ibm.com>
 *
 * This header is BSD licensed so anyone can use the definitions to implement
 * compatible drivers/servers.
 *

 *    notice, this list of conditions and the following disclaimer.
 *    disclaimer.
--

 * 3. Neither the name of IBM nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.

 * $FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.h 326022 2017-11-20 19:36:21Z pfg $
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * $FreeBSD$
--

/* VirtIO PCI vendor/device ID. */
#define VIRTIO_PCI_VENDORID	0x1AF4
#define VIRTIO_PCI_DEVICEID_MIN	0x1000
#define VIRTIO_PCI_DEVICEID_MAX	0x103F
};
--

/* VirtIO ABI version, this must match exactly. */
#define VIRTIO_PCI_ABI_VERSION	0
	int			 vtv_no_intr;
	int			 vtv_notify_offset;
};
--

/*
 * VirtIO Header, located in BAR 0.
 */
#define VIRTIO_PCI_HOST_FEATURES  0  /* host's supported features (32bit, RO)*/
#define VIRTIO_PCI_GUEST_FEATURES 4  /* guest's supported features (32, RW) */
#define VIRTIO_PCI_QUEUE_PFN      8  /* physical address of VQ (32, RW) */
#define VIRTIO_PCI_QUEUE_NUM      12 /* number of ring entries (16, RO) */
#define VIRTIO_PCI_QUEUE_SEL      14 /* current VQ selection (16, RW) */
#define VIRTIO_PCI_QUEUE_NOTIFY	  16 /* notify host regarding VQ (16, RW) */
#define VIRTIO_PCI_STATUS         18 /* device status register (8, RW) */
#define VIRTIO_PCI_ISR            19 /* interrupt status register, reading
				      * also clears the register (8, RO) */
/* Only if MSIX is enabled: */
#define VIRTIO_MSI_CONFIG_VECTOR  20 /* configuration change vector (16, RW) */
#define VIRTIO_MSI_QUEUE_VECTOR   22 /* vector for selected VQ notifications
					(16, RW) */

/* The bit of the ISR which indicates a device has an interrupt. */
#define VIRTIO_PCI_ISR_INTR	0x1
/* The bit of the ISR which indicates a device configuration change. */
#define VIRTIO_PCI_ISR_CONFIG	0x2
/* Vector value used to disable MSI for queue. */
#define VIRTIO_MSI_NO_VECTOR	0xFFFF
#define VTPCI_FLAG_MSIX			0x4000
#define VTPCI_FLAG_SHARED_MSIX		0x8000
#define VTPCI_FLAG_ITYPE_MASK		0xF000
--

/*
 * The remaining space is defined by each driver as the per-driver
 * configuration space.
 */
#define VIRTIO_PCI_CONFIG_OFF(msix_enabled)     ((msix_enabled) ? 24 : 20)

/*
 * How many bits to shift physical queue address written to QUEUE_PFN.
 * 12 is historical, and due to x86 page size.
 */
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT	12
	 * vector.
	 *
	 * If MSIX is not available, attempt to have the whole device share
	 * one MSI vector, and then, finally, one intx interrupt.
	 */
	struct vtpci_interrupt		 vtpci_device_interrupt;
	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
	int				 vtpci_nmsix_resources;
};
--

/* The alignment to use between consumer and producer parts of vring. */
#define VIRTIO_PCI_VRING_ALIGN	4096
static inline device_t
vtpci_child_device(struct vtpci_common *cn)
{
	return (cn->vtpci_child_dev);
}

static inline bool
vtpci_is_msix_available(struct vtpci_common *cn)
{
	return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0);
}

static inline bool
vtpci_is_msix_enabled(struct vtpci_common *cn)
{
	return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0);
}

static inline bool
vtpci_is_modern(struct vtpci_common *cn)
{
	return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0);
}

static inline int
vtpci_virtqueue_count(struct vtpci_common *cn)
{
	return (cn->vtpci_nvqs);
}

void	vtpci_init(struct vtpci_common *cn, device_t dev, bool modern);
int	vtpci_add_child(struct vtpci_common *cn);
int	vtpci_delete_child(struct vtpci_common *cn);
void	vtpci_child_detached(struct vtpci_common *cn);
int	vtpci_reinit(struct vtpci_common *cn);

uint64_t vtpci_negotiate_features(struct vtpci_common *cn,
	     uint64_t child_features, uint64_t host_features);
int	 vtpci_with_feature(struct vtpci_common *cn, uint64_t feature);

int	vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result);
int	vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value);

int	vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs,
	    struct vq_alloc_info *vq_info);
int	vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type);
void	vtpci_release_child_resources(struct vtpci_common *cn);
--

Line 32 Link Here

(-)/usr/src/sys/dev/virtio/random/virtio_random.c (-33 / +6 lines)
32	__FBSDID("$FreeBSD$");	32	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/random/virtio_random.c 338324 2018-08-26 12:51:46Z markm $");
33	--
Lines 61-62 Link Here
61	static int vtrnd_negotiate_features(struct vtrnd_softc *);	61	static void vtrnd_negotiate_features(struct vtrnd_softc *);
62	static int vtrnd_setup_features(struct vtrnd_softc *);
63	--
Line 89 Link Here
89	DRIVER_MODULE(virtio_random, vtpcil, vtrnd_driver, vtrnd_devclass,	88	DRIVER_MODULE(virtio_random, virtio_pci, vtrnd_driver, vtrnd_devclass,
90	--
Lines 91-92 Link Here
91	DRIVER_MODULE(virtio_random, vtpcim, vtrnd_driver, vtrnd_devclass,
92	vtrnd_modevent, 0);
Line 136 Link Here
136	virtio_set_feature_desc(dev, vtrnd_feature_desc);
Lines 140-144 Link Here
140	error = vtrnd_setup_features(sc);	136	virtio_set_feature_desc(dev, vtrnd_feature_desc);
141	if (error) {	137	vtrnd_negotiate_features(sc);
142	device_printf(dev, "cannot setup features\n");
143	goto fail;
144	}
145	--
Line 173 Link Here
173	static int	166	static void
174	--
Line 183 Link Here
183	return (virtio_finalize_features(dev));
Lines 187-198 Link Here
187	vtrnd_setup_features(struct vtrnd_softc *sc)
188	{
189	int error;
190
191	error = vtrnd_negotiate_features(sc);
192	if (error)
193	return (error);
194
195	return (0);
196	}
197
198	static int
Lines 240-241 Link Here
240	// random_harvest_queue(&value, sizeof(value), sizeof(value) * NBBY / 2,
241	// RANDOM_PURE_VIRTIO);

Line 32 Link Here

(-)/usr/src/sys/dev/virtio/scsi/virtio_scsi.c (-99 / +40 lines)
32	__FBSDID("$FreeBSD$");	32	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/scsi/virtio_scsi.c 349691 2019-07-03 19:52:24Z vangyzen $");
33	--
Lines 79-80 Link Here
79	static int vtscsi_negotiate_features(struct vtscsi_softc *);	79	static void vtscsi_negotiate_features(struct vtscsi_softc *);
80	static int vtscsi_setup_features(struct vtscsi_softc *);
81	--
Lines 139-142 Link Here
139	static void vtscsi_init_scsi_cmd_req(struct vtscsi_softc *,	138	static void vtscsi_init_scsi_cmd_req(struct ccb_scsiio *,
140	struct ccb_scsiio , struct virtio_scsi_cmd_req );	139	struct virtio_scsi_cmd_req *);
141	static void vtscsi_init_ctrl_tmf_req(struct vtscsi_softc , struct ccb_hdr ,	140	static void vtscsi_init_ctrl_tmf_req(struct ccb_hdr *, uint32_t,
142	uint32_t, uintptr_t, struct virtio_scsi_ctrl_tmf_req *);	141	uintptr_t, struct virtio_scsi_ctrl_tmf_req *);
143	--
Line 188 Link Here
188	static void vtscsi_setup_sysctl(struct vtscsi_softc *);	187	static void vtscsi_add_sysctl(struct vtscsi_softc *);
189	--
Lines 193-200 Link Here
193	#define vtscsi_modern(_sc) (((_sc)->vtscsi_features & VIRTIO_F_VERSION_1) != 0)
194	#define vtscsi_htog16(_sc, _val) virtio_htog16(vtscsi_modern(_sc), _val)
195	#define vtscsi_htog32(_sc, _val) virtio_htog32(vtscsi_modern(_sc), _val)
196	#define vtscsi_htog64(_sc, _val) virtio_htog64(vtscsi_modern(_sc), _val)
197	#define vtscsi_gtoh16(_sc, _val) virtio_gtoh16(vtscsi_modern(_sc), _val)
198	#define vtscsi_gtoh32(_sc, _val) virtio_gtoh32(vtscsi_modern(_sc), _val)
199	#define vtscsi_gtoh64(_sc, _val) virtio_gtoh64(vtscsi_modern(_sc), _val)
200
Lines 218-219 Link Here
218	{ VIRTIO_SCSI_F_CHANGE, "ChangeEvent" },
219	{ VIRTIO_SCSI_F_T10_PI, "T10PI" },
Line 242 Link Here
242	DRIVER_MODULE(virtio_scsi, vtpcil, vtscsi_driver, vtscsi_devclass,	231	DRIVER_MODULE(virtio_scsi, virtio_pci, vtscsi_driver, vtscsi_devclass,
243	--
Lines 244-245 Link Here
244	DRIVER_MODULE(virtio_scsi, vtpcim, vtscsi_driver, vtscsi_devclass,
245	vtscsi_modevent, 0);
Line 291 Link Here
291	virtio_set_feature_desc(dev, vtscsi_feature_desc);
Line 297 Link Here
297	vtscsi_setup_sysctl(sc);	283	vtscsi_add_sysctl(sc);
298	--
Lines 299-303 Link Here
299	error = vtscsi_setup_features(sc);	285	virtio_set_feature_desc(dev, vtscsi_feature_desc);
300	if (error) {	286	vtscsi_negotiate_features(sc);
301	device_printf(dev, "cannot setup features\n");
302	goto fail;
303	}
304	--
Line 305 Link Here
		288	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
		289	sc->vtscsi_flags \|= VTSCSI_FLAG_INDIRECT;
		290	if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
		291	sc->vtscsi_flags \|= VTSCSI_FLAG_BIDIRECTIONAL;
		292	if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
		293	sc->vtscsi_flags \|= VTSCSI_FLAG_HOTPLUG;
		294
Line 416 Link Here
416	static int	406	static void
417	--
Lines 423-426 Link Here
423	features = VTSCSI_FEATURES;	413	features = virtio_negotiate_features(dev, VTSCSI_FEATURES);
424		414	sc->vtscsi_features = features;
425	sc->vtscsi_features = virtio_negotiate_features(dev, features);
426	return (virtio_finalize_features(dev));
427	--
Lines 429-450 Link Here
429	static int
430	vtscsi_setup_features(struct vtscsi_softc *sc)
431	{
432	device_t dev;
433	int error;
434
435	dev = sc->vtscsi_dev;
436
437	error = vtscsi_negotiate_features(sc);
438	if (error)
439	return (error);
440
441	if (virtio_with_feature(dev, VIRTIO_RING_F_INDIRECT_DESC))
442	sc->vtscsi_flags \|= VTSCSI_FLAG_INDIRECT;
443	if (virtio_with_feature(dev, VIRTIO_SCSI_F_INOUT))
444	sc->vtscsi_flags \|= VTSCSI_FLAG_BIDIRECTIONAL;
445	if (virtio_with_feature(dev, VIRTIO_SCSI_F_HOTPLUG))
446	sc->vtscsi_flags \|= VTSCSI_FLAG_HOTPLUG;
447
448	return (0);
449	}
450
Line 568 Link Here
568	virtio_reinit_complete(dev);
Line 570 Link Here
		535	virtio_reinit_complete(dev);
Line 1123 Link Here
1123	vtscsi_init_scsi_cmd_req(sc, csio, cmd_req);	1089	vtscsi_init_scsi_cmd_req(csio, cmd_req);
1124	--
Line 1243 Link Here
1243	vtscsi_init_ctrl_tmf_req(sc, to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,	1209	vtscsi_init_ctrl_tmf_req(to_ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1244	--
Line 1351 Link Here
1351	uint32_t resp_sense_length;
Line 1355 Link Here
1355	csio->resid = vtscsi_htog32(sc, cmd_resp->resid);	1320	csio->resid = cmd_resp->resid;
1356	--
Lines 1362-1364 Link Here
1362	resp_sense_length = vtscsi_htog32(sc, cmd_resp->sense_len);	1327	if (cmd_resp->sense_len > 0) {
1363
1364	if (resp_sense_length > 0) {
1365	--
Lines 1367-1368 Link Here
1367	if (resp_sense_length < csio->sense_len)	1330	if (cmd_resp->sense_len < csio->sense_len)
1368	csio->sense_resid = csio->sense_len - resp_sense_length;	1331	csio->sense_resid = csio->sense_len -
1369	--	1332	cmd_resp->sense_len;
Lines 1372-1373 Link Here
1372	bzero(&csio->sense_data, sizeof(csio->sense_data));	1336	memcpy(&csio->sense_data, cmd_resp->sense,
1373	memcpy(cmd_resp->sense, &csio->sense_data,
1374	--
Line 1534 Link Here
1534	vtscsi_init_ctrl_tmf_req(sc, ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,	1497	vtscsi_init_ctrl_tmf_req(ccbh, VIRTIO_SCSI_T_TMF_ABORT_TASK,
1535	--
Line 1603 Link Here
1603	vtscsi_init_ctrl_tmf_req(sc, ccbh, subtype, 0, tmf_req);	1566	vtscsi_init_ctrl_tmf_req(ccbh, subtype, 0, tmf_req);
1604	--
Line 1640 Link Here
1640	vtscsi_init_scsi_cmd_req(struct vtscsi_softc sc, struct ccb_scsiio csio,	1603	vtscsi_init_scsi_cmd_req(struct ccb_scsiio *csio,
1641	--
Line 1661 Link Here
1661	cmd_req->tag = vtscsi_gtoh64(sc, (uintptr_t) csio);	1624	cmd_req->tag = (uintptr_t) csio;
1662	--
Lines 1671-1672 Link Here
1671	vtscsi_init_ctrl_tmf_req(struct vtscsi_softc sc, struct ccb_hdr ccbh,	1634	vtscsi_init_ctrl_tmf_req(struct ccb_hdr *ccbh, uint32_t subtype,
1672	uint32_t subtype, uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)	1635	uintptr_t tag, struct virtio_scsi_ctrl_tmf_req *tmf_req)
1673	--
Lines 1677-1679 Link Here
1677	tmf_req->type = vtscsi_gtoh32(sc, VIRTIO_SCSI_T_TMF);	1640	tmf_req->type = VIRTIO_SCSI_T_TMF;
1678	tmf_req->subtype = vtscsi_gtoh32(sc, subtype);	1641	tmf_req->subtype = subtype;
1679	tmf_req->tag = vtscsi_gtoh64(sc, tag);	1642	tmf_req->tag = tag;
1680	--
Line 2313 Link Here
2313	vtscsi_setup_sysctl(struct vtscsi_softc *sc)	2276	vtscsi_add_sysctl(struct vtscsi_softc *sc)
2314	--

Line 28 Link Here

(-)/usr/src/sys/dev/virtio/scsi/virtio_scsi.h (-35 / +9 lines)
28	* $FreeBSD$	28	* $FreeBSD: releng/12.1/sys/dev/virtio/scsi/virtio_scsi.h 326255 2017-11-27 14:52:40Z pfg $
29	--
Line 34 Link Here
34	/* Default values of the CDB and sense data size configuration fields */	34	/* Feature bits */
35	--	35	#define VIRTIO_SCSI_F_INOUT 0x0001 /* Single request can contain both
		36	* read and write buffers */
		37	#define VIRTIO_SCSI_F_HOTPLUG 0x0002 /* Host should enable hot plug/unplug
		38	* of new LUNs and targets.
		39	*/
		40
Line 43 Link Here
43	uint8_t prio; /* SAM command priority field */	49	uint8_t prio;
44	--
Lines 48-59 Link Here
48	/* SCSI command request, followed by protection information */
49	struct virtio_scsi_cmd_req_pi {
50	uint8_t lun[8]; /* Logical Unit Number */
51	uint64_t tag; /* Command identifier */
52	uint8_t task_attr; /* Task attribute */
53	uint8_t prio; /* SAM command priority field */
54	uint8_t crn;
55	uint32_t pi_bytesout; /* DataOUT PI Number of bytes */
56	uint32_t pi_bytesin; /* DataIN PI Number of bytes */
57	uint8_t cdb[VIRTIO_SCSI_CDB_SIZE];
58	} __packed;
59
Lines 113-128 Link Here
113	/* Feature bits */
114	#define VIRTIO_SCSI_F_INOUT 0x0001 /* Single request can contain both
115	* read and write buffers.
116	*/
117	#define VIRTIO_SCSI_F_HOTPLUG 0x0002 /* Host should enable hot plug/unplug
118	* of new LUNs and targets.
119	*/
120	#define VIRTIO_SCSI_F_CHANGE 0x0004 /* Host will report changes to LUN
121	* parameters via a
122	* VIRTIO_SCSI_T_PARAM_CHANGE event.
123	*/
124	#define VIRTIO_SCSI_F_T10_PI 0x0008 /* Extended fields for T10 protection
125	* information (DIF/DIX) are included
126	* in the SCSI request header.
127	*/
128
Line 165 Link Here
165	#define VIRTIO_SCSI_T_PARAM_CHANGE 3

Line 28 Link Here

(-)/usr/src/sys/dev/virtio/scsi/virtio_scsivar.h (-2 / +1 lines)
28	* $FreeBSD$	28	* $FreeBSD: releng/12.1/sys/dev/virtio/scsi/virtio_scsivar.h 326255 2017-11-27 14:52:40Z pfg $
29	--

Line 30 Link Here

(-)/usr/src/sys/dev/virtio/virtio.c (-111 / +33 lines)
30	__FBSDID("$FreeBSD$");	30	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/virtio.c 329601 2018-02-19 19:28:24Z bryanv $");
31	--
Line 63 Link Here
63	{ VIRTIO_ID_RPMSG, "Remote Processor Messaging" },	63	{ VIRTIO_ID_RPMSG, "Remote Processor Messaging" },
64	--
Lines 69-71 Link Here
69	{ VIRTIO_ID_INPUT, "Input" },	69	{ VIRTIO_ID_INPUT, "Input" },
70	{ VIRTIO_ID_VSOCK, "VSOCK Transport" },	70	{ VIRTIO_ID_VSOCK, "VSOCK Transport" },
71	{ VIRTIO_ID_CRYPTO, "Crypto" },	71	{ VIRTIO_ID_CRYPTO, "Crypto" },
72	--
Lines 78-84 Link Here
78	{ VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" }, /* Legacy */	78	{ VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty" },
79	{ VIRTIO_F_ANY_LAYOUT, "AnyLayout" }, /* Legacy */	79	{ VIRTIO_RING_F_INDIRECT_DESC, "RingIndirect" },
80	{ VIRTIO_RING_F_INDIRECT_DESC, "RingIndirectDesc" },	80	{ VIRTIO_RING_F_EVENT_IDX, "EventIdx" },
81	{ VIRTIO_RING_F_EVENT_IDX, "RingEventIdx" },	81	{ VIRTIO_F_BAD_FEATURE, "BadFeature" },
82	{ VIRTIO_F_BAD_FEATURE, "BadFeature" }, /* Legacy */
83	{ VIRTIO_F_VERSION_1, "Version1" },
84	{ VIRTIO_F_IOMMU_PLATFORM, "IOMMUPlatform" },
85	--
Lines 122-124 Link Here
122	int	119	void
123	virtio_describe_sbuf(struct sbuf *sb, uint64_t features,	120	virtio_describe(device_t dev, const char *msg,
124	struct virtio_feature_desc *desc)	121	uint64_t features, struct virtio_feature_desc *desc)
125	--
Line 126 Link Here
126	const char *name;	123	struct sbuf sb;
127	--
Line 128 Link Here
		125	char *buf;
		126	const char *name;
Line 130 Link Here
130	sbuf_printf(sb, "%#jx", (uintmax_t) features);	129	if ((buf = malloc(512, M_TEMP, M_NOWAIT)) == NULL) {
131	--	130	device_printf(dev, "%s features: %#jx\n", msg, (uintmax_t) features);
		131	return;
		132	}
Line 132 Link Here
		134	sbuf_new(&sb, buf, 512, SBUF_FIXEDLEN);
		135	sbuf_printf(&sb, "%s features: %#jx", msg, (uintmax_t) features);
		136
Line 141 Link Here
141	sbuf_cat(sb, " <");	146	sbuf_cat(&sb, " <");
142	--
Line 143 Link Here
143	sbuf_cat(sb, ",");	148	sbuf_cat(&sb, ",");
144	--
Line 147 Link Here
147	sbuf_printf(sb, "%#jx", (uintmax_t) val);	152	sbuf_printf(&sb, "%#jx", (uintmax_t) val);
148	--
Line 149 Link Here
149	sbuf_cat(sb, name);	154	sbuf_cat(&sb, name);
150	--
Line 153 Link Here
153	sbuf_cat(sb, ">");	158	sbuf_cat(&sb, ">");
154	--
Lines 155-175 Link Here
155	return (sbuf_finish(sb));	160	#if __FreeBSD_version < 900020
156	}	161	sbuf_finish(&sb);
157		162	if (sbuf_overflowed(&sb) == 0)
158	void	163	#else
159	virtio_describe(device_t dev, const char *msg, uint64_t features,	164	if (sbuf_finish(&sb) == 0)
160	struct virtio_feature_desc *desc)	165	#endif
161	{
162	struct sbuf sb;
163	char *buf;
164	int error;
165
166	if ((buf = malloc(1024, M_TEMP, M_NOWAIT)) == NULL) {
167	error = ENOMEM;
168	goto out;
169	}
170
171	sbuf_new(&sb, buf, 1024, SBUF_FIXEDLEN);
172	sbuf_printf(&sb, "%s features: ", msg);
173
174	error = virtio_describe_sbuf(&sb, features, desc);
175	if (error == 0)
176	--
Lines 180-185 Link Here
180
181	out:
182	if (error != 0) {
183	device_printf(dev, "%s features: %#jx\n", msg,
184	(uintmax_t) features);
185	}
Lines 188-229 Link Here
188	uint64_t
189	virtio_filter_transport_features(uint64_t features)
190	{
191	uint64_t transport, mask;
192
193	transport = (1ULL <<
194	(VIRTIO_TRANSPORT_F_END - VIRTIO_TRANSPORT_F_START)) - 1;
195	transport <<= VIRTIO_TRANSPORT_F_START;
196
197	mask = -1ULL & ~transport;
198	mask \|= VIRTIO_RING_F_INDIRECT_DESC;
199	mask \|= VIRTIO_RING_F_EVENT_IDX;
200	mask \|= VIRTIO_F_VERSION_1;
201
202	return (features & mask);
203	}
204
205	int
206	virtio_bus_is_modern(device_t dev)
207	{
208	uintptr_t modern;
209
210	virtio_read_ivar(dev, VIRTIO_IVAR_MODERN, &modern);
211	return (modern != 0);
212	}
213
214	void
215	virtio_read_device_config_array(device_t dev, bus_size_t offset, void *dst,
216	int size, int count)
217	{
218	int i, gen;
219
220	do {
221	gen = virtio_config_generation(dev);
222
223	for (i = 0; i < count; i++) {
224	virtio_read_device_config(dev, offset + i * size,
225	(uint8_t ) dst + i size, size);
226	}
227	} while (gen != virtio_config_generation(dev));
228	}
229
Lines 255-261 Link Here
255	}
256
257	int
258	virtio_finalize_features(device_t dev)
259	{
260
261	return (VIRTIO_BUS_FINALIZE_FEATURES(device_get_parent(dev)));

Line 28 Link Here

(-)/usr/src/sys/dev/virtio/virtio.h (-15 / +2 lines)
28	* $FreeBSD$	28	* $FreeBSD: releng/12.1/sys/dev/virtio/virtio.h 329601 2018-02-19 19:28:24Z bryanv $
29	--
Line 34 Link Here
34	#include <dev/virtio/virtio_endian.h>
Line 38 Link Here
38	struct sbuf;
Line 62 Link Here
62	#define VIRTIO_IVAR_MODERN 7
Lines 71-77 Link Here
71	uint64_t features, struct virtio_feature_desc *desc);	68	uint64_t features, struct virtio_feature_desc *feature_desc);
72	int virtio_describe_sbuf(struct sbuf *sb, uint64_t features,
73	struct virtio_feature_desc *desc);
74	uint64_t virtio_filter_transport_features(uint64_t features);
75	int virtio_bus_is_modern(device_t dev);
76	void virtio_read_device_config_array(device_t dev, bus_size_t offset,
77	void *dst, int size, int count);
78	--
Line 85 Link Here
85	int virtio_finalize_features(device_t dev);
Line 143 Link Here
143	VIRTIO_READ_IVAR(modern, VIRTIO_IVAR_MODERN);

Line 26 Link Here

(-)/usr/src/sys/dev/virtio/virtio_bus_if.m (-13 / +1 lines)
26	# $FreeBSD$	26	# $FreeBSD: releng/12.1/sys/dev/virtio/virtio_bus_if.m 329601 2018-02-19 19:28:24Z bryanv $
27	--
Lines 39-44 Link Here
39	virtio_bus_default_finalize_features(device_t dev)
40	{
41	return (0);
42	}
43
44	static int
Lines 56-59 Link Here
56	METHOD int finalize_features {
57	device_t dev;
58	} DEFAULT virtio_bus_default_finalize_features;
59
Line 93 Link Here
93	bus_size_t offset;

Line 28 Link Here

(-)/usr/src/sys/dev/virtio/virtio_config.h (-2 / +1 lines)
28	* $FreeBSD$	28	* $FreeBSD: releng/12.1/sys/dev/virtio/virtio_config.h 335305 2018-06-17 20:45:48Z bryanv $
29	--

Line 28 Link Here

(-)/usr/src/sys/dev/virtio/virtio_ids.h (-2 / +1 lines)
28	* $FreeBSD$	28	* $FreeBSD: releng/12.1/sys/dev/virtio/virtio_ids.h 327958 2018-01-14 06:03:40Z bryanv $
29	--

Line 26 Link Here

(-)/usr/src/sys/dev/virtio/virtio_if.m (-2 / +1 lines)
26	# $FreeBSD$	26	# $FreeBSD: releng/12.1/sys/dev/virtio/virtio_if.m 255110 2013-09-01 04:20:23Z bryanv $
27	--

Line 32 Link Here

(-)/usr/src/sys/dev/virtio/virtio_ring.h (-2 / +1 lines)
32	* $FreeBSD$	32	* $FreeBSD: releng/12.1/sys/dev/virtio/virtio_ring.h 335305 2018-06-17 20:45:48Z bryanv $
33	--

Line 35 Link Here

(-)/usr/src/sys/dev/virtio/virtqueue.c (-114 / +58 lines)
35	__FBSDID("$FreeBSD$");	35	__FBSDID("$FreeBSD: releng/12.1/sys/dev/virtio/virtqueue.c 329602 2018-02-19 19:31:18Z bryanv $");
36	--
Line 60 Link Here
		60	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
Lines 63-65 Link Here
63	#define VIRTQUEUE_FLAG_MODERN 0x0001	64	#define VIRTQUEUE_FLAG_INDIRECT 0x0001
64	#define VIRTQUEUE_FLAG_INDIRECT 0x0002	65	#define VIRTQUEUE_FLAG_EVENT_IDX 0x0002
65	#define VIRTQUEUE_FLAG_EVENT_IDX 0x0004
66	--
Line 67 Link Here
		67	int vq_alignment;
		68	int vq_ring_size;
		69	void *vq_ring_mem;
Line 68 Link Here
68	bus_size_t vq_notify_offset;	71	int vq_indirect_mem_size;
69	--
Lines 87-92 Link Here
87	void *vq_ring_mem;
88	int vq_indirect_mem_size;
89	int vq_alignment;
90	int vq_ring_size;
91	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
92
Lines 140-147 Link Here
140	#define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
141	#define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val)
142	#define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val)
143	#define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val)
144	#define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val)
145	#define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val)
146	#define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val)
147
Lines 161-163 Link Here
161	virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,	150	virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align,
162	bus_size_t notify_offset, int align, vm_paddr_t highaddr,	151	vm_paddr_t highaddr, struct vq_alloc_info info, struct virtqueue *vqp)
163	struct vq_alloc_info info, struct virtqueue *vqp)
164	--
Line 199 Link Here
199	vq->vq_notify_offset = notify_offset;
Lines 206-207 Link Here
206	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
207	vq->vq_flags \|= VIRTQUEUE_FLAG_MODERN;
Lines 312-313 Link Here
312	indirect[i].next = vq_gtoh16(vq, i + 1);	297	indirect[i].next = i + 1;
313	indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);	298	indirect[i].next = VQ_RING_DESC_CHAIN_END;
314	--
Line 411 Link Here
411
Line 460 Link Here
460	used_idx = vq_htog16(vq, vq->vq_ring.used->idx);	444	used_idx = vq->vq_ring.used->idx;
461	--
Line 472 Link Here
472	if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))	456	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
473	--
Line 499 Link Here
499	avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);	483	avail_idx = vq->vq_ring.avail->idx;
500	--
Lines 524-529 Link Here
524	vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,	508	vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx -
525	vq->vq_used_cons_idx - vq->vq_nentries - 1);	509	vq->vq_nentries - 1;
526	return;	510	} else
527	}	511	vq->vq_ring.avail->flags \|= VRING_AVAIL_F_NO_INTERRUPT;
528
529	vq->vq_ring.avail->flags \|= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
530	--
Line 592 Link Here
592	if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))	574	if (vq->vq_used_cons_idx == vq->vq_ring.used->idx)
593	--
Line 599 Link Here
599	desc_idx = (uint16_t) vq_htog32(vq, uep->id);	581	desc_idx = (uint16_t) uep->id;
600	--
Line 601 Link Here
601	*len = vq_htog32(vq, uep->len);	583	*len = uep->len;
602	--
Lines 659-665 Link Here
659	vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),	641	vq->vq_name, vq->vq_nentries, vq->vq_free_cnt,
660	vq->vq_queued_cnt, vq->vq_desc_head_idx,	642	virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx,
661	vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,	643	vq->vq_ring.avail->idx, vq->vq_used_cons_idx,
662	vq_htog16(vq, vq->vq_ring.used->idx),	644	vq->vq_ring.used->idx,
663	vq_htog16(vq, vring_used_event(&vq->vq_ring)),	645	vring_used_event(&vq->vq_ring),
664	vq_htog16(vq, vq->vq_ring.avail->flags),	646	vq->vq_ring.avail->flags,
665	vq_htog16(vq, vq->vq_ring.used->flags));	647	vq->vq_ring.used->flags);
666	--
Lines 682-683 Link Here
682	vr->desc[i].next = vq_gtoh16(vq, i + 1);	664	vr->desc[i].next = i + 1;
683	vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);	665	vr->desc[i].next = VQ_RING_DESC_CHAIN_END;
684	--
Line 689 Link Here
689	uint16_t avail_idx, avail_ring_idx;	671	uint16_t avail_idx;
690	--
Lines 698-700 Link Here
698	avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);	680	avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1);
699	avail_ring_idx = avail_idx & (vq->vq_nentries - 1);	681	vq->vq_ring.avail->ring[avail_idx] = desc_idx;
700	vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);
701	--
Line 703 Link Here
703	vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);	684	vq->vq_ring.avail->idx++;
704	--
Line 722 Link Here
722	i++, idx = vq_htog16(vq, dp->next), seg++) {	703	i++, idx = dp->next, seg++) {
723	--
Lines 727-728 Link Here
727	dp->addr = vq_gtoh64(vq, seg->ss_paddr);	708	dp->addr = seg->ss_paddr;
728	dp->len = vq_gtoh32(vq, seg->ss_len);	709	dp->len = seg->ss_len;
729	--
Line 732 Link Here
732	dp->flags \|= vq_gtoh16(vq, VRING_DESC_F_NEXT);	713	dp->flags \|= VRING_DESC_F_NEXT;
733	--
Line 734 Link Here
734	dp->flags \|= vq_gtoh16(vq, VRING_DESC_F_WRITE);	715	dp->flags \|= VRING_DESC_F_WRITE;
735	--
Lines 779-781 Link Here
779	dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);	760	dp->addr = dxp->indirect_paddr;
780	dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));	761	dp->len = needed * sizeof(struct vring_desc);
781	dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);	762	dp->flags = VRING_DESC_F_INDIRECT;
782	--
Line 786 Link Here
786	vq->vq_desc_head_idx = vq_htog16(vq, dp->next);	767	vq->vq_desc_head_idx = dp->next;
787	--
Lines 804-810 Link Here
804	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {	785	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX)
805	vring_used_event(&vq->vq_ring) =	786	vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc;
806	vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);	787	else
807	} else {	788	vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
808	vq->vq_ring.avail->flags &=
809	vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
810	}
811	--
Line 828 Link Here
828	uint16_t new_idx, prev_idx, event_idx, flags;	806	uint16_t new_idx, prev_idx, event_idx;
829	--
Line 831 Link Here
831	new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);	809	new_idx = vq->vq_ring.avail->idx;
832	--
Line 833 Link Here
833	event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));	811	event_idx = vring_avail_event(&vq->vq_ring);
834	--
Lines 838-839 Link Here
838	flags = vq->vq_ring.used->flags;	816	return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0);
839	return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
840	--
Lines 846-847 Link Here
846	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,	823	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index);
847	vq->vq_notify_offset);
848	--
Lines 866-870 Link Here
866	if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {	842	if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) {
867	while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {	843	while (dp->flags & VRING_DESC_F_NEXT) {
868	uint16_t next_idx = vq_htog16(vq, dp->next);	844	VQ_RING_ASSERT_VALID_IDX(vq, dp->next);
869	VQ_RING_ASSERT_VALID_IDX(vq, next_idx);	845	dp = &vq->vq_ring.desc[dp->next];
870	dp = &vq->vq_ring.desc[next_idx];
871	--
Line 883 Link Here
883	dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);	858	dp->next = vq->vq_desc_head_idx;
884	--

Line 28 Link Here

(-)/usr/src/sys/dev/virtio/virtqueue.h (-5 / +3 lines)
28	* $FreeBSD$	28	* $FreeBSD: releng/12.1/sys/dev/virtio/virtqueue.h 329602 2018-02-19 19:31:18Z bryanv $
29	--
Lines 73-74 Link Here
73	bus_size_t notify_offset, int align, vm_paddr_t highaddr,	73	int align, vm_paddr_t highaddr, struct vq_alloc_info *info,
74	struct vq_alloc_info info, struct virtqueue *vqp);	74	struct virtqueue **vqp);
75	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/Makefile 273515 2014-10-23 04:47:32Z bryanv $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/balloon/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/balloon/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/block/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/block/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/console/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/console/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/network/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/network/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/pci/Makefile (-6 / +3 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/pci/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--
Lines 29-31 Link Here
29	SRCS= virtio_pci.c virtio_pci_legacy.c virtio_pci_modern.c	29	SRCS= virtio_pci.c
30	SRCS+= virtio_pci_if.c virtio_pci_if.h	30	SRCS+= virtio_bus_if.h virtio_if.h
31	SRCS+= virtio_bus_if.h virtio_if.h
32	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/random/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/random/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/scsi/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/scsi/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Line 2 Link Here

(-)/usr/src/sys/modules/virtio/virtio/Makefile (-2 / +1 lines)
2	# $FreeBSD$	2	# $FreeBSD: releng/12.1/sys/modules/virtio/virtio/Makefile 314651 2017-03-04 10:10:17Z ngie $
3	--

Lines 47-48 Link Here

(-)/usr/src/sys/net/netmap_virt.h (-3 / +2 lines)
47	#define PTNETMAP_PCI_DEVICE_ID 0xcccc /* memory device */	47	#define PTNETMAP_PCI_DEVICE_ID 0x000c /* memory device */
48	#define PTNETMAP_PCI_NETIF_ID 0xcccd /* ptnet network interface */	48	#define PTNETMAP_PCI_NETIF_ID 0x000d /* ptnet network interface */
49	--

Line 450 Link Here

(-)/usr/src/sys/x86/include/specialreg.h (-1 lines)
450	#define IA32_ARCH_CAP_IF_PSCHANGE_MC_NO 0x00000040

Return to bug 236922

Lines 1792-1836 Link Here

(-)/usr/src/sys/amd64/amd64/pmap.c (-68 / +1 lines)
1792	static int pmap_allow_2m_x_ept;
1793	SYSCTL_INT(_vm_pmap, OID_AUTO, allow_2m_x_ept, CTLFLAG_RWTUN \| CTLFLAG_NOFETCH,
1794	&pmap_allow_2m_x_ept, 0,
1795	"Allow executable superpage mappings in EPT");
1796
1797	void
1798	pmap_allow_2m_x_ept_recalculate(void)
1799	{
1800	/*
1801	* SKL002, SKL012S. Since the EPT format is only used by
1802	* Intel CPUs, the vendor check is merely a formality.
1803	*/
1804	if (!(cpu_vendor_id != CPU_VENDOR_INTEL \|\|
1805	(cpu_ia32_arch_caps & IA32_ARCH_CAP_IF_PSCHANGE_MC_NO) != 0 \|\|
1806	(CPUID_TO_FAMILY(cpu_id) == 0x6 &&
1807	(CPUID_TO_MODEL(cpu_id) == 0x26 \|\| /* Atoms */
1808	CPUID_TO_MODEL(cpu_id) == 0x27 \|\|
1809	CPUID_TO_MODEL(cpu_id) == 0x35 \|\|
1810	CPUID_TO_MODEL(cpu_id) == 0x36 \|\|
1811	CPUID_TO_MODEL(cpu_id) == 0x37 \|\|
1812	CPUID_TO_MODEL(cpu_id) == 0x86 \|\|
1813	CPUID_TO_MODEL(cpu_id) == 0x1c \|\|
1814	CPUID_TO_MODEL(cpu_id) == 0x4a \|\|
1815	CPUID_TO_MODEL(cpu_id) == 0x4c \|\|
1816	CPUID_TO_MODEL(cpu_id) == 0x4d \|\|
1817	CPUID_TO_MODEL(cpu_id) == 0x5a \|\|
1818	CPUID_TO_MODEL(cpu_id) == 0x5c \|\|
1819	CPUID_TO_MODEL(cpu_id) == 0x5d \|\|
1820	CPUID_TO_MODEL(cpu_id) == 0x5f \|\|
1821	CPUID_TO_MODEL(cpu_id) == 0x6e \|\|
1822	CPUID_TO_MODEL(cpu_id) == 0x7a \|\|
1823	CPUID_TO_MODEL(cpu_id) == 0x57 \|\| /* Knights */
1824	CPUID_TO_MODEL(cpu_id) == 0x85))))
1825	pmap_allow_2m_x_ept = 1;
1826	TUNABLE_INT_FETCH("hw.allow_2m_x_ept", &pmap_allow_2m_x_ept);
1827	}
1828
1829	static bool
1830	pmap_allow_2m_x_page(pmap_t pmap, bool executable)
1831	{
1832
1833	return (pmap->pm_type != PT_EPT \|\| !executable \|\|
1834	!pmap_allow_2m_x_ept);
1835	}
1836
Lines 1881-1883 Link Here
1881	/* IFU */
1882	pmap_allow_2m_x_ept_recalculate();
1883
Lines 5453-5461 Link Here
5453	static bool
5454	pmap_pde_ept_executable(pmap_t pmap, pd_entry_t pde)
5455	{
5456
5457	if (pmap->pm_type != PT_EPT)
5458	return (false);
5459	return ((pde & EPT_PG_EXECUTE) != 0);
5460	}
5461
Lines 5497-5499 Link Here
5497	if ((newpde & ((PG_FRAME & PDRMASK) \| PG_A \| PG_V)) != (PG_A \| PG_V) \|\|	5440	if ((newpde & ((PG_FRAME & PDRMASK) \| PG_A \| PG_V)) != (PG_A \| PG_V)) {
5498	!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
5499	newpde))) {
5500	--
Lines 5929-5934 Link Here
5929	if (!pmap_allow_2m_x_page(pmap, pmap_pde_ept_executable(pmap,
5930	newpde))) {
5931	CTR2(KTR_PMAP, "pmap_enter_pde: 2m x blocked for va %#lx"
5932	" in pmap %p", va, pmap);
5933	return (KERN_FAILURE);
5934	}
Line 6081 Link Here
6081	pmap_allow_2m_x_page(pmap, (prot & VM_PROT_EXECUTE) != 0) &&

Line 2 Link Here

(-)/usr/src/sys/dev/virtio/pci/virtio_pci.h (-119 / +64 lines)
2	* SPDX-License-Identifier: BSD-2-Clause-FreeBSD	2	* SPDX-License-Identifier: BSD-3-Clause
3	--
Lines 4-5 Link Here
4	* Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>	4	* Copyright IBM Corp. 2007
5	* All rights reserved.
6	--
Line 7 Link Here
		6	* Authors:
		7	* Anthony Liguori <aliguori@us.ibm.com>
		8	*
		9	* This header is BSD licensed so anyone can use the definitions to implement
		10	* compatible drivers/servers.
		11	*
Lines 11-12 Link Here
11	* notice unmodified, this list of conditions, and the following	16	* notice, this list of conditions and the following disclaimer.
12	* disclaimer.
13	--
Line 16 Link Here
		20	* 3. Neither the name of IBM nor the names of its contributors
		21	* may be used to endorse or promote products derived from this software
		22	* without specific prior written permission.
		23	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
		24	* ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
		25	* TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
		26	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE
		27	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
		28	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
		29	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
		30	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
		31	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
		32	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
		33	* SUCH DAMAGE.
Lines 17-28 Link Here
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	35	* $FreeBSD: releng/12.1/sys/dev/virtio/pci/virtio_pci.h 326022 2017-11-20 19:36:21Z pfg $
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*
28	* $FreeBSD$
29	--
Lines 34-38 Link Here
34	struct vtpci_interrupt {	41	/* VirtIO PCI vendor/device ID. */
35	struct resource *vti_irq;	42	#define VIRTIO_PCI_VENDORID 0x1AF4
36	int vti_rid;	43	#define VIRTIO_PCI_DEVICEID_MIN 0x1000
37	void *vti_handler;	44	#define VIRTIO_PCI_DEVICEID_MAX 0x103F
38	};
39	--
Lines 40-44 Link Here
40	struct vtpci_virtqueue {	46	/* VirtIO ABI version, this must match exactly. */
41	struct virtqueue *vtv_vq;	47	#define VIRTIO_PCI_ABI_VERSION 0
42	int vtv_no_intr;
43	int vtv_notify_offset;
44	};
45	--
Lines 46-51 Link Here
46	struct vtpci_common {	49	/*
47	device_t vtpci_dev;	50	* VirtIO Header, located in BAR 0.
48	uint64_t vtpci_host_features;	51	*/
49	uint64_t vtpci_features;	52	#define VIRTIO_PCI_HOST_FEATURES 0 /* host's supported features (32bit, RO)*/
50	struct vtpci_virtqueue *vtpci_vqs;	53	#define VIRTIO_PCI_GUEST_FEATURES 4 /* guest's supported features (32, RW) */
51	int vtpci_nvqs;	54	#define VIRTIO_PCI_QUEUE_PFN 8 /* physical address of VQ (32, RW) */
52	--	55	#define VIRTIO_PCI_QUEUE_NUM 12 /* number of ring entries (16, RO) */
		56	#define VIRTIO_PCI_QUEUE_SEL 14 /* current VQ selection (16, RW) */
		57	#define VIRTIO_PCI_QUEUE_NOTIFY 16 /* notify host regarding VQ (16, RW) */
		58	#define VIRTIO_PCI_STATUS 18 /* device status register (8, RW) */
		59	#define VIRTIO_PCI_ISR 19 /* interrupt status register, reading
		60	* also clears the register (8, RO) */
		61	/* Only if MSIX is enabled: */
		62	#define VIRTIO_MSI_CONFIG_VECTOR 20 /* configuration change vector (16, RW) */
		63	#define VIRTIO_MSI_QUEUE_VECTOR 22 /* vector for selected VQ notifications
		64	(16, RW) */
Lines 53-61 Link Here
53	uint32_t vtpci_flags;	66	/* The bit of the ISR which indicates a device has an interrupt. */
54	#define VTPCI_FLAG_NO_MSI 0x0001	67	#define VIRTIO_PCI_ISR_INTR 0x1
55	#define VTPCI_FLAG_NO_MSIX 0x0002	68	/* The bit of the ISR which indicates a device configuration change. */
56	#define VTPCI_FLAG_MODERN 0x0004	69	#define VIRTIO_PCI_ISR_CONFIG 0x2
57	#define VTPCI_FLAG_INTX 0x1000	70	/* Vector value used to disable MSI for queue. */
58	#define VTPCI_FLAG_MSI 0x2000	71	#define VIRTIO_MSI_NO_VECTOR 0xFFFF
59	#define VTPCI_FLAG_MSIX 0x4000
60	#define VTPCI_FLAG_SHARED_MSIX 0x8000
61	#define VTPCI_FLAG_ITYPE_MASK 0xF000
62	--
Lines 63-65 Link Here
63	/* The VirtIO PCI "bus" will only ever have one child. */	73	/*
64	device_t vtpci_child_dev;	74	* The remaining space is defined by each driver as the per-driver
65	struct virtio_feature_desc *vtpci_child_feat_desc;	75	* configuration space.
66	--	76	*/
		77	#define VIRTIO_PCI_CONFIG_OFF(msix_enabled) ((msix_enabled) ? 24 : 20)
Lines 67-80 Link Here
67	/*	79	/*
68	* Ideally, each virtqueue that the driver provides a callback for will	80	* How many bits to shift physical queue address written to QUEUE_PFN.
69	* receive its own MSIX vector. If there are not sufficient vectors	81	* 12 is historical, and due to x86 page size.
70	* available, then attempt to have all the VQs share one vector. For	82	*/
71	* MSIX, the configuration changed notifications must be on their own	83	#define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12
72	* vector.
73	*
74	* If MSIX is not available, attempt to have the whole device share
75	* one MSI vector, and then, finally, one intx interrupt.
76	*/
77	struct vtpci_interrupt vtpci_device_interrupt;
78	struct vtpci_interrupt *vtpci_msix_vq_interrupts;
79	int vtpci_nmsix_resources;
80	};
81	--
Lines 82-130 Link Here
82	extern int vtpci_disable_msix;	85	/* The alignment to use between consumer and producer parts of vring. */
83		86	#define VIRTIO_PCI_VRING_ALIGN 4096
84	static inline device_t
85	vtpci_child_device(struct vtpci_common *cn)
86	{
87	return (cn->vtpci_child_dev);
88	}
89
90	static inline bool
91	vtpci_is_msix_available(struct vtpci_common *cn)
92	{
93	return ((cn->vtpci_flags & VTPCI_FLAG_NO_MSIX) == 0);
94	}
95
96	static inline bool
97	vtpci_is_msix_enabled(struct vtpci_common *cn)
98	{
99	return ((cn->vtpci_flags & VTPCI_FLAG_MSIX) != 0);
100	}
101
102	static inline bool
103	vtpci_is_modern(struct vtpci_common *cn)
104	{
105	return ((cn->vtpci_flags & VTPCI_FLAG_MODERN) != 0);
106	}
107
108	static inline int
109	vtpci_virtqueue_count(struct vtpci_common *cn)
110	{
111	return (cn->vtpci_nvqs);
112	}
113
114	void vtpci_init(struct vtpci_common *cn, device_t dev, bool modern);
115	int vtpci_add_child(struct vtpci_common *cn);
116	int vtpci_delete_child(struct vtpci_common *cn);
117	void vtpci_child_detached(struct vtpci_common *cn);
118	int vtpci_reinit(struct vtpci_common *cn);
119
120	uint64_t vtpci_negotiate_features(struct vtpci_common *cn,
121	uint64_t child_features, uint64_t host_features);
122	int vtpci_with_feature(struct vtpci_common *cn, uint64_t feature);
123
124	int vtpci_read_ivar(struct vtpci_common cn, int index, uintptr_t result);
125	int vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value);
126
127	int vtpci_alloc_virtqueues(struct vtpci_common *cn, int flags, int nvqs,
128	struct vq_alloc_info *vq_info);
129	int vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type);
130	void vtpci_release_child_resources(struct vtpci_common *cn);
131	--