diff --git i/sys/cam/cam_xpt.c w/sys/cam/cam_xpt.c
index fb5d041..cda56e7 100644
--- i/sys/cam/cam_xpt.c
+++ w/sys/cam/cam_xpt.c
@@ -2901,7 +2901,24 @@ call_sim:
 				 */
 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
 				callout_stop(&dev->callout);
+				/*
+				 * If CAM_DEV_REL_TIMEOUT_PENDING is still
+				 * set, then the callout is still
+				 * cancelable, since we hold the callout
+				 * mutex and the callout function clears the
+				 * flag before it ever unlocks/relocks the
+				 * mutex.  Since we already acquired a
+				 * device ref for the callout we just
+				 * stopped, we don't need another ref.
+				 */
 			} else {
+				/*
+				 * Since the callout gets a pointer to the
+				 * device, acquire a reference to make sure
+				 * the device can't go away until the
+				 * callout finishes or is stopped.
+				 */
+				xpt_acquire_device(dev);
 
 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
 			}
@@ -4355,7 +4372,20 @@ xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
 		 * to release this queue.
 		 */
 		if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
-			callout_stop(&dev->callout);
+			/*
+			 * If CAM_DEV_REL_TIMEOUT_PENDING is still set,
+			 * either the callout is still cancelable, or the
+			 * callout function (xpt_release_devq_timeout) is
+			 * our caller.  It's too early for the callout
+			 * itself to release its ref, so only release the
+			 * device ref if we stop the callout.  If we
+			 * can't stop the callout, it will release
+			 * its ref when it finishes.
+			 */
+			if (callout_stop(&dev->callout) != 0) {
+				/* Release the callout's device reference */
+				xpt_release_device(dev);
+			}
 			dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
 		}
 		/*
@@ -4646,6 +4676,38 @@ static void
 xpt_destroy_device(void *context, int pending)
 {
 	struct cam_ed	*device = context;
+	struct cam_eb *bus = device->target->bus;
+	struct cam_devq *devq;
+
+	callout_drain(&device->callout);
+
+	/* Release our slot in the devq */
+	devq = bus->sim->devq;
+	mtx_lock(&devq->send_mtx);
+	KASSERT((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) == 0,
+	    ("device %p has timeout pending", device));
+	cam_devq_resize(devq, devq->send_queue.array_size - 1);
+	mtx_unlock(&devq->send_mtx);
+
+	KASSERT(SLIST_EMPTY(&device->periphs),
+	    ("destroying device, but periphs list is not empty"));
+	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
+	    ("destroying device while still queued for ccbs"));
+
+	xpt_release_target(device->target);
+
+	cam_ccbq_fini(&device->ccbq);
+	/*
+	 * Free allocated memory.  free(9) does nothing if the
+	 * supplied pointer is NULL, so it is safe to call without
+	 * checking.
+	 */
+	free(device->supported_vpds, M_CAMXPT);
+	free(device->device_id, M_CAMXPT);
+	free(device->ext_inq, M_CAMXPT);
+	free(device->physpath, M_CAMXPT);
+	free(device->rcap_buf, M_CAMXPT);
+	free(device->serial_num, M_CAMXPT);
 
 	mtx_lock(&device->device_mtx);
 	mtx_destroy(&device->device_mtx);
@@ -4723,7 +4785,6 @@ void
 xpt_release_device(struct cam_ed *device)
 {
 	struct cam_eb *bus = device->target->bus;
-	struct cam_devq *devq;
 
 	mtx_lock(&bus->eb_mtx);
 	if (--device->refcount > 0) {
@@ -4735,34 +4796,6 @@ xpt_release_device(struct cam_ed *device)
 	device->target->generation++;
 	mtx_unlock(&bus->eb_mtx);
 
-	/* Release our slot in the devq */
-	devq = bus->sim->devq;
-	mtx_lock(&devq->send_mtx);
-	cam_devq_resize(devq, devq->send_queue.array_size - 1);
-	mtx_unlock(&devq->send_mtx);
-
-	KASSERT(SLIST_EMPTY(&device->periphs),
-	    ("destroying device, but periphs list is not empty"));
-	KASSERT(device->devq_entry.index == CAM_UNQUEUED_INDEX,
-	    ("destroying device while still queued for ccbs"));
-
-	if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
-		callout_stop(&device->callout);
-
-	xpt_release_target(device->target);
-
-	cam_ccbq_fini(&device->ccbq);
-	/*
-	 * Free allocated memory.  free(9) does nothing if the
-	 * supplied pointer is NULL, so it is safe to call without
-	 * checking.
-	 */
-	free(device->supported_vpds, M_CAMXPT);
-	free(device->device_id, M_CAMXPT);
-	free(device->ext_inq, M_CAMXPT);
-	free(device->physpath, M_CAMXPT);
-	free(device->rcap_buf, M_CAMXPT);
-	free(device->serial_num, M_CAMXPT);
 	taskqueue_enqueue(xsoftc.xpt_taskq, &device->device_destroy_task);
 }