Attachment #173190 for bug #211525

View | Details | Raw Unified | Return to bug 211525
Collapse All | Expand All




#define VCHIQ_MINOR 0

/* Some per-instance constants */
#define MAX_COMPLETIONS 128
#define MAX_SERVICES 64
#define MAX_ELEMENTS 8
#define MSG_QUEUE_SIZE 128

#define KEEPALIVE_VER 1
#define KEEPALIVE_VER_MIN KEEPALIVE_VER

	void *bulk_userdata)
{
	VCHIQ_COMPLETION_DATA_T *completion;
	int insert;
	DEBUG_INITIALISE(g_state.local)

	insert = instance->completion_insert;
	while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) {
		/* Out of space - wait for the client */
		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
		vchiq_log_trace(vchiq_arm_log_level,
			"add_completion - completion queue full");
		DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);

		if (down_interruptible(&instance->remove_event) != 0) {
			vchiq_log_info(vchiq_arm_log_level,
				"service_callback interrupted");
			return VCHIQ_RETRY;
		}

		if (instance->closing) {
			vchiq_log_info(vchiq_arm_log_level,
				"service_callback closing");
			return VCHIQ_SUCCESS;
		}
		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
	}

	completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)];
		 &instance->completions[instance->completion_insert &
		 (MAX_COMPLETIONS - 1)];

	completion->header = header;
	completion->reason = reason;

	wmb();

	if (reason == VCHIQ_MESSAGE_AVAILABLE)
		user_service->message_available_pos = insert;

	instance->completion_insert = ++insert;

	up(&instance->insert_event);


	USER_SERVICE_T *user_service;
	VCHIQ_SERVICE_T *service;
	VCHIQ_INSTANCE_T instance;
	int skip_completion = 0;
	DEBUG_INITIALISE(g_state.local)

	DEBUG_TRACE(SERVICE_CALLBACK_LINE);

		user_service->msg_queue[user_service->msg_insert &
			(MSG_QUEUE_SIZE - 1)] = header;
		user_service->msg_insert++;
		spin_unlock(&msg_queue_spinlock);

		up(&user_service->insert_event);

		/* If there is a thread waiting in DEQUEUE_MESSAGE, or if
		** there is a MESSAGE_AVAILABLE in the completion queue then

		if (((user_service->message_available_pos -
			instance->completion_remove) >= 0) ||
			user_service->dequeue_pending) {
			DEBUG_TRACE(SERVICE_CALLBACK_LINE);
			user_service->dequeue_pending = 0;
			skip_completion = 1;
		}

		spin_unlock(&msg_queue_spinlock);

		up(&user_service->insert_event);

		header = NULL;
	}

	if (skip_completion) {
		DEBUG_TRACE(SERVICE_CALLBACK_LINE);
		return VCHIQ_SUCCESS;
	}

	DEBUG_TRACE(SERVICE_CALLBACK_LINE);

	return add_completion(instance, reason, header, user_service,

			break;
		}

                memcpy(&args, (const void*)arg, sizeof(args));

		lmutex_lock(&instance->completion_mutex);


		while ((instance->completion_remove ==
			instance->completion_insert)
			&& !instance->closing) {

			DEBUG_TRACE(AWAIT_COMPLETION_LINE);
			lmutex_unlock(&instance->completion_mutex);
			rc = down_interruptible(&instance->insert_event);

		}
		DEBUG_TRACE(AWAIT_COMPLETION_LINE);

		/* A read memory barrier is needed to stop prefetch of a stale
		** completion record
		*/
		rmb();

		if (ret == 0) {
			int msgbufcount = args.msgbufcount;
			int remove;

			remove = instance->completion_remove;

			for (count = 0; count < args.count; count++) {
				VCHIQ_COMPLETION_DATA_T *completion;
				VCHIQ_SERVICE_T *service1;
				USER_SERVICE_T *user_service;
				VCHIQ_HEADER_T *header;

				if (remove == instance->completion_insert)
					break;

				completion = &instance->completions[
					remove & (MAX_COMPLETIONS - 1)];


				/* A read memory barrier is needed to prevent
				** the prefetch of a stale completion record
				*/
				rmb();

				service1 = completion->service_userdata;
				user_service = service1->base.userdata;

					break;
				}

				/* Ensure that the above copy has completed
				** before advancing the remove pointer. */
				mb();

				instance->completion_remove = ++remove;
			}

			if (msgbufcount != args.msgbufcount) {

Lines 622-636 process_free_queue(VCHIQ_STATE_T *state) Link Here

(-)b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_core.c (-23 / +22 lines)
622	BITSET_T service_found[BITSET_SIZE(VCHIQ_MAX_SERVICES)];	622	BITSET_T service_found[BITSET_SIZE(VCHIQ_MAX_SERVICES)];
623	int slot_queue_available;	623	int slot_queue_available;
624		624
625	/* Use a read memory barrier to ensure that any state that may have
626	** been modified by another thread is not masked by stale prefetched
627	** values. */
628	rmb();
629
630	/* Find slots which have been freed by the other side, and return them	625	/* Find slots which have been freed by the other side, and return them
631	** to the available queue. */	626	** to the available queue. */
632	slot_queue_available = state->slot_queue_available;	627	slot_queue_available = state->slot_queue_available;
633		628
		629	/* Use a memory barrier to ensure that any state that may have been
		630	** modified by another thread is not masked by stale prefetched
		631	** values. */
		632	mb();
		633
634	while (slot_queue_available != local->slot_queue_recycle) {	634	while (slot_queue_available != local->slot_queue_recycle) {
635	unsigned int pos;	635	unsigned int pos;
636	int slot_index = local->slot_queue[slot_queue_available++ &	636	int slot_index = local->slot_queue[slot_queue_available++ &
Lines 638-643 process_free_queue(VCHIQ_STATE_T *state) Link Here
638	char data = (char )SLOT_DATA_FROM_INDEX(state, slot_index);	638	char data = (char )SLOT_DATA_FROM_INDEX(state, slot_index);
639	int data_found = 0;	639	int data_found = 0;
640		640
		641	rmb();
		642
641	vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%x %x %x",	643	vchiq_log_trace(vchiq_core_log_level, "%d: pfq %d=%x %x %x",
642	state->id, slot_index, (unsigned int)data,	644	state->id, slot_index, (unsigned int)data,
643	local->slot_queue_recycle, slot_queue_available);	645	local->slot_queue_recycle, slot_queue_available);
Lines 753-758 process_free_queue(VCHIQ_STATE_T *state) Link Here
753	up(&state->data_quota_event);	755	up(&state->data_quota_event);
754	}	756	}
755		757
		758	mb();
		759
756	state->slot_queue_available = slot_queue_available;	760	state->slot_queue_available = slot_queue_available;
757	up(&state->slot_available_event);	761	up(&state->slot_available_event);
758	}	762	}
Lines 904-919 queue_message(VCHIQ_STATE_T state, VCHIQ_SERVICE_T service, Link Here
904	error_count);	908	error_count);
905	return VCHIQ_ERROR;	909	return VCHIQ_ERROR;
906	}	910	}
907	if (i == 0) {
908	if (SRVTRACE_ENABLED(service,
909	VCHIQ_LOG_INFO))
910	vchiq_log_dump_mem("Sent", 0,
911	header->data + pos,
912	min(64u,
913	elements[0].size));
914	}
915	}	911	}
916		912
		913	if (SRVTRACE_ENABLED(service,
		914	VCHIQ_LOG_INFO))
		915	vchiq_log_dump_mem("Sent", 0,
		916	header->data,
		917	min(16, pos));
		918
917	spin_lock(&quota_spinlock);	919	spin_lock(&quota_spinlock);
918	service_quota->message_use_count++;	920	service_quota->message_use_count++;
919		921
Lines 1052-1067 queue_message_sync(VCHIQ_STATE_T state, VCHIQ_SERVICE_T service, Link Here
1052	error_count);	1054	error_count);
1053	return VCHIQ_ERROR;	1055	return VCHIQ_ERROR;
1054	}	1056	}
1055	if (i == 0) {
1056	if (vchiq_sync_log_level >=
1057	VCHIQ_LOG_TRACE)
1058	vchiq_log_dump_mem("Sent Sync",
1059	0, header->data + pos,
1060	min(64u,
1061	elements[0].size));
1062	}
1063	}	1057	}
1064		1058
		1059	if (vchiq_sync_log_level >= VCHIQ_LOG_TRACE)
		1060	vchiq_log_dump_mem("Sent Sync",
		1061	0, header->data,
		1062	min(16, pos));
		1063
1065	VCHIQ_SERVICE_STATS_INC(service, ctrl_tx_count);	1064	VCHIQ_SERVICE_STATS_INC(service, ctrl_tx_count);
1066	VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);	1065	VCHIQ_SERVICE_STATS_ADD(service, ctrl_tx_bytes, size);
1067	} else {	1066	} else {
Lines 1732-1738 parse_rx_slots(VCHIQ_STATE_T *state) Link Here
1732	remoteport, localport, size);	1731	remoteport, localport, size);
1733	if (size > 0)	1732	if (size > 0)
1734	vchiq_log_dump_mem("Rcvd", 0, header->data,	1733	vchiq_log_dump_mem("Rcvd", 0, header->data,
1735	min(64, size));	1734	min(16, size));
1736	}	1735	}
1737		1736
1738	if (((unsigned int)header & VCHIQ_SLOT_MASK) + calc_stride(size)	1737	if (((unsigned int)header & VCHIQ_SLOT_MASK) + calc_stride(size)
Lines 2202-2208 sync_func(void *v) Link Here
2202	remoteport, localport, size);	2201	remoteport, localport, size);
2203	if (size > 0)	2202	if (size > 0)
2204	vchiq_log_dump_mem("Rcvd", 0, header->data,	2203	vchiq_log_dump_mem("Rcvd", 0, header->data,
2205	min(64, size));	2204	min(16, size));
2206	}	2205	}
2207		2206
2208	switch (type) {	2207	switch (type) {

Return to bug 211525

Lines 46-55 Link Here

(-)b/sys/contrib/vchiq/interface/vchiq_arm/vchiq_arm.c (-28 / +47 lines)
46	#define VCHIQ_MINOR 0	46	#define VCHIQ_MINOR 0
47		47
48	/* Some per-instance constants */	48	/* Some per-instance constants */
49	#define MAX_COMPLETIONS 16	49	#define MAX_COMPLETIONS 128
50	#define MAX_SERVICES 64	50	#define MAX_SERVICES 64
51	#define MAX_ELEMENTS 8	51	#define MAX_ELEMENTS 8
52	#define MSG_QUEUE_SIZE 64	52	#define MSG_QUEUE_SIZE 128
53		53
54	#define KEEPALIVE_VER 1	54	#define KEEPALIVE_VER 1
55	#define KEEPALIVE_VER_MIN KEEPALIVE_VER	55	#define KEEPALIVE_VER_MIN KEEPALIVE_VER
Lines 208-237 add_completion(VCHIQ_INSTANCE_T instance, VCHIQ_REASON_T reason, Link Here
208	void *bulk_userdata)	208	void *bulk_userdata)
209	{	209	{
210	VCHIQ_COMPLETION_DATA_T *completion;	210	VCHIQ_COMPLETION_DATA_T *completion;
		211	int insert;
211	DEBUG_INITIALISE(g_state.local)	212	DEBUG_INITIALISE(g_state.local)
212		213
213	while (instance->completion_insert ==	214	insert = instance->completion_insert;
214	(instance->completion_remove + MAX_COMPLETIONS)) {	215	while ((insert - instance->completion_remove) >= MAX_COMPLETIONS) {
215	/* Out of space - wait for the client */	216	/* Out of space - wait for the client */
216	DEBUG_TRACE(SERVICE_CALLBACK_LINE);	217	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
217	vchiq_log_trace(vchiq_arm_log_level,	218	vchiq_log_trace(vchiq_arm_log_level,
218	"add_completion - completion queue full");	219	"add_completion - completion queue full");
219	DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);	220	DEBUG_COUNT(COMPLETION_QUEUE_FULL_COUNT);
		221
220	if (down_interruptible(&instance->remove_event) != 0) {	222	if (down_interruptible(&instance->remove_event) != 0) {
221	vchiq_log_info(vchiq_arm_log_level,	223	vchiq_log_info(vchiq_arm_log_level,
222	"service_callback interrupted");	224	"service_callback interrupted");
223	return VCHIQ_RETRY;	225	return VCHIQ_RETRY;
224	} else if (instance->closing) {	226	}
		227
		228	if (instance->closing) {
225	vchiq_log_info(vchiq_arm_log_level,	229	vchiq_log_info(vchiq_arm_log_level,
226	"service_callback closing");	230	"service_callback closing");
227	return VCHIQ_ERROR;	231	return VCHIQ_SUCCESS;
228	}	232	}
229	DEBUG_TRACE(SERVICE_CALLBACK_LINE);	233	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
230	}	234	}
231		235
232	completion =	236	completion = &instance->completions[insert & (MAX_COMPLETIONS - 1)];
233	&instance->completions[instance->completion_insert &
234	(MAX_COMPLETIONS - 1)];
235		237
236	completion->header = header;	238	completion->header = header;
237	completion->reason = reason;	239	completion->reason = reason;
Lines 252-260 add_completion(VCHIQ_INSTANCE_T instance, VCHIQ_REASON_T reason, Link Here
252	wmb();	254	wmb();
253		255
254	if (reason == VCHIQ_MESSAGE_AVAILABLE)	256	if (reason == VCHIQ_MESSAGE_AVAILABLE)
255	user_service->message_available_pos =	257	user_service->message_available_pos = insert;
256	instance->completion_insert;	258
257	instance->completion_insert++;	259	instance->completion_insert = ++insert;
258		260
259	up(&instance->insert_event);	261	up(&instance->insert_event);
260		262
Lines 279-284 service_callback(VCHIQ_REASON_T reason, VCHIQ_HEADER_T *header, Link Here
279	USER_SERVICE_T *user_service;	281	USER_SERVICE_T *user_service;
280	VCHIQ_SERVICE_T *service;	282	VCHIQ_SERVICE_T *service;
281	VCHIQ_INSTANCE_T instance;	283	VCHIQ_INSTANCE_T instance;
		284	int skip_completion = 0;
282	DEBUG_INITIALISE(g_state.local)	285	DEBUG_INITIALISE(g_state.local)
283		286
284	DEBUG_TRACE(SERVICE_CALLBACK_LINE);	287	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
Lines 345-353 service_callback(VCHIQ_REASON_T reason, VCHIQ_HEADER_T *header, Link Here
345	user_service->msg_queue[user_service->msg_insert &	348	user_service->msg_queue[user_service->msg_insert &
346	(MSG_QUEUE_SIZE - 1)] = header;	349	(MSG_QUEUE_SIZE - 1)] = header;
347	user_service->msg_insert++;	350	user_service->msg_insert++;
348	spin_unlock(&msg_queue_spinlock);
349
350	up(&user_service->insert_event);
351		351
352	/* If there is a thread waiting in DEQUEUE_MESSAGE, or if	352	/* If there is a thread waiting in DEQUEUE_MESSAGE, or if
353	** there is a MESSAGE_AVAILABLE in the completion queue then	353	** there is a MESSAGE_AVAILABLE in the completion queue then
Lines 356-368 service_callback(VCHIQ_REASON_T reason, VCHIQ_HEADER_T *header, Link Here
356	if (((user_service->message_available_pos -	356	if (((user_service->message_available_pos -
357	instance->completion_remove) >= 0) \|\|	357	instance->completion_remove) >= 0) \|\|
358	user_service->dequeue_pending) {	358	user_service->dequeue_pending) {
359	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
360	user_service->dequeue_pending = 0;	359	user_service->dequeue_pending = 0;
361	return VCHIQ_SUCCESS;	360	skip_completion = 1;
362	}	361	}
363		362
		363	spin_unlock(&msg_queue_spinlock);
		364
		365	up(&user_service->insert_event);
		366
364	header = NULL;	367	header = NULL;
365	}	368	}
		369
		370	if (skip_completion) {
		371	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
		372	return VCHIQ_SUCCESS;
		373	}
		374
366	DEBUG_TRACE(SERVICE_CALLBACK_LINE);	375	DEBUG_TRACE(SERVICE_CALLBACK_LINE);
367		376
368	return add_completion(instance, reason, header, user_service,	377	return add_completion(instance, reason, header, user_service,
Lines 789-795 vchiq_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, Link Here
789	break;	798	break;
790	}	799	}
791		800
792	memcpy(&args, (const void*)arg, sizeof(args));	801	memcpy(&args, (const void*)arg, sizeof(args));
793		802
794	lmutex_lock(&instance->completion_mutex);	803	lmutex_lock(&instance->completion_mutex);
795		804
Lines 797-802 vchiq_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, Link Here
797	while ((instance->completion_remove ==	806	while ((instance->completion_remove ==
798	instance->completion_insert)	807	instance->completion_insert)
799	&& !instance->closing) {	808	&& !instance->closing) {
		809
800	DEBUG_TRACE(AWAIT_COMPLETION_LINE);	810	DEBUG_TRACE(AWAIT_COMPLETION_LINE);
801	lmutex_unlock(&instance->completion_mutex);	811	lmutex_unlock(&instance->completion_mutex);
802	rc = down_interruptible(&instance->insert_event);	812	rc = down_interruptible(&instance->insert_event);
Lines 811-834 vchiq_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, Link Here
811	}	821	}
812	DEBUG_TRACE(AWAIT_COMPLETION_LINE);	822	DEBUG_TRACE(AWAIT_COMPLETION_LINE);
813		823
814	/* A read memory barrier is needed to stop prefetch of a stale
815	** completion record
816	*/
817	rmb();
818
819	if (ret == 0) {	824	if (ret == 0) {
820	int msgbufcount = args.msgbufcount;	825	int msgbufcount = args.msgbufcount;
		826	int remove;
		827
		828	remove = instance->completion_remove;
		829
821	for (count = 0; count < args.count; count++) {	830	for (count = 0; count < args.count; count++) {
822	VCHIQ_COMPLETION_DATA_T *completion;	831	VCHIQ_COMPLETION_DATA_T *completion;
823	VCHIQ_SERVICE_T *service1;	832	VCHIQ_SERVICE_T *service1;
824	USER_SERVICE_T *user_service;	833	USER_SERVICE_T *user_service;
825	VCHIQ_HEADER_T *header;	834	VCHIQ_HEADER_T *header;
826	if (instance->completion_remove ==	835
827	instance->completion_insert)	836	if (remove == instance->completion_insert)
828	break;	837	break;
		838
829	completion = &instance->completions[	839	completion = &instance->completions[
830	instance->completion_remove &	840	remove & (MAX_COMPLETIONS - 1)];
831	(MAX_COMPLETIONS - 1)];	841
		842
		843	/* A read memory barrier is needed to prevent
		844	** the prefetch of a stale completion record
		845	*/
		846	rmb();
832		847
833	service1 = completion->service_userdata;	848	service1 = completion->service_userdata;
834	user_service = service1->base.userdata;	849	user_service = service1->base.userdata;
Lines 905-911 vchiq_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int fflag, Link Here
905	break;	920	break;
906	}	921	}
907		922
908	instance->completion_remove++;	923	/* Ensure that the above copy has completed
		924	** before advancing the remove pointer. */
		925	mb();
		926
		927	instance->completion_remove = ++remove;
909	}	928	}
910		929
911	if (msgbufcount != args.msgbufcount) {	930	if (msgbufcount != args.msgbufcount) {