The manpage of pthread_mutex_lock mentions EINVAL and EDEADLK as possible error codes, but not EBUSY.  However, threads not owning the mutex will get EBUSY when compiled with -lpthread:

from src/lib/libpthread/thread/thr_mutex.c:

        /* case PTHREAD_MUTEX_DEFAULT: */
        case PTHREAD_MUTEX_ERRORCHECK:
        case PTHREAD_MUTEX_NORMAL:
                /*
                 * POSIX specifies that mutexes should return EDEADLK if a 
                 * recursive lock is detected.
                 */
                if (m->m_owner == curthread)
                        ret = EDEADLK;
                else
                        ret = EBUSY;
                break;

Fix: Something like this:
How-To-Repeat: Compile and execute the following program:

#include <sys/types.h>

#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>

pthread_mutex_t mutex;

void
trylock(const char* tname)
{
    int rc;

    rc = pthread_mutex_trylock(&mutex);
    if (rc != 0) 
        fprintf(stderr, "%s: Could not lock mutex again: %s\n", 
                tname, strerror(rc));
    else {
        fprintf(stderr, "Could lock mutex again!");
        exit(1);
    }
}

void*
start(void* data)
{
    /* try to lock the mutex again */
    trylock("Second thread");
}

int
main(int argc, const char* argv[])
{
    pthread_t       thread;
    int             rc;

    /* create a mutex and acquire a lock on it for the main thread */
    if (pthread_mutex_init(&mutex, NULL) != 0) {
        perror("Could not initialize mutex");
        exit(1);
    }
    if (pthread_mutex_lock(&mutex) != 0) {
        perror("Could not lock mutex");
        exit(1);
    }

    /* spawn a second thread */
    if (pthread_create(&thread, NULL, start, NULL) != 0) {
        perror("Could not create thread");
        exit(1);
    }
    
    /* try to lock the mutex again */
    trylock("Main thread");

    if (pthread_join(thread, NULL) != 0)  {
        perror("Could not join threads");
        exit(1);
    }

    exit(0);
}