# Created by: Michal Bielicki <m.bielicki@llizardfs.com>

# $FreeBSD$

PORTNAME=	isa-l_crypto

PORTVERSION=	2.21.0

DISTVERSIONPREFIX=	v

CATEGORIES=	security

PKGNAMEPREFIX=	ktls_

PKGNAMESUFFIX=	-kmod

MAINTAINER=	gallatin@FreeBSD.org

COMMENT=	KTLS module using Intel(R) ISA-L crypto

LICENSE=	BSD3CLAUSE

LICENSE_FILE=	${WRKSRC}/LICENSE

ONLY_FOR_ARCHS=	amd64

BUILD_DEPENDS=	yasm:devel/yasm

USES=		kmod uidfix

USE_GITHUB=	yes

GH_ACCOUNT=	01org

MAKE_ENV+=	ISASRC=${WRKSRC}

PLIST_FILES=	${KMODDIR}/ktls_intel-isa-l.ko

.include <bsd.port.pre.mk>

.if !exists(${SRC_BASE}/sys/sys/ktls.h)

IGNORE=		requires KTLS support

.endif

post-extract:

	${CP} ${FILESDIR}/* ${WRKSRC}/

	${ECHO} "#include <sys/stdint.h>" > ${WRKSRC}/include/stdint.h

.include <bsd.port.post.mk>

TIMESTAMP = 1566856170

SHA256 (01org-isa-l_crypto-v2.21.0_GH0.tar.gz) = 58284d7e5e60c37d6d4c1bbc2a876e95d14ca153443775a491f91c21a4243171

SIZE (01org-isa-l_crypto-v2.21.0_GH0.tar.gz) = 502413

#

# Copyright (c) 2016-2019 Netflix, Inc

#

# Redistribution and use in source and binary forms, with or without

# modification, are permitted provided that the following conditions

# are met:

# 1. Redistributions of source code must retain the above copyright

#    notice, this list of conditions and the following disclaimer,

#    without modification.

# 2. Redistributions in binary form must reproduce at minimum a disclaimer

#    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any

#    redistribution must be conditioned upon including a substantially

#    similar Disclaimer requirement for further binary redistribution.

#

# NO WARRANTY

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS

# ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT

# LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY

# AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL

# THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,

# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF

# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS

# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER

# IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)

# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF

# THE POSSIBILITY OF SUCH DAMAGES.

#

# $FreeBSD$

#

ISAINC=${ISASRC}/include

ISAAES=${ISASRC}/aes

INCS=-I${ISAINC} -I${ISAAES}

CFLAGS+=${INCS}

YASM=${LOCALBASE}/bin/yasm

.PATH: ${ISASRC}/aes

.SUFFIXES: .asm

.asm.o:

	${YASM} -g dwarf2 -f elf64 ${INCS} -o ${.TARGET} ${.IMPSRC}

KMOD=	ktls_intel-isa-l

SRCS=	gcm128_avx_gen2.asm \

	gcm128_avx_gen4.asm \

	gcm128_sse.asm \

	gcm256_avx_gen2.asm \

	gcm256_avx_gen4.asm \

	gcm256_sse.asm \

	gcm_multibinary.asm \

	gcm128_avx_gen2_nt.asm \

	gcm128_avx_gen4_nt.asm \

	gcm128_sse_nt.asm \

	gcm256_avx_gen2_nt.asm \

	gcm256_avx_gen4_nt.asm \

	gcm256_sse_nt.asm \

	gcm_multibinary_nt.asm \

	keyexp_128.asm \

	keyexp_192.asm \

	keyexp_256.asm \

	keyexp_multibinary.asm \

	gcm_pre.c \

	intelisa_kern.c

.include <bsd.kmod.mk>

/*-

 * SPDX-License-Identifier: BSD-2-Clause

 *

 * Copyright (c) 2014-2018  Netflix Inc.

 * All rights reserved.

 *

 * Redistribution and use in source and binary forms, with or without

 * modification, are permitted provided that the following conditions

 * are met:

 * 1. Redistributions of source code must retain the above copyright

 *    notice, this list of conditions and the following disclaimer.

 * 2. Redistributions in binary form must reproduce the above copyright

 *    notice, this list of conditions and the following disclaimer in the

 *    documentation and/or other materials provided with the distribution.

 *

 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND

 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE

 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE

 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE

 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL

 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS

 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)

 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY

 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF

 * SUCH DAMAGE.

 *

 *

 */

#include <sys/cdefs.h>

__FBSDID("$FreeBSD$");

#include <sys/types.h>

#include <sys/param.h>

#include <sys/kernel.h>

#include <sys/ktls.h>

#include <sys/lock.h>

#include <sys/malloc.h>

#include <sys/mutex.h>

#include <sys/proc.h>

#include <sys/sockbuf.h>

#include <sys/filedesc.h>

#include <sys/sysctl.h>

#include <sys/counter.h>

#include <sys/uio.h>

#include <sys/module.h>

#include <opencrypto/xform.h>

#include <machine/fpu.h>

#include "aes_gcm.h"

#define KTLS_INTELISA_AEAD_TAGLEN	16

struct isa_gcm_struct {

	struct gcm_key_data key_data;

	struct gcm_context_data ctx_data;

	void (*gcm_pre) (const void *key, struct gcm_key_data *); /* Done once per key */

	void (*gcm_init) (const struct gcm_key_data *key_data,

	    struct gcm_context_data *context_data,

	    uint8_t *iv,

	    uint8_t const *aad,

	    uint64_t aad_len);	/* Done at start of crypt */

	void (*gcm_upd) (const struct gcm_key_data *key_data,

	    struct gcm_context_data *context_data,

	    uint8_t *out,

	    const uint8_t *in,

	    uint64_t len);	/* With each block of data */

	void (*gcm_upd_nt) (const struct gcm_key_data *key_data,

	    struct gcm_context_data *context_data,

	    uint8_t *out,

	    const uint8_t *in,

	    uint64_t len);	/* With each block of data */

	void (*gcm_final) (const struct gcm_key_data *key_data,

	    struct gcm_context_data *context_data,

	    uint8_t *tag,

	    uint64_t tag_len);	/* Pulls out the tag */

};

SYSCTL_DECL(_kern_ipc_tls);

static int ktls_use_intel_isa_gcm = 1;

SYSCTL_INT(_kern_ipc_tls, OID_AUTO, isa_gcm, CTLFLAG_RW,

    &ktls_use_intel_isa_gcm, 1,

    "Should we use the Intel ISA GCM if available");

SYSCTL_DECL(_kern_ipc_tls_stats);

static counter_u64_t ktls_offload_isa_aead;

SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aead_crypts,

    CTLFLAG_RD, &ktls_offload_isa_aead,

    "Total number of Intel ISA TLS AEAD encrypts called");

static counter_u64_t intelisa_unaligned_mem_b;

SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned_bytes,

    CTLFLAG_RD, &intelisa_unaligned_mem_b,

    "Byte cnt of intel isa unaligned");

static counter_u64_t intelisa_aligned_mem_b;

SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned_bytes,

    CTLFLAG_RD, &intelisa_aligned_mem_b,

    "Byte cnt of intel isa aligned");

static counter_u64_t intelisa_unaligned_mem;

SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_unaligned,

    CTLFLAG_RD, &intelisa_unaligned_mem,

    "Call cnt of intel isa unaligned");

static counter_u64_t intelisa_aligned_mem;

SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, isa_aligned,

    CTLFLAG_RD, &intelisa_aligned_mem,

    "Call cnt of intel isa aligned");

static MALLOC_DEFINE(M_INTEL_ISA, "isal_tls", "Intel ISA-L TLS");

static int

intel_isa_seal(struct isa_gcm_struct *isa,

    struct iovec *outiov, int numiovs,

    uint8_t * nd, int noncelen,

    struct iovec *iniov,

    uint8_t * ad, int adlen,

    uint8_t * tagout, size_t *taglen)

{

	int i;

	bool nt = true;

	bool misaligned_len, misaligned_start;

	int fixup = 0;

	uint8_t *in;

	uint8_t *out;

	uint64_t len;

	uint8_t iv[32];

	uint8_t const IVend[] = GCM_IV_END_MARK;

	if (noncelen > 28) {

		return (-1);

	}

	memcpy(iv, nd, noncelen);

	memcpy(&iv[noncelen], IVend, sizeof(IVend));

	isa->gcm_init(&isa->key_data, &isa->ctx_data, iv, ad, (size_t)adlen);

	for (i = 0; i < numiovs; i++) {

		in = iniov[i].iov_base;

		out = outiov[i].iov_base;

		len = iniov[i].iov_len;

		misaligned_start = ((uintptr_t)in & 0xf) != 0;

		misaligned_len = (len & 0xf) != 0;

		if (misaligned_start || misaligned_len) {

			/*

			 * Try to do as much of a page using

			 * non-temporals as we possibly can, and leave

			 * a ragged tail as a separate chunk.

			 */

			if (nt && !misaligned_start && len > 0xf) {

				len = len & ~0xf;

				fixup = iniov[i].iov_len - len;

			} else {

				nt = false;

			}

		}

fixup_done:

		if (nt) {

			isa->gcm_upd_nt(&isa->key_data, &isa->ctx_data, out, in, len);

			counter_u64_add(intelisa_aligned_mem, 1);

			counter_u64_add(intelisa_aligned_mem_b, len);

		} else {

			isa->gcm_upd(&isa->key_data, &isa->ctx_data, out, in, len);

			counter_u64_add(intelisa_unaligned_mem, 1);

			counter_u64_add(intelisa_unaligned_mem_b, len);

		}

		if (fixup) {

			in += len;

			out += len;

			len = fixup;

			fixup = 0;

			nt = false;

			goto fixup_done;

		}

	}

	isa->gcm_final(&isa->key_data, &isa->ctx_data, tagout, *taglen);

	return (0);

}

static int

ktls_intelisa_aead_encrypt(struct ktls_session *tls,

    const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,

    struct iovec *outiov, int iovcnt, uint64_t seqno)

{

	struct isa_gcm_struct *isa;

	struct tls_aead_data ad;

	struct tls_nonce_data nd;

	size_t noncelen, adlen, taglen;

	int ret;

	uint16_t tls_comp_len;

	isa = (struct isa_gcm_struct *)tls->cipher;

	KASSERT(isa != NULL, ("Null cipher"));

	counter_u64_add(ktls_offload_isa_aead, 1);

	taglen = KTLS_INTELISA_AEAD_TAGLEN;

	/* Setup the nonce */

	memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN);

	memcpy(&nd.seq, hdr + 1, sizeof(nd.seq));

	noncelen = sizeof(nd);

	/* Setup the associated data */

	tls_comp_len = ntohs(hdr->tls_length) -

	    (KTLS_INTELISA_AEAD_TAGLEN + sizeof(nd.seq));

	ad.seq = htobe64(seqno);

	ad.type = hdr->tls_type;

	ad.tls_vmajor = hdr->tls_vmajor;

	ad.tls_vminor = hdr->tls_vminor;

	ad.tls_length = htons(tls_comp_len);

	adlen = sizeof(ad);

	ret = intel_isa_seal(isa, outiov, iovcnt,

	    (uint8_t *) & nd, noncelen, iniov,

	    (uint8_t *) & ad, adlen, trailer, &taglen);

	return(ret);

}

static int

ktls_intelisa_setup_cipher(struct isa_gcm_struct *isa, uint8_t *key)

{

	struct fpu_kern_ctx *fpu_ctx;

	if (key == NULL) {

		return (EINVAL);

	}

	fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NOWAIT);

	if (fpu_ctx == NULL) {

		return (ENOMEM);

	}

	fpu_kern_enter(curthread, fpu_ctx, FPU_KERN_NORMAL);

	isa->gcm_pre(key, &isa->key_data);

	fpu_kern_leave(curthread, fpu_ctx);

	fpu_kern_free_ctx(fpu_ctx);

	return (0);

}

static void

ktls_intelisa_free(struct ktls_session *tls)

{

	struct isa_gcm_struct *isa;

	isa = tls->cipher;

	explicit_bzero(isa, sizeof(*isa));

	free(isa, M_INTEL_ISA);

}

static int

ktls_intelisa_try(struct socket *so, struct ktls_session *tls)

{

	struct isa_gcm_struct *isa;

	int error;

	if (ktls_use_intel_isa_gcm &&

	    tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {

		isa = malloc(sizeof (*isa), M_INTEL_ISA, M_NOWAIT | M_ZERO);

		if (isa == NULL) {

			return (ENOMEM);

		}

		switch (tls->params.cipher_key_len) {

		case 16:

			isa->gcm_pre = aes_gcm_pre_128;

			isa->gcm_init = aes_gcm_init_128;

			isa->gcm_upd = aes_gcm_enc_128_update;

			isa->gcm_upd_nt = aes_gcm_enc_128_update_nt;

			isa->gcm_final = aes_gcm_enc_128_finalize;

			break;

		case 32:

			isa->gcm_pre = aes_gcm_pre_256;

			isa->gcm_init = aes_gcm_init_256;

			isa->gcm_upd = aes_gcm_enc_256_update;

			isa->gcm_upd_nt = aes_gcm_enc_256_update_nt;

			isa->gcm_final = aes_gcm_enc_256_finalize;

			break;

		default:

			free(isa, M_INTEL_ISA);

			return (EOPNOTSUPP);

		}

		error = ktls_intelisa_setup_cipher(isa, tls->params.cipher_key);

		if (error) {

			free(isa, M_INTEL_ISA);

			return (error);

		}

		tls->cipher = isa;

		tls->sw_encrypt = ktls_intelisa_aead_encrypt;

		tls->free = ktls_intelisa_free;

		return (0);

	}

	return (EOPNOTSUPP);

}

struct ktls_crypto_backend intelisa_backend  = {

	.name = "Intel ISA-L",

	.prio = 20,

	.api_version = KTLS_API_VERSION,

	.try = ktls_intelisa_try,

};

static int

intelisa_init(void)

{

	ktls_offload_isa_aead = counter_u64_alloc(M_WAITOK);

	intelisa_aligned_mem = counter_u64_alloc(M_WAITOK);

	intelisa_aligned_mem_b = counter_u64_alloc(M_WAITOK);

	intelisa_unaligned_mem = counter_u64_alloc(M_WAITOK);

	intelisa_unaligned_mem_b = counter_u64_alloc(M_WAITOK);

	return (ktls_crypto_backend_register(&intelisa_backend));

}

static int

intelisa_unload(void)

{

	int error;

	error = ktls_crypto_backend_deregister(&intelisa_backend);

	if (error)

		return (error);

	counter_u64_free(ktls_offload_isa_aead);

	counter_u64_free(intelisa_aligned_mem);

	counter_u64_free(intelisa_aligned_mem_b);

	counter_u64_free(intelisa_unaligned_mem);

	counter_u64_free(intelisa_unaligned_mem_b);

	return (0);

}

static int

intelisa_module_event_handler(module_t mod, int evt, void *arg)

{

	switch (evt) {

	case MOD_LOAD:

		return (intelisa_init());

	case MOD_UNLOAD:

		return (intelisa_unload());

	default:

		return (EOPNOTSUPP);

	}

}

static moduledata_t intelisa_moduledata = {

	"intelisa",

	intelisa_module_event_handler,

	NULL

};

DECLARE_MODULE(intelisa, intelisa_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);

ISA-L crypto is a collection of optimized low-level functions targeting

cryptography applications.

This port provides a software encryption module for in-kernel TLS

(KTLS) that uses routines in the ISA-L crypto library to encrypt TLS

records.

WWW: https://github.com/01org/isa-l_crypto