|
Lines 111-117
Link Here
|
| 111 |
#endif /* CRC32_ARM64 */ |
111 |
#endif /* CRC32_ARM64 */ |
| 112 |
|
112 |
|
| 113 |
#ifdef CRC32_ARM64_DEFAULT |
113 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
114 |
#ifndef __FreeBSD__ |
| 114 |
#include <asm/hwcap.h> |
115 |
#include <asm/hwcap.h> |
|
|
116 |
#endif |
| 115 |
#include <sys/auxv.h> |
117 |
#include <sys/auxv.h> |
| 116 |
#endif /* CRC32_ARM64_DEFAULT */ |
118 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 117 |
|
119 |
|
|
Lines 360-367
Link Here
|
| 360 |
#endif /* CRC32_ARM64_APPLE */ |
362 |
#endif /* CRC32_ARM64_APPLE */ |
| 361 |
|
363 |
|
| 362 |
#ifdef CRC32_ARM64_DEFAULT |
364 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
365 |
#ifdef __FreeBSD__ |
| 366 |
bool can_use_crc32() { |
| 367 |
unsigned long capabilities; |
| 368 |
|
| 369 |
if (elf_aux_info(AT_HWCAP, &capabilities, sizeof(unsigned long))) |
| 370 |
return false; |
| 371 |
return capabilities & HWCAP_CRC32; |
| 372 |
} |
| 373 |
bool can_use_poly_mul() { |
| 374 |
unsigned long capabilities; |
| 375 |
|
| 376 |
if (elf_aux_info(AT_HWCAP, &capabilities, sizeof(unsigned long))) |
| 377 |
return false; |
| 378 |
return capabilities & HWCAP_CRC32; |
| 379 |
} |
| 380 |
#else |
| 363 |
bool can_use_crc32() { return getauxval(AT_HWCAP) & HWCAP_CRC32; } |
381 |
bool can_use_crc32() { return getauxval(AT_HWCAP) & HWCAP_CRC32; } |
| 364 |
bool can_use_poly_mul() { return getauxval(AT_HWCAP) & HWCAP_PMULL; } |
382 |
bool can_use_poly_mul() { return getauxval(AT_HWCAP) & HWCAP_PMULL; } |
|
|
383 |
#endif |
| 365 |
#endif /* CRC32_ARM64_DEFAULT */ |
384 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 366 |
|
385 |
|
| 367 |
/** A helper template to statically unroll a loop with a fixed number of |
386 |
/** A helper template to statically unroll a loop with a fixed number of |
|
Lines 470-494
Link Here
|
| 470 |
|
489 |
|
| 471 |
#ifdef CRC32_ARM64 |
490 |
#ifdef CRC32_ARM64 |
| 472 |
#ifdef CRC32_ARM64_DEFAULT |
491 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
492 |
#ifndef __clang__ |
| 473 |
MY_ATTRIBUTE((target("+crc"))) |
493 |
MY_ATTRIBUTE((target("+crc"))) |
|
|
494 |
#endif |
| 474 |
#endif /* CRC32_ARM64_DEFAULT */ |
495 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 475 |
uint32_t crc32_impl::update(uint32_t crc, unsigned char data) { |
496 |
uint32_t crc32_impl::update(uint32_t crc, unsigned char data) { |
| 476 |
return __crc32cb(crc, data); |
497 |
return __crc32cb(crc, data); |
| 477 |
} |
498 |
} |
| 478 |
#ifdef CRC32_ARM64_DEFAULT |
499 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
500 |
#ifndef __clang__ |
| 479 |
MY_ATTRIBUTE((target("+crc"))) |
501 |
MY_ATTRIBUTE((target("+crc"))) |
|
|
502 |
#endif |
| 480 |
#endif /* CRC32_ARM64_DEFAULT */ |
503 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 481 |
uint32_t crc32_impl::update(uint32_t crc, uint16_t data) { |
504 |
uint32_t crc32_impl::update(uint32_t crc, uint16_t data) { |
| 482 |
return __crc32ch(crc, data); |
505 |
return __crc32ch(crc, data); |
| 483 |
} |
506 |
} |
| 484 |
#ifdef CRC32_ARM64_DEFAULT |
507 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
508 |
#ifndef __clang__ |
| 485 |
MY_ATTRIBUTE((target("+crc"))) |
509 |
MY_ATTRIBUTE((target("+crc"))) |
|
|
510 |
#endif |
| 486 |
#endif /* CRC32_ARM64_DEFAULT */ |
511 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 487 |
uint32_t crc32_impl::update(uint32_t crc, uint32_t data) { |
512 |
uint32_t crc32_impl::update(uint32_t crc, uint32_t data) { |
| 488 |
return __crc32cw(crc, data); |
513 |
return __crc32cw(crc, data); |
| 489 |
} |
514 |
} |
| 490 |
#ifdef CRC32_ARM64_DEFAULT |
515 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
516 |
#ifndef __clang__ |
| 491 |
MY_ATTRIBUTE((target("+crc"))) |
517 |
MY_ATTRIBUTE((target("+crc"))) |
|
|
518 |
#endif |
| 492 |
#endif /* CRC32_ARM64_DEFAULT */ |
519 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 493 |
uint64_t crc32_impl::update(uint64_t crc, uint64_t data) { |
520 |
uint64_t crc32_impl::update(uint64_t crc, uint64_t data) { |
| 494 |
return (uint64_t)__crc32cd((uint32_t)crc, data); |
521 |
return (uint64_t)__crc32cd((uint32_t)crc, data); |
|
Lines 534-540
Link Here
|
| 534 |
} |
561 |
} |
| 535 |
template <uint32_t w> |
562 |
template <uint32_t w> |
| 536 |
#ifdef CRC32_ARM64_DEFAULT |
563 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
564 |
#ifndef __clang__ |
| 537 |
MY_ATTRIBUTE((target("+crypto"))) |
565 |
MY_ATTRIBUTE((target("+crypto"))) |
|
|
566 |
#endif |
| 538 |
#endif /* CRC32_ARM64_DEFAULT */ |
567 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 539 |
uint64_t use_pclmul::polynomial_mul_rev(uint32_t rev_u) { |
568 |
uint64_t use_pclmul::polynomial_mul_rev(uint32_t rev_u) { |
| 540 |
constexpr uint64_t flipped_w = flip_at_32(w); |
569 |
constexpr uint64_t flipped_w = flip_at_32(w); |
|
Lines 777-783
Link Here
|
| 777 |
MY_ATTRIBUTE((flatten)) |
806 |
MY_ATTRIBUTE((flatten)) |
| 778 |
#endif /* CRC32_ARM64_APPLE */ |
807 |
#endif /* CRC32_ARM64_APPLE */ |
| 779 |
#ifdef CRC32_ARM64_DEFAULT |
808 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
809 |
#ifndef __clang__ |
| 780 |
MY_ATTRIBUTE((target("+crc+crypto"), flatten)) |
810 |
MY_ATTRIBUTE((target("+crc+crypto"), flatten)) |
|
|
811 |
#endif |
| 781 |
#endif /* CRC32_ARM64_DEFAULT */ |
812 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 782 |
uint32_t crc32_using_pclmul(const byte *data, size_t len) { |
813 |
uint32_t crc32_using_pclmul(const byte *data, size_t len) { |
| 783 |
return crc32<use_pclmul>(0, data, len); |
814 |
return crc32<use_pclmul>(0, data, len); |
|
Lines 797-803
Link Here
|
| 797 |
MY_ATTRIBUTE((flatten)) |
828 |
MY_ATTRIBUTE((flatten)) |
| 798 |
#endif /* CRC32_ARM64_APPLE */ |
829 |
#endif /* CRC32_ARM64_APPLE */ |
| 799 |
#ifdef CRC32_ARM64_DEFAULT |
830 |
#ifdef CRC32_ARM64_DEFAULT |
|
|
831 |
#ifndef __clang__ |
| 800 |
MY_ATTRIBUTE((target("+crc"), flatten)) |
832 |
MY_ATTRIBUTE((target("+crc"), flatten)) |
|
|
833 |
#endif |
| 801 |
#endif /* CRC32_ARM64_DEFAULT */ |
834 |
#endif /* CRC32_ARM64_DEFAULT */ |
| 802 |
uint32_t crc32_using_unrolled_loop_poly_mul(const byte *data, size_t len) { |
835 |
uint32_t crc32_using_unrolled_loop_poly_mul(const byte *data, size_t len) { |
| 803 |
return crc32<use_unrolled_loop_poly_mul>(0, data, len); |
836 |
return crc32<use_unrolled_loop_poly_mul>(0, data, len); |