From 8b62bff364523bd20ae45dab4fd549c7035ed1bd Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Sat, 1 Jul 2023 20:56:53 -0700 Subject: [PATCH] Introduce xxhash into third_party (#837) --- Makefile | 1 + third_party/third_party.mk | 1 + third_party/xxhash/LICENSE | 27 + third_party/xxhash/README.cosmo | 14 + third_party/xxhash/cli/COPYING | 340 + third_party/xxhash/cli/README.md | 5 + third_party/xxhash/cli/xsum_arch.h | 160 + third_party/xxhash/cli/xsum_bench.c | 474 + third_party/xxhash/cli/xsum_bench.h | 52 + third_party/xxhash/cli/xsum_config.h | 243 + third_party/xxhash/cli/xsum_os_specific.c | 576 + third_party/xxhash/cli/xsum_os_specific.h | 96 + third_party/xxhash/cli/xsum_output.c | 67 + third_party/xxhash/cli/xsum_output.h | 63 + third_party/xxhash/cli/xsum_sanity_check.c | 712 + third_party/xxhash/cli/xsum_sanity_check.h | 61 + third_party/xxhash/cli/xxhsum.1 | 107 + third_party/xxhash/cli/xxhsum.1.md | 151 + third_party/xxhash/cli/xxhsum.c | 1338 + third_party/xxhash/tests/Makefile | 132 + .../xxhash/tests/bench/.clang_complete | 2 + third_party/xxhash/tests/bench/.gitignore | 12 + third_party/xxhash/tests/bench/LICENSE | 340 + third_party/xxhash/tests/bench/Makefile | 69 + third_party/xxhash/tests/bench/benchHash.c | 184 + third_party/xxhash/tests/bench/benchHash.h | 68 + third_party/xxhash/tests/bench/benchfn.c | 277 + third_party/xxhash/tests/bench/benchfn.h | 184 + third_party/xxhash/tests/bench/bhDisplay.c | 182 + third_party/xxhash/tests/bench/bhDisplay.h | 62 + third_party/xxhash/tests/bench/hashes.h | 119 + third_party/xxhash/tests/bench/main.c | 232 + third_party/xxhash/tests/bench/timefn.c | 211 + third_party/xxhash/tests/bench/timefn.h | 108 + third_party/xxhash/tests/cli-comment-line.sh | 33 + .../xxhash/tests/collisions/.gitignore | 3 + third_party/xxhash/tests/collisions/LICENSE | 340 + third_party/xxhash/tests/collisions/Makefile | 76 + third_party/xxhash/tests/collisions/README.md | 123 + .../tests/collisions/allcodecs/README.md | 2 + .../xxhash/tests/collisions/allcodecs/dummy.c | 39 + .../xxhash/tests/collisions/allcodecs/dummy.h | 46 + third_party/xxhash/tests/collisions/hashes.h | 130 + third_party/xxhash/tests/collisions/main.c | 1165 + third_party/xxhash/tests/collisions/pool.c | 362 + third_party/xxhash/tests/collisions/pool.h | 81 + third_party/xxhash/tests/collisions/sort.cc | 75 + third_party/xxhash/tests/collisions/sort.hh | 43 + .../xxhash/tests/collisions/threading.c | 83 + .../xxhash/tests/collisions/threading.h | 152 + third_party/xxhash/tests/filename-escape.sh | 22 + .../xxhash/tests/generate_unicode_test.c | 161 + third_party/xxhash/tests/multiInclude.c | 85 + third_party/xxhash/tests/ppc_define.c | 63 + third_party/xxhash/tests/sanity_test.c | 765 + .../xxhash/tests/sanity_test_vectors.h | 45833 ++++++++++++++++ .../tests/sanity_test_vectors_generator.c | 465 + third_party/xxhash/tests/unicode_lint.sh | 44 + third_party/xxhash/xxh3.h | 56 + third_party/xxhash/xxh_x86dispatch.c | 808 + third_party/xxhash/xxh_x86dispatch.h | 86 + third_party/xxhash/xxhash.c | 44 + third_party/xxhash/xxhash.h | 6487 +++ third_party/xxhash/xxhash.mk | 55 + 64 files changed, 64397 insertions(+) create mode 100644 third_party/xxhash/LICENSE create mode 100644 third_party/xxhash/README.cosmo create mode 100644 third_party/xxhash/cli/COPYING create mode 100644 third_party/xxhash/cli/README.md create mode 100644 third_party/xxhash/cli/xsum_arch.h create mode 100644 third_party/xxhash/cli/xsum_bench.c create mode 100644 third_party/xxhash/cli/xsum_bench.h create mode 100644 third_party/xxhash/cli/xsum_config.h create mode 100644 third_party/xxhash/cli/xsum_os_specific.c create mode 100644 third_party/xxhash/cli/xsum_os_specific.h create mode 100644 third_party/xxhash/cli/xsum_output.c create mode 100644 third_party/xxhash/cli/xsum_output.h create mode 100644 third_party/xxhash/cli/xsum_sanity_check.c create mode 100644 third_party/xxhash/cli/xsum_sanity_check.h create mode 100644 third_party/xxhash/cli/xxhsum.1 create mode 100644 third_party/xxhash/cli/xxhsum.1.md create mode 100644 third_party/xxhash/cli/xxhsum.c create mode 100644 third_party/xxhash/tests/Makefile create mode 100644 third_party/xxhash/tests/bench/.clang_complete create mode 100644 third_party/xxhash/tests/bench/.gitignore create mode 100644 third_party/xxhash/tests/bench/LICENSE create mode 100644 third_party/xxhash/tests/bench/Makefile create mode 100644 third_party/xxhash/tests/bench/benchHash.c create mode 100644 third_party/xxhash/tests/bench/benchHash.h create mode 100644 third_party/xxhash/tests/bench/benchfn.c create mode 100644 third_party/xxhash/tests/bench/benchfn.h create mode 100644 third_party/xxhash/tests/bench/bhDisplay.c create mode 100644 third_party/xxhash/tests/bench/bhDisplay.h create mode 100644 third_party/xxhash/tests/bench/hashes.h create mode 100644 third_party/xxhash/tests/bench/main.c create mode 100644 third_party/xxhash/tests/bench/timefn.c create mode 100644 third_party/xxhash/tests/bench/timefn.h create mode 100644 third_party/xxhash/tests/cli-comment-line.sh create mode 100644 third_party/xxhash/tests/collisions/.gitignore create mode 100644 third_party/xxhash/tests/collisions/LICENSE create mode 100644 third_party/xxhash/tests/collisions/Makefile create mode 100644 third_party/xxhash/tests/collisions/README.md create mode 100644 third_party/xxhash/tests/collisions/allcodecs/README.md create mode 100644 third_party/xxhash/tests/collisions/allcodecs/dummy.c create mode 100644 third_party/xxhash/tests/collisions/allcodecs/dummy.h create mode 100644 third_party/xxhash/tests/collisions/hashes.h create mode 100644 third_party/xxhash/tests/collisions/main.c create mode 100644 third_party/xxhash/tests/collisions/pool.c create mode 100644 third_party/xxhash/tests/collisions/pool.h create mode 100644 third_party/xxhash/tests/collisions/sort.cc create mode 100644 third_party/xxhash/tests/collisions/sort.hh create mode 100644 third_party/xxhash/tests/collisions/threading.c create mode 100644 third_party/xxhash/tests/collisions/threading.h create mode 100755 third_party/xxhash/tests/filename-escape.sh create mode 100644 third_party/xxhash/tests/generate_unicode_test.c create mode 100644 third_party/xxhash/tests/multiInclude.c create mode 100644 third_party/xxhash/tests/ppc_define.c create mode 100644 third_party/xxhash/tests/sanity_test.c create mode 100644 third_party/xxhash/tests/sanity_test_vectors.h create mode 100644 third_party/xxhash/tests/sanity_test_vectors_generator.c create mode 100755 third_party/xxhash/tests/unicode_lint.sh create mode 100644 third_party/xxhash/xxh3.h create mode 100644 third_party/xxhash/xxh_x86dispatch.c create mode 100644 third_party/xxhash/xxh_x86dispatch.h create mode 100644 third_party/xxhash/xxhash.c create mode 100644 third_party/xxhash/xxhash.h create mode 100644 third_party/xxhash/xxhash.mk diff --git a/Makefile b/Makefile index 4e0cb77c1..0fec469b0 100644 --- a/Makefile +++ b/Makefile @@ -196,6 +196,7 @@ include third_party/mbedtls/test/test.mk include third_party/quickjs/quickjs.mk include third_party/lz4cli/lz4cli.mk include third_party/zip/zip.mk +include third_party/xxhash/xxhash.mk include third_party/unzip/unzip.mk include tool/build/lib/buildlib.mk include third_party/chibicc/chibicc.mk diff --git a/third_party/third_party.mk b/third_party/third_party.mk index 39c079157..59de16f56 100644 --- a/third_party/third_party.mk +++ b/third_party/third_party.mk @@ -23,6 +23,7 @@ o/$(MODE)/third_party: \ o/$(MODE)/third_party/make \ o/$(MODE)/third_party/maxmind \ o/$(MODE)/third_party/mbedtls \ + o/$(MODE)/third_party/xxhash \ o/$(MODE)/third_party/musl \ o/$(MODE)/third_party/nsync \ o/$(MODE)/third_party/puff \ diff --git a/third_party/xxhash/LICENSE b/third_party/xxhash/LICENSE new file mode 100644 index 000000000..55c4fc095 --- /dev/null +++ b/third_party/xxhash/LICENSE @@ -0,0 +1,27 @@ +// clang-format off +xxHash Library +Copyright (c) 2012-2021 Yann Collet +All rights reserved. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/xxhash/README.cosmo b/third_party/xxhash/README.cosmo new file mode 100644 index 000000000..74bdc464e --- /dev/null +++ b/third_party/xxhash/README.cosmo @@ -0,0 +1,14 @@ +DESCRIPTION + + Extremely fast non-cryptographic hash algorithm + +SOURCE + + https://github.com/Cyan4973/xxHash + + commit 5b45185d0419f63eb5911133f3dc578a163d162f + Author: Yann Collet + Date: Mon Jun 19 23:58:07 2023 -0700 + + fix minor typo in comment + diff --git a/third_party/xxhash/cli/COPYING b/third_party/xxhash/cli/COPYING new file mode 100644 index 000000000..0d83b0427 --- /dev/null +++ b/third_party/xxhash/cli/COPYING @@ -0,0 +1,340 @@ +// clang-format off + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/third_party/xxhash/cli/README.md b/third_party/xxhash/cli/README.md new file mode 100644 index 000000000..c8f33f0c5 --- /dev/null +++ b/third_party/xxhash/cli/README.md @@ -0,0 +1,5 @@ +// clang-format off +This directory contains source code dedicated to the `xxhsum` command line utility, +which is a user program of `libxxhash`. + +Note that, in contrast with the library `libxxhash`, the command line utility `xxhsum` ships with GPLv2 license. diff --git a/third_party/xxhash/cli/xsum_arch.h b/third_party/xxhash/cli/xsum_arch.h new file mode 100644 index 000000000..ed7ee0aa4 --- /dev/null +++ b/third_party/xxhash/cli/xsum_arch.h @@ -0,0 +1,160 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/* + * Checks for predefined macros by the compiler to try and get both the arch + * and the compiler version. + */ +#ifndef XSUM_ARCH_H +#define XSUM_ARCH_H + +#include "third_party/xxhash/cli/xsum_config.h" + +#define XSUM_LIB_VERSION XXH_VERSION_MAJOR.XXH_VERSION_MINOR.XXH_VERSION_RELEASE +#define XSUM_QUOTE(str) #str +#define XSUM_EXPAND_AND_QUOTE(str) XSUM_QUOTE(str) +#define XSUM_PROGRAM_VERSION XSUM_EXPAND_AND_QUOTE(XSUM_LIB_VERSION) + + +/* Show compiler versions in WELCOME_MESSAGE. XSUM_CC_VERSION_FMT will return the printf specifiers, + * and VERSION will contain the comma separated list of arguments to the XSUM_CC_VERSION_FMT string. */ +#if defined(__clang_version__) +/* Clang does its own thing. */ +# ifdef __apple_build_version__ +# define XSUM_CC_VERSION_FMT "Apple Clang %s" +# else +# define XSUM_CC_VERSION_FMT "Clang %s" +# endif +# define XSUM_CC_VERSION __clang_version__ +#elif defined(__VERSION__) +/* GCC and ICC */ +# define XSUM_CC_VERSION_FMT "%s" +# ifdef __INTEL_COMPILER /* icc adds its prefix */ +# define XSUM_CC_VERSION __VERSION__ +# else /* assume GCC */ +# define XSUM_CC_VERSION "GCC " __VERSION__ +# endif +#elif defined(_MSC_FULL_VER) && defined(_MSC_BUILD) +/* + * MSVC + * "For example, if the version number of the Visual C++ compiler is + * 15.00.20706.01, the _MSC_FULL_VER macro evaluates to 150020706." + * + * https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros?view=vs-2017 + */ +# define XSUM_CC_VERSION_FMT "MSVC %02i.%02i.%05i.%02i" +# define XSUM_CC_VERSION _MSC_FULL_VER / 10000000 % 100, _MSC_FULL_VER / 100000 % 100, _MSC_FULL_VER % 100000, _MSC_BUILD +#elif defined(_MSC_VER) /* old MSVC */ +# define XSUM_CC_VERSION_FMT "MSVC %02i.%02i" +# define XSUM_CC_VERSION _MSC_VER / 100, _MSC_VER % 100 +#elif defined(__TINYC__) +/* tcc stores its version in the __TINYC__ macro. */ +# define XSUM_CC_VERSION_FMT "tcc %i.%i.%i" +# define XSUM_CC_VERSION __TINYC__ / 10000 % 100, __TINYC__ / 100 % 100, __TINYC__ % 100 +#else +# define XSUM_CC_VERSION_FMT "%s" +# define XSUM_CC_VERSION "unknown compiler" +#endif + +/* makes the next part easier */ +#if (defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)) && !defined(_M_ARM64EC) +# define XSUM_ARCH_X64 1 +# define XSUM_ARCH_X86 "x86_64" +#elif defined(__i386__) || defined(_M_IX86) || defined(_M_IX86_FP) +# define XSUM_ARCH_X86 "i386" +#endif + +/* Try to detect the architecture. */ +#if defined(XSUM_ARCH_X86) +# if defined(XXHSUM_DISPATCH) +# define XSUM_ARCH XSUM_ARCH_X86 " autoVec" +# elif defined(__AVX512F__) +# define XSUM_ARCH XSUM_ARCH_X86 " + AVX512" +# elif defined(__AVX2__) +# define XSUM_ARCH XSUM_ARCH_X86 " + AVX2" +# elif defined(__AVX__) +# define XSUM_ARCH XSUM_ARCH_X86 " + AVX" +# elif defined(_M_X64) || defined(_M_AMD64) || defined(__x86_64__) \ + || defined(__SSE2__) || (defined(_M_IX86_FP) && _M_IX86_FP == 2) +# define XSUM_ARCH XSUM_ARCH_X86 " + SSE2" +# else +# define XSUM_ARCH XSUM_ARCH_X86 +# endif +#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC) +# define XSUM_ARCH "aarch64 + NEON" +#elif defined(__arm__) || defined(__thumb__) || defined(__thumb2__) || defined(_M_ARM) +/* ARM has a lot of different features that can change xxHash significantly. */ +# ifdef __ARM_ARCH +# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(__ARM_ARCH) +# else +# define XSUM_ARCH_ARM_VER XSUM_EXPAND_AND_QUOTE(_M_ARM) +# endif +# if defined(_M_ARM) /* windows arm is always thumb-2 */ \ + || defined(__thumb2__) || (defined(__thumb__) && (__thumb__ == 2 || __ARM_ARCH >= 7)) +# define XSUM_ARCH_THUMB " Thumb-2" +# elif defined(__thumb__) +# define XSUM_ARCH_THUMB " Thumb-1" +# else +# define XSUM_ARCH_THUMB "" +# endif +/* ARMv7 has unaligned by default */ +# if defined(__ARM_FEATURE_UNALIGNED) || __ARM_ARCH >= 7 || defined(_M_ARM) +# define XSUM_ARCH_UNALIGNED " + unaligned" +# else +# define XSUM_ARCH_UNALIGNED "" +# endif +# if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(_M_ARM) +# define XSUM_ARCH_NEON " + NEON" +# else +# define XSUM_ARCH_NEON "" +# endif +# define XSUM_ARCH "ARMv" XSUM_ARCH_ARM_VER XSUM_ARCH_THUMB XSUM_ARCH_NEON XSUM_ARCH_UNALIGNED +#elif defined(__powerpc64__) || defined(__ppc64__) || defined(__PPC64__) +# if defined(__GNUC__) && defined(__POWER9_VECTOR__) +# define XSUM_ARCH "ppc64 + POWER9 vector" +# elif defined(__GNUC__) && defined(__POWER8_VECTOR__) +# define XSUM_ARCH "ppc64 + POWER8 vector" +# else +# define XSUM_ARCH "ppc64" +# endif +#elif defined(__powerpc__) || defined(__ppc__) || defined(__PPC__) +# define XSUM_ARCH "ppc" +#elif defined(__AVR) +# define XSUM_ARCH "AVR" +#elif defined(__mips64) +# define XSUM_ARCH "mips64" +#elif defined(__mips) +# define XSUM_ARCH "mips" +#elif defined(__s390x__) +# define XSUM_ARCH "s390x" +#elif defined(__s390__) +# define XSUM_ARCH "s390" +#else +# define XSUM_ARCH "unknown" +#endif + + +#endif /* XSUM_ARCH_H */ diff --git a/third_party/xxhash/cli/xsum_bench.c b/third_party/xxhash/cli/xsum_bench.c new file mode 100644 index 000000000..4ce2cdfad --- /dev/null +++ b/third_party/xxhash/cli/xsum_bench.c @@ -0,0 +1,474 @@ +// clang-format off +/* + * xsum_bench - Benchmark functions for xxhsum + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#include "third_party/xxhash/cli/xsum_output.h" /* XSUM_logLevel */ +#include "third_party/xxhash/cli/xsum_bench.h" +#include "third_party/xxhash/cli/xsum_sanity_check.h" /* XSUM_fillTestBuffer */ +#include "third_party/xxhash/cli/xsum_os_specific.h" /* XSUM_getFileSize */ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +#endif +#include "third_party/xxhash/xxhash.h" +#ifdef XXHSUM_DISPATCH +#include "third_party/xxhash/xxh_x86dispatch.h" /* activate _dispatch() redirectors */ +#endif + +#include "libc/calls/calls.h" +#include "libc/calls/termios.h" +#include "libc/fmt/conv.h" +#include "libc/limits.h" +#include "libc/mem/alg.h" +#include "libc/mem/alloca.h" +#include "libc/mem/mem.h" +#include "libc/runtime/runtime.h" +#include "libc/stdio/dprintf.h" +#include "libc/stdio/rand.h" +#include "libc/stdio/temp.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/exit.h" +#include "third_party/getopt/getopt.h" +#include "third_party/musl/crypt.h" +#include "third_party/musl/rand48.h" /* malloc, free */ +#include "libc/assert.h" +#include "libc/mem/alg.h" +#include "libc/mem/mem.h" +#include "libc/str/str.h" /* strlen, memcpy */ +#include "libc/calls/calls.h" +#include "libc/calls/struct/timespec.h" +#include "libc/calls/struct/timeval.h" +#include "libc/calls/weirdtypes.h" +#include "libc/sysv/consts/clock.h" +#include "libc/sysv/consts/sched.h" +#include "libc/sysv/consts/timer.h" +#include "libc/time/struct/tm.h" +#include "libc/time/time.h" /* clock_t, clock, CLOCKS_PER_SEC */ +#include "libc/errno.h" /* errno */ + +#define TIMELOOP_S 1 +#define TIMELOOP (TIMELOOP_S * CLOCKS_PER_SEC) /* target timing per iteration */ +#define TIMELOOP_MIN (TIMELOOP / 2) /* minimum timing to validate a result */ + +/* Each benchmark iteration attempts to match TIMELOOP (1 second). + * The nb of loops is adjusted at each iteration to reach that target. + * However, initially, there is no information, so 1st iteration blindly targets an arbitrary speed. + * If it's too small, it will be adjusted, and a new attempt will be made. + * But if it's too large, the first iteration can be very long, + * before being fixed at second attempt. + * So prefer starting with small speed targets. + * XXH_1ST_SPEED_TARGET is defined in MB/s */ +#ifndef XXH_1ST_SPEED_TARGET +# define XXH_1ST_SPEED_TARGET 10 +#endif + +#define MAX_MEM (2 GB - 64 MB) + +static clock_t XSUM_clockSpan( clock_t start ) +{ + return clock() - start; /* works even if overflow; Typical max span ~ 30 mn */ +} + +static size_t XSUM_findMaxMem(XSUM_U64 requiredMem) +{ + size_t const step = 64 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += 2*step; + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + while (!testmem) { + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + testmem = malloc ((size_t)requiredMem); + } + free (testmem); + + /* keep some space available */ + if (requiredMem > step) requiredMem -= step; + else requiredMem >>= 1; + + return (size_t)requiredMem; +} + +/* + * A secret buffer used for benchmarking XXH3's withSecret variants. + * + * In order for the bench to be realistic, the secret buffer would need to be + * pre-generated. + * + * Adding a pointer to the parameter list would be messy. + */ +static XSUM_U8 g_benchSecretBuf[XXH3_SECRET_SIZE_MIN]; + +/* + * Wrappers for the benchmark. + * + * If you would like to add other hashes to the bench, create a wrapper and add + * it to the g_hashesToBench table. It will automatically be added. + */ +typedef XSUM_U32 (*hashFunction)(const void* buffer, size_t bufferSize, XSUM_U32 seed); + +static XSUM_U32 localXXH32(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return XXH32(buffer, bufferSize, seed); +} +static XSUM_U32 localXXH32_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH32_state_t state; + (void)seed; + XXH32_reset(&state, seed); + XXH32_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH32_digest(&state); +} +static XSUM_U32 localXXH64(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return (XSUM_U32)XXH64(buffer, bufferSize, seed); +} +static XSUM_U32 localXXH64_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH64_state_t state; + (void)seed; + XXH64_reset(&state, seed); + XXH64_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH64_digest(&state); +} +static XSUM_U32 localXXH3_64b(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)XXH3_64bits(buffer, bufferSize); +} +static XSUM_U32 localXXH3_64b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return (XSUM_U32)XXH3_64bits_withSeed(buffer, bufferSize, seed); +} +static XSUM_U32 localXXH3_64b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)XXH3_64bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf)); +} +static XSUM_U32 localXXH3_128b(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)(XXH3_128bits(buffer, bufferSize).low64); +} +static XSUM_U32 localXXH3_128b_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + return (XSUM_U32)(XXH3_128bits_withSeed(buffer, bufferSize, seed).low64); +} +static XSUM_U32 localXXH3_128b_secret(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + (void)seed; + return (XSUM_U32)(XXH3_128bits_withSecret(buffer, bufferSize, g_benchSecretBuf, sizeof(g_benchSecretBuf)).low64); +} +static XSUM_U32 localXXH3_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + (void)seed; + XXH3_64bits_reset(&state); + XXH3_64bits_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH3_64bits_digest(&state); +} +static XSUM_U32 localXXH3_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_64bits_reset_withSeed(&state, (XXH64_hash_t)seed); + XXH3_64bits_update(&state, buffer, bufferSize); + return (XSUM_U32)XXH3_64bits_digest(&state); +} +static XSUM_U32 localXXH128_stream(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + (void)seed; + XXH3_128bits_reset(&state); + XXH3_128bits_update(&state, buffer, bufferSize); + return (XSUM_U32)(XXH3_128bits_digest(&state).low64); +} +static XSUM_U32 localXXH128_stream_seeded(const void* buffer, size_t bufferSize, XSUM_U32 seed) +{ + XXH3_state_t state; + XXH3_INITSTATE(&state); + XXH3_128bits_reset_withSeed(&state, (XXH64_hash_t)seed); + XXH3_128bits_update(&state, buffer, bufferSize); + return (XSUM_U32)(XXH3_128bits_digest(&state).low64); +} + + +typedef struct { + const char* name; + hashFunction func; +} hashInfo; + +static const hashInfo g_hashesToBench[] = { + { "XXH32", &localXXH32 }, + { "XXH64", &localXXH64 }, + { "XXH3_64b", &localXXH3_64b }, + { "XXH3_64b w/seed", &localXXH3_64b_seeded }, + { "XXH3_64b w/secret", &localXXH3_64b_secret }, + { "XXH128", &localXXH3_128b }, + { "XXH128 w/seed", &localXXH3_128b_seeded }, + { "XXH128 w/secret", &localXXH3_128b_secret }, + { "XXH32_stream", &localXXH32_stream }, + { "XXH64_stream", &localXXH64_stream }, + { "XXH3_stream", &localXXH3_stream }, + { "XXH3_stream w/seed",&localXXH3_stream_seeded }, + { "XXH128_stream", &localXXH128_stream }, + { "XXH128_stream w/seed",&localXXH128_stream_seeded }, +}; +#define NB_HASHFUNC (sizeof(g_hashesToBench) / sizeof(*g_hashesToBench)) + +#define NB_TESTFUNC (1 + 2 * NB_HASHFUNC) +int const g_nbTestFunctions = NB_TESTFUNC; +char g_testIDs[NB_TESTFUNC] = { 0 }; +const char k_testIDs_default[NB_TESTFUNC] = { 0, + 1 /*XXH32*/, 0, + 1 /*XXH64*/, 0, + 1 /*XXH3*/, 0, 0, 0, 0, 0, + 1 /*XXH128*/ }; + +int g_nbIterations = NBLOOPS_DEFAULT; +#define HASHNAME_MAX 29 +static void XSUM_benchHash(hashFunction h, const char* hName, int testID, + const void* buffer, size_t bufferSize) +{ + XSUM_U32 nbh_perIteration = (XSUM_U32)((XXH_1ST_SPEED_TARGET MB) / (bufferSize+1)) + 1; + int iterationNb, nbIterations = g_nbIterations + !g_nbIterations /* min 1 */; + double fastestH = 100000000.; + assert(HASHNAME_MAX > 2); + XSUM_logVerbose(2, "\r%80s\r", ""); /* Clean display line */ + + for (iterationNb = 1; iterationNb <= nbIterations; iterationNb++) { + XSUM_U32 r=0; + clock_t cStart; + + XSUM_logVerbose(2, "%2i-%-*.*s : %10u ->\r", + iterationNb, + HASHNAME_MAX, HASHNAME_MAX, hName, + (unsigned)bufferSize); + cStart = clock(); + while (clock() == cStart); /* starts clock() at its exact beginning */ + cStart = clock(); + + { XSUM_U32 u; + for (u=0; u (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */ + nbh_perIteration = (XSUM_U32)nbh_perSecond; + } + /* g_nbIterations==0 => quick evaluation, no claim of accuracy */ + if (g_nbIterations>0) { + iterationNb--; /* new round for a more accurate speed evaluation */ + continue; + } + } + if (ticksPerHash < fastestH) fastestH = ticksPerHash; + if (fastestH>0.) { /* avoid div by zero */ + XSUM_logVerbose(2, "%2i-%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \r", + iterationNb, + HASHNAME_MAX, HASHNAME_MAX, hName, + (unsigned)bufferSize, + (double)1 / fastestH, + ((double)bufferSize / (1 MB)) / fastestH); + } } + { double nbh_perSecond = (1 / fastestH) + 1; + if (nbh_perSecond > (double)(4000U<<20)) nbh_perSecond = (double)(4000U<<20); /* avoid overflow */ + nbh_perIteration = (XSUM_U32)nbh_perSecond; + } + } + XSUM_logVerbose(1, "%2i#%-*.*s : %10u -> %8.0f it/s (%7.1f MB/s) \n", + testID, + HASHNAME_MAX, HASHNAME_MAX, hName, + (unsigned)bufferSize, + (double)1 / fastestH, + ((double)bufferSize / (1 MB)) / fastestH); + if (XSUM_logLevel<1) + XSUM_logVerbose(0, "%u, ", (unsigned)((double)1 / fastestH)); +} + + +/* + * Allocates a string containing s1 and s2 concatenated. Acts like strdup. + * The result must be freed. + */ +static char* XSUM_strcatDup(const char* s1, const char* s2) +{ + assert(s1 != NULL); + assert(s2 != NULL); + { size_t len1 = strlen(s1); + size_t len2 = strlen(s2); + char* buf = (char*)malloc(len1 + len2 + 1); + if (buf != NULL) { + /* strcpy(buf, s1) */ + memcpy(buf, s1, len1); + /* strcat(buf, s2) */ + memcpy(buf + len1, s2, len2 + 1); + } + return buf; + } +} + + +/*! + * XSUM_benchMem(): + * buffer: Must be 16-byte aligned. + * The real allocated size of buffer is supposed to be >= (bufferSize+3). + * returns: 0 on success, 1 if error (invalid mode selected) + */ +static void XSUM_benchMem(const void* buffer, size_t bufferSize) +{ + assert((((size_t)buffer) & 15) == 0); /* ensure alignment */ + XSUM_fillTestBuffer(g_benchSecretBuf, sizeof(g_benchSecretBuf)); + { int i; + for (i = 1; i < (int)NB_TESTFUNC; i++) { + int const hashFuncID = (i-1) / 2; + assert(g_hashesToBench[hashFuncID].name != NULL); + if (g_testIDs[i] == 0) continue; + /* aligned */ + if ((i % 2) == 1) { + XSUM_benchHash(g_hashesToBench[hashFuncID].func, g_hashesToBench[hashFuncID].name, i, buffer, bufferSize); + } + /* unaligned */ + if ((i % 2) == 0) { + /* Append "unaligned". */ + char* const hashNameBuf = XSUM_strcatDup(g_hashesToBench[hashFuncID].name, " unaligned"); + assert(hashNameBuf != NULL); + XSUM_benchHash(g_hashesToBench[hashFuncID].func, hashNameBuf, i, ((const char*)buffer)+3, bufferSize); + free(hashNameBuf); + } + } } +} + +static size_t XSUM_selectBenchedSize(const char* fileName) +{ + XSUM_U64 const inFileSize = XSUM_getFileSize(fileName); + size_t benchedSize = (size_t) XSUM_findMaxMem(inFileSize); + if ((XSUM_U64)benchedSize > inFileSize) benchedSize = (size_t)inFileSize; + if (benchedSize < inFileSize) { + XSUM_log("Not enough memory for '%s' full size; testing %i MB only...\n", fileName, (int)(benchedSize>>20)); + } + return benchedSize; +} + + +int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles) +{ + int fileIdx; + for (fileIdx=0; fileIdx 10 KB) { + XSUM_logVerbose(1, "%u KB", (unsigned)(keySize >> 10)); + } else { + XSUM_logVerbose(1, "%u bytes", (unsigned)keySize); + } + XSUM_logVerbose(1, "... \n"); + + XSUM_benchMem(alignedBuffer, keySize); + free(buffer); + } + return 0; +} diff --git a/third_party/xxhash/cli/xsum_bench.h b/third_party/xxhash/cli/xsum_bench.h new file mode 100644 index 000000000..894895bb3 --- /dev/null +++ b/third_party/xxhash/cli/xsum_bench.h @@ -0,0 +1,52 @@ +// clang-format off +/* + * xsum_bench - Benchmark functions for xxhsum + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XSUM_BENCH_H +#define XSUM_BENCH_H + + /* size_t */ + +#define NBLOOPS_DEFAULT 3 /* Default number of benchmark iterations */ + +extern int const g_nbTestFunctions; +extern char g_testIDs[]; /* size : g_nbTestFunctions */ +extern const char k_testIDs_default[]; +extern int g_nbIterations; + +int XSUM_benchInternal(size_t keySize); +int XSUM_benchFiles(const char* fileNamesTable[], int nbFiles); + + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef __cplusplus +} +#endif + +#endif /* XSUM_BENCH_H */ diff --git a/third_party/xxhash/cli/xsum_config.h b/third_party/xxhash/cli/xsum_config.h new file mode 100644 index 000000000..7fae8d0a5 --- /dev/null +++ b/third_party/xxhash/cli/xsum_config.h @@ -0,0 +1,243 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +/* + * This contains various configuration parameters and feature detection for + * xxhsum. + * + * Similar to config.h in Autotools, this should be the first header included. + */ + +#ifndef XSUM_CONFIG_H +#define XSUM_CONFIG_H + + +/* ************************************ + * Compiler Options + **************************************/ +/* + * Disable Visual C's warnings when using the "insecure" CRT functions instead + * of the "secure" _s functions. + * + * These functions are not portable, and aren't necessary if you are using the + * original functions properly. + */ +#if defined(_MSC_VER) || defined(_WIN32) +# ifndef _CRT_SECURE_NO_WARNINGS +# define _CRT_SECURE_NO_WARNINGS +# endif +#endif + +/* Under Linux at least, pull in the *64 commands */ +#ifndef _LARGEFILE64_SOURCE +# define _LARGEFILE64_SOURCE +#endif +#ifndef _FILE_OFFSET_BITS +# define _FILE_OFFSET_BITS 64 +#endif + +/* + * So we can use __attribute__((__format__)) + */ +#ifdef __GNUC__ +# define XSUM_ATTRIBUTE(x) __attribute__(x) +#else +# define XSUM_ATTRIBUTE(x) +#endif + +#if !defined(_WIN32) && (defined(__unix__) || defined(__unix) || (defined(__APPLE__) && defined(__MACH__)) /* UNIX-like OS */ \ + || defined(__midipix__) || defined(__VMS)) +# if (defined(__APPLE__) && defined(__MACH__)) || defined(__SVR4) || defined(_AIX) || defined(__hpux) /* POSIX.1-2001 (SUSv3) conformant */ \ + || defined(__DragonFly__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) /* BSD distros */ +# define XSUM_PLATFORM_POSIX_VERSION 200112L +# else +# if defined(__linux__) || defined(__linux) +# ifndef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200112L /* use feature test macro */ +# endif +# endif +#include "libc/calls/calls.h" +#include "libc/calls/weirdtypes.h" +#include "libc/runtime/pathconf.h" +#include "libc/runtime/runtime.h" +#include "libc/runtime/sysconf.h" +#include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/fileno.h" +#include "libc/sysv/consts/o.h" +#include "libc/sysv/consts/ok.h" +#include "libc/time/time.h" +#include "third_party/getopt/getopt.h" +#include "third_party/musl/crypt.h" +#include "third_party/musl/lockf.h" /* declares _POSIX_VERSION */ +# if defined(_POSIX_VERSION) /* POSIX compliant */ +# define XSUM_PLATFORM_POSIX_VERSION _POSIX_VERSION +# else +# define XSUM_PLATFORM_POSIX_VERSION 0 +# endif +# endif +#endif +#if !defined(XSUM_PLATFORM_POSIX_VERSION) +# define XSUM_PLATFORM_POSIX_VERSION -1 +#endif + +/** + * + * fzakaria: remove this + * + * #if !defined(S_ISREG) + * # define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) + * #endif + */ + + +/* ************************************ + * Windows helpers + **************************************/ + +/* + * Whether to use the Windows UTF-16 APIs instead of the portable libc 8-bit + * ("ANSI") APIs. + * + * Windows is not UTF-8 clean by default, and the only way to access every file + * on the OS is to use UTF-16. + * + * Do note that xxhsum uses UTF-8 internally and only uses UTF-16 for command + * line arguments, console I/O, and opening files. + * + * Additionally, this guarantees all piped output is UTF-8. + */ +#if defined(XSUM_WIN32_USE_WCHAR) && !defined(_WIN32) +/* We use Windows APIs, only use this on Windows. */ +# undef XSUM_WIN32_USE_WCHAR +#endif + +#ifndef XSUM_WIN32_USE_WCHAR +# if defined(_WIN32) +#include "libc/fmt/conv.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "libc/str/unicode.h" +#include "libc/time/time.h" +# if WCHAR_MAX == 0xFFFFU /* UTF-16 wchar_t */ +# define XSUM_WIN32_USE_WCHAR 1 +# else +# define XSUM_WIN32_USE_WCHAR 0 +# endif +# else +# define XSUM_WIN32_USE_WCHAR 0 +# endif +#endif + +#if !XSUM_WIN32_USE_WCHAR +/* + * It doesn't make sense to have one without the other. + * Due to XSUM_WIN32_USE_WCHAR being undef'd, this also handles + * non-WIN32 platforms. + */ +# undef XSUM_WIN32_USE_WMAIN +# define XSUM_WIN32_USE_WMAIN 0 +#else +/* + * Whether to use wmain() or main(). + * + * wmain() is preferred because we don't have to mess with internal hidden + * APIs. + * + * It always works on MSVC, but in MinGW, it only works on MinGW-w64 with the + * -municode flag. + * + * Therefore we have to use main() -- there is no better option. + */ +# ifndef XSUM_WIN32_USE_WMAIN +# if defined(_UNICODE) || defined(UNICODE) /* MinGW -municode */ \ + || defined(_MSC_VER) /* MSVC */ +# define XSUM_WIN32_USE_WMAIN 1 +# else +# define XSUM_WIN32_USE_WMAIN 0 +# endif +# endif +/* + * It is always good practice to define these to prevent accidental use of the + * ANSI APIs, even if the program primarily uses UTF-8. + */ +# ifndef _UNICODE +# define _UNICODE +# endif +# ifndef UNICODE +# define UNICODE +# endif +#endif /* XSUM_WIN32_USE_WCHAR */ + +#ifndef XSUM_API +# ifdef XXH_INLINE_ALL +# define XSUM_API static +# else +# define XSUM_API +# endif +#endif + +#ifndef XSUM_NO_TESTS +# define XSUM_NO_TESTS 0 +#endif + +/* *************************** + * Basic types + * ***************************/ + +#if defined(__cplusplus) /* C++ */ \ + || (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L) /* C99 */ +#include "libc/inttypes.h" +#include "libc/limits.h" +#include "libc/literal.h" + typedef uint8_t XSUM_U8; + typedef uint32_t XSUM_U32; + typedef uint64_t XSUM_U64; +# else +#include "libc/limits.h" +#include "libc/sysv/consts/_posix.h" +#include "libc/sysv/consts/iov.h" +#include "libc/sysv/consts/limits.h" +#include "libc/sysv/consts/xopen.h" +#include "libc/thread/thread.h" + typedef unsigned char XSUM_U8; +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XSUM_U32; +# else + typedef unsigned long XSUM_U32; +# endif + typedef unsigned long long XSUM_U64; +#endif /* not C++/C99 */ + +/* *************************** + * Common constants + * ***************************/ + +#define KB *( 1<<10) +#define MB *( 1<<20) +#define GB *(1U<<30) + + +#endif /* XSUM_CONFIG_H */ diff --git a/third_party/xxhash/cli/xsum_os_specific.c b/third_party/xxhash/cli/xsum_os_specific.c new file mode 100644 index 000000000..ef7ca0676 --- /dev/null +++ b/third_party/xxhash/cli/xsum_os_specific.c @@ -0,0 +1,576 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#include "third_party/xxhash/cli/xsum_os_specific.h" /* XSUM_API */ +#include "libc/calls/calls.h" +#include "libc/calls/struct/stat.h" +#include "libc/calls/struct/stat.macros.h" +#include "libc/calls/struct/timespec.h" +#include "libc/calls/weirdtypes.h" +#include "libc/sysv/consts/s.h" +#include "libc/sysv/consts/utime.h" +#include "libc/time/time.h" /* stat() / _stat64() */ + +/* + * This file contains all of the ugly boilerplate to make xxhsum work across + * platforms. + */ +#if defined(_MSC_VER) || XSUM_WIN32_USE_WCHAR + typedef struct __stat64 XSUM_stat_t; +# if defined(_MSC_VER) + typedef int mode_t; +# endif +#else + typedef struct stat XSUM_stat_t; +#endif + +#if (defined(__linux__) && (XSUM_PLATFORM_POSIX_VERSION >= 1)) \ + || (XSUM_PLATFORM_POSIX_VERSION >= 200112L) \ + || defined(__DJGPP__) \ + || defined(__MSYS__) \ + || defined(__HAIKU__) +#include "libc/calls/calls.h" +#include "libc/calls/weirdtypes.h" +#include "libc/runtime/pathconf.h" +#include "libc/runtime/runtime.h" +#include "libc/runtime/sysconf.h" +#include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/fileno.h" +#include "libc/sysv/consts/o.h" +#include "libc/sysv/consts/ok.h" +#include "libc/time/time.h" +#include "third_party/getopt/getopt.h" +#include "third_party/musl/crypt.h" +#include "third_party/musl/lockf.h" /* isatty */ +# define XSUM_IS_CONSOLE(stdStream) isatty(fileno(stdStream)) +#elif defined(MSDOS) || defined(OS2) +// MISSING #include /* _isatty */ +# define XSUM_IS_CONSOLE(stdStream) _isatty(_fileno(stdStream)) +#elif defined(WIN32) || defined(_WIN32) +// MISSING #include /* _isatty */ +#include "libc/nt/accounting.h" +#include "libc/nt/automation.h" +#include "libc/nt/console.h" +#include "libc/nt/debug.h" +#include "libc/nt/dll.h" +#include "libc/nt/enum/keyaccess.h" +#include "libc/nt/enum/regtype.h" +#include "libc/nt/errors.h" +#include "libc/nt/events.h" +#include "libc/nt/files.h" +#include "libc/nt/ipc.h" +#include "libc/nt/memory.h" +#include "libc/nt/paint.h" +#include "libc/nt/process.h" +#include "libc/nt/registry.h" +#include "libc/nt/synchronization.h" +#include "libc/nt/thread.h" +#include "libc/nt/windows.h" +#include "libc/nt/winsock.h" /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +#include "libc/calls/calls.h" +#include "libc/calls/weirdtypes.h" +#include "libc/fmt/fmt.h" +#include "libc/stdio/dprintf.h" +#include "libc/stdio/stdio.h" +#include "libc/stdio/temp.h" +#include "third_party/musl/tempnam.h" /* FILE */ +static __inline int XSUM_IS_CONSOLE(FILE* stdStream) +{ + DWORD dummy; + return _isatty(_fileno(stdStream)) && GetConsoleMode((HANDLE)_get_osfhandle(_fileno(stdStream)), &dummy); +} +#else +# define XSUM_IS_CONSOLE(stdStream) 0 +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(_WIN32) +#include "libc/calls/calls.h" +#include "libc/calls/struct/flock.h" +#include "libc/calls/weirdtypes.h" +#include "libc/sysv/consts/at.h" +#include "libc/sysv/consts/f.h" +#include "libc/sysv/consts/fd.h" +#include "libc/sysv/consts/o.h" +#include "libc/sysv/consts/posix.h" +#include "libc/sysv/consts/s.h" +#include "libc/sysv/consts/splice.h" /* _O_BINARY */ +// MISSING #include /* _setmode, _fileno, _get_osfhandle */ +# if !defined(__DJGPP__) +#include "libc/nt/accounting.h" +#include "libc/nt/automation.h" +#include "libc/nt/console.h" +#include "libc/nt/debug.h" +#include "libc/nt/dll.h" +#include "libc/nt/enum/keyaccess.h" +#include "libc/nt/enum/regtype.h" +#include "libc/nt/errors.h" +#include "libc/nt/events.h" +#include "libc/nt/files.h" +#include "libc/nt/ipc.h" +#include "libc/nt/memory.h" +#include "libc/nt/paint.h" +#include "libc/nt/process.h" +#include "libc/nt/registry.h" +#include "libc/nt/synchronization.h" +#include "libc/nt/thread.h" +#include "libc/nt/windows.h" +#include "libc/nt/winsock.h" /* DeviceIoControl, HANDLE, FSCTL_SET_SPARSE */ +// MISSING #include /* FSCTL_SET_SPARSE */ +# define XSUM_SET_BINARY_MODE(file) { int const unused=_setmode(_fileno(file), _O_BINARY); (void)unused; } +# else +# define XSUM_SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +# endif +#else +# define XSUM_SET_BINARY_MODE(file) ((void)file) +#endif + +XSUM_API int XSUM_isConsole(FILE* stream) +{ + return XSUM_IS_CONSOLE(stream); +} + +XSUM_API void XSUM_setBinaryMode(FILE* stream) +{ + XSUM_SET_BINARY_MODE(stream); +} + +#if !XSUM_WIN32_USE_WCHAR + +XSUM_API FILE* XSUM_fopen(const char* filename, const char* mode) +{ + return fopen(filename, mode); +} +XSUM_ATTRIBUTE((__format__(__printf__, 2, 0))) +XSUM_API int XSUM_vfprintf(FILE* stream, const char* format, va_list ap) +{ + return vfprintf(stream, format, ap); +} + +static int XSUM_stat(const char* infilename, XSUM_stat_t* statbuf) +{ +#if defined(_MSC_VER) + return _stat64(infilename, statbuf); +#else + return stat(infilename, statbuf); +#endif +} + +#ifndef XSUM_NO_MAIN +int main(int argc, const char* argv[]) +{ + return XSUM_main(argc, argv); +} +#endif + +/* Unicode helpers for Windows to make UTF-8 act as it should. */ +#else +#include "libc/nt/accounting.h" +#include "libc/nt/automation.h" +#include "libc/nt/console.h" +#include "libc/nt/debug.h" +#include "libc/nt/dll.h" +#include "libc/nt/enum/keyaccess.h" +#include "libc/nt/enum/regtype.h" +#include "libc/nt/errors.h" +#include "libc/nt/events.h" +#include "libc/nt/files.h" +#include "libc/nt/ipc.h" +#include "libc/nt/memory.h" +#include "libc/nt/paint.h" +#include "libc/nt/process.h" +#include "libc/nt/registry.h" +#include "libc/nt/synchronization.h" +#include "libc/nt/thread.h" +#include "libc/nt/windows.h" +#include "libc/nt/winsock.h" +#include "libc/fmt/conv.h" +#include "libc/stdio/stdio.h" +#include "libc/str/str.h" +#include "libc/str/unicode.h" +#include "libc/time/time.h" + +/***************************************************************************** + * Unicode conversion tools + *****************************************************************************/ + +/* + * Converts a UTF-8 string to UTF-16. Acts like strdup. The string must be freed afterwards. + * This version allows keeping the output length. + */ +static wchar_t* XSUM_widenString(const char* str, int* lenOut) +{ + int const len = MultiByteToWideChar(CP_UTF8, 0, str, -1, NULL, 0); + if (lenOut != NULL) *lenOut = len; + if (len == 0) return NULL; + { wchar_t* buf = (wchar_t*)malloc((size_t)len * sizeof(wchar_t)); + if (buf != NULL) { + if (MultiByteToWideChar(CP_UTF8, 0, str, -1, buf, len) == 0) { + free(buf); + return NULL; + } } + return buf; + } +} + +/* + * Converts a UTF-16 string to UTF-8. Acts like strdup. The string must be freed afterwards. + * This version allows keeping the output length. + */ +static char* XSUM_narrowString(const wchar_t *str, int *lenOut) +{ + int len = WideCharToMultiByte(CP_UTF8, 0, str, -1, NULL, 0, NULL, NULL); + if (lenOut != NULL) *lenOut = len; + if (len == 0) return NULL; + { char* const buf = (char*)malloc((size_t)len * sizeof(char)); + if (buf != NULL) { + if (WideCharToMultiByte(CP_UTF8, 0, str, -1, buf, len, NULL, NULL) == 0) { + free(buf); + return NULL; + } } + return buf; + } +} + + + +/***************************************************************************** + * File helpers + *****************************************************************************/ +/* + * fopen wrapper that supports UTF-8 + * + * fopen will only accept ANSI filenames, which means that we can't open Unicode filenames. + * + * In order to open a Unicode filename, we need to convert filenames to UTF-16 and use _wfopen. + */ +XSUM_API FILE* XSUM_fopen(const char* filename, const char* mode) +{ + FILE* f = NULL; + wchar_t* const wide_filename = XSUM_widenString(filename, NULL); + if (wide_filename != NULL) { + wchar_t* const wide_mode = XSUM_widenString(mode, NULL); + if (wide_mode != NULL) { + f = _wfopen(wide_filename, wide_mode); + free(wide_mode); + } + free(wide_filename); + } + return f; +} + +/* + * stat() wrapper which supports UTF-8 filenames. + */ +static int XSUM_stat(const char* infilename, XSUM_stat_t* statbuf) +{ + int r = -1; + wchar_t* const wide_filename = XSUM_widenString(infilename, NULL); + if (wide_filename != NULL) { + r = _wstat64(wide_filename, statbuf); + free(wide_filename); + } + return r; +} + +/* + * In case it isn't available, this is what MSVC 2019 defines in stdarg.h. + */ +#if defined(_MSC_VER) && !defined(__clang__) && !defined(va_copy) +# define XSUM_va_copy(destination, source) ((destination) = (source)) +#else +# define XSUM_va_copy(destination, source) va_copy(destination, source) +#endif + +/* + * vasprintf for Windows. + */ +XSUM_ATTRIBUTE((__format__(__printf__, 2, 0))) +static int XSUM_vasprintf(char** strp, const char* format, va_list ap) +{ + int size; + va_list copy; + /* + * To be safe, make a va_copy. + * + * Note that Microsoft doesn't use va_copy in its sample code: + * https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/vsprintf-vsprintf-l-vswprintf-vswprintf-l-vswprintf-l?view=vs-2019 + */ + XSUM_va_copy(copy, ap); + /* Calculate how many characters we need */ + size = _vscprintf(format, ap); + va_end(copy); + + if (size < 0) { + *strp = NULL; + return size; + } else { + int ret; + *strp = (char*) malloc((size_t)size + 1); + if (*strp == NULL) { + return -1; + } + /* vsprintf into the new buffer */ + ret = vsprintf(*strp, format, ap); + if (ret < 0) { + free(*strp); + *strp = NULL; + } + return ret; + } +} + +/* + * fprintf wrapper that supports UTF-8. + * + * fprintf doesn't properly handle Unicode on Windows. + * + * Additionally, it is codepage sensitive on console and may crash the program. + * + * Instead, we use vsnprintf, and either print with fwrite or convert to UTF-16 + * for console output and use the codepage-independent WriteConsoleW. + * + * Credit to t-mat: https://github.com/t-mat/xxHash/commit/5691423 + */ +XSUM_ATTRIBUTE((__format__(__printf__, 2, 0))) +XSUM_API int XSUM_vfprintf(FILE *stream, const char *format, va_list ap) +{ + int result; + char* u8_str = NULL; + + /* + * Generate the UTF-8 output string with vasprintf. + */ + result = XSUM_vasprintf(&u8_str, format, ap); + + if (result >= 0) { + const size_t nchar = (size_t)result + 1; + + /* + * Check if we are outputting to a console. Don't use XSUM_isConsole + * directly -- we don't need to call _get_osfhandle twice. + */ + int fileNb = _fileno(stream); + intptr_t handle_raw = _get_osfhandle(fileNb); + HANDLE handle = (HANDLE)handle_raw; + DWORD dwTemp; + + if (handle_raw < 0) { + result = -1; + } else if (_isatty(fileNb) && GetConsoleMode(handle, &dwTemp)) { + /* + * Convert to UTF-16 and output with WriteConsoleW. + * + * This is codepage independent and works on Windows XP's default + * msvcrt.dll. + */ + int len; + wchar_t* const u16_buf = XSUM_widenString(u8_str, &len); + if (u16_buf == NULL) { + result = -1; + } else { + if (WriteConsoleW(handle, u16_buf, (DWORD)len - 1, &dwTemp, NULL)) { + result = (int)dwTemp; + } else { + result = -1; + } + free(u16_buf); + } + } else { + /* fwrite the UTF-8 string if we are printing to a file */ + result = (int)fwrite(u8_str, 1, nchar - 1, stream); + if (result == 0) { + result = -1; + } + } + free(u8_str); + } + return result; +} + +#ifndef XSUM_NO_MAIN +/***************************************************************************** + * Command Line argument parsing + *****************************************************************************/ + +/* Converts a UTF-16 argv to UTF-8. */ +static char** XSUM_convertArgv(int argc, wchar_t* utf16_argv[]) +{ + char** const utf8_argv = (char**)malloc((size_t)(argc + 1) * sizeof(char*)); + if (utf8_argv != NULL) { + int i; + for (i = 0; i < argc; i++) { + utf8_argv[i] = XSUM_narrowString(utf16_argv[i], NULL); + if (utf8_argv[i] == NULL) { + /* Out of memory, whoops. */ + while (i-- > 0) { + free(utf8_argv[i]); + } + free(utf8_argv); + return NULL; + } + } + utf8_argv[argc] = NULL; + } + return utf8_argv; +} +/* Frees arguments returned by XSUM_convertArgv */ +static void XSUM_freeArgv(int argc, char** argv) +{ + int i; + if (argv == NULL) { + return; + } + for (i = 0; i < argc; i++) { + free(argv[i]); + } + free(argv); +} + +static int XSUM_wmain(int argc, wchar_t* utf16_argv[]) +{ + /* Convert the UTF-16 arguments to UTF-8. */ + char** utf8_argv = XSUM_convertArgv(argc, utf16_argv); + + if (utf8_argv == NULL) { + /* An unfortunate but incredibly unlikely error. */ + fprintf(stderr, "xxhsum: error converting command line arguments!\n"); + abort(); + } else { + int ret; + + /* + * MinGW's terminal uses full block buffering for stderr. + * + * This is nonstandard behavior and causes text to not display until + * the buffer fills. + * + * `setvbuf()` can easily correct this to make text display instantly. + */ + setvbuf(stderr, NULL, _IONBF, 0); + + /* Call our real main function */ + ret = XSUM_main(argc, (void*)utf8_argv); + + /* Cleanup */ + XSUM_freeArgv(argc, utf8_argv); + return ret; + } +} + +#if XSUM_WIN32_USE_WMAIN + +/* + * The preferred method of obtaining the real UTF-16 arguments. Always works + * on MSVC, sometimes works on MinGW-w64 depending on the compiler flags. + */ +#ifdef __cplusplus +extern "C" +#endif +int __cdecl wmain(int argc, wchar_t* utf16_argv[]) +{ + return XSUM_wmain(argc, utf16_argv); +} +#else /* !XSUM_WIN32_USE_WMAIN */ + +/* + * Wrap `XSUM_wmain()` using `main()` and `__wgetmainargs()` on MinGW without + * Unicode support. + * + * `__wgetmainargs()` is used in the CRT startup to retrieve the arguments for + * `wmain()`, so we use it on MinGW to emulate `wmain()`. + * + * It is an internal function and not declared in any public headers, so we + * have to declare it manually. + * + * An alternative that doesn't mess with internal APIs is `GetCommandLineW()` + * with `CommandLineToArgvW()`, but the former doesn't expand wildcards and the + * latter requires linking to Shell32.dll and its numerous dependencies. + * + * This method keeps our dependencies to kernel32.dll and the CRT. + * + * https://docs.microsoft.com/en-us/cpp/c-runtime-library/getmainargs-wgetmainargs?view=vs-2019 + */ +typedef struct { + int newmode; +} _startupinfo; + +#ifdef __cplusplus +extern "C" +#endif +int __cdecl __wgetmainargs( + int* Argc, + wchar_t*** Argv, + wchar_t*** Env, + int DoWildCard, + _startupinfo* StartInfo +); + +int main(int ansi_argc, const char* ansi_argv[]) +{ + int utf16_argc; + wchar_t** utf16_argv; + wchar_t** utf16_envp; /* Unused but required */ + _startupinfo startinfo = {0}; /* 0 == don't change new mode */ + + /* Get wmain's UTF-16 arguments. Make sure we expand wildcards. */ + if (__wgetmainargs(&utf16_argc, &utf16_argv, &utf16_envp, 1, &startinfo) < 0) + /* In the very unlikely case of an error, use the ANSI arguments. */ + return XSUM_main(ansi_argc, ansi_argv); + + /* Call XSUM_wmain with our UTF-16 arguments */ + return XSUM_wmain(utf16_argc, utf16_argv); +} + +#endif /* !XSUM_WIN32_USE_WMAIN */ +#endif /* !XSUM_NO_MAIN */ +#endif /* XSUM_WIN32_USE_WCHAR */ + + +/* + * Determines whether the file at filename is a directory. + */ +XSUM_API int XSUM_isDirectory(const char* filename) +{ + XSUM_stat_t statbuf; + int r = XSUM_stat(filename, &statbuf); +#ifdef _MSC_VER + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + +/* + * Returns the filesize of the file at filename. + */ +XSUM_API XSUM_U64 XSUM_getFileSize(const char* filename) +{ + XSUM_stat_t statbuf; + int r = XSUM_stat(filename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ + return (XSUM_U64)statbuf.st_size; +} diff --git a/third_party/xxhash/cli/xsum_os_specific.h b/third_party/xxhash/cli/xsum_os_specific.h new file mode 100644 index 000000000..c890bd638 --- /dev/null +++ b/third_party/xxhash/cli/xsum_os_specific.h @@ -0,0 +1,96 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XSUM_OS_SPECIFIC_H +#define XSUM_OS_SPECIFIC_H + +#include "third_party/xxhash/cli/xsum_config.h" +#include "libc/calls/calls.h" +#include "libc/calls/weirdtypes.h" +#include "libc/fmt/fmt.h" +#include "libc/stdio/dprintf.h" +#include "libc/stdio/stdio.h" +#include "libc/stdio/temp.h" +#include "third_party/musl/tempnam.h" + + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Declared here to be implemented in user code. + * + * Functions like main(), but is passed UTF-8 arguments even on Windows. + */ +XSUM_API int XSUM_main(int argc, const char* argv[]); + +/* + * Returns whether stream is a console. + * + * Functionally equivalent to isatty(fileno(stream)). + */ +XSUM_API int XSUM_isConsole(FILE* stream); + +/* + * Sets stream to pure binary mode (a.k.a. no CRLF conversions). + */ +XSUM_API void XSUM_setBinaryMode(FILE* stream); + +/* + * Returns whether the file at filename is a directory. + */ +XSUM_API int XSUM_isDirectory(const char* filename); + +/* + * Returns the file size of the file at filename. + */ +XSUM_API XSUM_U64 XSUM_getFileSize(const char* filename); + +/* + * UTF-8 stdio wrappers primarily for Windows + */ + +/* + * fopen() wrapper. Accepts UTF-8 filenames on Windows. + * + * Specifically, on Windows, the arguments will be converted to UTF-16 + * and passed to _wfopen(). + */ +XSUM_API FILE* XSUM_fopen(const char* filename, const char* mode); + +/* + * vfprintf() wrapper which prints UTF-8 strings to Windows consoles + * if applicable. + */ +XSUM_ATTRIBUTE((__format__(__printf__, 2, 0))) +XSUM_API int XSUM_vfprintf(FILE* stream, const char* format, va_list ap); + +#ifdef __cplusplus +} +#endif + +#endif /* XSUM_OS_SPECIFIC_H */ diff --git a/third_party/xxhash/cli/xsum_output.c b/third_party/xxhash/cli/xsum_output.c new file mode 100644 index 000000000..71da757bf --- /dev/null +++ b/third_party/xxhash/cli/xsum_output.c @@ -0,0 +1,67 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#include "third_party/xxhash/cli/xsum_output.h" +#include "third_party/xxhash/cli/xsum_os_specific.h" /* XSUM_API */ + +int XSUM_logLevel = 2; + +XSUM_ATTRIBUTE((__format__(__printf__, 1, 2))) +XSUM_API int XSUM_log(const char* format, ...) +{ + int ret; + va_list ap; + va_start(ap, format); + ret = XSUM_vfprintf(stderr, format, ap); + va_end(ap); + return ret; +} + + +XSUM_ATTRIBUTE((__format__(__printf__, 1, 2))) +XSUM_API int XSUM_output(const char* format, ...) +{ + int ret; + va_list ap; + va_start(ap, format); + ret = XSUM_vfprintf(stdout, format, ap); + va_end(ap); + return ret; +} + +XSUM_ATTRIBUTE((__format__(__printf__, 2, 3))) +XSUM_API int XSUM_logVerbose(int minLevel, const char* format, ...) +{ + if (XSUM_logLevel >= minLevel) { + int ret; + va_list ap; + va_start(ap, format); + ret = XSUM_vfprintf(stderr, format, ap); + va_end(ap); + return ret; + } + return 0; +} diff --git a/third_party/xxhash/cli/xsum_output.h b/third_party/xxhash/cli/xsum_output.h new file mode 100644 index 000000000..1f681d8f3 --- /dev/null +++ b/third_party/xxhash/cli/xsum_output.h @@ -0,0 +1,63 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XSUM_OUTPUT_H +#define XSUM_OUTPUT_H + +#include "third_party/xxhash/cli/xsum_config.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * How verbose the output is. + */ +extern int XSUM_logLevel; + +/* + * Same as fprintf(stderr, format, ...) + */ +XSUM_ATTRIBUTE((__format__(__printf__, 1, 2))) +XSUM_API int XSUM_log(const char *format, ...); + +/* + * Like XSUM_log, but only outputs if XSUM_logLevel >= minLevel. + */ +XSUM_ATTRIBUTE((__format__(__printf__, 2, 3))) +XSUM_API int XSUM_logVerbose(int minLevel, const char *format, ...); + +/* + * Same as printf(format, ...) + */ +XSUM_ATTRIBUTE((__format__(__printf__, 1, 2))) +XSUM_API int XSUM_output(const char *format, ...); + +#ifdef __cplusplus +} +#endif + +#endif /* XSUM_OUTPUT_H */ diff --git a/third_party/xxhash/cli/xsum_sanity_check.c b/third_party/xxhash/cli/xsum_sanity_check.c new file mode 100644 index 000000000..74377526f --- /dev/null +++ b/third_party/xxhash/cli/xsum_sanity_check.c @@ -0,0 +1,712 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#include "third_party/xxhash/cli/xsum_sanity_check.h" +#include "third_party/xxhash/cli/xsum_output.h" /* XSUM_log */ +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +#endif +#include "third_party/xxhash/xxhash.h" + +#include "libc/calls/calls.h" +#include "libc/calls/termios.h" +#include "libc/fmt/conv.h" +#include "libc/limits.h" +#include "libc/mem/alg.h" +#include "libc/mem/alloca.h" +#include "libc/mem/mem.h" +#include "libc/runtime/runtime.h" +#include "libc/stdio/dprintf.h" +#include "libc/stdio/rand.h" +#include "libc/stdio/temp.h" +#include "libc/str/str.h" +#include "libc/sysv/consts/exit.h" +#include "third_party/getopt/getopt.h" +#include "third_party/musl/crypt.h" +#include "third_party/musl/rand48.h" /* exit */ +#include "libc/assert.h" +#include "libc/mem/alg.h" +#include "libc/mem/mem.h" +#include "libc/str/str.h" /* memcmp */ + +/* use #define to make them constant, required for initialization */ +#define PRIME32 2654435761U +#define PRIME64 11400714785074694797ULL + +/* + * Fills a test buffer with pseudorandom data. + * + * This is used in the sanity check - its values must not be changed. + */ +XSUM_API void XSUM_fillTestBuffer(XSUM_U8* buffer, size_t len) +{ + XSUM_U64 byteGen = PRIME32; + size_t i; + + assert(buffer != NULL); + + for (i=0; i>56); + byteGen *= PRIME64; + } +} + + + +/* ************************************************ + * Self-test: + * ensure results consistency across platforms + *********************************************** */ +#if XSUM_NO_TESTS +XSUM_API void XSUM_sanityCheck(void) +{ + XSUM_log("This version of xxhsum is not verified.\n"); +} +#else + +/* + * Test data vectors + */ +typedef struct { + XSUM_U32 len; + XSUM_U32 seed; + XSUM_U32 Nresult; +} XSUM_testdata32_t; + +typedef struct { + XSUM_U32 len; + XSUM_U64 seed; + XSUM_U64 Nresult; +} XSUM_testdata64_t; + +typedef struct { + XSUM_U32 len; + XSUM_U64 seed; + XXH128_hash_t Nresult; +} XSUM_testdata128_t; + +#define SECRET_SAMPLE_NBBYTES 5 +typedef struct { + XSUM_U32 seedLen; + XSUM_U32 secretLen; + XSUM_U8 byte[SECRET_SAMPLE_NBBYTES]; +} XSUM_testdata_sample_t; + +/* XXH32 */ +static const XSUM_testdata32_t XSUM_XXH32_testdata[] = { + { 0, 0, 0x02CC5D05U }, + { 0, PRIME32, 0x36B78AE7U }, + { 1, 0, 0xCF65B03EU }, + { 1, PRIME32, 0xB4545AA4U }, + { 14, 0, 0x1208E7E2U }, + { 14, PRIME32, 0x6AF1D1FEU }, + { 222, 0, 0x5BD11DBDU }, + { 222, PRIME32, 0x58803C5FU } +}; + +/* XXH64 */ +static const XSUM_testdata64_t XSUM_XXH64_testdata[] = { + { 0, 0, 0xEF46DB3751D8E999ULL }, + { 0, PRIME32, 0xAC75FDA2929B17EFULL }, + { 1, 0, 0xE934A84ADB052768ULL }, + { 1, PRIME32, 0x5014607643A9B4C3ULL }, + { 4, 0, 0x9136A0DCA57457EEULL }, + { 14, 0, 0x8282DCC4994E35C8ULL }, + { 14, PRIME32, 0xC3BD6BF63DEB6DF0ULL }, + { 222, 0, 0xB641AE8CB691C174ULL }, + { 222, PRIME32, 0x20CB8AB7AE10C14AULL } +}; +/* + * XXH3: + * Due to being a more complex hash function with specializations for certain + * lengths, a more extensive test is used for XXH3. + */ + +/* XXH3_64bits, seeded */ +static const XSUM_testdata64_t XSUM_XXH3_testdata[] = { + { 0, 0, 0x2D06800538D394C2ULL }, /* empty string */ + { 0, PRIME64, 0xA8A6B918B2F0364AULL }, + { 1, 0, 0xC44BDFF4074EECDBULL }, /* 1 - 3 */ + { 1, PRIME64, 0x032BE332DD766EF8ULL }, + { 6, 0, 0x27B56A84CD2D7325ULL }, /* 4 - 8 */ + { 6, PRIME64, 0x84589C116AB59AB9ULL }, + { 12, 0, 0xA713DAF0DFBB77E7ULL }, /* 9 - 16 */ + { 12, PRIME64, 0xE7303E1B2336DE0EULL }, + { 24, 0, 0xA3FE70BF9D3510EBULL }, /* 17 - 32 */ + { 24, PRIME64, 0x850E80FC35BDD690ULL }, + { 48, 0, 0x397DA259ECBA1F11ULL }, /* 33 - 64 */ + { 48, PRIME64, 0xADC2CBAA44ACC616ULL }, + { 80, 0, 0xBCDEFBBB2C47C90AULL }, /* 65 - 96 */ + { 80, PRIME64, 0xC6DD0CB699532E73ULL }, + { 195, 0, 0xCD94217EE362EC3AULL }, /* 129-240 */ + { 195, PRIME64, 0xBA68003D370CB3D9ULL }, + + { 403, 0, 0xCDEB804D65C6DEA4ULL }, /* one block, last stripe is overlapping */ + { 403, PRIME64, 0x6259F6ECFD6443FDULL }, + { 512, 0, 0x617E49599013CB6BULL }, /* one block, finishing at stripe boundary */ + { 512, PRIME64, 0x3CE457DE14C27708ULL }, + { 2048, 0, 0xDD59E2C3A5F038E0ULL }, /* 2 blocks, finishing at block boundary */ + { 2048, PRIME64, 0x66F81670669ABABCULL }, + { 2099, 0, 0xC6B9D9B3FC9AC765ULL }, /* 2 blocks + 1 partial block, to detect off-by-one scrambling issues, like #816 */ + { 2099, PRIME64, 0x184F316843663974ULL }, + { 2240, 0, 0x6E73A90539CF2948ULL }, /* 3 blocks, finishing at stripe boundary */ + { 2240, PRIME64, 0x757BA8487D1B5247ULL }, + { 2367, 0, 0xCB37AEB9E5D361EDULL }, /* 3 blocks, last stripe is overlapping */ + { 2367, PRIME64, 0xD2DB3415B942B42AULL } +}; +/* XXH3_64bits, custom secret */ +static const XSUM_testdata64_t XSUM_XXH3_withSecret_testdata[] = { + { 0, 0, 0x3559D64878C5C66CULL }, /* empty string */ + { 1, 0, 0x8A52451418B2DA4DULL }, /* 1 - 3 */ + { 6, 0, 0x82C90AB0519369ADULL }, /* 4 - 8 */ + { 12, 0, 0x14631E773B78EC57ULL }, /* 9 - 16 */ + { 24, 0, 0xCDD5542E4A9D9FE8ULL }, /* 17 - 32 */ + { 48, 0, 0x33ABD54D094B2534ULL }, /* 33 - 64 */ + { 80, 0, 0xE687BA1684965297ULL }, /* 65 - 96 */ + { 195, 0, 0xA057273F5EECFB20ULL }, /* 129-240 */ + + { 403, 0, 0x14546019124D43B8ULL }, /* one block, last stripe is overlapping */ + { 512, 0, 0x7564693DD526E28DULL }, /* one block, finishing at stripe boundary */ + { 2048, 0, 0xD32E975821D6519FULL }, /* >= 2 blodcks, at least one scrambling */ + { 2367, 0, 0x293FA8E5173BB5E7ULL }, /* >= 2 blocks, at least one scrambling, last stripe unaligned */ + + { 64*10*3, 0, 0x751D2EC54BC6038BULL } /* exactly 3 full blocks, not a multiple of 256 */ +}; +/* XXH3_128bits, seeded */ +static const XSUM_testdata128_t XSUM_XXH128_testdata[] = { + { 0, 0, { 0x6001C324468D497FULL, 0x99AA06D3014798D8ULL } }, /* empty string */ + { 0, PRIME32, { 0x5444F7869C671AB0ULL, 0x92220AE55E14AB50ULL } }, + { 1, 0, { 0xC44BDFF4074EECDBULL, 0xA6CD5E9392000F6AULL } }, /* 1 - 3 */ + { 1, PRIME32, { 0xB53D5557E7F76F8DULL, 0x89B99554BA22467CULL } }, + { 6, 0, { 0x3E7039BDDA43CFC6ULL, 0x082AFE0B8162D12AULL } }, /* 4 - 8 */ + { 6, PRIME32, { 0x269D8F70BE98856EULL, 0x5A865B5389ABD2B1ULL } }, + { 12, 0, { 0x061A192713F69AD9ULL, 0x6E3EFD8FC7802B18ULL } }, /* 9 - 16 */ + { 12, PRIME32, { 0x9BE9F9A67F3C7DFBULL, 0xD7E09D518A3405D3ULL } }, + { 24, 0, { 0x1E7044D28B1B901DULL, 0x0CE966E4678D3761ULL } }, /* 17 - 32 */ + { 24, PRIME32, { 0xD7304C54EBAD40A9ULL, 0x3162026714A6A243ULL } }, + { 48, 0, { 0xF942219AED80F67BULL, 0xA002AC4E5478227EULL } }, /* 33 - 64 */ + { 48, PRIME32, { 0x7BA3C3E453A1934EULL, 0x163ADDE36C072295ULL } }, + { 81, 0, { 0x5E8BAFB9F95FB803ULL, 0x4952F58181AB0042ULL } }, /* 65 - 96 */ + { 81, PRIME32, { 0x703FBB3D7A5F755CULL, 0x2724EC7ADC750FB6ULL } }, + { 222, 0, { 0xF1AEBD597CEC6B3AULL, 0x337E09641B948717ULL } }, /* 129-240 */ + { 222, PRIME32, { 0xAE995BB8AF917A8DULL, 0x91820016621E97F1ULL } }, + + { 403, 0, { 0xCDEB804D65C6DEA4ULL, 0x1B6DE21E332DD73DULL } }, /* one block, last stripe is overlapping */ + { 403, PRIME64, { 0x6259F6ECFD6443FDULL, 0xBED311971E0BE8F2ULL } }, + { 512, 0, { 0x617E49599013CB6BULL, 0x18D2D110DCC9BCA1ULL } }, /* one block, finishing at stripe boundary */ + { 512, PRIME64, { 0x3CE457DE14C27708ULL, 0x925D06B8EC5B8040ULL } }, + { 2048, 0, { 0xDD59E2C3A5F038E0ULL, 0xF736557FD47073A5ULL } }, /* 2 blocks, finishing at block boundary */ + { 2048, PRIME32, { 0x230D43F30206260BULL, 0x7FB03F7E7186C3EAULL } }, + { 2240, 0, { 0x6E73A90539CF2948ULL, 0xCCB134FBFA7CE49DULL } }, /* 3 blocks, finishing at stripe boundary */ + { 2240, PRIME32, { 0xED385111126FBA6FULL, 0x50A1FE17B338995FULL } }, + { 2367, 0, { 0xCB37AEB9E5D361EDULL, 0xE89C0F6FF369B427ULL } }, /* 3 blocks, last stripe is overlapping */ + { 2367, PRIME32, { 0x6F5360AE69C2F406ULL, 0xD23AAE4B76C31ECBULL } } +}; + +/* XXH128, custom secret */ +static const XSUM_testdata128_t XSUM_XXH128_withSecret_testdata[] = { + { 0, 0, { 0x005923CCEECBE8AEULL, 0x5F70F4EA232F1D38ULL } }, /* empty string */ + { 1, 0, { 0x8A52451418B2DA4DULL, 0x3A66AF5A9819198EULL } }, /* 1 - 3 */ + { 6, 0, { 0x0B61C8ACA7D4778FULL, 0x376BD91B6432F36DULL } }, /* 4 - 8 */ + { 12, 0, { 0xAF82F6EBA263D7D8ULL, 0x90A3C2D839F57D0FULL } } /* 9 - 16 */ +}; + +#define SECRET_SIZE_MAX 9867 +static const XSUM_testdata_sample_t XSUM_XXH3_generateSecret_testdata[] = { + { 0, 192, { 0xE7, 0x8C, 0x77, 0x77, 0x00 } }, + { 1, 240, { 0x2B, 0x3E, 0xDE, 0xC1, 0x00 } }, + { XXH3_SECRET_SIZE_MIN - 1, 277, { 0xE8, 0x39, 0x6C, 0xCC, 0x7B } }, + { XXH3_SECRET_DEFAULT_SIZE + 500, SECRET_SIZE_MAX, { 0xD6, 0x1C, 0x41, 0x17, 0xB3 } } +}; + +static void XSUM_checkResult32(XXH32_hash_t r1, XXH32_hash_t r2) +{ + static int nbTests = 1; + if (r1!=r2) { + XSUM_log("\rError: 32-bit hash test %i: Internal sanity check failed!\n", nbTests); + XSUM_log("\rGot 0x%08X, expected 0x%08X.\n", (unsigned)r1, (unsigned)r2); + XSUM_log("\rNote: If you modified the hash functions, make sure to either update the values\n" + "or temporarily recompile with XSUM_NO_TESTS=1.\n"); + exit(1); + } + nbTests++; +} + +static void XSUM_checkResult64(XXH64_hash_t r1, XXH64_hash_t r2) +{ + static int nbTests = 1; + if (r1!=r2) { + XSUM_log("\rError: 64-bit hash test %i: Internal sanity check failed!\n", nbTests); + XSUM_log("\rGot 0x%08X%08XULL, expected 0x%08X%08XULL.\n", + (unsigned)(r1>>32), (unsigned)r1, (unsigned)(r2>>32), (unsigned)r2); + XSUM_log("\rNote: If you modified the hash functions, make sure to either update the values\n" + "or temporarily recompile with XSUM_NO_TESTS=1.\n"); + exit(1); + } + nbTests++; +} + +static void XSUM_checkResult128(XXH128_hash_t r1, XXH128_hash_t r2) +{ + static int nbTests = 1; + if ((r1.low64 != r2.low64) || (r1.high64 != r2.high64)) { + XSUM_log("\rError: 128-bit hash test %i: Internal sanity check failed.\n", nbTests); + XSUM_log("\rGot { 0x%08X%08XULL, 0x%08X%08XULL }, expected { 0x%08X%08XULL, 0x%08X%08XULL } \n", + (unsigned)(r1.low64>>32), (unsigned)r1.low64, (unsigned)(r1.high64>>32), (unsigned)r1.high64, + (unsigned)(r2.low64>>32), (unsigned)r2.low64, (unsigned)(r2.high64>>32), (unsigned)r2.high64 ); + XSUM_log("\rNote: If you modified the hash functions, make sure to either update the values\n" + "or temporarily recompile with XSUM_NO_TESTS=1.\n"); + exit(1); + } + nbTests++; +} + + +static void XSUM_testXXH32(const void* data, const XSUM_testdata32_t* testData) +{ + XXH32_state_t *state = XXH32_createState(); + size_t pos; + + size_t len = testData->len; + XSUM_U32 seed = testData->seed; + XSUM_U32 Nresult = testData->Nresult; + + if (len == 0) { + data = NULL; + } else { + assert(data != NULL); + } + + assert(state != NULL); + + XSUM_checkResult32(XXH32(data, len, seed), Nresult); + + (void)XXH32_reset(state, seed); + (void)XXH32_update(state, data, len); + XSUM_checkResult32(XXH32_digest(state), Nresult); + + (void)XXH32_reset(state, seed); + for (pos=0; poslen; + XSUM_U64 seed = testData->seed; + XSUM_U64 Nresult = testData->Nresult; + + if (len == 0) { + data = NULL; + } else { + assert(data != NULL); + } + + assert(state != NULL); + + XSUM_checkResult64(XXH64(data, len, seed), Nresult); + + (void)XXH64_reset(state, seed); + (void)XXH64_update(state, data, len); + XSUM_checkResult64(XXH64_digest(state), Nresult); + + (void)XXH64_reset(state, seed); + for (pos=0; pos> 40); +} + +/* + * Technically, XXH3_64bits_update is identical to XXH3_128bits_update as of + * v0.8.0, but we treat them as separate. + */ +typedef XXH_errorcode (*XSUM_XXH3_update_t)(XXH3_state_t* state, const void* input, size_t length); + +/* + * Runs the passed XXH3_update variant on random lengths. This is to test the + * more complex logic of the update function, catching bugs like this one: + * https://github.com/Cyan4973/xxHash/issues/378 + */ +static void XSUM_XXH3_randomUpdate(XXH3_state_t* state, const void* data, + size_t len, XSUM_XXH3_update_t update_fn) +{ + size_t p = 0; + while (p < len) { + size_t const modulo = len > 2 ? len : 2; + size_t l = (size_t)(XSUM_rand()) % modulo; + if (p + l > len) l = len - p; + (void)update_fn(state, (const char*)data+p, l); + p += l; + } +} + +static void XSUM_testXXH3(const void* data, const XSUM_testdata64_t* testData) +{ + size_t len = testData->len; + XSUM_U64 seed = testData->seed; + XSUM_U64 Nresult = testData->Nresult; + if (len == 0) { + data = NULL; + } else { + assert(data != NULL); + } + { XSUM_U64 const Dresult = XXH3_64bits_withSeed(data, len, seed); + XSUM_checkResult64(Dresult, Nresult); + } + + /* check that the no-seed variant produces same result as seed==0 */ + if (seed == 0) { + XSUM_U64 const Dresult = XXH3_64bits(data, len); + XSUM_checkResult64(Dresult, Nresult); + } + + /* check that the combination of + * XXH3_generateSecret_fromSeed() and XXH3_64bits_withSecretandSeed() + * results in exactly the same hash generation as XXH3_64bits_withSeed() */ + { char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1]; + char* const secret = secretBuffer + 1; /* intentional unalignment */ + XXH3_generateSecret_fromSeed(secret, seed); + { XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed); + XSUM_checkResult64(Dresult, Nresult); + } } + + /* streaming API test */ + { XXH3_state_t* const state = XXH3_createState(); + assert(state != NULL); + /* single ingestion */ + (void)XXH3_64bits_reset_withSeed(state, seed); + (void)XXH3_64bits_update(state, data, len); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + + /* random ingestion */ + (void)XXH3_64bits_reset_withSeed(state, seed); + XSUM_XXH3_randomUpdate(state, data, len, &XXH3_64bits_update); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + + /* byte by byte ingestion */ + { size_t pos; + (void)XXH3_64bits_reset_withSeed(state, seed); + for (pos=0; poslen; + XSUM_U64 Nresult = testData->Nresult; + + if (len == 0) { + data = NULL; + } else { + assert(data != NULL); + } + { XSUM_U64 const Dresult = XXH3_64bits_withSecret(data, len, secret, secretSize); + XSUM_checkResult64(Dresult, Nresult); + } + + /* check that XXH3_64bits_withSecretandSeed() + * results in exactly the same return value as XXH3_64bits_withSecret() */ + if (len > XXH3_MIDSIZE_MAX) + { XSUM_U64 const Dresult = XXH3_64bits_withSecretandSeed(data, len, secret, secretSize, 0); + XSUM_checkResult64(Dresult, Nresult); + } + + /* streaming API test */ + { XXH3_state_t *state = XXH3_createState(); + assert(state != NULL); + (void)XXH3_64bits_reset_withSecret(state, secret, secretSize); + (void)XXH3_64bits_update(state, data, len); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + + /* random ingestion */ + (void)XXH3_64bits_reset_withSecret(state, secret, secretSize); + XSUM_XXH3_randomUpdate(state, data, len, &XXH3_64bits_update); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + + /* byte by byte ingestion */ + { size_t pos; + (void)XXH3_64bits_reset_withSecret(state, secret, secretSize); + for (pos=0; pos XXH3_MIDSIZE_MAX) { + /* single ingestion */ + (void)XXH3_64bits_reset_withSecretandSeed(state, secret, secretSize, 0); + (void)XXH3_64bits_update(state, data, len); + XSUM_checkResult64(XXH3_64bits_digest(state), Nresult); + } + + XXH3_freeState(state); + } +} + +static void XSUM_testXXH128(const void* data, const XSUM_testdata128_t* testData) +{ + size_t len = (size_t)testData->len; + XSUM_U64 seed = testData->seed; + XXH128_hash_t const Nresult = testData->Nresult; + if (len == 0) { + data = NULL; + } else { + assert(data != NULL); + } + + { XXH128_hash_t const Dresult = XXH3_128bits_withSeed(data, len, seed); + XSUM_checkResult128(Dresult, Nresult); + } + + /* check that XXH128() is identical to XXH3_128bits_withSeed() */ + { XXH128_hash_t const Dresult2 = XXH128(data, len, seed); + XSUM_checkResult128(Dresult2, Nresult); + } + + /* check that the no-seed variant produces same result as seed==0 */ + if (seed == 0) { + XXH128_hash_t const Dresult = XXH3_128bits(data, len); + XSUM_checkResult128(Dresult, Nresult); + } + + /* check that the combination of + * XXH3_generateSecret_fromSeed() and XXH3_128bits_withSecretandSeed() + * results in exactly the same hash generation as XXH3_64bits_withSeed() */ + { char secretBuffer[XXH3_SECRET_DEFAULT_SIZE+1]; + char* const secret = secretBuffer + 1; /* intentional unalignment */ + XXH3_generateSecret_fromSeed(secret, seed); + { XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, XXH3_SECRET_DEFAULT_SIZE, seed); + XSUM_checkResult128(Dresult, Nresult); + } } + + /* streaming API test */ + { XXH3_state_t *state = XXH3_createState(); + assert(state != NULL); + + /* single ingestion */ + (void)XXH3_128bits_reset_withSeed(state, seed); + (void)XXH3_128bits_update(state, data, len); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + + /* random ingestion */ + (void)XXH3_128bits_reset_withSeed(state, seed); + XSUM_XXH3_randomUpdate(state, data, len, &XXH3_128bits_update); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + + /* byte by byte ingestion */ + { size_t pos; + (void)XXH3_128bits_reset_withSeed(state, seed); + for (pos=0; poslen; + XXH128_hash_t Nresult = testData->Nresult; + if (len == 0) { + data = NULL; + } else { + assert(data != NULL); + } + { XXH128_hash_t const Dresult = XXH3_128bits_withSecret(data, len, secret, secretSize); + XSUM_checkResult128(Dresult, Nresult); + } + + /* check that XXH3_128bits_withSecretandSeed() + * results in exactly the same return value as XXH3_128bits_withSecret() */ + if (len > XXH3_MIDSIZE_MAX) + { XXH128_hash_t const Dresult = XXH3_128bits_withSecretandSeed(data, len, secret, secretSize, 0); + XSUM_checkResult128(Dresult, Nresult); + } + + /* streaming API test */ + { XXH3_state_t* const state = XXH3_createState(); + assert(state != NULL); + (void)XXH3_128bits_reset_withSecret(state, secret, secretSize); + (void)XXH3_128bits_update(state, data, len); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + + /* random ingestion */ + (void)XXH3_128bits_reset_withSecret(state, secret, secretSize); + XSUM_XXH3_randomUpdate(state, data, len, &XXH3_128bits_update); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + + /* byte by byte ingestion */ + { size_t pos; + (void)XXH3_128bits_reset_withSecret(state, secret, secretSize); + for (pos=0; pos XXH3_MIDSIZE_MAX) { + /* single ingestion */ + (void)XXH3_128bits_reset_withSecretandSeed(state, secret, secretSize, 0); + (void)XXH3_128bits_update(state, data, len); + XSUM_checkResult128(XXH3_128bits_digest(state), Nresult); + } + + XXH3_freeState(state); + } +} + +static void XSUM_testSecretGenerator(const void* customSeed, const XSUM_testdata_sample_t* testData) +{ + static int nbTests = 1; + const int sampleIndex[SECRET_SAMPLE_NBBYTES] = { 0, 62, 131, 191, 241 }; /* position of sampled bytes */ + XSUM_U8 secretBuffer[SECRET_SIZE_MAX] = {0}; + XSUM_U8 samples[SECRET_SAMPLE_NBBYTES]; + int i; + + assert(testData->secretLen <= SECRET_SIZE_MAX); + XXH3_generateSecret(secretBuffer, testData->secretLen, customSeed, testData->seedLen); + for (i=0; ibyte, sizeof(testData->byte))) { + XSUM_log("\rError: Secret generation test %i: Internal sanity check failed. \n", nbTests); + XSUM_log("\rGot { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X }, expected { 0x%02X, 0x%02X, 0x%02X, 0x%02X, 0x%02X } \n", + samples[0], samples[1], samples[2], samples[3], samples[4], + testData->byte[0], testData->byte[1], testData->byte[2], testData->byte[3], testData->byte[4] ); + exit(1); + } + nbTests++; +} + +/*! + * XSUM_sanityCheck(): + * Runs a sanity check before the benchmark. + * + * Exits on an incorrect output. + */ +XSUM_API void XSUM_sanityCheck(void) +{ + size_t i; +#define SANITY_BUFFER_SIZE 2367 + XSUM_U8 sanityBuffer[SANITY_BUFFER_SIZE]; + const void* const secret = sanityBuffer + 7; + const size_t secretSize = XXH3_SECRET_SIZE_MIN + 11; + assert(sizeof(sanityBuffer) >= 7 + secretSize); + + XSUM_fillTestBuffer(sanityBuffer, sizeof(sanityBuffer)); + + /* XXH32 */ + for (i = 0; i < (sizeof(XSUM_XXH32_testdata)/sizeof(XSUM_XXH32_testdata[0])); i++) { + XSUM_testXXH32(sanityBuffer, &XSUM_XXH32_testdata[i]); + } + /* XXH64 */ + for (i = 0; i < (sizeof(XSUM_XXH64_testdata)/sizeof(XSUM_XXH64_testdata[0])); i++) { + XSUM_testXXH64(sanityBuffer, &XSUM_XXH64_testdata[i]); + } + /* XXH3_64bits, seeded */ + for (i = 0; i < (sizeof(XSUM_XXH3_testdata)/sizeof(XSUM_XXH3_testdata[0])); i++) { + XSUM_testXXH3(sanityBuffer, &XSUM_XXH3_testdata[i]); + } + /* XXH3_64bits, custom secret */ + for (i = 0; i < (sizeof(XSUM_XXH3_withSecret_testdata)/sizeof(XSUM_XXH3_withSecret_testdata[0])); i++) { + XSUM_testXXH3_withSecret(sanityBuffer, secret, secretSize, &XSUM_XXH3_withSecret_testdata[i]); + } + /* XXH128 */ + for (i = 0; i < (sizeof(XSUM_XXH128_testdata)/sizeof(XSUM_XXH128_testdata[0])); i++) { + XSUM_testXXH128(sanityBuffer, &XSUM_XXH128_testdata[i]); + } + /* XXH128 with custom Secret */ + for (i = 0; i < (sizeof(XSUM_XXH128_withSecret_testdata)/sizeof(XSUM_XXH128_withSecret_testdata[0])); i++) { + XSUM_testXXH128_withSecret(sanityBuffer, secret, secretSize, &XSUM_XXH128_withSecret_testdata[i]); + } + /* secret generator */ + for (i = 0; i < (sizeof(XSUM_XXH3_generateSecret_testdata)/sizeof(XSUM_XXH3_generateSecret_testdata[0])); i++) { + assert(XSUM_XXH3_generateSecret_testdata[i].seedLen <= SANITY_BUFFER_SIZE); + XSUM_testSecretGenerator(sanityBuffer, &XSUM_XXH3_generateSecret_testdata[i]); + } + + XSUM_logVerbose(3, "\r%70s\r", ""); /* Clean display line */ + XSUM_logVerbose(3, "Sanity check -- all tests ok\n"); +} + +#endif /* !XSUM_NO_TESTS */ diff --git a/third_party/xxhash/cli/xsum_sanity_check.h b/third_party/xxhash/cli/xsum_sanity_check.h new file mode 100644 index 000000000..29ada2d42 --- /dev/null +++ b/third_party/xxhash/cli/xsum_sanity_check.h @@ -0,0 +1,61 @@ +// clang-format off +/* + * xxhsum - Command line interface for xxhash algorithms + * Copyright (C) 2013-2021 Yann Collet + * + * GPL v2 License + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along + * with this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + * + * You can contact the author at: + * - xxHash homepage: https://www.xxhash.com + * - xxHash source repository: https://github.com/Cyan4973/xxHash + */ + +#ifndef XSUM_SANITY_CHECK_H +#define XSUM_SANITY_CHECK_H + +#include "third_party/xxhash/cli/xsum_config.h" /* XSUM_API, XSUM_U8 */ + + /* size_t */ + + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Runs a series of self-tests. + * + * Exits if any of these tests fail, printing a message to stderr. + * + * If XSUM_NO_TESTS is defined to non-zero, + * this will instead print a warning if this is called (e.g. via xxhsum -b). + */ +XSUM_API void XSUM_sanityCheck(void); + +/* + * Fills a test buffer with pseudorandom data. + * + * This is used in the sanity check and the benchmarks. + * Its values must not be changed. + */ +XSUM_API void XSUM_fillTestBuffer(XSUM_U8* buffer, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif /* XSUM_SANITY_CHECK_H */ diff --git a/third_party/xxhash/cli/xxhsum.1 b/third_party/xxhash/cli/xxhsum.1 new file mode 100644 index 000000000..672a63c57 --- /dev/null +++ b/third_party/xxhash/cli/xxhsum.1 @@ -0,0 +1,107 @@ +// clang-format off +.TH "XXHSUM" "1" "December 2021" "xxhsum 0.8.1" "User Commands" +.SH "NAME" +\fBxxhsum\fR \- print or check xxHash non\-cryptographic checksums +.SH "SYNOPSIS" +\fBxxhsum [