From 3ae9347ad2d98ef6a315690bb6589e9d9bce3960 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 29 Jan 2025 10:00:08 +0100
Subject: [PATCH] bench: Cache benchmark results + better re-run filtering

---
 FU/Benchmarks.pod | 130 +++++++++++++++++++++++++++++++---------------
 Makefile.PL       |   5 ++
 bench.PL          | 121 ++++++++++++++++++++++++++++--------------
 3 files changed, 174 insertions(+), 82 deletions(-)

diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index 125373f..35c16c3 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -4,28 +4,35 @@ FU::Benchmarks - A bunch of automated benchmark results.
 
 =head1 DESCRIPTION
 
-This file is automatically generated from 'bench.pl' in the L<FU> distribution.
+This file is automatically generated from 'bench.PL' in the L<FU> distribution.
 These benchmarks compare performance of some FU functionality against similar
 modules found on CPAN.
 
-=head1 CONTEXT
+B<DISCLAIMER#1:> Obtaining accurate measurements is notoriously hard. Take the
+numbers below with a few buckets of salt, any difference below 10% is most
+likely noise.
 
-These benchmarks were performed on 2025-01-28 with perl v5.40.0 on x86_64-linux.
+B<DISCLAIMER#2:> Goodhart's law: "When a measure becomes a target, it ceases to
+be a good measure". I've used these benchmarks to find and optimize hotspots in
+FU, which in turn means these numbers may look better than they are in
+real-world use.
+
+=head1 MODULE VERSIONS
 
 The following module versions were used:
 
 =over
 
-=item L<FU> 0.1
-
 =item L<Cpanel::JSON::XS> 4.38
 
+=item L<FU> 0.1
+
 =item L<JSON::PP> 4.16
 
-=item L<JSON::XS> 4.03
-
 =item L<JSON::SIMD> 1.06
 
+=item L<JSON::XS> 4.03
+
 
 
 =back
@@ -44,58 +51,97 @@ SIMD parts are only used for parsing.
 
 API object from L<JSON::XS> documentation.
 
-            JSON::PP      5370/s
-    Cpanel::JSON::XS    112211/s
-          JSON::SIMD    128743/s
-            JSON::XS    130606/s
-            FU::Util    130813/s
+          JSON::PP      5342/s
+  Cpanel::JSON::XS    110660/s
+        JSON::SIMD    128161/s
+          JSON::XS    130434/s
+          FU::Util    129117/s
 
 Small integers
 
-            JSON::PP       113/s
-    Cpanel::JSON::XS      7262/s
-          JSON::SIMD      8217/s
-            JSON::XS      8142/s
-            FU::Util      9154/s
+          JSON::PP       117/s
+  Cpanel::JSON::XS      7370/s
+        JSON::SIMD      8191/s
+          JSON::XS      8143/s
+          FU::Util      9188/s
 
 Large integers
 
-            JSON::PP      2136/s
-    Cpanel::JSON::XS     29220/s
-          JSON::SIMD     35834/s
-            JSON::XS     35879/s
-            FU::Util    117838/s
+          JSON::PP      2208/s
+  Cpanel::JSON::XS     29299/s
+        JSON::SIMD     37344/s
+          JSON::XS     35873/s
+          FU::Util    114084/s
 
 ASCII strings
 
-            JSON::PP      2893/s
-    Cpanel::JSON::XS    118698/s
-          JSON::SIMD    137235/s
-            JSON::XS    135933/s
-            FU::Util    172207/s
+          JSON::PP      2798/s
+  Cpanel::JSON::XS    116754/s
+        JSON::SIMD    134130/s
+          JSON::XS    133137/s
+          FU::Util    166142/s
 
 Unicode strings
 
-            JSON::PP      5186/s
-    Cpanel::JSON::XS     97154/s
-          JSON::SIMD    109441/s
-            JSON::XS    105691/s
-            FU::Util    106058/s
+          JSON::PP      5067/s
+  Cpanel::JSON::XS     95453/s
+        JSON::SIMD    107955/s
+          JSON::XS    105367/s
+          FU::Util    103071/s
 
 String escaping (few)
 
-            JSON::PP      4280/s
-    Cpanel::JSON::XS    140105/s
-          JSON::SIMD    161231/s
-            JSON::XS    160077/s
-            FU::Util    182074/s
+          JSON::PP      4275/s
+  Cpanel::JSON::XS    138030/s
+        JSON::SIMD    157735/s
+          JSON::XS    159066/s
+          FU::Util    171426/s
 
 String escaping (many)
 
-            JSON::PP      2235/s
-    Cpanel::JSON::XS    144829/s
-          JSON::SIMD    161006/s
-            JSON::XS    161246/s
-            FU::Util    136568/s
+          JSON::PP      2231/s
+  Cpanel::JSON::XS    140657/s
+        JSON::SIMD    154850/s
+          JSON::XS    154280/s
+          FU::Util    132514/s
 
 
+
+=cut
+
+# Cached data used by bench.PL.
+jsonfmt/api	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	110660
+jsonfmt/api	FU::Util	FU	0.1	129117
+jsonfmt/api	JSON::PP	JSON::PP	4.16	5342
+jsonfmt/api	JSON::SIMD	JSON::SIMD	1.06	128161
+jsonfmt/api	JSON::XS	JSON::XS	4.03	130434
+jsonfmt/intl	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	29299
+jsonfmt/intl	FU::Util	FU	0.1	114084
+jsonfmt/intl	JSON::PP	JSON::PP	4.16	2208
+jsonfmt/intl	JSON::SIMD	JSON::SIMD	1.06	37344
+jsonfmt/intl	JSON::XS	JSON::XS	4.03	35873
+jsonfmt/ints	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	7370
+jsonfmt/ints	FU::Util	FU	0.1	9188
+jsonfmt/ints	JSON::PP	JSON::PP	4.16	117
+jsonfmt/ints	JSON::SIMD	JSON::SIMD	1.06	8191
+jsonfmt/ints	JSON::XS	JSON::XS	4.03	8143
+jsonfmt/strel	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	140657
+jsonfmt/strel	FU::Util	FU	0.1	132514
+jsonfmt/strel	JSON::PP	JSON::PP	4.16	2231
+jsonfmt/strel	JSON::SIMD	JSON::SIMD	1.06	154850
+jsonfmt/strel	JSON::XS	JSON::XS	4.03	154280
+jsonfmt/stres	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	138030
+jsonfmt/stres	FU::Util	FU	0.1	171426
+jsonfmt/stres	JSON::PP	JSON::PP	4.16	4275
+jsonfmt/stres	JSON::SIMD	JSON::SIMD	1.06	157735
+jsonfmt/stres	JSON::XS	JSON::XS	4.03	159066
+jsonfmt/strs	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	116754
+jsonfmt/strs	FU::Util	FU	0.1	166142
+jsonfmt/strs	JSON::PP	JSON::PP	4.16	2798
+jsonfmt/strs	JSON::SIMD	JSON::SIMD	1.06	134130
+jsonfmt/strs	JSON::XS	JSON::XS	4.03	133137
+jsonfmt/stru	Cpanel::JSON::XS	Cpanel::JSON::XS	4.38	95453
+jsonfmt/stru	FU::Util	FU	0.1	103071
+jsonfmt/stru	JSON::PP	JSON::PP	4.16	5067
+jsonfmt/stru	JSON::SIMD	JSON::SIMD	1.06	107955
+jsonfmt/stru	JSON::XS	JSON::XS	4.03	105367
diff --git a/Makefile.PL b/Makefile.PL
index f2d48be..29553b7 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -1,4 +1,9 @@
 use ExtUtils::MakeMaker;
+use Config;
+
+os_unsupported if $Config{ivsize} < 8;
+os_unsupported if $Config{usequadmath};
+
 WriteMakefile(
     NAME => 'FU',
     VERSION_FROM => 'FU.pm',
diff --git a/bench.PL b/bench.PL
index 9d95c83..0319540 100755
--- a/bench.PL
+++ b/bench.PL
@@ -1,17 +1,21 @@
 #!/usr/bin/perl
 
-exit if @ARGV && @ARGV[0] eq 'bench';
-
 # Can be invoked as:
-#   ./bench.PL          # (or 'make bench') generates FU/Benchmarks.pod
-#   ./bench.PL regex    # run benchmark(s) matching the regex
+#   ./bench.PL          # generates FU/Benchmarks.pod, running new benchmarks as necessary
+#   ./bench.PL id func  # invalidate cache for the (regex-)matching benchmark IDs and funcs and re-run them
+#
+# This script obviously has more dependencies than the FU distribution itself.
+# It's supposed to be used by maintainers, not users.
+
+
+# MakeMaker automatically runs this script as a default built step, but that's not very useful.
+BEGIN { exit if @ARGV && @ARGV[0] eq 'bench'; }
 
 use v5.36;
 use builtin 'true', 'false';
 use Benchmark ':hireswallclock', 'timethis';
-use Config;
 
-my $modules = join '', map sprintf("=item L<%s> %s\n\n", $_, eval "require $_; \$${_}::VERSION"), qw/
+my %modules = map +($_, eval "require $_; \$${_}::VERSION"), qw/
     FU
     Cpanel::JSON::XS
     JSON::PP
@@ -19,26 +23,43 @@ my $modules = join '', map sprintf("=item L<%s> %s\n\n", $_, eval "require $_; \
     JSON::SIMD
 /;
 
-
-my(%bench, @bench);
-sub bench($name, @arg) {
-    push @bench, $name;
-    $bench{$name} = \@arg;
-}
-
-sub runbench($text, @f) {
-    print "$text\n\n";
-
-    # TODO: Should include variance; factor-compared-to-slowest might be cool too
-    for my ($t, $f) (@f) {
-        my $o = timethis -1, $f, 0, 'none';
-        printf "  %18s%10d/s\n", $t, $o->iters/$o->real;
+my %data; # "id func modver" => { id func module modver rate exists }
+{
+    my $indata;
+    if (open my $F, '<', 'FU/Benchmarks.pod') {
+        while (<$F>) {
+            chomp;
+            $indata = 1 if /^# Cached data used by bench\.PL/;
+            next if !$indata || !$_ || /^#/;
+            my %d;
+            @d{qw/id func module modver rate/} = split /\t/;
+            $data{"$d{id} $d{func} $d{modver}"} = \%d;
+        }
     }
-    print "\n";
 }
 
-sub runbenches($re) {
-    runbench $bench{$_}->@* for grep /$re/, @bench;
+if (@ARGV) {
+    my $idre = qr/$ARGV[0]/i;
+    my $funcre = $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/;
+    delete $_->{rate} for grep $_->{id} =~ /$idre/ && $_->{func} =~ /$funcre/, values %data;
+}
+
+
+my @bench; # [ id, text, [ func_1, funcmodule_1, funcsub_n, .. ] ]
+sub def($id, $text, @f) {
+    for my ($f, $m, $sub) (@f) {
+        $m ||= $f;
+        my $d = "$id $f $modules{$m}";
+        $data{$d} ||= { id => $id, func => $f, module => $m, modver => $modules{$m} };
+        $d = $data{$d};
+        $d->{exists} = 1;
+        if (!exists $d->{rate}) {
+            my $o = timethis -1, $sub, 0, 'none';
+            $d->{rate} = sprintf '%.0f', $o->iters/$o->real;
+            printf "%-20s%-20s%10d/s\n", $d->{id}, $d->{func}, $d->{rate};
+        }
+    }
+    push @bench, [ $id, $text, \@f ];
 }
 
 
@@ -52,12 +73,12 @@ my $j_si = JSON::SIMD->new->allow_nonref->core_bools->convert_blessed;
 use FU::Util 'json_format';
 
 sub jsonfmt($name, $text, $data) {
-    bench "jsonfmt/$name", $text,
-        'JSON::PP',        sub { $j_pp->encode($data) },
-        'Cpanel::JSON::XS',sub { $j_cp->encode($data) },
-        'JSON::SIMD',      sub { $j_si->encode($data) },
-        'JSON::XS',        sub { $j_xs->encode($data) },
-        'FU::Util',        sub { json_format $data };
+    def "jsonfmt/$name", $text,
+        'JSON::PP',         undef, sub { $j_pp->encode($data) },
+        'Cpanel::JSON::XS', undef, sub { $j_cp->encode($data) },
+        'JSON::SIMD',       undef, sub { $j_si->encode($data) },
+        'JSON::XS',         undef, sub { $j_xs->encode($data) },
+        'FU::Util',         'FU',  sub { json_format $data };
 }
 
 # From JSON::XS POD.
@@ -77,20 +98,29 @@ jsonfmt strel => 'String escaping (many)', [ map "This \" \\ needs \b\x01\x02\x0
 
 
 
+delete @data{ grep !$data{$_}{exists}, keys %data };
 
-if (!@ARGV || $ARGV[0] eq 'bench') {
-    chomp(my $date = `date +%F`);
-    print "Writing to FU/Benchmarks.pod...\n";
+sub fmtbench($id, $text, $fs) {
+    my $r = "$text\n\n";
+    for my ($f, $m, $sub) (@$fs) {
+        $m ||= $f;
+        $r .= sprintf "%18s%10d/s\n", $f, $data{"$id $f $modules{$m}"}{rate};
+    }
+    "$r\n"
+}
+
+{
     open my $F, '>FU/Benchmarks.pod' or die $!;
     select $F;
     while (<DATA>) {
-        s/^:modules/$modules/;
-        s/^:benches (.+)/runbenches $1/e;
-        s/^:context/These benchmarks were performed on $date with perl $^V on $Config{archname}./;
+        s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
+        s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
         print;
     }
-} else {
-    runbenches $_ for @ARGV;
+    for (sort keys %data) {
+        my $b = $data{$_};
+        print join("\t", @{$b}{qw/ id func module modver rate /})."\n";
+    }
 }
 
 __DATA__
@@ -100,13 +130,20 @@ FU::Benchmarks - A bunch of automated benchmark results.
 
 =head1 DESCRIPTION
 
-This file is automatically generated from 'bench.pl' in the L<FU> distribution.
+This file is automatically generated from 'bench.PL' in the L<FU> distribution.
 These benchmarks compare performance of some FU functionality against similar
 modules found on CPAN.
 
-=head1 CONTEXT
+B<DISCLAIMER#1:> Obtaining accurate measurements is notoriously hard. Take the
+numbers below with a few buckets of salt, any difference below 10% is most
+likely noise.
 
-:context
+B<DISCLAIMER#2:> Goodhart's law: "When a measure becomes a target, it ceases to
+be a good measure". I've used these benchmarks to find and optimize hotspots in
+FU, which in turn means these numbers may look better than they are in
+real-world use.
+
+=head1 MODULE VERSIONS
 
 The following module versions were used:
 
@@ -129,3 +166,7 @@ Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
 SIMD parts are only used for parsing.
 
 :benches ^jsonfmt
+
+=cut
+
+# Cached data used by bench.PL.