vndb/fu
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bench: Cache benchmark results + better re-run filtering

Browse source
This commit is contained in:
Yorhel 2025-01-29 10:00:08 +01:00
parent 12326ca8e4
commit 3ae9347ad2
3 changed files with 174 additions and 82 deletions
Download patch file Download diff file Expand all files Collapse all files

130
FU/Benchmarks.pod
View file

@ -4,28 +4,35 @@ FU::Benchmarks - A bunch of automated benchmark results.
=head1 DESCRIPTION =head1 DESCRIPTION
This file is automatically generated from 'bench.pl' in the L<FU> distribution. This file is automatically generated from 'bench.PL' in the L<FU> distribution.
These benchmarks compare performance of some FU functionality against similar These benchmarks compare performance of some FU functionality against similar
modules found on CPAN. modules found on CPAN.
=head1 CONTEXT B<DISCLAIMER#1:> Obtaining accurate measurements is notoriously hard. Take the
numbers below with a few buckets of salt, any difference below 10% is most
likely noise.
These benchmarks were performed on 2025-01-28 with perl v5.40.0 on x86_64-linux. B<DISCLAIMER#2:> Goodhart's law: "When a measure becomes a target, it ceases to
be a good measure". I've used these benchmarks to find and optimize hotspots in
FU, which in turn means these numbers may look better than they are in
real-world use.
=head1 MODULE VERSIONS
The following module versions were used: The following module versions were used:
=over =over
=item L<FU> 0.1
=item L<Cpanel::JSON::XS> 4.38 =item L<Cpanel::JSON::XS> 4.38
=item L<FU> 0.1
=item L<JSON::PP> 4.16 =item L<JSON::PP> 4.16
=item L<JSON::XS> 4.03
=item L<JSON::SIMD> 1.06 =item L<JSON::SIMD> 1.06
=item L<JSON::XS> 4.03
=back =back
@ -44,58 +51,97 @@ SIMD parts are only used for parsing.
API object from L<JSON::XS> documentation. API object from L<JSON::XS> documentation.
JSON::PP 5370/s JSON::PP 5342/s
Cpanel::JSON::XS 112211/s Cpanel::JSON::XS 110660/s
JSON::SIMD 128743/s JSON::SIMD 128161/s
JSON::XS 130606/s JSON::XS 130434/s
FU::Util 130813/s FU::Util 129117/s
Small integers Small integers
JSON::PP 113/s JSON::PP 117/s
Cpanel::JSON::XS 7262/s Cpanel::JSON::XS 7370/s
JSON::SIMD 8217/s JSON::SIMD 8191/s
JSON::XS 8142/s JSON::XS 8143/s
FU::Util 9154/s FU::Util 9188/s
Large integers Large integers
JSON::PP 2136/s JSON::PP 2208/s
Cpanel::JSON::XS 29220/s Cpanel::JSON::XS 29299/s
JSON::SIMD 35834/s JSON::SIMD 37344/s
JSON::XS 35879/s JSON::XS 35873/s
FU::Util 117838/s FU::Util 114084/s
ASCII strings ASCII strings
JSON::PP 2893/s JSON::PP 2798/s
Cpanel::JSON::XS 118698/s Cpanel::JSON::XS 116754/s
JSON::SIMD 137235/s JSON::SIMD 134130/s
JSON::XS 135933/s JSON::XS 133137/s
FU::Util 172207/s FU::Util 166142/s
Unicode strings Unicode strings
JSON::PP 5186/s JSON::PP 5067/s
Cpanel::JSON::XS 97154/s Cpanel::JSON::XS 95453/s
JSON::SIMD 109441/s JSON::SIMD 107955/s
JSON::XS 105691/s JSON::XS 105367/s
FU::Util 106058/s FU::Util 103071/s
String escaping (few) String escaping (few)
JSON::PP 4280/s JSON::PP 4275/s
Cpanel::JSON::XS 140105/s Cpanel::JSON::XS 138030/s
JSON::SIMD 161231/s JSON::SIMD 157735/s
JSON::XS 160077/s JSON::XS 159066/s
FU::Util 182074/s FU::Util 171426/s
String escaping (many) String escaping (many)
JSON::PP 2235/s JSON::PP 2231/s
Cpanel::JSON::XS 144829/s Cpanel::JSON::XS 140657/s
JSON::SIMD 161006/s JSON::SIMD 154850/s
JSON::XS 161246/s JSON::XS 154280/s
FU::Util 136568/s FU::Util 132514/s
=cut
# Cached data used by bench.PL.
jsonfmt/api Cpanel::JSON::XS Cpanel::JSON::XS 4.38 110660
jsonfmt/api FU::Util FU 0.1 129117
jsonfmt/api JSON::PP JSON::PP 4.16 5342
jsonfmt/api JSON::SIMD JSON::SIMD 1.06 128161
jsonfmt/api JSON::XS JSON::XS 4.03 130434
jsonfmt/intl Cpanel::JSON::XS Cpanel::JSON::XS 4.38 29299
jsonfmt/intl FU::Util FU 0.1 114084
jsonfmt/intl JSON::PP JSON::PP 4.16 2208
jsonfmt/intl JSON::SIMD JSON::SIMD 1.06 37344
jsonfmt/intl JSON::XS JSON::XS 4.03 35873
jsonfmt/ints Cpanel::JSON::XS Cpanel::JSON::XS 4.38 7370
jsonfmt/ints FU::Util FU 0.1 9188
jsonfmt/ints JSON::PP JSON::PP 4.16 117
jsonfmt/ints JSON::SIMD JSON::SIMD 1.06 8191
jsonfmt/ints JSON::XS JSON::XS 4.03 8143
jsonfmt/strel Cpanel::JSON::XS Cpanel::JSON::XS 4.38 140657
jsonfmt/strel FU::Util FU 0.1 132514
jsonfmt/strel JSON::PP JSON::PP 4.16 2231
jsonfmt/strel JSON::SIMD JSON::SIMD 1.06 154850
jsonfmt/strel JSON::XS JSON::XS 4.03 154280
jsonfmt/stres Cpanel::JSON::XS Cpanel::JSON::XS 4.38 138030
jsonfmt/stres FU::Util FU 0.1 171426
jsonfmt/stres JSON::PP JSON::PP 4.16 4275
jsonfmt/stres JSON::SIMD JSON::SIMD 1.06 157735
jsonfmt/stres JSON::XS JSON::XS 4.03 159066
jsonfmt/strs Cpanel::JSON::XS Cpanel::JSON::XS 4.38 116754
jsonfmt/strs FU::Util FU 0.1 166142
jsonfmt/strs JSON::PP JSON::PP 4.16 2798
jsonfmt/strs JSON::SIMD JSON::SIMD 1.06 134130
jsonfmt/strs JSON::XS JSON::XS 4.03 133137
jsonfmt/stru Cpanel::JSON::XS Cpanel::JSON::XS 4.38 95453
jsonfmt/stru FU::Util FU 0.1 103071
jsonfmt/stru JSON::PP JSON::PP 4.16 5067
jsonfmt/stru JSON::SIMD JSON::SIMD 1.06 107955
jsonfmt/stru JSON::XS JSON::XS 4.03 105367

5
Makefile.PL
View file

@ -1,4 +1,9 @@
use ExtUtils::MakeMaker; use ExtUtils::MakeMaker;
use Config;
os_unsupported if $Config{ivsize} < 8;
os_unsupported if $Config{usequadmath};
WriteMakefile( WriteMakefile(
NAME => 'FU', NAME => 'FU',
VERSION_FROM => 'FU.pm', VERSION_FROM => 'FU.pm',

121
bench.PL
View file

@ -1,17 +1,21 @@
#!/usr/bin/perl #!/usr/bin/perl
exit if @ARGV && @ARGV[0] eq 'bench';
# Can be invoked as: # Can be invoked as:
# ./bench.PL # (or 'make bench') generates FU/Benchmarks.pod # ./bench.PL # generates FU/Benchmarks.pod, running new benchmarks as necessary
# ./bench.PL regex # run benchmark(s) matching the regex # ./bench.PL id func # invalidate cache for the (regex-)matching benchmark IDs and funcs and re-run them
#
# This script obviously has more dependencies than the FU distribution itself.
# It's supposed to be used by maintainers, not users.
# MakeMaker automatically runs this script as a default built step, but that's not very useful.
BEGIN { exit if @ARGV && @ARGV[0] eq 'bench'; }
use v5.36; use v5.36;
use builtin 'true', 'false'; use builtin 'true', 'false';
use Benchmark ':hireswallclock', 'timethis'; use Benchmark ':hireswallclock', 'timethis';
use Config;
my $modules = join '', map sprintf("=item L<%s> %s\n\n", $_, eval "require $_; \$${_}::VERSION"), qw/ my %modules = map +($_, eval "require $_; \$${_}::VERSION"), qw/
FU FU
Cpanel::JSON::XS Cpanel::JSON::XS
JSON::PP JSON::PP
@ -19,26 +23,43 @@ my $modules = join '', map sprintf("=item L<%s> %s\n\n", $_, eval "require $_; \
JSON::SIMD JSON::SIMD
/; /;
my %data; # "id func modver" => { id func module modver rate exists }
my(%bench, @bench); {
sub bench($name, @arg) { my $indata;
push @bench, $name; if (open my $F, '<', 'FU/Benchmarks.pod') {
$bench{$name} = \@arg; while (<$F>) {
} chomp;
$indata = 1 if /^# Cached data used by bench\.PL/;
sub runbench($text, @f) { next if !$indata || !$_ || /^#/;
print "$text\n\n"; my %d;
@d{qw/id func module modver rate/} = split /\t/;
# TODO: Should include variance; factor-compared-to-slowest might be cool too $data{"$d{id} $d{func} $d{modver}"} = \%d;
for my ($t, $f) (@f) { }
my $o = timethis -1, $f, 0, 'none';
printf " %18s%10d/s\n", $t, $o->iters/$o->real;
} }
print "\n";
} }
sub runbenches($re) { if (@ARGV) {
runbench $bench{$_}->@* for grep /$re/, @bench; my $idre = qr/$ARGV[0]/i;
my $funcre = $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/;
delete $_->{rate} for grep $_->{id} =~ /$idre/ && $_->{func} =~ /$funcre/, values %data;
}
my @bench; # [ id, text, [ func_1, funcmodule_1, funcsub_n, .. ] ]
sub def($id, $text, @f) {
for my ($f, $m, $sub) (@f) {
$m ||= $f;
my $d = "$id $f $modules{$m}";
$data{$d} ||= { id => $id, func => $f, module => $m, modver => $modules{$m} };
$d = $data{$d};
$d->{exists} = 1;
if (!exists $d->{rate}) {
my $o = timethis -1, $sub, 0, 'none';
$d->{rate} = sprintf '%.0f', $o->iters/$o->real;
printf "%-20s%-20s%10d/s\n", $d->{id}, $d->{func}, $d->{rate};
}
}
push @bench, [ $id, $text, \@f ];
} }
@ -52,12 +73,12 @@ my $j_si = JSON::SIMD->new->allow_nonref->core_bools->convert_blessed;
use FU::Util 'json_format'; use FU::Util 'json_format';
sub jsonfmt($name, $text, $data) { sub jsonfmt($name, $text, $data) {
bench "jsonfmt/$name", $text, def "jsonfmt/$name", $text,
'JSON::PP', sub { $j_pp->encode($data) }, 'JSON::PP', undef, sub { $j_pp->encode($data) },
'Cpanel::JSON::XS',sub { $j_cp->encode($data) }, 'Cpanel::JSON::XS', undef, sub { $j_cp->encode($data) },
'JSON::SIMD', sub { $j_si->encode($data) }, 'JSON::SIMD', undef, sub { $j_si->encode($data) },
'JSON::XS', sub { $j_xs->encode($data) }, 'JSON::XS', undef, sub { $j_xs->encode($data) },
'FU::Util', sub { json_format $data }; 'FU::Util', 'FU', sub { json_format $data };
} }
# From JSON::XS POD. # From JSON::XS POD.
@ -77,20 +98,29 @@ jsonfmt strel => 'String escaping (many)', [ map "This \" \\ needs \b\x01\x02\x0
delete @data{ grep !$data{$_}{exists}, keys %data };
if (!@ARGV || $ARGV[0] eq 'bench') { sub fmtbench($id, $text, $fs) {
chomp(my $date = `date +%F`); my $r = "$text\n\n";
print "Writing to FU/Benchmarks.pod...\n"; for my ($f, $m, $sub) (@$fs) {
$m ||= $f;
$r .= sprintf "%18s%10d/s\n", $f, $data{"$id $f $modules{$m}"}{rate};
}
"$r\n"
}
{
open my $F, '>FU/Benchmarks.pod' or die $!; open my $F, '>FU/Benchmarks.pod' or die $!;
select $F; select $F;
while (<DATA>) { while (<DATA>) {
s/^:modules/$modules/; s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
s/^:benches (.+)/runbenches $1/e; s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
s/^:context/These benchmarks were performed on $date with perl $^V on $Config{archname}./;
print; print;
} }
} else { for (sort keys %data) {
runbenches $_ for @ARGV; my $b = $data{$_};
print join("\t", @{$b}{qw/ id func module modver rate /})."\n";
}
} }
__DATA__ __DATA__
@ -100,13 +130,20 @@ FU::Benchmarks - A bunch of automated benchmark results.
=head1 DESCRIPTION =head1 DESCRIPTION
This file is automatically generated from 'bench.pl' in the L<FU> distribution. This file is automatically generated from 'bench.PL' in the L<FU> distribution.
These benchmarks compare performance of some FU functionality against similar These benchmarks compare performance of some FU functionality against similar
modules found on CPAN. modules found on CPAN.
=head1 CONTEXT B<DISCLAIMER#1:> Obtaining accurate measurements is notoriously hard. Take the
numbers below with a few buckets of salt, any difference below 10% is most
likely noise.
:context B<DISCLAIMER#2:> Goodhart's law: "When a measure becomes a target, it ceases to
be a good measure". I've used these benchmarks to find and optimize hotspots in
FU, which in turn means these numbers may look better than they are in
real-world use.
=head1 MODULE VERSIONS
The following module versions were used: The following module versions were used:
@ -129,3 +166,7 @@ Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
SIMD parts are only used for parsing. SIMD parts are only used for parsing.
:benches ^jsonfmt :benches ^jsonfmt
=cut
# Cached data used by bench.PL.