bench: Cache benchmark results + better re-run filtering
This commit is contained in:
Yorhel
parent
12326ca8e4
commit
3ae9347ad2
3 changed files with 174 additions and 82 deletions
|
|
@ -4,28 +4,35 @@ FU::Benchmarks - A bunch of automated benchmark results.
|
|||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
This file is automatically generated from 'bench.pl' in the L<FU> distribution.
|
||||
This file is automatically generated from 'bench.PL' in the L<FU> distribution.
|
||||
These benchmarks compare performance of some FU functionality against similar
|
||||
modules found on CPAN.
|
||||
|
||||
=head1 CONTEXT
|
||||
B<DISCLAIMER#1:> Obtaining accurate measurements is notoriously hard. Take the
|
||||
numbers below with a few buckets of salt, any difference below 10% is most
|
||||
likely noise.
|
||||
|
||||
These benchmarks were performed on 2025-01-28 with perl v5.40.0 on x86_64-linux.
|
||||
B<DISCLAIMER#2:> Goodhart's law: "When a measure becomes a target, it ceases to
|
||||
be a good measure". I've used these benchmarks to find and optimize hotspots in
|
||||
FU, which in turn means these numbers may look better than they are in
|
||||
real-world use.
|
||||
|
||||
=head1 MODULE VERSIONS
|
||||
|
||||
The following module versions were used:
|
||||
|
||||
=over
|
||||
|
||||
=item L<FU> 0.1
|
||||
|
||||
=item L<Cpanel::JSON::XS> 4.38
|
||||
|
||||
=item L<FU> 0.1
|
||||
|
||||
=item L<JSON::PP> 4.16
|
||||
|
||||
=item L<JSON::XS> 4.03
|
||||
|
||||
=item L<JSON::SIMD> 1.06
|
||||
|
||||
=item L<JSON::XS> 4.03
|
||||
|
||||
|
||||
|
||||
=back
|
||||
|
|
@ -44,58 +51,97 @@ SIMD parts are only used for parsing.
|
|||
|
||||
API object from L<JSON::XS> documentation.
|
||||
|
||||
JSON::PP 5370/s
|
||||
Cpanel::JSON::XS 112211/s
|
||||
JSON::SIMD 128743/s
|
||||
JSON::XS 130606/s
|
||||
FU::Util 130813/s
|
||||
JSON::PP 5342/s
|
||||
Cpanel::JSON::XS 110660/s
|
||||
JSON::SIMD 128161/s
|
||||
JSON::XS 130434/s
|
||||
FU::Util 129117/s
|
||||
|
||||
Small integers
|
||||
|
||||
JSON::PP 113/s
|
||||
Cpanel::JSON::XS 7262/s
|
||||
JSON::SIMD 8217/s
|
||||
JSON::XS 8142/s
|
||||
FU::Util 9154/s
|
||||
JSON::PP 117/s
|
||||
Cpanel::JSON::XS 7370/s
|
||||
JSON::SIMD 8191/s
|
||||
JSON::XS 8143/s
|
||||
FU::Util 9188/s
|
||||
|
||||
Large integers
|
||||
|
||||
JSON::PP 2136/s
|
||||
Cpanel::JSON::XS 29220/s
|
||||
JSON::SIMD 35834/s
|
||||
JSON::XS 35879/s
|
||||
FU::Util 117838/s
|
||||
JSON::PP 2208/s
|
||||
Cpanel::JSON::XS 29299/s
|
||||
JSON::SIMD 37344/s
|
||||
JSON::XS 35873/s
|
||||
FU::Util 114084/s
|
||||
|
||||
ASCII strings
|
||||
|
||||
JSON::PP 2893/s
|
||||
Cpanel::JSON::XS 118698/s
|
||||
JSON::SIMD 137235/s
|
||||
JSON::XS 135933/s
|
||||
FU::Util 172207/s
|
||||
JSON::PP 2798/s
|
||||
Cpanel::JSON::XS 116754/s
|
||||
JSON::SIMD 134130/s
|
||||
JSON::XS 133137/s
|
||||
FU::Util 166142/s
|
||||
|
||||
Unicode strings
|
||||
|
||||
JSON::PP 5186/s
|
||||
Cpanel::JSON::XS 97154/s
|
||||
JSON::SIMD 109441/s
|
||||
JSON::XS 105691/s
|
||||
FU::Util 106058/s
|
||||
JSON::PP 5067/s
|
||||
Cpanel::JSON::XS 95453/s
|
||||
JSON::SIMD 107955/s
|
||||
JSON::XS 105367/s
|
||||
FU::Util 103071/s
|
||||
|
||||
String escaping (few)
|
||||
|
||||
JSON::PP 4280/s
|
||||
Cpanel::JSON::XS 140105/s
|
||||
JSON::SIMD 161231/s
|
||||
JSON::XS 160077/s
|
||||
FU::Util 182074/s
|
||||
JSON::PP 4275/s
|
||||
Cpanel::JSON::XS 138030/s
|
||||
JSON::SIMD 157735/s
|
||||
JSON::XS 159066/s
|
||||
FU::Util 171426/s
|
||||
|
||||
String escaping (many)
|
||||
|
||||
JSON::PP 2235/s
|
||||
Cpanel::JSON::XS 144829/s
|
||||
JSON::SIMD 161006/s
|
||||
JSON::XS 161246/s
|
||||
FU::Util 136568/s
|
||||
JSON::PP 2231/s
|
||||
Cpanel::JSON::XS 140657/s
|
||||
JSON::SIMD 154850/s
|
||||
JSON::XS 154280/s
|
||||
FU::Util 132514/s
|
||||
|
||||
|
||||
|
||||
=cut
|
||||
|
||||
# Cached data used by bench.PL.
|
||||
jsonfmt/api Cpanel::JSON::XS Cpanel::JSON::XS 4.38 110660
|
||||
jsonfmt/api FU::Util FU 0.1 129117
|
||||
jsonfmt/api JSON::PP JSON::PP 4.16 5342
|
||||
jsonfmt/api JSON::SIMD JSON::SIMD 1.06 128161
|
||||
jsonfmt/api JSON::XS JSON::XS 4.03 130434
|
||||
jsonfmt/intl Cpanel::JSON::XS Cpanel::JSON::XS 4.38 29299
|
||||
jsonfmt/intl FU::Util FU 0.1 114084
|
||||
jsonfmt/intl JSON::PP JSON::PP 4.16 2208
|
||||
jsonfmt/intl JSON::SIMD JSON::SIMD 1.06 37344
|
||||
jsonfmt/intl JSON::XS JSON::XS 4.03 35873
|
||||
jsonfmt/ints Cpanel::JSON::XS Cpanel::JSON::XS 4.38 7370
|
||||
jsonfmt/ints FU::Util FU 0.1 9188
|
||||
jsonfmt/ints JSON::PP JSON::PP 4.16 117
|
||||
jsonfmt/ints JSON::SIMD JSON::SIMD 1.06 8191
|
||||
jsonfmt/ints JSON::XS JSON::XS 4.03 8143
|
||||
jsonfmt/strel Cpanel::JSON::XS Cpanel::JSON::XS 4.38 140657
|
||||
jsonfmt/strel FU::Util FU 0.1 132514
|
||||
jsonfmt/strel JSON::PP JSON::PP 4.16 2231
|
||||
jsonfmt/strel JSON::SIMD JSON::SIMD 1.06 154850
|
||||
jsonfmt/strel JSON::XS JSON::XS 4.03 154280
|
||||
jsonfmt/stres Cpanel::JSON::XS Cpanel::JSON::XS 4.38 138030
|
||||
jsonfmt/stres FU::Util FU 0.1 171426
|
||||
jsonfmt/stres JSON::PP JSON::PP 4.16 4275
|
||||
jsonfmt/stres JSON::SIMD JSON::SIMD 1.06 157735
|
||||
jsonfmt/stres JSON::XS JSON::XS 4.03 159066
|
||||
jsonfmt/strs Cpanel::JSON::XS Cpanel::JSON::XS 4.38 116754
|
||||
jsonfmt/strs FU::Util FU 0.1 166142
|
||||
jsonfmt/strs JSON::PP JSON::PP 4.16 2798
|
||||
jsonfmt/strs JSON::SIMD JSON::SIMD 1.06 134130
|
||||
jsonfmt/strs JSON::XS JSON::XS 4.03 133137
|
||||
jsonfmt/stru Cpanel::JSON::XS Cpanel::JSON::XS 4.38 95453
|
||||
jsonfmt/stru FU::Util FU 0.1 103071
|
||||
jsonfmt/stru JSON::PP JSON::PP 4.16 5067
|
||||
jsonfmt/stru JSON::SIMD JSON::SIMD 1.06 107955
|
||||
jsonfmt/stru JSON::XS JSON::XS 4.03 105367
|
||||
|
|
|
|||
|
|
@ -1,4 +1,9 @@
|
|||
use ExtUtils::MakeMaker;
|
||||
use Config;
|
||||
|
||||
os_unsupported if $Config{ivsize} < 8;
|
||||
os_unsupported if $Config{usequadmath};
|
||||
|
||||
WriteMakefile(
|
||||
NAME => 'FU',
|
||||
VERSION_FROM => 'FU.pm',
|
||||
|
|
|
|||
121
bench.PL
121
bench.PL
|
|
@ -1,17 +1,21 @@
|
|||
#!/usr/bin/perl
|
||||
|
||||
exit if @ARGV && @ARGV[0] eq 'bench';
|
||||
|
||||
# Can be invoked as:
|
||||
# ./bench.PL # (or 'make bench') generates FU/Benchmarks.pod
|
||||
# ./bench.PL regex # run benchmark(s) matching the regex
|
||||
# ./bench.PL # generates FU/Benchmarks.pod, running new benchmarks as necessary
|
||||
# ./bench.PL id func # invalidate cache for the (regex-)matching benchmark IDs and funcs and re-run them
|
||||
#
|
||||
# This script obviously has more dependencies than the FU distribution itself.
|
||||
# It's supposed to be used by maintainers, not users.
|
||||
|
||||
|
||||
# MakeMaker automatically runs this script as a default built step, but that's not very useful.
|
||||
BEGIN { exit if @ARGV && @ARGV[0] eq 'bench'; }
|
||||
|
||||
use v5.36;
|
||||
use builtin 'true', 'false';
|
||||
use Benchmark ':hireswallclock', 'timethis';
|
||||
use Config;
|
||||
|
||||
my $modules = join '', map sprintf("=item L<%s> %s\n\n", $_, eval "require $_; \$${_}::VERSION"), qw/
|
||||
my %modules = map +($_, eval "require $_; \$${_}::VERSION"), qw/
|
||||
FU
|
||||
Cpanel::JSON::XS
|
||||
JSON::PP
|
||||
|
|
@ -19,26 +23,43 @@ my $modules = join '', map sprintf("=item L<%s> %s\n\n", $_, eval "require $_; \
|
|||
JSON::SIMD
|
||||
/;
|
||||
|
||||
|
||||
my(%bench, @bench);
|
||||
sub bench($name, @arg) {
|
||||
push @bench, $name;
|
||||
$bench{$name} = \@arg;
|
||||
}
|
||||
|
||||
sub runbench($text, @f) {
|
||||
print "$text\n\n";
|
||||
|
||||
# TODO: Should include variance; factor-compared-to-slowest might be cool too
|
||||
for my ($t, $f) (@f) {
|
||||
my $o = timethis -1, $f, 0, 'none';
|
||||
printf " %18s%10d/s\n", $t, $o->iters/$o->real;
|
||||
my %data; # "id func modver" => { id func module modver rate exists }
|
||||
{
|
||||
my $indata;
|
||||
if (open my $F, '<', 'FU/Benchmarks.pod') {
|
||||
while (<$F>) {
|
||||
chomp;
|
||||
$indata = 1 if /^# Cached data used by bench\.PL/;
|
||||
next if !$indata || !$_ || /^#/;
|
||||
my %d;
|
||||
@d{qw/id func module modver rate/} = split /\t/;
|
||||
$data{"$d{id} $d{func} $d{modver}"} = \%d;
|
||||
}
|
||||
}
|
||||
print "\n";
|
||||
}
|
||||
|
||||
sub runbenches($re) {
|
||||
runbench $bench{$_}->@* for grep /$re/, @bench;
|
||||
if (@ARGV) {
|
||||
my $idre = qr/$ARGV[0]/i;
|
||||
my $funcre = $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/;
|
||||
delete $_->{rate} for grep $_->{id} =~ /$idre/ && $_->{func} =~ /$funcre/, values %data;
|
||||
}
|
||||
|
||||
|
||||
my @bench; # [ id, text, [ func_1, funcmodule_1, funcsub_n, .. ] ]
|
||||
sub def($id, $text, @f) {
|
||||
for my ($f, $m, $sub) (@f) {
|
||||
$m ||= $f;
|
||||
my $d = "$id $f $modules{$m}";
|
||||
$data{$d} ||= { id => $id, func => $f, module => $m, modver => $modules{$m} };
|
||||
$d = $data{$d};
|
||||
$d->{exists} = 1;
|
||||
if (!exists $d->{rate}) {
|
||||
my $o = timethis -1, $sub, 0, 'none';
|
||||
$d->{rate} = sprintf '%.0f', $o->iters/$o->real;
|
||||
printf "%-20s%-20s%10d/s\n", $d->{id}, $d->{func}, $d->{rate};
|
||||
}
|
||||
}
|
||||
push @bench, [ $id, $text, \@f ];
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -52,12 +73,12 @@ my $j_si = JSON::SIMD->new->allow_nonref->core_bools->convert_blessed;
|
|||
use FU::Util 'json_format';
|
||||
|
||||
sub jsonfmt($name, $text, $data) {
|
||||
bench "jsonfmt/$name", $text,
|
||||
'JSON::PP', sub { $j_pp->encode($data) },
|
||||
'Cpanel::JSON::XS',sub { $j_cp->encode($data) },
|
||||
'JSON::SIMD', sub { $j_si->encode($data) },
|
||||
'JSON::XS', sub { $j_xs->encode($data) },
|
||||
'FU::Util', sub { json_format $data };
|
||||
def "jsonfmt/$name", $text,
|
||||
'JSON::PP', undef, sub { $j_pp->encode($data) },
|
||||
'Cpanel::JSON::XS', undef, sub { $j_cp->encode($data) },
|
||||
'JSON::SIMD', undef, sub { $j_si->encode($data) },
|
||||
'JSON::XS', undef, sub { $j_xs->encode($data) },
|
||||
'FU::Util', 'FU', sub { json_format $data };
|
||||
}
|
||||
|
||||
# From JSON::XS POD.
|
||||
|
|
@ -77,20 +98,29 @@ jsonfmt strel => 'String escaping (many)', [ map "This \" \\ needs \b\x01\x02\x0
|
|||
|
||||
|
||||
|
||||
delete @data{ grep !$data{$_}{exists}, keys %data };
|
||||
|
||||
if (!@ARGV || $ARGV[0] eq 'bench') {
|
||||
chomp(my $date = `date +%F`);
|
||||
print "Writing to FU/Benchmarks.pod...\n";
|
||||
sub fmtbench($id, $text, $fs) {
|
||||
my $r = "$text\n\n";
|
||||
for my ($f, $m, $sub) (@$fs) {
|
||||
$m ||= $f;
|
||||
$r .= sprintf "%18s%10d/s\n", $f, $data{"$id $f $modules{$m}"}{rate};
|
||||
}
|
||||
"$r\n"
|
||||
}
|
||||
|
||||
{
|
||||
open my $F, '>FU/Benchmarks.pod' or die $!;
|
||||
select $F;
|
||||
while (<DATA>) {
|
||||
s/^:modules/$modules/;
|
||||
s/^:benches (.+)/runbenches $1/e;
|
||||
s/^:context/These benchmarks were performed on $date with perl $^V on $Config{archname}./;
|
||||
s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
|
||||
s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
|
||||
print;
|
||||
}
|
||||
} else {
|
||||
runbenches $_ for @ARGV;
|
||||
for (sort keys %data) {
|
||||
my $b = $data{$_};
|
||||
print join("\t", @{$b}{qw/ id func module modver rate /})."\n";
|
||||
}
|
||||
}
|
||||
|
||||
__DATA__
|
||||
|
|
@ -100,13 +130,20 @@ FU::Benchmarks - A bunch of automated benchmark results.
|
|||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
This file is automatically generated from 'bench.pl' in the L<FU> distribution.
|
||||
This file is automatically generated from 'bench.PL' in the L<FU> distribution.
|
||||
These benchmarks compare performance of some FU functionality against similar
|
||||
modules found on CPAN.
|
||||
|
||||
=head1 CONTEXT
|
||||
B<DISCLAIMER#1:> Obtaining accurate measurements is notoriously hard. Take the
|
||||
numbers below with a few buckets of salt, any difference below 10% is most
|
||||
likely noise.
|
||||
|
||||
:context
|
||||
B<DISCLAIMER#2:> Goodhart's law: "When a measure becomes a target, it ceases to
|
||||
be a good measure". I've used these benchmarks to find and optimize hotspots in
|
||||
FU, which in turn means these numbers may look better than they are in
|
||||
real-world use.
|
||||
|
||||
=head1 MODULE VERSIONS
|
||||
|
||||
The following module versions were used:
|
||||
|
||||
|
|
@ -129,3 +166,7 @@ Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
|
|||
SIMD parts are only used for parsing.
|
||||
|
||||
:benches ^jsonfmt
|
||||
|
||||
=cut
|
||||
|
||||
# Cached data used by bench.PL.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue