From f09a103c530030ce2b2718619194f5c077144cf5 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 2 Mar 2025 10:06:33 +0100
Subject: [PATCH 01/86] Some test portability fixes again + minor changes

---
 FU/Pg.pm        |  4 ++--
 Makefile.PL     | 16 ++++++++++------
 bench.PL        | 20 +++++++++++---------
 c/pgconn.c      |  9 +++------
 c/pgtypes.c     |  4 ++--
 t/json_format.t |  2 +-
 t/json_parse.t  |  6 ++++--
 7 files changed, 33 insertions(+), 28 deletions(-)

diff --git a/FU/Pg.pm b/FU/Pg.pm
index 175bba3..7b9de5d 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -666,8 +666,8 @@ module does not.
 
 Converted between floating point seconds since C<00:00:00>, supporting
 microsecond precision. This format allows for easy comparison against Unix
-timestamps (time of day = C<$timestamp % 86400>) and can be added to an integer
-date value to form a complete timestamp.
+timestamps (time of day in UTC = C<$timestamp % 86400>) and can be added to an
+integer date value to form a complete timestamp.
 
 (There's no support for the string format yet)
 
diff --git a/Makefile.PL b/Makefile.PL
index 87362fc..326fdb4 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -2,7 +2,7 @@ use ExtUtils::MakeMaker;
 use Config;
 
 os_unsupported if $^O eq 'MSWin32'; # I don't know on which OS'es the code will work exactly, but this one I can easily rule out.
-os_unsupported if $Config{ivsize} < 8;
+os_unsupported if $Config{ptrsize} < 8;
 os_unsupported if $Config{usequadmath};
 
 WriteMakefile(
@@ -15,12 +15,16 @@ WriteMakefile(
     MIN_PERL_VERSION => 'v5.36',
     META_MERGE => {
         dynamic_config => 0,
+        'meta-spec' => { version => 2 },
         resources => {
-            repository => 'https://code.blicky.net/yorhel/fu',
-            bugtracker => 'https://code.blicky.net/yorhel/fu/issues',
-        },
-        no_index => {
-            file => 'bench.PL',
+            repository => {
+                web => 'https://code.blicky.net/yorhel/fu',
+                type => 'git',
+            },
+            bugtracker => {
+                web => 'https://code.blicky.net/yorhel/fu/issues',
+                mailto => 'projects@yorhel.nl',
+            },
         },
     },
     depend => { '$(OBJECT)', 'c/*.c' },
diff --git a/bench.PL b/bench.PL
index d8c13d3..fa53068 100755
--- a/bench.PL
+++ b/bench.PL
@@ -223,6 +223,7 @@ sub fmtbench($id, $text, $xs, $ys) {
     open my $F, '>FU/Benchmarks.pod' or die $!;
     select $F;
     while (<DATA>) {
+        s/^%/=/;
         s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
         s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
         print;
@@ -233,12 +234,13 @@ sub fmtbench($id, $text, $xs, $ys) {
     }
 }
 
+# s/^=/%/  to prevent tools from interpreting the below as POD
 __DATA__
-=head1 NAME
+%head1 NAME
 
 FU::Benchmarks - A bunch of automated benchmark results.
 
-=head1 DESCRIPTION
+%head1 DESCRIPTION
 
 This file is automatically generated from 'bench.PL' in the L<FU> distribution.
 These benchmarks compare performance of some FU functionality against similar
@@ -256,19 +258,19 @@ real-world use.
 B<DISCLAIMER#3:> Many of these benchmarks exists solely to test edge case
 performance, these numbers are not representative for real-world use.
 
-=head1 MODULE VERSIONS
+%head1 MODULE VERSIONS
 
 The following module versions were used:
 
-=over
+%over
 
 :modules
 
-=back
+%back
 
-=head1 BENCHMARKS
+%head1 BENCHMARKS
 
-=head2 JSON Parsing & Formatting
+%head2 JSON Parsing & Formatting
 
 These benchmarks run on large-ish arrays with repeated values. JSON encoding is
 sufficiently fast that Perl function calling overhead tends to dominate for
@@ -279,10 +281,10 @@ SIMD parts are only used for parsing.
 
 :benches ^json
 
-=head2 XML Writing
+%head2 XML Writing
 
 :benches ^xml
 
-=cut
+%cut
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.
diff --git a/c/pgconn.c b/c/pgconn.c
index ace6ace..4b84463 100644
--- a/c/pgconn.c
+++ b/c/pgconn.c
@@ -488,7 +488,7 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) {
             t->send = fupg_send_text;
             t->recv = fupg_recv_text;
         } else {
-            /* TODO: (multi)ranges, custom overrides, by-name lookup for dynamic-oid types */
+            /* TODO: (multi)ranges, by-name lookup for dynamic-oid types */
             const fupg_type *builtin = fupg_builtin_byoid(t->oid);
             if (builtin) {
                 t->send = builtin->send;
@@ -571,11 +571,8 @@ static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid
         return;
     }
 
-    /* Minor wart? When the type is overridden by oid, the name & oid in error
-     * messages will be that of the builtin type.  When overridden by name, the
-     * name will be correct but the oid is still of the builtin type.
-     * Some send/recv functions have slightly different behavior based on oid,
-     * in those cases this behavior is useful. */
+    /* Minor wart? When the type is overridden by oid, its name in error
+     * messages will be that of the builtin type instead of the actual type. */
 
     SV *cb = NULL;
     const fupg_type *e, *t;
diff --git a/c/pgtypes.c b/c/pgtypes.c
index 481cd03..f9c5b00 100644
--- a/c/pgtypes.c
+++ b/c/pgtypes.c
@@ -456,8 +456,8 @@ RECVFN(perlcb) {
     call_sv(ctx->cb, G_SCALAR);
     SPAGAIN;
 
-    SV *ret = newSV(0);
-    sv_setsv(ret, POPs);
+    SV *ret = POPs;
+    SvREFCNT_inc(ret);
     PUTBACK;
 
     FREETMPS;
diff --git a/t/json_format.t b/t/json_format.t
index 0797a4f..5288e1a 100644
--- a/t/json_format.t
+++ b/t/json_format.t
@@ -55,7 +55,7 @@ my @tests = (
     ''.$$, '"'.$$.'"',
     do { my $x = 12; utf8::decode($x); $x }, '"12"',
     do { no warnings 'numeric'; my $x = '19a'; $x += 0; $x }, '19',
-    1844674407370955161 / 10, $Config{uselongdouble} ? 184467440737095516 : '1.84467440737096e+17',
+    $Config{uselongdouble} ? () : ( 1844674407370955161 / 10, '1.84467440737096e+17' ),
 );
 
 my @errors = (
diff --git a/t/json_parse.t b/t/json_parse.t
index 901d01f..0c26dff 100644
--- a/t/json_parse.t
+++ b/t/json_parse.t
@@ -97,8 +97,10 @@ num ' -0 ', 0;
 num '-9223372036854775808';
 num '9223372036854775807';
 num '18446744073709551615';
-num '-9223372036854775809', $Config{uselongdouble} ? -9.22337203685477581e+18 : -9.22337203685478e+18;
-num '18446744073709551616', $Config{uselongdouble} ? 1.84467440737095516e+19 : 1.84467440737096e+19;
+if (!$Config{uselongdouble}) { # Behavior of longdouble is architecture-dependent
+    num '-9223372036854775809', -9.22337203685478e+18;
+    num '18446744073709551616', 1.84467440737096e+19;
+}
 num '1.234';
 num '1e5', 100000;
 num '1e+5', 100000;

From 7839e7df787331a01447bd3d31329f9261bb5414 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 2 Mar 2025 21:09:59 +0100
Subject: [PATCH 02/86] FU: Fix typo in error handling

---
 FU.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index 78c0fce..a0cd38e 100644
--- a/FU.pm
+++ b/FU.pm
@@ -288,7 +288,7 @@ sub _read_req($c) {
 
     # The HTTP reader above and the FastCGI XS reader operate on bytes.
     # Decode these into Unicode strings and check for special characters.
-    eval { FU::Util::utf8_decode($_); 1} || fu->err(400, $@)
+    eval { FU::Util::utf8_decode($_); 1} || fu->error(400, $@)
         for ($REQ->{path}, $REQ->{qs}, values $REQ->{hdr}->%*);
 
     ($REQ->{path}, my $qs) = split /\?/, $REQ->{path}//'', 2;

From cbebc3a21eb5cc832109671466676aff275a26b9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 5 Mar 2025 15:32:01 +0100
Subject: [PATCH 03/86] Validate: Rework API, ->validate() now throws error
 instead of result object

This is a slight simplification and removes the need to pass around
partially normalized data. I've never found a use for the unsafe_data()
method.
---
 FU.pm          |   5 +-
 FU/Validate.pm | 284 ++++++++++++++++-----------------------
 t/validate.t   | 353 +++++++++++++++++++++++++------------------------
 3 files changed, 295 insertions(+), 347 deletions(-)

diff --git a/FU.pm b/FU.pm
index a0cd38e..4d04562 100644
--- a/FU.pm
+++ b/FU.pm
@@ -639,8 +639,8 @@ sub _getfield($data, @a) {
     return $data->{$a[0]} if @a == 1 && !ref $a[0];
     require FU::Validate;
     my $schema = FU::Validate->compile(@a > 1 ? { keys => {@a} } : $a[0]);
-    my $res = $schema->validate($data);
-    fu->error(400, "Input validation failed") if !$res; # TODO: More detailed error message
+    my $res = eval { $schema->validate($data) };
+    fu->error(400, "Input validation failed") if $@; # TODO: More detailed error message
     return @a == 2 ? $res->data->{$a[0]} : $res->data;
 }
 
@@ -659,7 +659,6 @@ sub formdata {
             if (fu->header('content-type')||'') ne 'application/x-www-form-urlencoded';
         FU::Util::query_decode($FU::REQ->{data});
     } || fu->error(400, $@);
-    # TODO: Accept schema validation thing.
     _getfield $FU::REQ->{formdata}, @_;
 }
 
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 1538c5a..7f1cc1c 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -102,7 +102,7 @@ sub _compile($schema, $validations, $rec) {
     my @keys = keys $schema->{keys}->%* if $schema->{keys};
 
     for my($name, $val) (%$schema) {
-        if($builtin{$name}) {
+        if ($builtin{$name}) {
             $top{$name} = $schema->{$name};
             next;
         }
@@ -125,7 +125,7 @@ sub _compile($schema, $validations, $rec) {
 
     # Inherit some builtin options from validations
     for my $t (@val) {
-        if($top{type} && $t->{schema}{type} && $top{type} ne $t->{schema}{type}) {
+        if ($top{type} && $t->{schema}{type} && $top{type} ne $t->{schema}{type}) {
             confess "Incompatible types, the schema specifies '$top{type}' but validation '$t->{name}' requires '$t->{schema}{type}'" if $schema->{type};
             confess "Incompatible types, '$t->[0]' requires '$t->{schema}{type}', but another validation requires '$top{type}'";
         }
@@ -164,7 +164,7 @@ sub compile($pkg, $schema, $validations={}) {
 
     delete $c->{schema}{default} if ref $c->{schema}{default} eq 'SCALAR' && ${$c->{schema}{default}} eq 'required';
 
-    if(exists $c->{schema}{sort}) {
+    if (exists $c->{schema}{sort}) {
         my $s = $c->{schema}{sort};
         $c->{schema}{sort} =
             ref $s eq 'CODE' ? $s
@@ -178,186 +178,169 @@ sub compile($pkg, $schema, $validations={}) {
 }
 
 
-sub _validate_rec($c, $input) {
+sub _validate_rec {
+    my $c = $_[0];
+
     # hash keys
-    if($c->{schema}{keys}) {
+    if ($c->{schema}{keys}) {
         my @err;
         for my ($k, $s) ($c->{schema}{keys}->%*) {
-            if(!exists $input->{$k}) {
+            if (!exists $_[1]{$k}) {
                 next if $s->{schema}{missing} eq 'ignore';
-                return [$input, { validation => 'missing', key => $k }] if $s->{schema}{missing} eq 'reject';
-                $input->{$k} = ref $s->{schema}{default} eq 'CODE' ? $s->{schema}{default}->() : $s->{schema}{default} // undef;
+                return { validation => 'missing', key => $k } if $s->{schema}{missing} eq 'reject';
+                $_[1]{$k} = ref $s->{schema}{default} eq 'CODE' ? $s->{schema}{default}->() : $s->{schema}{default} // undef;
                 next if exists $s->{schema}{default};
             }
 
-            my $r = _validate($s, $input->{$k});
-            $input->{$k} = $r->[0];
-            if($r->[1]) {
-                $r->[1]{key} = $k;
-                push @err, $r->[1];
+            my $r = _validate($s, $_[1]{$k});
+            if ($r) {
+                $r->{key} = $k;
+                push @err, $r;
             }
         }
-        return [$input, { validation => 'keys', errors => \@err }] if @err;
+        return { validation => 'keys', errors => \@err } if @err;
     }
 
     # array values
-    if($c->{schema}{values}) {
+    if ($c->{schema}{values}) {
         my @err;
-        for my $i (0..$#$input) {
-            my $r = _validate($c->{schema}{values}, $input->[$i]);
-            $input->[$i] = $r->[0];
-            if($r->[1]) {
-                $r->[1]{index} = $i;
-                push @err, $r->[1];
+        for my $i (0..$#{$_[1]}) {
+            my $r = _validate($c->{schema}{values}, $_[1][$i]);
+            if ($r) {
+                $r->{index} = $i;
+                push @err, $r;
             }
         }
-        return [$input, { validation => 'values', errors => \@err }] if @err;
+        return { validation => 'values', errors => \@err } if @err;
     }
 
     # validations
     for ($c->{validations}->@*) {
-        my $r = _validate_rec($_, $input);
-        $input = $r->[0];
-
-        return [$input, {
+        my $r = _validate_rec($_, $_[1]);
+        return {
             # If the error was a custom 'func' object, then make that the primary cause.
             # This makes it possible for validations to provide their own error objects.
-            $r->[1]{validation} eq 'func' && (!exists $r->[1]{result} || keys $r->[1]->%* > 2) ? $r->[1]->%* : (error => $r->[1]),
+            $r->{validation} eq 'func' && (!exists $r->{result} || keys $r->%* > 2) ? $r->%* : (error => $r),
             validation => $_->{name},
-        }] if $r->[1];
+        } if $r;
     }
 
     # func
-    if($c->{schema}{func}) {
-        my $r = $c->{schema}{func}->($input);
-        return [$input, { %$r, validation => 'func' }] if ref $r eq 'HASH';
-        return [$input, { validation => 'func', result => $r }] if !$r;
+    if ($c->{schema}{func}) {
+        my $r = $c->{schema}{func}->($_[1]);
+        return { %$r, validation => 'func' } if ref $r eq 'HASH';
+        return { validation => 'func', result => $r } if !$r;
     }
-
-    return [$input]
 }
 
 
-sub _validate_array($c, $input) {
-    return [$input] if $c->{schema}{type} ne 'array';
+sub _validate_array {
+    my $c = $_[0];
+    return if $c->{schema}{type} ne 'array';
 
-    $input = [sort { $c->{schema}{sort}->($a, $b) } @$input ] if $c->{schema}{sort};
+    $_[1] = [sort { $c->{schema}{sort}->($a, $b) } $_[1]->@* ] if $c->{schema}{sort};
 
     # Key-based uniqueness
-    if($c->{schema}{unique} && ref $c->{schema}{unique} eq 'CODE') {
+    if ($c->{schema}{unique} && ref $c->{schema}{unique} eq 'CODE') {
         my %h;
-        for my $i (0..$#$input) {
-            my $k = $c->{schema}{unique}->($input->[$i]);
-            return [$input, { validation => 'unique', index_a => $h{$k}, value_a => $input->[$h{$k}], index_b => $i, value_b => $input->[$i], key => $k }] if exists $h{$k};
+        for my $i (0..$#{$_[1]}) {
+            my $k = $c->{schema}{unique}->($_[1][$i]);
+            return { validation => 'unique', index_a => $h{$k}, value_a => $_[1][$h{$k}], index_b => $i, value_b => $_[1][$i], key => $k } if exists $h{$k};
             $h{$k} = $i;
         }
 
     # Comparison-based uniqueness
-    } elsif($c->{schema}{unique}) {
-        for my $i (0..$#$input-1) {
-            return [$input, { validation => 'unique', index_a => $i, value_a => $input->[$i], index_b => $i+1, value_b => $input->[$i+1] }]
-                if $c->{schema}{sort}->($input->[$i], $input->[$i+1]) == 0
+    } elsif ($c->{schema}{unique}) {
+        for my $i (0..$#{$_[1]}-1) {
+            return { validation => 'unique', index_a => $i, value_a => $_[1][$i], index_b => $i+1, value_b => $_[1][$i+1] }
+                if $c->{schema}{sort}->($_[1][$i], $_[1][$i+1]) == 0
         }
     }
-
-    return [$input]
 }
 
 
-sub _validate_input($c, $input) {
+sub _validate_input {
+    my $c = $_[0];
+
     # rmwhitespace (needs to be done before the 'default' test)
-    if(defined $input && !ref $input && $c->{schema}{type} eq 'scalar' && $c->{schema}{rmwhitespace}) {
-        $input =~ s/\r//g;
-        $input =~ s/^\s*//;
-        $input =~ s/\s*$//;
+    if (defined $_[1] && !ref $_[1] && $c->{schema}{type} eq 'scalar' && $c->{schema}{rmwhitespace}) {
+        $_[1] =~ s/\r//g;
+        $_[1] =~ s/^\s*//;
+        $_[1] =~ s/\s*$//;
     }
 
     # default
-    if(!defined $input || (!ref $input && $input eq '')) {
-        return [ref $c->{schema}{default} eq 'CODE' ? $c->{schema}{default}->($input) : $c->{schema}{default}] if exists $c->{schema}{default};
-        return [$input, { validation => 'required' }];
+    if (!defined $_[1] || (!ref $_[1] && $_[1] eq '')) {
+        if (exists $c->{schema}{default}) {
+            $_[1] = ref $c->{schema}{default} eq 'CODE' ? $c->{schema}{default}->($_[1]) : $c->{schema}{default};
+            return;
+        }
+        return { validation => 'required' };
     }
 
-    if($c->{schema}{type} eq 'scalar') {
-        return [$input, { validation => 'type', expected => 'scalar', got => lc ref $input }] if ref $input;
+    if ($c->{schema}{type} eq 'scalar') {
+        return { validation => 'type', expected => 'scalar', got => lc ref $_[1] } if ref $_[1];
 
-    } elsif($c->{schema}{type} eq 'hash') {
-        return [$input, { validation => 'type', expected => 'hash', got => lc ref $input || 'scalar' }] if ref $input ne 'HASH';
+    } elsif ($c->{schema}{type} eq 'hash') {
+        return { validation => 'type', expected => 'hash', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'HASH';
 
         # Each branch below makes a shallow copy of the hash, so that further
         # validations can perform in-place modifications without affecting the
         # input.
-        if($c->{schema}{unknown} eq 'remove') {
-            $input = { map +($_, $input->{$_}), grep $c->{known_keys}{$_}, keys %$input };
-        } elsif($c->{schema}{unknown} eq 'reject') {
-            my @err = grep !$c->{known_keys}{$_}, keys %$input;
-            return [$input, { validation => 'unknown', keys => \@err, expected => [ sort keys %{$c->{known_keys}} ] }] if @err;
-            $input = { %$input };
+        if ($c->{schema}{unknown} eq 'remove') {
+            $_[1] = { map +($_, $_[1]{$_}), grep $c->{known_keys}{$_}, keys $_[1]->%* };
+        } elsif ($c->{schema}{unknown} eq 'reject') {
+            my @err = grep !$c->{known_keys}{$_}, keys $_[1]->%*;
+            return { validation => 'unknown', keys => \@err, expected => [ sort keys %{$c->{known_keys}} ] } if @err;
+            $_[1] = { $_[1]->%* };
         } else {
-            $input = { %$input };
+            $_[1] = { $_[1]->%* };
         }
 
-    } elsif($c->{schema}{type} eq 'array') {
-        $input = [$input] if $c->{schema}{scalar} && !ref $input;
-        return [$input, { validation => 'type', expected => $c->{schema}{scalar} ? 'array or scalar' : 'array', got => lc ref $input || 'scalar' }] if ref $input ne 'ARRAY';
-        $input = [@$input]; # Create a shallow copy to prevent in-place modification.
+    } elsif ($c->{schema}{type} eq 'array') {
+        $_[1] = [$_[1]] if $c->{schema}{scalar} && !ref $_[1];
+        return { validation => 'type', expected => $c->{schema}{scalar} ? 'array or scalar' : 'array', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'ARRAY';
+        $_[1] = [$_[1]->@*]; # Create a shallow copy to prevent in-place modification.
 
-    } elsif($c->{schema}{type} eq 'any') {
+    } elsif ($c->{schema}{type} eq 'any') {
         # No need to do anything here.
 
     } else {
         confess "Unknown type '$c->{schema}{type}'"; # Already checked in compile(), but be extra safe
     }
 
-    my $r = _validate_rec($c, $input);
-    return $r if $r->[1];
-    $input = $r->[0];
-
-    _validate_array($c, $input);
+    &_validate_rec || &_validate_array;
 }
 
 
-sub _validate($c, $input) {
-    my $r = _validate_input($c, $input);
-    return $r if !$r->[1] || !exists $c->{schema}{onerror};
-    [ ref $c->{schema}{onerror} eq 'CODE' ? $c->{schema}{onerror}->(bless $r, 'FU::Validate::Result') : $c->{schema}{onerror} ]
+sub _validate {
+    my $c = $_[0];
+    my $r = &_validate_input;
+    ($r, $_[1]) = (undef, ref $c->{schema}{onerror} eq 'CODE' ? $c->{schema}{onerror}->($_[0], bless $r, 'FU::Validate::err') : $c->{schema}{onerror})
+        if $r && exists $c->{schema}{onerror};
+    $r
 }
 
 
 sub validate($c, $input) {
-    bless _validate($c, $input), 'FU::Validate::Result';
+    my $r = _validate($c, $input);
+    return $input if !$r;
+    die bless $r, 'FU::Validate::err';
+    $input
 }
 
 
 
 
-package FU::Validate::Result;
-
+package FU::Validate::err;
 use v5.36;
-use Carp 'confess';
 
-# A result object contains: [$data, $error]
+use overload '""' => sub {
+    # TODO: Better error message
+    require Data::Dumper;
+    Data::Dumper->new([{$_[0]->%*}])->Terse(1)->Pair(':')->Indent(0)->Sortkeys(1)->Dump."\n";
+};
 
-# In boolean context, returns whether the validation succeeded.
-use overload bool => sub { !$_[0][1] };
-
-# Returns the validation errors, or undef if validation succeeded
-sub err { $_[0][1] }
-
-# Returns the validated and normalized input, dies if validation didn't succeed.
-sub data {
-    if($_[0][1]) {
-        require Data::Dumper;
-        my $s = Data::Dumper->new([$_[0][1]])->Terse(1)->Pair(':')->Indent(0)->Sortkeys(1)->Dump;
-        confess "Validation failed: $s";
-    }
-    $_[0][0]
-}
-
-# Same as 'data', but returns partially validated and normalized data if validation failed.
-sub unsafe_data { $_[0][0] }
-
-# TODO: Human-readable error message formatting
 
 1;
 __END__
@@ -402,67 +385,22 @@ follows:
 
 C<$schema> is the schema that describes the data to be validated (see L</SCHEMA
 DEFINITION> below) and C<$validations> is an optional hashref containing
-L<custom validations|/Custom validations> that C<$schema> can refer to.
+L<custom validations|/Custom validations> that C<$schema> can refer to.  An
+error is thrown if the C<$validations> or C<$schema> are invalid.
 
 To validate input, run:
 
-  my $result = $validator->validate($input);
+  my $validated_input = $validator->validate($input);
 
-C<$input> is the data to be validated, and the C<$result> object is L<described
-below|/Result object>.
+C<validate()> returns a validated and (depending on the schema) normalized copy
+of C<$input>. Great care is taken that C<$input> is not being modified
+in-place, even if data normalization is being performed.
 
-Both C<compile()> and C<validate()> may throw an error if the C<$validations>
-or C<$schema> are invalid. Errors in the C<$input> should never cause an error
-to be thrown, since these are always reported in the C<$result> object.
-
-This module takes great care that C<$input> is not being modified in place,
-even if data normalization is being performed. The normalized data can be read
-from the C<$result> object.
-
-=head2 Result object
-
-The C<$result> object returned by C<validate()> overloads boolean context, so
-you can check if the validation succeeded with a simple if statement:
-
-  my $result = $validator->validate(..);
-  if($result) {
-    # Success!
-    my $data = $result->data;
-  } else {
-    # Input failed to validate...
-    my $error = $result->err;
-  }
-
-In addition, the result object implements the following methods:
-
-=over
-
-=item data()
-
-Returns the validated and normalized data. This method throws an error if
-validation failed, so if you're lazy and don't want to bother too much with
-proper error reporting, you can safely I<validate-and-die> in a single step:
-
-  my $validated_data = $v->validate(..)->data;
-
-(Note regarding reference semantics: The returned data will usually be a
-(possibly modified) copy of C<$input>, but may in some cases still have nested
-references to data in C<$input> - so if you are working with nested hashrefs,
-arrayrefs or other objects and are going to make modifications to the values
-embedded within them, these changes may or may not also affect the values in
-the original C<$input>. Make a deep copy of the data if you're concerned about
-this).
-
-=item err()
-
-Returns I<undef> if validation succeeded, an error object otherwise.
-
-An error object is a hashref containing at least one key: I<validation>, which
-indicates the name of the validation that failed. Additional keys with more
-detailed information may be present, depending on the validation. These are
-documented in L</SCHEMA DEFINITION> below.
-
-=back
+An error is thrown if the input does not validate. The error object is a
+C<FU::Validate::err>-blessed hashref containing at least one key:
+I<validation>, which indicates the name of the validation that failed.
+Additional keys with more detailed information may be present, depending on the
+validation. These are documented in L</SCHEMA DEFINITION> below.
 
 
 =head1 SCHEMA DEFINITION
@@ -519,9 +457,9 @@ Instead of reporting an error, return C<$val> if this input fails validation
 for whatever reason. Setting this option in the top-level schema ensures that
 the validation will always succeed regardless of the input.
 
-If C<$val> is a CODE reference, the subroutine is called with the result object
-for this validation as its first argument. The return value of the subroutine
-is then returned for this validation.
+If C<$val> is a CODE reference, the subroutine is called with the (partially
+normalized) input as first argument and error object as second argument. The
+return value of the subroutine is then returned for this validation.
 
 =item rmwhitespace => 0/1
 
@@ -856,11 +794,11 @@ Here's a simple example that defines and uses a custom validation named
 I<stringbool>, which accepts either the string I<true> or I<false>.
 
   my $validations = {
-    stringbool => { enum => ['true', 'false'] }
+      stringbool => { enum => ['true', 'false'] }
   };
   my $schema = { stringbool => 1 };
   my $result = FU::Validate->compile($schema, $validations)->validate('true');
-  # $result->data() eq 'true'
+  # $result eq 'true'
 
 A custom validation can also be defined as a subroutine, in which case it can
 accept options. Here is an example of a I<prefix> custom validation, which
@@ -868,9 +806,9 @@ requires that the string starts with the given prefix. The subroutine returns a
 schema that contains the I<func> built-in option to do the actual validation.
 
   my $validations = {
-    prefix => sub($prefix) {
-      return { func => sub { $_[0] =~ /^\Q$prefix/ } }
-    }
+      prefix => sub($prefix) {
+          return { func => sub { $_[0] =~ /^\Q$prefix/ } }
+      }
   };
   my $schema = { prefix => 'Hello, ' };
   my $result = FU::Validate->compile($schema, $validations)->validate('Hello, World!');
@@ -891,10 +829,10 @@ mixes validations of different types. For example, the following throws an
 error:
 
   FU::Validate->compile({
-    # top-level schema says we expect a hash
-    type => 'hash',
-    # but the 'int' validation implies that the type is a scalar
-    int => 1
+      # top-level schema says we expect a hash
+      type => 'hash',
+      # but the 'int' validation implies that the type is a scalar
+      int => 1
   });
 
 The I<keys>, I<values> and C<func> built-in options are validated separately
diff --git a/t/validate.t b/t/validate.t
index c7eb459..fdcd416 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -14,7 +14,7 @@ my %validations = (
     setundef => { func => sub { $_[0] = undef; 1 } },
     defaultsub1 => { default => sub { 2 } },
     defaultsub2 => { default => sub { defined $_[0] } },
-    onerrorsub => { onerror => sub { ref $_[0] } },
+    onerrorsub => { onerror => sub { ref $_[1] } },
     collapsews => { rmwhitespace => 0, func => sub { $_[0] =~ s/\s+/ /g; 1 } },
     neverfails => { onerror => 'err' },
     revnum => { type => 'array', sort => sub($x,$y) { $y <=> $x } },
@@ -31,8 +31,7 @@ my %validations = (
 );
 
 
-sub t {
-    my($schema, $input, $output, $error) = @_;
+sub t($schema, $input, $output) {
     my $line = (caller)[2];
 
     my $schema_copy = dclone([$schema])->[0];
@@ -40,203 +39,215 @@ sub t {
 
     my $res = FU::Validate->compile($schema, \%validations)->validate($input);
     #diag explain FU::Validate->compile($schema, \%validations) if $line == 139;
-    is !$error, !!$res, "boolean context $line";
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
-    is_deeply $res->unsafe_data(), $output, "unsafe_data $line";
-    is_deeply $res->data(), $output, "data ok $line" if !$error;
-    ok !eval { $res->data; 1}, "data err $line" if $error;
-    is_deeply $res->err(), $error, "err $line";
+    is_deeply $res, $output, "data ok $line";
+}
+
+sub f($schema, $input, $error) {
+    my $line = (caller)[2];
+
+    my $schema_copy = dclone([$schema])->[0];
+    my $input_copy = dclone([$input])->[0];
+
+    ok !eval { FU::Validate->compile($schema, \%validations)->validate($input); 1 }, "eval $line";
+    is_deeply $schema, $schema_copy, "schema modification $line";
+    is_deeply $input,  $input_copy,  "input modification $line";
+    is_deeply { $@->%* }, $error, "err $line";
 }
 
 
 # default
-t {}, 0, 0, undef;
-t {}, '', '', { validation => 'required' };
-t {}, undef, undef, { validation => 'required' };
-t { default => undef }, undef, undef, undef;
-t { default => undef }, '', undef, undef;
-t { defaultsub1 => 1 }, undef, 2, undef;
-t { defaultsub2 => 1 }, undef, '', undef;
-t { defaultsub2 => 1 }, '', 1, undef;
-t { onerrorsub => 1 }, undef, 'FU::Validate::Result', undef;
+t {}, 0, 0;
+f {}, '', { validation => 'required' };
+f {}, undef, { validation => 'required' };
+t { default => undef }, undef, undef;
+t { default => undef }, '', undef;
+t { defaultsub1 => 1 }, undef, 2;
+t { defaultsub2 => 1 }, undef, '';
+t { defaultsub2 => 1 }, '', 1;
+t { onerrorsub => 1 }, undef, 'FU::Validate::err';
 
 # rmwhitespace
-t {}, " Va\rl id \n ", 'Val id', undef;
-t { rmwhitespace => 0 }, " Va\rl id \n ", " Va\rl id \n ", undef;
-t {}, '  ', '', { validation => 'required' };
-t { rmwhitespace => 0 }, '  ', '  ', undef;
+t {}, " Va\rl id \n ", 'Val id';
+t { rmwhitespace => 0 }, " Va\rl id \n ", " Va\rl id \n ";
+f {}, '  ', { validation => 'required' };
+t { rmwhitespace => 0 }, '  ', '  ';
 
 # arrays
-t {}, [], [], { validation => 'type', expected => 'scalar', got => 'array' };
-t { type => 'array' }, 1, 1, { validation => 'type', expected => 'array', got => 'scalar' };
-t { type => 'array' }, [], [], undef;
-t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}], undef;
-t { type => 'array', scalar => 1 }, 1, [1], undef;
-t { type => 'array', values => {} }, [undef], [undef], { validation => 'values', errors => [{ index => 0, validation => 'required' }] };
-t { type => 'array', values => {} }, [' a '], ['a'], undef;
-t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/], undef;
-t { type => 'array', sort => 'num' }, [qw/20 100 3/], [qw/3 20 100/], undef;
-t { revnum => 1 },                    [qw/20 100 3/], [qw/100 20 3/], undef;
-t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 1/], [qw/1 2 3/], undef;
-t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 3/], [qw/2 3 3/], { validation => 'unique', index_a => 1, value_a => 3, index_b => 2, value_b => 3 };
-t { type => 'array', unique => 1 }, [qw/3 1 2/], [qw/3 1 2/], undef;
-t { type => 'array', unique => 1 }, [qw/3 1 3/], [qw/3 1 3/], { validation => 'unique', index_a => 0, value_a => 3, index_b => 2, value_b => 3, key => 3 };
-t { uniquelength => 1 }, [[],[1],[1,2]], [[],[1],[1,2]], undef;
-t { uniquelength => 1 }, [[],[1],[2]], [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 };
-t { type => 'array', setundef => 1 }, [], undef, undef;
-t { type => 'array', values => { type => 'any', setundef => 1 } }, [[]], [undef], undef;
+f {}, [], { validation => 'type', expected => 'scalar', got => 'array' };
+f { type => 'array' }, 1, { validation => 'type', expected => 'array', got => 'scalar' };
+t { type => 'array' }, [], [];
+t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}];
+t { type => 'array', scalar => 1 }, 1, [1];
+f { type => 'array', values => {} }, [undef], { validation => 'values', errors => [{ index => 0, validation => 'required' }] };
+t { type => 'array', values => {} }, [' a '], ['a'];
+t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/];
+t { type => 'array', sort => 'num' }, [qw/20 100 3/], [qw/3 20 100/];
+t { revnum => 1 },                    [qw/20 100 3/], [qw/100 20 3/];
+t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 1/], [qw/1 2 3/];
+f { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 3/], { validation => 'unique', index_a => 1, value_a => 3, index_b => 2, value_b => 3 };
+t { type => 'array', unique => 1 }, [qw/3 1 2/], [qw/3 1 2/];
+f { type => 'array', unique => 1 }, [qw/3 1 3/], { validation => 'unique', index_a => 0, value_a => 3, index_b => 2, value_b => 3, key => 3 };
+t { uniquelength => 1 }, [[],[1],[1,2]], [[],[1],[1,2]];
+f { uniquelength => 1 }, [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 };
+t { type => 'array', setundef => 1 }, [], undef;
+t { type => 'array', values => { type => 'any', setundef => 1 } }, [[]], [undef];
 
 # hashes
-t { type => 'hash' }, [], [], { validation => 'type', expected => 'hash', got => 'array' };
-t { type => 'hash' }, 'a', 'a', { validation => 'type', expected => 'hash', got => 'scalar' };
-t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {}, undef;
-t { type => 'hash', keys => { a=>{} } }, {}, {a=>undef}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] }; # XXX: the key doesn't necessarily have to be created
-t { type => 'hash', keys => { a=>{missing=>'ignore'} } }, {}, {}, undef;
-t { type => 'hash', keys => { a=>{default=>undef} } }, {}, {a=>undef}, undef;
-t { type => 'hash', keys => { a=>{missing=>'create',default=>undef} } }, {}, {a=>undef}, undef;
-t { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {}, {key => 'a', validation => 'missing'};
+f { type => 'hash' }, [], { validation => 'type', expected => 'hash', got => 'array' };
+f { type => 'hash' }, 'a', { validation => 'type', expected => 'hash', got => 'scalar' };
+t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {};
+f { type => 'hash', keys => { a=>{} } }, {}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] };
+t { type => 'hash', keys => { a=>{missing=>'ignore'} } }, {}, {};
+t { type => 'hash', keys => { a=>{default=>undef} } }, {}, {a=>undef};
+t { type => 'hash', keys => { a=>{missing=>'create',default=>undef} } }, {}, {a=>undef};
+f { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {key => 'a', validation => 'missing'};
 
-t { type => 'hash', keys => { a=>{} } }, {a=>' a '}, {a=>'a'}, undef; # Test against in-place modification
-t { type => 'hash', keys => { a=>{} }, unknown => 'remove' }, { a=>1,b=>1 }, { a=>1 }, undef;
-t { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] };
-t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>1,b=>1 }, undef;
-t { type => 'hash', setundef => 1 }, {}, undef, undef;
-t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef}, undef;
+t { type => 'hash', keys => { a=>{} } }, {a=>' a '}, {a=>'a'}; # Test against in-place modification
+t { type => 'hash', keys => { a=>{} }, unknown => 'remove' }, { a=>1,b=>1 }, { a=>1 };
+f { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] };
+t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>1,b=>1 };
+t { type => 'hash', setundef => 1 }, {}, undef;
+t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef};
 
 # default validations
-t { minlength => 3 }, 'ab', 'ab', { validation => 'minlength', expected => 3, got => 2 };
-t { minlength => 3 }, 'abc', 'abc', undef;
-t { maxlength => 3 }, 'abcd', 'abcd', { validation => 'maxlength', expected => 3, got => 4 };
-t { maxlength => 3 }, 'abc', 'abc', undef;
-t { minlength => 3, maxlength => 3 }, 'abc', 'abc', undef;
-t { length => 3 }, 'ab',   'ab',   { validation => 'length', expected => 3, got => 2 };
-t { length => 3 }, 'abcd', 'abcd', { validation => 'length', expected => 3, got => 4 };
-t { length => 3 }, 'abc',  'abc',  undef;
-t { length => [1,3] }, 'abc',  'abc', undef;
-t { length => [1,3] }, 'abcd', 'abcd', { validation => 'length', expected => [1,3], got => 4 };;
-t { type => 'array', length => 0 }, [], [], undef;
-t { type => 'array', length => 1 }, [1,2], [1,2], { validation => 'length', expected => 1, got => 2 };
-t { type => 'hash', length => 0 }, {}, {}, undef;
-t { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 };
-t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1}, undef;
-t { regex => '^a' }, 'abc', 'abc', undef;  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
-t { regex => '^a' }, 'cba', 'cba', { validation => 'regex', regex => '^a', got => 'cba' };
-t { enum => [1,2] }, 1, 1, undef;
-t { enum => [1,2] }, 2, 2, undef;
-t { enum => [1,2] }, 3, 3, { validation => 'enum', expected => [1,2], got => 3 };
-t { enum => 1 }, 1, 1, undef;
-t { enum => 1 }, 2, 2, { validation => 'enum', expected => [1], got => 2 };
-t { enum => {a=>1,b=>2} }, 'a', 'a', undef;
-t { enum => {a=>1,b=>2} }, 'c', 'c', { validation => 'enum', expected => ['a','b'], got => 'c' };
-t { anybool => 1 }, 1, true, undef;
-t { anybool => 1 }, undef, false, undef;
-t { anybool => 1 }, '', false, undef;
-t { anybool => 1 }, {}, true, undef;
-t { anybool => 1 }, [], true, undef;
-t { anybool => 1 }, bless({}, 'test'), true, undef;
-t { bool => 1 }, 1, 1, { validation => 'bool' };
-t { bool => 1 }, \1, true, undef;
+f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 };
+t { minlength => 3 }, 'abc', 'abc';
+f { maxlength => 3 }, 'abcd', { validation => 'maxlength', expected => 3, got => 4 };
+t { maxlength => 3 }, 'abc', 'abc';
+t { minlength => 3, maxlength => 3 }, 'abc', 'abc';
+f { length => 3 }, 'ab',   { validation => 'length', expected => 3, got => 2 };
+f { length => 3 }, 'abcd', { validation => 'length', expected => 3, got => 4 };
+t { length => 3 }, 'abc',  'abc';
+t { length => [1,3] }, 'abc',  'abc';
+f { length => [1,3] }, 'abcd', { validation => 'length', expected => [1,3], got => 4 };;
+t { type => 'array', length => 0 }, [], [];
+f { type => 'array', length => 1 }, [1,2], { validation => 'length', expected => 1, got => 2 };
+t { type => 'hash', length => 0 }, {}, {};
+f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 };
+t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1};
+t { regex => '^a' }, 'abc', 'abc';  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
+f { regex => '^a' }, 'cba', { validation => 'regex', regex => '^a', got => 'cba' };
+t { enum => [1,2] }, 1, 1;
+t { enum => [1,2] }, 2, 2;
+f { enum => [1,2] }, 3, { validation => 'enum', expected => [1,2], got => 3 };
+t { enum => 1 }, 1, 1;
+f { enum => 1 }, 2, { validation => 'enum', expected => [1], got => 2 };
+t { enum => {a=>1,b=>2} }, 'a', 'a';
+f { enum => {a=>1,b=>2} }, 'c', { validation => 'enum', expected => ['a','b'], got => 'c' };
+t { anybool => 1 }, 1, true;
+t { anybool => 1 }, undef, false;
+t { anybool => 1 }, '', false;
+t { anybool => 1 }, {}, true;
+t { anybool => 1 }, [], true;
+t { anybool => 1 }, bless({}, 'test'), true;
+f { bool => 1 }, 1, { validation => 'bool' };
+t { bool => 1 }, \1, true;
 my($true, $false) = (1,0);
-t { bool => 1 }, bless(\$true, 'boolean'), true, undef;
-t { bool => 1 }, bless(\$false, 'boolean'), false, undef;
-t { bool => 1 }, bless(\$true, 'test'), bless(\$true, 'test'), { validation => 'bool' };
-t { ascii => 1 }, 'ab c', 'ab c', undef;
-t { ascii => 1 }, "a\nb", "a\nb", { validation => 'ascii', got => "a\nb" };
+t { bool => 1 }, bless(\$true, 'boolean'), true;
+t { bool => 1 }, bless(\$false, 'boolean'), false;
+f { bool => 1 }, bless(\$true, 'test'), { validation => 'bool' };
+t { ascii => 1 }, 'ab c', 'ab c';
+f { ascii => 1 }, "a\nb", { validation => 'ascii', got => "a\nb" };
 
 # custom validations
-t { hex => 1 }, 'DeadBeef', 'DeadBeef', undef;
-t { hex => 1 }, 'x', 'x', { validation => 'hex', error => { validation => 'regex', regex => "$validations{hex}{regex}", got => 'x' } };
-t { prefix => 'a' }, 'abc', 'abc', undef;
-t { prefix => 'a' }, 'cba', 'cba', { validation => 'prefix', error => { validation => 'func', result => '' } };
-t { mybool => 1 }, 'abc', 1, undef;
-t { mybool => 1 }, undef, 0, undef;
-t { mybool => 1 }, '', 0, undef;
-t { collapsews => 1 }, " \t\n ", ' ', undef;
-t { collapsews => 1 }, '   x  ', ' x ', undef;
-t { collapsews => 1, rmwhitespace => 1 }, '   x  ', 'x', undef;
-t { person => 1 }, 1, 1, { validation => 'type', expected => 'hash', got => 'scalar' };
-t { person => 1, default => 1 }, undef, 1, undef;
-t { person => 1 }, { sex => 1 }, { sex => 1, name => undef }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } };
-t { person => 1 }, { sex => undef, name => 'y' }, { sex => 1, name => 'y' }, undef;
-t { person => 1, keys => {age => {default => \'required'}} }, {name => 'x', sex => 'y'}, { name => 'x', sex => 'y', age => undef }, { validation => 'keys', errors => [{ key => 'age', validation => 'required' }] };
-t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => 1},  { name => 'x', sex => 'y', extra => 1 }, undef;
-t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { name => 'x', sex => 'y', extra => '' }, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] };
-t { person => 1 }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y', extra => 1}, undef;
-t { person => 1, unknown => 'remove' }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y'}, undef;
-t { neverfails => 1, int => 1 }, undef, 'err', undef;
-t { neverfails => 1, int => 1 }, 'x', 'err', undef;
-t { neverfails => 1, int => 1, onerror => undef }, 'x', undef, undef; # XXX: no way to 'unset' an inherited onerror clause, hmm.
+t { hex => 1 }, 'DeadBeef', 'DeadBeef';
+f { hex => 1 }, 'x', { validation => 'hex', error => { validation => 'regex', regex => "$validations{hex}{regex}", got => 'x' } };
+t { prefix => 'a' }, 'abc', 'abc';
+f { prefix => 'a' }, 'cba', { validation => 'prefix', error => { validation => 'func', result => '' } };
+t { mybool => 1 }, 'abc', 1;
+t { mybool => 1 }, undef, 0;
+t { mybool => 1 }, '', 0;
+t { collapsews => 1 }, " \t\n ", ' ';
+t { collapsews => 1 }, '   x  ', ' x ';
+t { collapsews => 1, rmwhitespace => 1 }, '   x  ', 'x';
+f { person => 1 }, 1, { validation => 'type', expected => 'hash', got => 'scalar' };
+t { person => 1, default => 1 }, undef, 1;
+f { person => 1 }, { sex => 1 }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } };
+t { person => 1 }, { sex => undef, name => 'y' }, { sex => 1, name => 'y' };
+f { person => 1, keys => {age => {default => \'required'}} }, {name => 'x', sex => 'y'}, { validation => 'keys', errors => [{ key => 'age', validation => 'required' }] };
+t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => 1},  { name => 'x', sex => 'y', extra => 1 };
+f { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] };
+t { person => 1 }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y', extra => 1};
+t { person => 1, unknown => 'remove' }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y'};
+t { neverfails => 1, int => 1 }, undef, 'err';
+t { neverfails => 1, int => 1 }, 'x', 'err';
+t { neverfails => 1, int => 1, onerror => undef }, 'x', undef; # XXX: no way to 'unset' an inherited onerror clause, hmm.
 
 # numbers
 sub nerr { +{ validation => 'num', got => $_[0] } }
-t { num => 1 }, 0, 0, undef;
-t { num => 1 }, '-', '-', nerr '-';
-t { num => 1 }, '00', '00', nerr '00';
-t { num => 1 }, '1', '1', undef;
-t { num => 1 }, '1.1.', '1.1.', nerr '1.1.';
-t { num => 1 }, '1.-1', '1.-1', nerr '1.-1';
-t { num => 1 }, '.1', '.1', nerr '.1';
-t { num => 1 }, '0.1e5', '0.1e5', undef;
-t { num => 1 }, '0.1e+5', '0.1e+5', undef;
-t { num => 1 }, '0.1e5.1', '0.1e5.1', nerr '0.1e5.1';
-t { int => 1 }, 0, 0, undef;
-t { int => 1 }, -123, -123, undef;
-t { int => 1 }, -123.1, -123.1, { validation => 'int', got => -123.1 };
-t { uint => 1 }, 0, 0, undef;
-t { uint => 1 }, 123, 123, undef;
-t { uint => 1 }, -123, -123, { validation => 'uint', got => -123 };
-t { min => 1 }, 1, 1, undef;
-t { min => 1 }, 0.9, 0.9, { validation => 'min', expected => 1, got => 0.9 };
-t { min => 1 }, 'a', 'a', { validation => 'min', error => nerr 'a' };
-t { max => 1 }, 1, 1, undef;
-t { max => 1 }, 1.1, 1.1, { validation => 'max', expected => 1, got => 1.1 };
-t { max => 1 }, 'a', 'a', { validation => 'max', error => nerr 'a' };
-t { range => [1,2] }, 1, 1, undef;
-t { range => [1,2] }, 2, 2, undef;
-t { range => [1,2] }, 0.9, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } };
-t { range => [1,2] }, 2.1, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } };
+t { num => 1 }, 0, 0;
+f { num => 1 }, '-', nerr '-';
+f { num => 1 }, '00', nerr '00';
+t { num => 1 }, '1', '1';
+f { num => 1 }, '1.1.', nerr '1.1.';
+f { num => 1 }, '1.-1', nerr '1.-1';
+f { num => 1 }, '.1', nerr '.1';
+t { num => 1 }, '0.1e5', '0.1e5';
+t { num => 1 }, '0.1e+5', '0.1e+5';
+f { num => 1 }, '0.1e5.1', nerr '0.1e5.1';
+t { int => 1 }, 0, 0;
+t { int => 1 }, -123, -123;
+f { int => 1 }, -123.1, { validation => 'int', got => -123.1 };
+t { uint => 1 }, 0, 0;
+t { uint => 1 }, 123, 123;
+f { uint => 1 }, -123, { validation => 'uint', got => -123 };
+t { min => 1 }, 1, 1;
+f { min => 1 }, 0.9, { validation => 'min', expected => 1, got => 0.9 };
+f { min => 1 }, 'a', { validation => 'min', error => nerr 'a' };
+t { max => 1 }, 1, 1;
+f { max => 1 }, 1.1, { validation => 'max', expected => 1, got => 1.1 };
+f { max => 1 }, 'a', { validation => 'max', error => nerr 'a' };
+t { range => [1,2] }, 1, 1;
+t { range => [1,2] }, 2, 2;
+f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } };
+f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } };
 #t { range => [1,2] }, 'a', 'a', { validation => 'range', error => { validation => 'max', error => nerr 'a' } }; # XXX: Error validation type depends on evaluation order
 
 # email template
 use utf8;
-t { email => 1 }, $_->[1], $_->[1], $_->[0] ? undef : { validation => 'email', got => $_->[1] } for (
-  [ 0, 'abc.com' ],
-  [ 0, 'abc@localhost' ],
-  [ 0, 'abc@10.0.0.' ],
-  [ 0, 'abc@256.0.0.1' ],
-  [ 0, '<whoami>@blicky.net' ],
-  [ 0, 'a @a.com' ],
-  [ 0, 'a"@a.com' ],
-  [ 0, 'a@[:]' ],
-  [ 0, 'a@127.0.0.1' ],
-  [ 0, 'a@[::1]' ],
-  [ 1, 'a@a.com' ],
-  [ 1, 'a@a.com.' ],
-  [ 1, 'é@yörhel.nl' ],
-  [ 1, 'a+_0-c@yorhel.nl' ],
-  [ 1, 'é@x-y_z.example' ],
-  [ 1, 'abc@x-y_z.example' ],
+f { email => 1 }, $_, { validation => 'email', got => $_ } for (
+  'abc.com',
+  'abc@localhost',
+  'abc@10.0.0.',
+  'abc@256.0.0.1',
+  '<whoami>@blicky.net',
+  'a @a.com',
+  'a"@a.com',
+  'a@[:]',
+  'a@127.0.0.1',
+  'a@[::1]',
+);
+t { email => 1 }, $_, $_ for (
+  'a@a.com',
+  'a@a.com.',
+  'é@yörhel.nl',
+  'a+_0-c@yorhel.nl',
+  'é@x-y_z.example',
+  'abc@x-y_z.example',
 );
 my $long = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxxxxxxxxxx.xxxxx';
-t { email => 1 }, $long, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } };
+f { email => 1 }, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } };
 
 # weburl template
-t { weburl => 1 }, $_->[1], $_->[1], $_->[0] ? undef : { validation => 'weburl', got => $_->[1] } for (
-  [ 0, 'http' ],
-  [ 0, 'http://' ],
-  [ 0, 'http:///' ],
-  [ 0, 'http://x/' ],
-  [ 0, 'http://x/' ],
-  [ 0, 'http://256.0.0.1/' ],
-  [ 0, 'http://blicky.net:050/' ],
-  [ 0, 'ftp//blicky.net/' ],
-  [ 1, 'http://blicky.net/' ],
-  [ 1, 'http://blicky.net:50/' ],
-  [ 1, 'https://blicky.net/' ],
-  [ 1, 'https://[::1]:80/' ],
-  [ 1, 'https://l-n.x_.example.com/' ],
-  [ 1, 'https://blicky.net/?#Who\'d%20ever%22makeaurl_like-this/!idont.know' ],
+f { weburl => 1 }, $_, { validation => 'weburl', got => $_ } for (
+  'http',
+  'http://',
+  'http:///',
+  'http://x/',
+  'http://x/',
+  'http://256.0.0.1/',
+  'http://blicky.net:050/',
+  'ftp//blicky.net/',
+);
+t { weburl => 1}, $_, $_ for (
+  'http://blicky.net/',
+  'http://blicky.net:50/',
+  'https://blicky.net/',
+  'https://[::1]:80/',
+  'https://l-n.x_.example.com/',
+  'https://blicky.net/?#Who\'d%20ever%22makeaurl_like-this/!idont.know',
 );
 
 

From e4b6b77e1bd0390c675c67e26cfb2de86f6f29c0 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 5 Mar 2025 15:39:46 +0100
Subject: [PATCH 04/86] Validate: rename rmwhitespace to trim and use
 builtin::trim()

---
 FU/Validate.pm | 45 ++++++++++++++++++++-------------------------
 t/validate.t   | 10 +++++-----
 2 files changed, 25 insertions(+), 30 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 7f1cc1c..296d78f 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -2,7 +2,7 @@ package FU::Validate 0.2;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
-use builtin qw/true false blessed/;
+use builtin qw/true false blessed trim/;
 use Carp 'confess';
 use FU::Util 'to_bool';
 
@@ -12,7 +12,7 @@ my %builtin = map +($_,1), qw/
     type
     default
     onerror
-    rmwhitespace
+    trim
     values scalar sort unique
     keys unknown missing
     func
@@ -130,7 +130,7 @@ sub _compile($schema, $validations, $rec) {
             confess "Incompatible types, '$t->[0]' requires '$t->{schema}{type}', but another validation requires '$top{type}'";
         }
         exists $t->{schema}{$_} and !exists $top{$_} and $top{$_} = delete $t->{schema}{$_}
-            for qw/default onerror rmwhitespace type scalar unknown missing sort unique/;
+            for qw/default onerror trim type scalar unknown missing sort unique/;
 
         push @keys, keys %{ delete $t->{known_keys} };
         push @keys, keys %{ $t->{schema}{keys} } if $t->{schema}{keys};
@@ -155,7 +155,7 @@ sub compile($pkg, $schema, $validations={}) {
 
     $c->{schema}{type} //= 'scalar';
     $c->{schema}{missing} //= 'create';
-    $c->{schema}{rmwhitespace} //= 1 if $c->{schema}{type} eq 'scalar';
+    $c->{schema}{trim} //= 1 if $c->{schema}{type} eq 'scalar';
     $c->{schema}{unknown} //= 'remove' if $c->{schema}{type} eq 'hash';
 
     confess "Invalid value for 'type': $c->{schema}{type}" if !$type_vals{$c->{schema}{type}};
@@ -262,12 +262,8 @@ sub _validate_array {
 sub _validate_input {
     my $c = $_[0];
 
-    # rmwhitespace (needs to be done before the 'default' test)
-    if (defined $_[1] && !ref $_[1] && $c->{schema}{type} eq 'scalar' && $c->{schema}{rmwhitespace}) {
-        $_[1] =~ s/\r//g;
-        $_[1] =~ s/^\s*//;
-        $_[1] =~ s/\s*$//;
-    }
+    # trim (needs to be done before the 'default' test)
+    $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $c->{schema}{type} eq 'scalar' && $c->{schema}{trim};
 
     # default
     if (!defined $_[1] || (!ref $_[1] && $_[1] eq '')) {
@@ -410,10 +406,10 @@ validation to be performed. None of the options or validations are required,
 but some built-ins have default values. This means that the empty schema C<{}>
 is actually equivalent to:
 
-  { type         => 'scalar',
-    rmwhitespace => 1,
-    default      => \'required',
-    missing      => 'create',
+  { type    => 'scalar',
+    trim    => 1,
+    default => \'required',
+    missing => 'create',
   }
 
 =head2 Built-in options
@@ -444,12 +440,11 @@ C<$val> is returned instead. If C<$val> is a CODE reference, the subroutine is
 called with the original value (which is either no argument, undef or an empty
 string) and the return value of the subroutine is used as value instead.
 
-The empty check is performed after I<rmwhitespace> and before any other
-validations. So a string containing only whitespace is considered an empty
-string and will be treated according to this I<default> option. As an
-additional side effect, other validations will never get to validate undef or
-an empty string, as these values are either rejected or substituted with a
-default.
+The empty check is performed after I<trim> and before any other validations. So
+a string containing only whitespace is considered an empty string and will be
+treated according to this I<default> option. As an additional side effect,
+other validations will never get to validate undef or an empty string, as these
+values are either rejected or substituted with a default.
 
 =item onerror => $val
 
@@ -461,11 +456,11 @@ If C<$val> is a CODE reference, the subroutine is called with the (partially
 normalized) input as first argument and error object as second argument. The
 return value of the subroutine is then returned for this validation.
 
-=item rmwhitespace => 0/1
+=item trim => 0/1
 
 By default, any whitespace around scalar-type input is removed before testing
-any other validations. Setting I<rmwhitespace> to a false value will disable
-this behavior.
+any other validations. Setting I<trim> to a false value will disable this
+behavior.
 
 =item keys => $hashref
 
@@ -818,8 +813,8 @@ schema that contains the I<func> built-in option to do the actual validation.
 Custom validations can also set built-in options, but the semantics differ a
 little depending on the option. First, be aware that many of the built-in
 options apply to the whole schema and not just to the custom validation.  For
-example, if the top-level schema sets C<< rmwhitespace => 0 >>, then all
-validations used in that schema may get input with whitespace around it.
+example, if the top-level schema sets C<< trim => 0 >>, then all validations
+used in that schema may get input with whitespace around it.
 
 All validations used in a schema need to agree upon a single I<type> option.
 If a custom validation does not specify a I<type> option (and no type is
diff --git a/t/validate.t b/t/validate.t
index fdcd416..1bf0b35 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -15,7 +15,7 @@ my %validations = (
     defaultsub1 => { default => sub { 2 } },
     defaultsub2 => { default => sub { defined $_[0] } },
     onerrorsub => { onerror => sub { ref $_[1] } },
-    collapsews => { rmwhitespace => 0, func => sub { $_[0] =~ s/\s+/ /g; 1 } },
+    collapsews => { trim => 0, func => sub { $_[0] =~ s/\s+/ /g; 1 } },
     neverfails => { onerror => 'err' },
     revnum => { type => 'array', sort => sub($x,$y) { $y <=> $x } },
     uniquelength => { type => 'array', values => { type => 'array' }, unique => sub { scalar @{$_[0]} } },
@@ -68,11 +68,11 @@ t { defaultsub2 => 1 }, undef, '';
 t { defaultsub2 => 1 }, '', 1;
 t { onerrorsub => 1 }, undef, 'FU::Validate::err';
 
-# rmwhitespace
+# trim
 t {}, " Va\rl id \n ", 'Val id';
-t { rmwhitespace => 0 }, " Va\rl id \n ", " Va\rl id \n ";
+t { trim => 0 }, " Va\rl id \n ", " Va\rl id \n ";
 f {}, '  ', { validation => 'required' };
-t { rmwhitespace => 0 }, '  ', '  ';
+t { trim => 0 }, '  ', '  ';
 
 # arrays
 f {}, [], { validation => 'type', expected => 'scalar', got => 'array' };
@@ -161,7 +161,7 @@ t { mybool => 1 }, undef, 0;
 t { mybool => 1 }, '', 0;
 t { collapsews => 1 }, " \t\n ", ' ';
 t { collapsews => 1 }, '   x  ', ' x ';
-t { collapsews => 1, rmwhitespace => 1 }, '   x  ', 'x';
+t { collapsews => 1, trim => 1 }, '   x  ', 'x';
 f { person => 1 }, 1, { validation => 'type', expected => 'hash', got => 'scalar' };
 t { person => 1, default => 1 }, undef, 1;
 f { person => 1 }, { sex => 1 }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } };

From e5755ddd80cf44dc5ba8a76b4495afb33e9bac0f Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 6 Mar 2025 10:16:12 +0100
Subject: [PATCH 05/86] Validate: Human-readable error messages

---
 FU.pm          |  4 +--
 FU/Validate.pm | 35 ++++++++++++++----
 t/validate.t   | 96 ++++++++++++++++++++++++++------------------------
 3 files changed, 79 insertions(+), 56 deletions(-)

diff --git a/FU.pm b/FU.pm
index 4d04562..57123a5 100644
--- a/FU.pm
+++ b/FU.pm
@@ -640,8 +640,8 @@ sub _getfield($data, @a) {
     require FU::Validate;
     my $schema = FU::Validate->compile(@a > 1 ? { keys => {@a} } : $a[0]);
     my $res = eval { $schema->validate($data) };
-    fu->error(400, "Input validation failed") if $@; # TODO: More detailed error message
-    return @a == 2 ? $res->data->{$a[0]} : $res->data;
+    fu->error(400, "Input validation failed: $@") if $@;
+    return @a == 2 ? $res->{$a[0]} : $res;
 }
 
 sub query {
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 296d78f..1c7c94b 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -321,8 +321,10 @@ sub _validate {
 sub validate($c, $input) {
     my $r = _validate($c, $input);
     return $input if !$r;
-    die bless $r, 'FU::Validate::err';
-    $input
+    $r = bless $r, 'FU::Validate::err';;
+    my @e = $r->errors;
+    $r->{longmess} = Carp::longmess(@e > 1 ? join("\n",@e)."\n" : $e[0]);
+    die $r;
 }
 
 
@@ -330,12 +332,31 @@ sub validate($c, $input) {
 
 package FU::Validate::err;
 use v5.36;
+use FU::Util;
 
-use overload '""' => sub {
-    # TODO: Better error message
-    require Data::Dumper;
-    Data::Dumper->new([{$_[0]->%*}])->Terse(1)->Pair(':')->Indent(0)->Sortkeys(1)->Dump."\n";
-};
+use overload '""' => sub { $_[0]{longmess} || join "\n", $_[0]->errors };
+
+sub _fmtkey($k) {
+    $k =~ /^[a-zA-Z0-9_-]+$/ ? $k : FU::Util::json_format($k);
+}
+
+sub _fmtval($v) {
+    eval { $v = FU::Util::json_format($v) }; "$v"
+}
+
+sub errors($e, $prefix='') {
+    my $val = $e->{validation};
+    my $p = $prefix ? "$prefix: " : '';
+    $val eq 'keys'     ? map errors($_, $prefix.'.'._fmtkey($_->{key})), $e->{errors}->@* :
+    $val eq 'missing'  ? $prefix.'.'._fmtkey($e->{key}).': required key missing' :
+    $val eq 'values'   ? map errors($_, $prefix."[$_->{index}]"), $e->{errors}->@* :
+    $val eq 'unique'   ? $prefix."[$e->{index_b}] value '"._fmtval($e->{value_a})."' duplicated" :
+    $val eq 'required' ? "${p}required value missing" :
+    $val eq 'type'     ? "${p}invalid type, expected '$e->{expected}' but got '$e->{got}'" :
+    $val eq 'unknown'  ? ($e->{keys}->@* > 1 ? "${p}unknown keys: ".join(', ', _fmtkey($e->{keys})) : "${p}unknown key '"._fmtkey($e->{keys}[0])."'") :
+    $e->{error}        ? errors($e->{error}, "${p}validation '$val'") :
+                         "${p}failed validation '$val'";
+}
 
 
 1;
diff --git a/t/validate.t b/t/validate.t
index 1bf0b35..b01352b 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -44,7 +44,7 @@ sub t($schema, $input, $output) {
     is_deeply $res, $output, "data ok $line";
 }
 
-sub f($schema, $input, $error) {
+sub f($schema, $input, $error, @msg) {
     my $line = (caller)[2];
 
     my $schema_copy = dclone([$schema])->[0];
@@ -53,14 +53,16 @@ sub f($schema, $input, $error) {
     ok !eval { FU::Validate->compile($schema, \%validations)->validate($input); 1 }, "eval $line";
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
+    delete $@->{longmess};
     is_deeply { $@->%* }, $error, "err $line";
+    is_deeply [$@->errors], \@msg, "errmsg $line";
 }
 
 
 # default
 t {}, 0, 0;
-f {}, '', { validation => 'required' };
-f {}, undef, { validation => 'required' };
+f {}, '', { validation => 'required' }, 'required value missing';
+f {}, undef, { validation => 'required' }, 'required value missing';
 t { default => undef }, undef, undef;
 t { default => undef }, '', undef;
 t { defaultsub1 => 1 }, undef, 2;
@@ -71,104 +73,104 @@ t { onerrorsub => 1 }, undef, 'FU::Validate::err';
 # trim
 t {}, " Va\rl id \n ", 'Val id';
 t { trim => 0 }, " Va\rl id \n ", " Va\rl id \n ";
-f {}, '  ', { validation => 'required' };
+f {}, '  ', { validation => 'required' }, 'required value missing';
 t { trim => 0 }, '  ', '  ';
 
 # arrays
-f {}, [], { validation => 'type', expected => 'scalar', got => 'array' };
-f { type => 'array' }, 1, { validation => 'type', expected => 'array', got => 'scalar' };
+f {}, [], { validation => 'type', expected => 'scalar', got => 'array' }, "invalid type, expected 'scalar' but got 'array'";
+f { type => 'array' }, 1, { validation => 'type', expected => 'array', got => 'scalar' }, "invalid type, expected 'array' but got 'scalar'";
 t { type => 'array' }, [], [];
 t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}];
 t { type => 'array', scalar => 1 }, 1, [1];
-f { type => 'array', values => {} }, [undef], { validation => 'values', errors => [{ index => 0, validation => 'required' }] };
+f { type => 'array', values => {} }, [undef], { validation => 'values', errors => [{ index => 0, validation => 'required' }] }, "[0]: required value missing";
 t { type => 'array', values => {} }, [' a '], ['a'];
 t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/];
 t { type => 'array', sort => 'num' }, [qw/20 100 3/], [qw/3 20 100/];
 t { revnum => 1 },                    [qw/20 100 3/], [qw/100 20 3/];
 t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 1/], [qw/1 2 3/];
-f { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 3/], { validation => 'unique', index_a => 1, value_a => 3, index_b => 2, value_b => 3 };
+f { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 3/], { validation => 'unique', index_a => 1, value_a => 3, index_b => 2, value_b => 3 }, q{[2] value '"3"' duplicated};
 t { type => 'array', unique => 1 }, [qw/3 1 2/], [qw/3 1 2/];
-f { type => 'array', unique => 1 }, [qw/3 1 3/], { validation => 'unique', index_a => 0, value_a => 3, index_b => 2, value_b => 3, key => 3 };
+f { type => 'array', unique => 1 }, [qw/3 1 3/], { validation => 'unique', index_a => 0, value_a => 3, index_b => 2, value_b => 3, key => 3 }, q{[2] value '"3"' duplicated};
 t { uniquelength => 1 }, [[],[1],[1,2]], [[],[1],[1,2]];
-f { uniquelength => 1 }, [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 };
+f { uniquelength => 1 }, [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 }, q{[2] value '[1]' duplicated};
 t { type => 'array', setundef => 1 }, [], undef;
 t { type => 'array', values => { type => 'any', setundef => 1 } }, [[]], [undef];
 
 # hashes
-f { type => 'hash' }, [], { validation => 'type', expected => 'hash', got => 'array' };
-f { type => 'hash' }, 'a', { validation => 'type', expected => 'hash', got => 'scalar' };
+f { type => 'hash' }, [], { validation => 'type', expected => 'hash', got => 'array' }, "invalid type, expected 'hash' but got 'array'";
+f { type => 'hash' }, 'a', { validation => 'type', expected => 'hash', got => 'scalar' }, "invalid type, expected 'hash' but got 'scalar'";
 t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {};
-f { type => 'hash', keys => { a=>{} } }, {}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] };
+f { type => 'hash', keys => { a=>{} } }, {}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] }, '.a: required value missing';
 t { type => 'hash', keys => { a=>{missing=>'ignore'} } }, {}, {};
 t { type => 'hash', keys => { a=>{default=>undef} } }, {}, {a=>undef};
 t { type => 'hash', keys => { a=>{missing=>'create',default=>undef} } }, {}, {a=>undef};
-f { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {key => 'a', validation => 'missing'};
+f { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {key => 'a', validation => 'missing'}, '.a: required key missing';
 
 t { type => 'hash', keys => { a=>{} } }, {a=>' a '}, {a=>'a'}; # Test against in-place modification
 t { type => 'hash', keys => { a=>{} }, unknown => 'remove' }, { a=>1,b=>1 }, { a=>1 };
-f { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] };
+f { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] }, "unknown key 'b'";
 t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>1,b=>1 };
 t { type => 'hash', setundef => 1 }, {}, undef;
 t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef};
 
 # default validations
-f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 };
+f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 }, "failed validation 'minlength'";
 t { minlength => 3 }, 'abc', 'abc';
-f { maxlength => 3 }, 'abcd', { validation => 'maxlength', expected => 3, got => 4 };
+f { maxlength => 3 }, 'abcd', { validation => 'maxlength', expected => 3, got => 4 }, "failed validation 'maxlength'";
 t { maxlength => 3 }, 'abc', 'abc';
 t { minlength => 3, maxlength => 3 }, 'abc', 'abc';
-f { length => 3 }, 'ab',   { validation => 'length', expected => 3, got => 2 };
-f { length => 3 }, 'abcd', { validation => 'length', expected => 3, got => 4 };
+f { length => 3 }, 'ab',   { validation => 'length', expected => 3, got => 2 }, "failed validation 'length'";
+f { length => 3 }, 'abcd', { validation => 'length', expected => 3, got => 4 }, "failed validation 'length'";
 t { length => 3 }, 'abc',  'abc';
 t { length => [1,3] }, 'abc',  'abc';
-f { length => [1,3] }, 'abcd', { validation => 'length', expected => [1,3], got => 4 };;
+f { length => [1,3] }, 'abcd', { validation => 'length', expected => [1,3], got => 4 }, "failed validation 'length'";
 t { type => 'array', length => 0 }, [], [];
-f { type => 'array', length => 1 }, [1,2], { validation => 'length', expected => 1, got => 2 };
+f { type => 'array', length => 1 }, [1,2], { validation => 'length', expected => 1, got => 2 }, "failed validation 'length'";
 t { type => 'hash', length => 0 }, {}, {};
-f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 };
+f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 }, "failed validation 'length'";
 t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1};
 t { regex => '^a' }, 'abc', 'abc';  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
-f { regex => '^a' }, 'cba', { validation => 'regex', regex => '^a', got => 'cba' };
+f { regex => '^a' }, 'cba', { validation => 'regex', regex => '^a', got => 'cba' }, "failed validation 'regex'";
 t { enum => [1,2] }, 1, 1;
 t { enum => [1,2] }, 2, 2;
-f { enum => [1,2] }, 3, { validation => 'enum', expected => [1,2], got => 3 };
+f { enum => [1,2] }, 3, { validation => 'enum', expected => [1,2], got => 3 }, "failed validation 'enum'";
 t { enum => 1 }, 1, 1;
-f { enum => 1 }, 2, { validation => 'enum', expected => [1], got => 2 };
+f { enum => 1 }, 2, { validation => 'enum', expected => [1], got => 2 }, "failed validation 'enum'";
 t { enum => {a=>1,b=>2} }, 'a', 'a';
-f { enum => {a=>1,b=>2} }, 'c', { validation => 'enum', expected => ['a','b'], got => 'c' };
+f { enum => {a=>1,b=>2} }, 'c', { validation => 'enum', expected => ['a','b'], got => 'c' }, "failed validation 'enum'";
 t { anybool => 1 }, 1, true;
 t { anybool => 1 }, undef, false;
 t { anybool => 1 }, '', false;
 t { anybool => 1 }, {}, true;
 t { anybool => 1 }, [], true;
 t { anybool => 1 }, bless({}, 'test'), true;
-f { bool => 1 }, 1, { validation => 'bool' };
+f { bool => 1 }, 1, { validation => 'bool' }, "failed validation 'bool'";
 t { bool => 1 }, \1, true;
 my($true, $false) = (1,0);
 t { bool => 1 }, bless(\$true, 'boolean'), true;
 t { bool => 1 }, bless(\$false, 'boolean'), false;
-f { bool => 1 }, bless(\$true, 'test'), { validation => 'bool' };
+f { bool => 1 }, bless(\$true, 'test'), { validation => 'bool' }, "failed validation 'bool'";
 t { ascii => 1 }, 'ab c', 'ab c';
-f { ascii => 1 }, "a\nb", { validation => 'ascii', got => "a\nb" };
+f { ascii => 1 }, "a\nb", { validation => 'ascii', got => "a\nb" }, "failed validation 'ascii'";
 
 # custom validations
 t { hex => 1 }, 'DeadBeef', 'DeadBeef';
-f { hex => 1 }, 'x', { validation => 'hex', error => { validation => 'regex', regex => "$validations{hex}{regex}", got => 'x' } };
+f { hex => 1 }, 'x', { validation => 'hex', error => { validation => 'regex', regex => "$validations{hex}{regex}", got => 'x' } }, "validation 'hex': failed validation 'regex'";
 t { prefix => 'a' }, 'abc', 'abc';
-f { prefix => 'a' }, 'cba', { validation => 'prefix', error => { validation => 'func', result => '' } };
+f { prefix => 'a' }, 'cba', { validation => 'prefix', error => { validation => 'func', result => '' } }, "validation 'prefix': failed validation 'func'";
 t { mybool => 1 }, 'abc', 1;
 t { mybool => 1 }, undef, 0;
 t { mybool => 1 }, '', 0;
 t { collapsews => 1 }, " \t\n ", ' ';
 t { collapsews => 1 }, '   x  ', ' x ';
 t { collapsews => 1, trim => 1 }, '   x  ', 'x';
-f { person => 1 }, 1, { validation => 'type', expected => 'hash', got => 'scalar' };
+f { person => 1 }, 1, { validation => 'type', expected => 'hash', got => 'scalar' }, "invalid type, expected 'hash' but got 'scalar'";
 t { person => 1, default => 1 }, undef, 1;
-f { person => 1 }, { sex => 1 }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } };
+f { person => 1 }, { sex => 1 }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } }, "validation 'person'.name: required value missing";
 t { person => 1 }, { sex => undef, name => 'y' }, { sex => 1, name => 'y' };
-f { person => 1, keys => {age => {default => \'required'}} }, {name => 'x', sex => 'y'}, { validation => 'keys', errors => [{ key => 'age', validation => 'required' }] };
+f { person => 1, keys => {age => {default => \'required'}} }, {name => 'x', sex => 'y'}, { validation => 'keys', errors => [{ key => 'age', validation => 'required' }] }, '.age: required value missing';
 t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => 1},  { name => 'x', sex => 'y', extra => 1 };
-f { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] };
+f { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] }, '.extra: required value missing';
 t { person => 1 }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y', extra => 1};
 t { person => 1, unknown => 'remove' }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y'};
 t { neverfails => 1, int => 1 }, undef, 'err';
@@ -176,7 +178,7 @@ t { neverfails => 1, int => 1 }, 'x', 'err';
 t { neverfails => 1, int => 1, onerror => undef }, 'x', undef; # XXX: no way to 'unset' an inherited onerror clause, hmm.
 
 # numbers
-sub nerr { +{ validation => 'num', got => $_[0] } }
+sub nerr { ({ validation => 'num', got => $_[0] }, "failed validation 'num'") }
 t { num => 1 }, 0, 0;
 f { num => 1 }, '-', nerr '-';
 f { num => 1 }, '00', nerr '00';
@@ -189,25 +191,25 @@ t { num => 1 }, '0.1e+5', '0.1e+5';
 f { num => 1 }, '0.1e5.1', nerr '0.1e5.1';
 t { int => 1 }, 0, 0;
 t { int => 1 }, -123, -123;
-f { int => 1 }, -123.1, { validation => 'int', got => -123.1 };
+f { int => 1 }, -123.1, { validation => 'int', got => -123.1 }, "failed validation 'int'";
 t { uint => 1 }, 0, 0;
 t { uint => 1 }, 123, 123;
-f { uint => 1 }, -123, { validation => 'uint', got => -123 };
+f { uint => 1 }, -123, { validation => 'uint', got => -123 }, "failed validation 'uint'";
 t { min => 1 }, 1, 1;
-f { min => 1 }, 0.9, { validation => 'min', expected => 1, got => 0.9 };
-f { min => 1 }, 'a', { validation => 'min', error => nerr 'a' };
+f { min => 1 }, 0.9, { validation => 'min', expected => 1, got => 0.9 }, "failed validation 'min'";
+f { min => 1 }, 'a', { validation => 'min', error => (nerr 'a')[0] }, "validation 'min': failed validation 'num'";
 t { max => 1 }, 1, 1;
-f { max => 1 }, 1.1, { validation => 'max', expected => 1, got => 1.1 };
-f { max => 1 }, 'a', { validation => 'max', error => nerr 'a' };
+f { max => 1 }, 1.1, { validation => 'max', expected => 1, got => 1.1 }, "failed validation 'max'";
+f { max => 1 }, 'a', { validation => 'max', error => (nerr 'a')[0] }, "validation 'max': failed validation 'num'";
 t { range => [1,2] }, 1, 1;
 t { range => [1,2] }, 2, 2;
-f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } };
-f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } };
+f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } }, "validation 'range': failed validation 'min'";
+f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } }, "validation 'range': failed validation 'max'";
 #t { range => [1,2] }, 'a', 'a', { validation => 'range', error => { validation => 'max', error => nerr 'a' } }; # XXX: Error validation type depends on evaluation order
 
 # email template
 use utf8;
-f { email => 1 }, $_, { validation => 'email', got => $_ } for (
+f { email => 1 }, $_, { validation => 'email', got => $_ }, "failed validation 'email'" for (
   'abc.com',
   'abc@localhost',
   'abc@10.0.0.',
@@ -228,10 +230,10 @@ t { email => 1 }, $_, $_ for (
   'abc@x-y_z.example',
 );
 my $long = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxxxxxxxxxx.xxxxx';
-f { email => 1 }, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } };
+f { email => 1 }, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } }, "validation 'email': failed validation 'maxlength'";
 
 # weburl template
-f { weburl => 1 }, $_, { validation => 'weburl', got => $_ } for (
+f { weburl => 1 }, $_, { validation => 'weburl', got => $_ }, "failed validation 'weburl'" for (
   'http',
   'http://',
   'http:///',

From 17176738a0d942f5925e76afee4414349d9a6fbd Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 8 Mar 2025 14:02:51 +0100
Subject: [PATCH 06/86] FU: Support multipart file uploads + some doc fixes

API is not super convenient and implementation is lousy, but uploading
files is not a super common operation so that should be fine.

At least it supports large files with only a single in-memory copy.
---
 FU.pm                   |  25 +++--
 FU/MultipartFormData.pm | 209 ++++++++++++++++++++++++++++++++++++++++
 FU/Pg.pm                |   4 +-
 FU/SQL.pm               |   6 +-
 FU/XMLWriter.pm         |   2 +-
 t/multipart.t           |  47 +++++++++
 6 files changed, 280 insertions(+), 13 deletions(-)
 create mode 100644 FU/MultipartFormData.pm
 create mode 100644 t/multipart.t

diff --git a/FU.pm b/FU.pm
index 57123a5..4807e0b 100644
--- a/FU.pm
+++ b/FU.pm
@@ -253,8 +253,9 @@ sub _read_req_http($sock, $req) {
 
     $req->{body} = '';
     while ($len > 0) {
-        my $r = $sock->read($req->{body}, $len, -1);
-        fu->error(400, 'Client disconnect before request was read') if !$r
+        my $r = $sock->read($req->{body}, $len, length $req->{body});
+        fu->error(400, 'Client disconnect before request was read') if !$r;
+        $len -= $r;
     }
 }
 
@@ -654,7 +655,6 @@ sub query {
 sub formdata {
     shift;
     $FU::REQ->{formdata} ||= eval {
-        # TODO: Support multipart encoding
         confess "Invalid content type for form data"
             if (fu->header('content-type')||'') ne 'application/x-www-form-urlencoded';
         FU::Util::query_decode($FU::REQ->{data});
@@ -662,6 +662,13 @@ sub formdata {
     _getfield $FU::REQ->{formdata}, @_;
 }
 
+sub multipart {
+    require FU::MultipartFormData;
+    $FU::REQ->{multipart} ||= eval {
+        FU::MultipartFormData->parse(fu->header('content-type')||'', $FU::REQ->{body})
+    } || fu->error(400, $@);
+}
+
 
 
 
@@ -1239,12 +1246,18 @@ Parse, validate and return multiple query parameters.
 
 =item fu->formdata($schema)
 
-Like C<< fu->query() >> but returns data from the POST request body.
+Like C<< fu->query() >> but returns data from the POST request body. This
+method only supports form data encoded as C<application/x-www-form-urlencoded>,
+which is the default for HTML C<< <form> >>s. To handle multipart form data,
+use C<< fu->multipart >> instead.
+
+=item fu->multipart
+
+Parse the request body as C<multipart/form-data> and return an array of fields.
+Refer to L<FU::MultipartFormData> for more information.
 
 =back
 
-I<TODO:> Support C<multipart/form-data> and file uploads.
-
 I<TODO:> Support JSON bodies.
 
 I<TODO:> Cookie parsing.
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
new file mode 100644
index 0000000..da415f8
--- /dev/null
+++ b/FU/MultipartFormData.pm
@@ -0,0 +1,209 @@
+package FU::MultipartFormData;
+use v5.36;
+use Carp 'confess';
+use FU::Util 'utf8_decode';
+
+sub _arg($d) { $d =~ s/^\s+//r =~ s/\s+$//r =~ s/^"(.+)"$/$1/r }
+
+sub parse($pkg, $header, $data) {
+    confess "Invalid multipart header '$header'"
+        if $header !~ m{^multipart/form-data\s*;\s*boundary\s*=(.+)$};
+    my $boundary = _arg $1;
+    confess "Invalid multipart boundary '$boundary'" if $boundary !~ /^[\x21-\x7e]+$/;
+    utf8::encode($boundary);
+
+    my @a;
+    while ($data =~ m{--\Q$boundary\E(?:--\r\n|\r\n((?:.+\r\n)+)\r\n)}xg) {
+        my $hdrs = $1;
+        $a[$#a]{length} = $-[0] - 2 - $a[$#a]{start} if @a;
+        if (!$hdrs) {
+            confess "Trailing garbage" if pos $data != length $data;
+            last;
+        }
+
+        my $d = bless {
+            data => $data,
+            start => pos $data,
+        }, $pkg;
+
+        confess "Missing content-disposition header" if $hdrs !~ /content-disposition:\s*form-data;(.+)/i;
+        my $v = $1;
+        confess "Missing 'name' parameter" if $v !~ /[;\s]name=([^[;\s]+)/;
+        $d->{name} = utf8_decode _arg $1;
+        $d->{filename} = utf8_decode _arg $1 if $v =~ /[;\s]filename=([^;\s]+)/;
+
+        if ($hdrs =~ /content-type:\s*([^;\s]+)(?:\s*;\s*charset=([^;\s]+))?/i) {
+            $d->{mime} = utf8_decode _arg $1;
+            $d->{charset} = utf8_decode _arg $2 if $2;
+        }
+        push @a, $d;
+    }
+    confess "Missing end-of-data marker" if @a && !defined $a[$#a]{length};
+    \@a
+}
+
+sub name     { $_[0]{name} }
+sub filename { $_[0]{filename} }
+sub mime     { $_[0]{mime} }
+sub charset  { $_[0]{charset} }
+sub length   { $_[0]{length} }
+
+sub substr($o,$off,$len=undef) {
+    $off += $o->{length} if $off < 0;
+    $off = 0 if $off < 0;
+    $off = $o->{length} if $off > $o->{length};
+
+    $len //= $o->{length} - $off;
+    $len += $o->{length} - 1 if $len < 0;
+    $len = 0 if $len < 0;
+    $len = $o->{length} - $off if $len > $o->{length} - $off;
+
+    substr $o->{data}, $o->{start} + $off, $len;
+}
+
+sub data     { $_[0]->substr(0) }
+sub value    { utf8_decode $_[0]->data }
+
+sub syswrite($o, $fh) {
+    my $off = $o->{start};
+    my $end = $o->{start} + $o->{length};
+    while ($off < $end) {
+        my $r = syswrite $fh, $o->{data}, $end-$off, $off;
+        return if !defined $r;
+        $off += $r;
+    }
+    $o->{length};
+}
+
+sub save($o, $fn) {
+    open my $F, '>', $fn or confess "Error opening '$fn': $!";
+    $o->syswrite($F) or confess "Error writing to '$fn': $!";
+}
+
+sub describe($o) {
+    my $head = eval { utf8_decode $o->substr(0, 100) };
+    if (defined $head && $head =~ /\n/) {
+        ($head) = split /\n/, $head, 2;
+        $head .= '...';
+    } elsif (defined $head && $o->{length} > 100) {
+        $head .= '...';
+    }
+    $o->{name}.': '.join ' ',
+        $o->{filename} ? "filename=$o->{filename}" : (),
+        $o->{mime} ? "mime=$o->{mime}" : (),
+        $o->{charset} ? "charset=$o->{charset}" : (),
+        "length=$o->{length}",
+        defined $head ? "value=$head" : ();
+}
+
+1;
+__END__
+
+=head1 NAME
+
+FU::MultipartFormData - Parse multipart/form-data
+
+=head1 SYNOPSIS
+
+  my $fields = FU::MultipartFormData->parse($content_type_header, $request_body);
+
+  for my $f (@$fields) {
+      print "%s   %d\n", $f->name, $f->length;
+
+      $f->save('file.png') if $f->name eq 'image';
+  }
+
+=head1 DESCRIPTION
+
+This is a tiny module to parse an HTTP request body encoded as
+C<multipart/form-data>, which is typically used to handle file uploads.
+
+The entire request body is assumed to be in memory as a Perl string, but this
+module makes an attempt to avoid any further copies of data values.
+
+=head2 Parsing
+
+=over FU::MultipartFormData->parse($header, $body)
+
+Returns an array of field objects from the given C<$header>, which must be a
+valid value for the C<Content-Type> request header, and the given C<$body>,
+which must hold the request body as a byte string. An error is thrown if the
+header is not valid or parsing failed.
+
+This module is pretty lousy and does not fully comform to any HTTP standards,
+but it does happen to be able to parse POST data from any browser that I've
+tried.
+
+=back
+
+=head2 Field Object
+
+Each field is parsed into a field object that supports the following methods:
+
+=over
+
+=item name
+
+Returns the field name as a Perl Unicode string.
+
+=item filename
+
+Returns the filename as a Perl Unicode string, or C<undef> if no filename was
+provided.
+
+=item mime
+
+Returns the mime type extracted from the field's C<Content-Type> header, or
+C<undef> if none was present.
+
+=item charset
+
+Returns the charset extracted from the field's C<Content-Type> header, or
+C<undef> if none was present.
+
+=item length
+
+Returns the byte length of the field value.
+
+=item data
+
+Returns a copy of the field value as a byte string. You'll want to avoid using
+this on large fields.
+
+=item value
+
+Returns a copy of the field value as a Unicode string. Uses C<utf8_decode()>
+from L<FU::Util>, so also throws an error if the value contains control
+characters.
+
+=item substr($off, $len)
+
+Equivalent to calling C<substr()> on the string returned by C<data>, but avoids
+a copy of the entire field value.
+
+=item syswrite($fh)
+
+Write the field value to C<$fh> using Perl's C<syswrite()>, returns C<undef> on
+error or the number of bytes written on success.
+
+Can be used to write uploaded file data to a file or send it over a socket or
+pipe, without making a full in-memory copy of the data.
+
+=item save($fn)
+
+Save the field value to the file C<$fn>, throws an error on failure.
+
+=item describe
+
+Returns a human-readable string to describe this field. Mainly for debugging
+purposes, the exact format is subject to change.
+
+=back
+
+=head1 COPYRIGHT
+
+MIT.
+
+=head1 AUTHOR
+
+Yorhel <projects@yorhel.nl>
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 7b9de5d..176a8ac 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -400,9 +400,7 @@ results into Perl values.
 
 Observed query preparation time, in seconds, including network round-trip.
 Returns 0 if a cached prepared statement was used or C<undef> if the query was
-executed without a separate preparation phase (currently only happens with C<<
-$conn->exec() >>, but support for direct query execution may be added for other
-queries in the future as well).
+executed without a separate preparation phase.
 
 =item $st->get_cache
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 73f5613..c86a3fb 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -356,12 +356,12 @@ values. This function results in different SQL depending on the C<in_style>
 option given to C<compile()>. The default C<'dbi'> style passes each value as a
 bind parameter:
 
-  SQL 'WHERE id', IN([1, 2, 3, 4]);
+  SQL 'WHERE id', IN [1, 2, 3, 4];
   # 'WHERE id IN(?, ?, ?, ?)', parameters: 1, 2, 3, 4
 
 The C<'pg'> style passes the entire array as a single bind parameter instead:
 
-  SQL 'WHERE id', IN([1, 2, 3, 4]);
+  SQL 'WHERE id', IN [1, 2, 3, 4];
   # 'WHERE id = ANY(?)', parameter: [1, 2, 3, 4]
 
 The C<'pg'> style allows for more efficient re-use of cached prepared
@@ -372,7 +372,7 @@ with L<DBD::Pg> or L<Pg::PQ>.
 
 Can be used in the C<$hashref> versions of C<AND>, C<OR> and C<WHERE> as well:
 
-  WHERE { id => IN([1, 2]) }
+  WHERE { id => IN [1, 2] }
   # 'WHERE id IN(?, ?)'
 
 =back
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 0911ced..08fc54d 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -146,7 +146,7 @@ These functions all return a byte string with (UTF-8) encoded XML.
 
 =item fragment($block)
 
-Executes C<$block> and captures the output of all I</"Output functions">
+Executes C<$block> and captures the output of all L</"Output functions">
 called within the same scope into a string. This function can be safely nested:
 
   my $string = fragment {
diff --git a/t/multipart.t b/t/multipart.t
new file mode 100644
index 0000000..842b9cd
--- /dev/null
+++ b/t/multipart.t
@@ -0,0 +1,47 @@
+use v5.36;
+use Test::More;
+use FU::MultipartFormData;
+
+# Example based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST
+my $t = <<'_' =~ s/\n/\r\n/rg;
+--delimiter12345
+Content-Disposition: form-data; name="field1"
+content-type: hello; charset=x
+
+value1
+--delimiter12345
+Content-Type: text
+Content-Disposition: form-data; filename="example.txt"; name=field2
+
+value2
+--delimiter12345--
+_
+
+
+my $l = FU::MultipartFormData->parse('multipart/form-data;boundary="delimiter12345"', $t);
+is scalar @$l, 2;
+
+my $v = $l->[0];
+is $v->name, 'field1';
+is $v->filename, undef;
+is $v->mime, 'hello';
+is $v->charset, 'x';
+is $v->length, 6;
+is $v->data, 'value1';
+
+is $v->substr(4), 'e1';
+is $v->substr(1, 2), 'al';
+is $v->substr(-2, 1), 'e';
+is $v->substr(-2, 5), 'e1';
+is $v->substr(-100, 2), 'va';
+is $v->substr(1, -3), 'al';
+
+$v = $l->[1];
+is $v->name, 'field2';
+is $v->filename, 'example.txt';
+is $v->mime, 'text';
+is $v->charset, undef;
+is $v->length, 6;
+is $v->data, 'value2';
+
+done_testing;

From 70c5199df493bf3ec83925e83ccd385d5a092f7d Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 8 Mar 2025 15:49:43 +0100
Subject: [PATCH 07/86] FU: Add JSON reading & writing methods

---
 FU.pm | 43 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/FU.pm b/FU.pm
index 4807e0b..1b3f6aa 100644
--- a/FU.pm
+++ b/FU.pm
@@ -637,7 +637,10 @@ sub headers { $FU::REQ->{hdr} }
 sub ip { $FU::REQ->{ip} }
 
 sub _getfield($data, @a) {
-    return $data->{$a[0]} if @a == 1 && !ref $a[0];
+    if (@a == 1 && !ref $a[0]) {
+        fu->error(400, "Expected top-level to be a hash") if ref $data ne 'HASH';
+        return $data->{$a[0]};
+    }
     require FU::Validate;
     my $schema = FU::Validate->compile(@a > 1 ? { keys => {@a} } : $a[0]);
     my $res = eval { $schema->validate($data) };
@@ -652,6 +655,15 @@ sub query {
     _getfield $FU::REQ->{qs_parsed}, @_;
 }
 
+sub json {
+    shift;
+    $FU::REQ->{json} ||= eval {
+        FU::Util::json_parse($FU::REQ->{data}, utf8 => 1)
+    } || fu->error(400, "JSON parse error: $@");
+    return $FU::REQ->{json} if !@_;
+    _getfield $FU::REQ->{json}, @_;
+}
+
 sub formdata {
     shift;
     $FU::REQ->{formdata} ||= eval {
@@ -715,6 +727,12 @@ sub set_header($, $hdr, $val=undef) {
     $FU::REQ->{reshdr}{ lc $hdr } = $val;
 }
 
+sub send_json($, $data) {
+    fu->set_header('content-type', 'application/json');
+    fu->set_body(FU::Util::json_format($data, canonical => 1, utf8 => 1));
+    fu->done;
+}
+
 sub send_file($, $root, $path) {
     # This also catches files with '..' somewhere in the middle of the name.
     # Let's just disallow that to simplify this check, I'd err on the side of
@@ -1242,9 +1260,12 @@ Parse, validate and return multiple query parameters.
   # Or, more concisely:
   my $data = fu->query(a => {anybool => 1}, b => {});
 
-=item fu->formdata($name)
+=item fu->json(@args)
 
-=item fu->formdata($schema)
+Like C<< fu->query() >> but parses the request body as JSON. Returns the
+decoded JSON value if C<@args> is empty.
+
+=item fu->formdata(@args)
 
 Like C<< fu->query() >> but returns data from the POST request body. This
 method only supports form data encoded as C<application/x-www-form-urlencoded>,
@@ -1253,13 +1274,11 @@ use C<< fu->multipart >> instead.
 
 =item fu->multipart
 
-Parse the request body as C<multipart/form-data> and return an array of fields.
-Refer to L<FU::MultipartFormData> for more information.
+Parse the request body as C<multipart/form-data> and return an array of field
+objects.  Refer to L<FU::MultipartFormData> for more information.
 
 =back
 
-I<TODO:> Support JSON bodies.
-
 I<TODO:> Cookie parsing.
 
 
@@ -1321,6 +1340,14 @@ templating system or L<FU::XMLWriter>:
       };
   });
 
+=item fu->send_json($data)
+
+Encode C<$data> as JSON (using C<json_format> in L<FU::Util>), set an
+appropriate C<Content-Type> header and send it to the client. Calls C<<
+fu->done >>.
+
+I<TODO:> Support schema-based normalization.
+
 =item fu->send_file($root, $path)
 
 If a file identified by C<"$root/$path"> exists, set that as response and call
@@ -1374,8 +1401,6 @@ one of the following status codes or an alias:
 
 I<TODO:> Setting cookies.
 
-I<TODO:> JSON output.
-
 
 =head2 Running the Site
 

From dc752e2a23d029b0ac9c81e77f7e41472567616d Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 9 Mar 2025 10:23:48 +0100
Subject: [PATCH 08/86] Pg: Support dynamic-oid types + vndbtag/vndbid

---
 c/pgconn.c          | 14 ++++---
 c/pgtypes.c         | 94 +++++++++++++++++++++++++++++++++++++++++++++
 t/pgtypes-dynamic.t | 22 +++++++++++
 3 files changed, 124 insertions(+), 6 deletions(-)

diff --git a/c/pgconn.c b/c/pgconn.c
index 4b84463..3a20b56 100644
--- a/c/pgconn.c
+++ b/c/pgconn.c
@@ -471,6 +471,7 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) {
         snprintf(t->name.n, sizeof(t->name.n), "%s", PQgetvalue(r, i, 1));
         char typ = *PQgetvalue(r, i, 2);
         t->elemoid = fu_frombeU(32, PQgetvalue(r, i, 3));
+        const fupg_type *builtin;
 
         if (t->elemoid) {
             if (typ == 'd') { /* domain */
@@ -487,13 +488,14 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) {
             /* enum, can use text send/recv */
             t->send = fupg_send_text;
             t->recv = fupg_recv_text;
+        } else if ((builtin = fupg_builtin_byoid(t->oid))) {
+            t->send = builtin->send;
+            t->recv = builtin->recv;
+        } else if ((builtin = fupg_dynoid_byname(t->name.n))) {
+            t->send = builtin->send;
+            t->recv = builtin->recv;
         } else {
-            /* TODO: (multi)ranges, by-name lookup for dynamic-oid types */
-            const fupg_type *builtin = fupg_builtin_byoid(t->oid);
-            if (builtin) {
-                t->send = builtin->send;
-                t->recv = builtin->recv;
-            }
+            /* TODO: (multi)ranges */
         }
     }
     PQclear(r);
diff --git a/c/pgtypes.c b/c/pgtypes.c
index f9c5b00..6fb8835 100644
--- a/c/pgtypes.c
+++ b/c/pgtypes.c
@@ -638,6 +638,72 @@ SENDFN(time) {
     fustr_writebeI(64, out, SvNV(val) * 1000000);
 }
 
+
+
+/* VNDB types */
+
+const char vndbtag_alpha[] = "\0""abcdefghijklmnopqrstuvwxyz?????";
+
+static I16 vndbtag_parse(char **str) {
+    I16 tag = 0;
+    if (**str >= 'a' && **str <= 'z') {
+        tag = (**str - 'a' + 1) << 10;
+        (*str)++;
+        if (**str >= 'a' && **str <= 'z') {
+            tag |= (**str - 'a' + 1) << 5;
+            (*str)++;
+            if (**str >= 'a' && **str <= 'z') {
+                tag |= **str - 'a' + 1;
+                (*str)++;
+            }
+        }
+    }
+    return tag;
+}
+
+void vndbtag_fmt(I16 tag, char *out) {
+    out[0] = vndbtag_alpha[(tag >> 10) & 31];
+    out[1] = vndbtag_alpha[(tag >>  5) & 31];
+    out[2] = vndbtag_alpha[(tag >>  0) & 31];
+    out[3] = 0;
+}
+
+RECVFN(vndbtag) {
+    RLEN(2);
+    SV *r = newSV(4);
+    SvPOK_only(r);
+    vndbtag_fmt(fu_frombeI(16, buf), SvPVX(r));
+    SvCUR_set(r, strlen(SvPVX(r)));
+    return r;
+}
+
+SENDFN(vndbtag) {
+    char *t = SvPV_nolen(val);
+    I16 v = vndbtag_parse(&t);
+    if (*t) SERR("Invalid vndbtag: '%s'", SvPV_nolen(val));
+    fustr_writebeI(16, out, v);
+}
+
+
+#define VNDBID2_MAXNUM (((I64)1<<48)-1)
+
+RECVFN(vndbid) {
+    RLEN(8);
+    I64 v = fu_frombeI(64, buf);
+    char tbuf[4];
+    vndbtag_fmt(v >> 48, tbuf);
+    return newSVpvf("%s%"UVuf, tbuf, (UV)(v & VNDBID2_MAXNUM));
+}
+
+SENDFN(vndbid) {
+    char *ostr = SvPV_nolen(val), *str = ostr;
+    UV num;
+    I16 tag = vndbtag_parse(&str);
+    if (!grok_atoUV(str, &num, NULL) || num > VNDBID2_MAXNUM) SERR("invalid vndbid '%s'", ostr);
+    fustr_writebeI(64, out, ((I64)tag)<<48 | num);
+}
+
+
 #undef SIV
 #undef RLEN
 #undef RECVFN
@@ -818,7 +884,24 @@ static const fupg_type fupg_builtin[] = {
 #define FUPG_BUILTIN (sizeof(fupg_builtin) / sizeof(fupg_type))
 
 
+/* List of types identified by name */
+
+#define DYNOID\
+    T("vndbtag",   vndbtag)\
+    T("vndbid",    vndbid)
+
+static const fupg_type fupg_dynoid[] = {
+#define T(name, fun) { 0, 0, {name"\0"}, fupg_send_##fun, fupg_recv_##fun },
+    DYNOID
+#undef T
+};
+
+#undef DYNOID
+#define FUPG_DYNOID (sizeof(fupg_dynoid) / sizeof(fupg_type))
+
+
 /* List of special types for use with set_type() */
+
 #define SPECIALS\
     T("$date_str",   date_str)\
     T("$hex",        hex     )
@@ -851,8 +934,19 @@ static const fupg_type *fupg_builtin_byoid(Oid oid) {
     return fupg_type_byoid(fupg_builtin, FUPG_BUILTIN, oid);
 }
 
+static const fupg_type *fupg_dynoid_byname(const char *name) {
+    size_t i;
+    for (i=0; i<FUPG_DYNOID; i++)
+        if (strcmp(fupg_dynoid[i].name.n, name) == 0)
+            return fupg_dynoid+i;
+    return NULL;
+}
+
 static const fupg_type *fupg_builtin_byname(const char *name) {
     size_t i;
+    const fupg_type *r = fupg_dynoid_byname(name);
+    if (r) return r;
+
     /* XXX: Can use binary search here if the list of specials grows.
      * That list does not have to be ordered by oid. */
     for (i=0; i<FUPG_SPECIALS; i++)
diff --git a/t/pgtypes-dynamic.t b/t/pgtypes-dynamic.t
index 9d034ad..b6954b5 100644
--- a/t/pgtypes-dynamic.t
+++ b/t/pgtypes-dynamic.t
@@ -121,4 +121,26 @@ subtest 'custom types', sub {
     is $txn->q("SELECT dom FROM fupg_test_table")->val, 0x6262;
 };
 
+
+subtest 'vndbid', sub {
+    plan skip_all => 'type not loaded in the database' if !$conn->q("SELECT 1 FROM pg_type WHERE typname = 'vndbtag'")->val;
+
+    for my $t (qw/a zz xxx/) {
+        is $conn->q('SELECT $1::vndbtag', $t)->val, $t;
+        is $conn->q('SELECT $1::vndbtag', $t)->text_params->val, $t;
+        is $conn->q('SELECT $1::vndbtag', $t)->text_results->val, $t;
+    }
+    ok !eval { $conn->q('SELECT $1::vndbtag', '')->val };
+    ok !eval { $conn->q('SELECT $1::vndbtag', 'abcd')->val };
+
+    for my $t (qw/a123 zz992883231 xxx18388123/) {
+        is $conn->q('SELECT $1::vndbid', $t)->val, $t;
+        is $conn->q('SELECT $1::vndbid', $t)->text_params->val, $t;
+        is $conn->q('SELECT $1::vndbid', $t)->text_results->val, $t;
+    }
+    ok !eval { $conn->q('SELECT $1::vndbid', '')->val };
+    ok !eval { $conn->q('SELECT $1::vndbid', 'ab')->val };
+    ok !eval { $conn->q('SELECT $1::vndbid', 'ab1219229999999999')->val };
+};
+
 done_testing;

From d9d2ad0434d8272fe32893825dad2538ac41bf78 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 10 Mar 2025 12:24:52 +0100
Subject: [PATCH 09/86] Pg: Add COPY support

---
 FU.xs      |  38 +++++++++++++++++
 FU/Pg.pm   | 117 ++++++++++++++++++++++++++++++++++++++++-------------
 c/libpq.h  |   6 ++-
 c/pgst.c   |  79 ++++++++++++++++++++++++++++++++++++
 t/pgcopy.t |  90 +++++++++++++++++++++++++++++++++++++++++
 5 files changed, 301 insertions(+), 29 deletions(-)
 create mode 100644 t/pgcopy.t

diff --git a/FU.xs b/FU.xs
index 3c6084a..1d63ffe 100644
--- a/FU.xs
+++ b/FU.xs
@@ -68,6 +68,7 @@ fuxmlwr *     FUXMLWR
 fupg_conn *   FUPG_CONN
 fupg_txn *    FUPG_TXN
 fupg_st *     FUPG_ST
+fupg_copy *   FUPG_COPY
 
 INPUT
 FUFCGI
@@ -89,6 +90,10 @@ FUPG_TXN
 FUPG_ST
     if (sv_derived_from($arg, \"FU::Pg::st\")) $var = (fupg_st *)SvIVX(SvRV($arg));
     else fu_confess(\"invalid statement object\");
+
+FUPG_COPY
+    if (sv_derived_from($arg, \"FU::Pg::copy\")) $var = (fupg_copy *)SvIVX(SvRV($arg));
+    else fu_confess(\"invalid COPY object\");
 #"
 EOT
 
@@ -233,6 +238,11 @@ void q(fupg_conn *c, SV *sv, ...)
     FUPG_CONN_COOKIE;
     ST(0) = fupg_q(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items);
 
+void copy(fupg_conn *c, SV *sv)
+  CODE:
+    FUPG_CONN_COOKIE;
+    ST(0) = fupg_copy_exec(aTHX_ c, SvPVutf8_nolen(sv));
+
 void _set_type(fupg_conn *c, SV *name, SV *sendsv, SV *recvsv)
   CODE:
     fupg_set_type(aTHX_ c, name, sendsv, recvsv);
@@ -282,6 +292,12 @@ void q(fupg_txn *t, SV *sv, ...)
     FUPG_TXN_COOKIE;
     ST(0) = fupg_q(aTHX_ t->conn, t->stflags, SvPVutf8_nolen(sv), ax, items);
 
+# XXX: The copy object should probably keep a ref on the transaction
+void copy(fupg_txn *t, SV *sv)
+  CODE:
+    FUPG_TXN_COOKIE;
+    ST(0) = fupg_copy_exec(aTHX_ t->conn, SvPVutf8_nolen(sv));
+
 
 
 MODULE = FU   PACKAGE = FU::Pg::st
@@ -393,6 +409,28 @@ void DESTROY(fupg_st *st)
     fupg_st_destroy(aTHX_ st);
 
 
+MODULE = FU   PACKAGE = FU::Pg::copy
+
+void write(fupg_copy *c, SV *sv)
+  CODE:
+    fupg_copy_write(aTHX_ c, sv);
+
+void read(fupg_copy *c)
+  CODE:
+    ST(0) = fupg_copy_read(aTHX_ c, 0);
+
+void is_binary(fupg_copy *c)
+  CODE:
+    ST(0) = c->bin ? &PL_sv_yes : &PL_sv_no;
+
+void close(fupg_copy *c)
+  CODE:
+    fupg_copy_close(aTHX_ c, 0);
+
+void DESTROY(fupg_copy *c)
+  CODE:
+    fupg_copy_destroy(aTHX_ c);
+
 
 MODULE = FU   PACKAGE = FU::XMLWriter
 
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 176a8ac..4bd4b23 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -112,6 +112,11 @@ Inside a transaction that is in an error state. The transaction must be rolled
 back in order to recover to a usable state. This happens automatically when the
 transaction object goes out of scope.
 
+=item active
+
+Currently executing a query. This state can only be observed during a L<COPY
+operation|/"COPY support">.
+
 =item bad
 
 Connection is dead or otherwise unusable.
@@ -155,10 +160,11 @@ executing the query, but I<before> the query results have been returned.
 
 The subroutine is (currently) only called for queries executed through C<<
 $conn->exec >>, C<< $conn->q >>, C<< $conn->Q >> and their C<$txn> variants;
-internal queries performed by this module (such as for transaction management,
-querying type information, etc) do not trigger the callback. Statements that
-result in an error being thrown during or before execution are also not
-traceable this way. This behavior might change in the future.
+C<< $conn->copy >> statements and internal queries performed by this module
+(such as for transaction management, querying type information, etc) do not
+trigger the callback. Statements that result in an error being thrown during or
+before execution are also not traceable this way. This behavior might change in
+the future.
 
 =item $conn->disconnect
 
@@ -519,6 +525,11 @@ current implementation does not track subtransactions that closely)
 
 A subtransaction is in error state and awaiting to be rolled back.
 
+=item active
+
+Currently executing a query. This state can only be observed during a L<COPY
+operation|/"COPY support">.
+
 =item bad
 
 Connection is dead or otherwise unusable.
@@ -740,6 +751,71 @@ I<TODO:> Methods to convert between the various formats.
 
 I<TODO:> Methods to query type info.
 
+
+=head2 COPY support
+
+You can use L<COPY
+statements|https://www.postgresql.org/docs/current/sql-copy.html> for efficient
+bulk data transfers between your application and the PostgreSQL server:
+
+=over
+
+=item $copy = $conn->copy($statement)
+
+=item $copy = $txn->copy($statement)
+
+Execute C<$statement> and return a C<FU::Pg::copy> object that lets you
+transfer data to or from Postgres.
+
+It is not possible to execute any other queries on the same connection while a
+copy operation is in progress. When used on a transaction object, C<$txn> must
+be kept alive long enough to finish the copy operation.
+
+=back
+
+A C<$copy> object supports the following methods:
+
+=over
+
+=item $copy->is_binary
+
+Returns true if the transfer is performed in the binary format, false for text.
+
+=item $copy->write($data)
+
+Send C<$data> to the server. An error is thrown if this is not a C<COPY FROM
+STDIN> operation. An error may be thrown if C<$data> is not a valid format
+understood by Postgres, but such errors can also be deferred to C<close()>.
+
+C<$data> is interpreted as a Perl Unicode string for textual transfers and as a
+binary string for binary transfers.
+
+=item $copy->read
+
+Return the next row read from the Postgres server, or C<undef> if no more data
+is coming. In the text format, a single line - including trailing newline - is
+returned as a Perl Unicode string. In the binary format, a single row is
+returned as a byte string. An error is thrown if this is not a C<COPY TO
+STDOUT> operation.
+
+=item $copy->close
+
+Marks the end of the copy operation. Does not return anything but throws an
+error if something went wrong.
+
+It is possible to close a read-copy operation before all data has been
+consumed, but that causes all data to still be read and discarded during
+C<close()>. If you really want to interrupt a large read operation, a more
+efficient approach is to call C<< $conn->close >> and discard the entire
+connection.
+
+It is not I<necessary> to call this method, simply letting the C<$copy> object
+run out of scope will do the trick as well, but in that case errors are
+silently discarded. An explicit C<close()> is recommended to catch errors.
+
+=back
+
+
 =head2 Errors
 
 All methods can throw an exception on error. When possible, the error message
@@ -823,32 +899,17 @@ to it after C<connect()> is always safe:
 
 =item * Only works with blocking (synchronous) calls, not very suitable for use
 in asynchronous frameworks unless you know your queries are fast and you have a
-low-latency connection with the Postgres server.
+low-latency connection with the Postgres server. This is unlikely to improve in
+future versions, Perl's async story is somewhat awkward in general, and fully
+supporting async operation might require a fundamental redesign of how this
+module works.
 
-=back
+=item * LISTEN support is still missing. May be added in a future version, as
+this seems doable without supporting full async.
 
-Missing features:
-
-=over
-
-=item COPY support
-
-I hope to implement this someday.
-
-=item LISTEN support
-
-Would be nice to have, most likely doable without going full async.
-
-=item Asynchronous calls
-
-Probably won't happen. Perl's async story is slightly awkward in general, and
-fully supporting async operation might require a fundamental redesign of how
-this module works. It certainly won't I<simplify> the implementation.
-
-=item Pipelining
-
-I have some ideas for an API, but doubt I'll ever implement it. Suffers from
-the same awkwardness and complexity as asynchronous calls.
+=item * Pipelining support is also missing. I have some ideas for an API, but
+doubt I'll ever implement it. Suffers from the same awkwardness and complexity
+as asynchronous calls.
 
 =back
 
diff --git a/c/libpq.h b/c/libpq.h
index f931016..94d817e 100644
--- a/c/libpq.h
+++ b/c/libpq.h
@@ -36,6 +36,7 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
 #define PG_DIAG_SOURCE_FUNCTION 'R'
 
 #define PG_FUNCS \
+    X(PQbinaryTuples, int, const PGresult *) \
     X(PQclear, void, PGresult *) \
     X(PQclosePrepared, PGresult *, PGconn *, const char *) \
     X(PQcmdTuples, char *, PGresult *) \
@@ -51,9 +52,10 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
     X(PQfname, char *, const PGresult *, int) \
     X(PQfreemem, void, void *) \
     X(PQftype, Oid, const PGresult *, int) \
+    X(PQgetCopyData, int, PGconn *, char **, int) \
+    X(PQgetResult, PGresult *, PGconn *) \
     X(PQgetisnull, int, const PGresult *, int, int) \
     X(PQgetlength, int, const PGresult *, int, int) \
-    X(PQgetResult, PGresult *, PGconn *) \
     X(PQgetvalue, char *, const PGresult *, int, int) \
     X(PQlibVersion, int, void) \
     X(PQnfields, int, const PGresult *) \
@@ -61,6 +63,8 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
     X(PQntuples, int, const PGresult *) \
     X(PQparamtype, Oid, const PGresult *, int) \
     X(PQpipelineSync, int, PGconn *) \
+    X(PQputCopyData, int, PGconn *, const char *, int) \
+    X(PQputCopyEnd, int, PGconn *, const char *) \
     X(PQresStatus, char *, ExecStatusType) \
     X(PQresultErrorField, char *, const PGresult *, int) \
     X(PQresultErrorMessage, char *, const PGresult *) \
diff --git a/c/pgst.c b/c/pgst.c
index 94c8072..1e01392 100644
--- a/c/pgst.c
+++ b/c/pgst.c
@@ -503,3 +503,82 @@ static SV *fupg_st_kvh(pTHX_ fupg_st *st) {
     }
     return sv;
 }
+
+
+
+
+/* COPY support */
+
+typedef struct {
+    SV *self;
+    fupg_conn *conn;
+    char in;
+    char bin;
+    char rddone;
+    char closed;
+} fupg_copy;
+
+static SV *fupg_copy_exec(pTHX_ fupg_conn *c, const char *sql) {
+    PGresult *r = PQexec(c->conn, sql);
+
+    if (!r) fupg_conn_croak(c, "exec");
+    int s = PQresultStatus(r);
+    switch (s) {
+        case PGRES_COPY_OUT:
+        case PGRES_COPY_IN:
+            break;
+        default: fupg_result_croak(r, "exec", sql);
+    }
+
+    fupg_copy *copy = safecalloc(1, sizeof(fupg_copy));
+    copy->conn = c;
+    SvREFCNT_inc(c->self);
+    copy->bin = !!PQbinaryTuples(r);
+    copy->in = s == PGRES_COPY_IN;
+    PQclear(r);
+    return fu_selfobj(copy, "FU::Pg::copy");
+}
+
+static void fupg_copy_write(pTHX_ fupg_copy *c, SV *data) {
+    STRLEN len;
+    const char *buf = c->bin ? SvPVbyte(data, len) : SvPVutf8(data, len);
+    if (PQputCopyData(c->conn->conn, buf, len) < 0) fupg_conn_croak(c->conn, "copy");
+}
+
+static SV *fupg_copy_read(pTHX_ fupg_copy *c, int discard) {
+    char *buf = NULL;
+    int len = PQgetCopyData(c->conn->conn, &buf, 0);
+    if (len == -1) {
+        c->rddone = 1;
+        return &PL_sv_undef;
+    } else if (len < 0) {
+        if (discard) c->rddone = 1;
+        else fupg_conn_croak(c->conn, "copy");
+    }
+    SV *r = discard ? &PL_sv_undef : newSVpvn_flags(buf, len, SVs_TEMP | (c->bin ? 0 : SVf_UTF8));
+    PQfreemem(buf);
+    return r;
+}
+
+static void fupg_copy_close(pTHX_ fupg_copy *c, int ignerror) {
+    if (c->closed) return;
+    c->closed = 1; /* Mark as closed even on error, a second attempt won't help anyway */
+
+    if (c->in && PQputCopyEnd(c->conn->conn, NULL) < 0 && !ignerror)
+        fupg_conn_croak(c->conn, "copyEnd");
+
+    while (!c->in && !c->rddone) fupg_copy_read(aTHX_ c, 1);
+
+    PGresult *r = PQgetResult(c->conn->conn);
+    if (!ignerror && !r) fupg_conn_croak(c->conn, "copyEnd");
+    if (!ignerror && PQresultStatus(r) != PGRES_COMMAND_OK) fupg_result_croak(r, "copy", "");
+    PQclear(r);
+
+    while ((r = PQgetResult(c->conn->conn))) PQclear(r);
+}
+
+static void fupg_copy_destroy(pTHX_ fupg_copy *c) {
+    fupg_copy_close(aTHX_ c, 1);
+    SvREFCNT_dec(c->conn->self);
+    safefree(c);
+}
diff --git a/t/pgcopy.t b/t/pgcopy.t
new file mode 100644
index 0000000..9c81349
--- /dev/null
+++ b/t/pgcopy.t
@@ -0,0 +1,90 @@
+use v5.36;
+use Test::More;
+
+plan skip_all => $@ if !eval { require FU::Pg; } && $@ =~ /Unable to load libpq/;
+die $@ if $@;
+plan skip_all => 'Please set FU_TEST_DB to a PostgreSQL connection string to run these tests' if !$ENV{FU_TEST_DB};
+
+my $conn = FU::Pg->connect($ENV{FU_TEST_DB});
+$conn->_debug_trace(0);
+
+ok !eval { $conn->copy('SELECT 1') };
+like $@, qr/unexpected status code/;
+
+ok !eval { $conn->copy('COPX') };
+like $@, qr/syntax error/;
+
+$conn->exec('CREATE TEMPORARY TABLE fupg_copy_test (v int)');
+
+is $conn->status, 'idle';
+{
+    my $c = $conn->copy('COPY (SELECT 1) TO STDOUT');
+    is $conn->status, 'active';
+    $c->close;
+}
+is $conn->status, 'idle';
+$conn->copy('COPY (SELECT 1) TO STDOUT');
+is $conn->status, 'idle';
+
+{
+    my $c = $conn->copy('COPY fupg_copy_test FROM STDIN');
+    is $conn->status, 'active';
+    $c->close;
+}
+is $conn->status, 'idle';
+$conn->copy('COPY fupg_copy_test FROM STDIN');
+is $conn->status, 'idle';
+
+{
+    my $c = $conn->copy('COPY fupg_copy_test FROM STDIN');
+    ok !$c->is_binary;
+    ok !eval { $c->{read} };
+    $c->write("1");
+    $c->write("\n2\n3\n");
+    $c->close;
+    ok !eval { $c->read };
+    ok !eval { $c->write('') };
+    $c->close;
+}
+is $conn->status, 'idle';
+
+{
+    my $c = $conn->copy('COPY (SELECT * FROM fupg_copy_test ORDER BY v) TO STDOUT');
+    ok !$c->is_binary;
+    ok !eval { $c->write('') };
+    is $c->read, "1\n";
+    is $c->read, "2\n";
+    is $c->read, "3\n";
+    is $c->read, undef;
+    $c->close;
+    ok !eval { $c->read };
+    ok !eval { $c->write('') };
+    $c->close;
+}
+is $conn->status, 'idle';
+
+my $bin = '';
+{
+    my $c = $conn->copy('COPY fupg_copy_test TO STDOUT (FORMAT binary)');
+    ok $c->is_binary;
+    while (my $d = $c->read) {
+        $bin .= $d;
+    }
+    $c->close;
+}
+is $conn->status, 'idle';
+
+{
+    my $txn = $conn->txn;
+    my $c = $txn->copy('COPY fupg_copy_test FROM STDIN (FORMAT binary)');
+    is $txn->status, 'active';
+    ok $c->is_binary;
+    $c->write($bin);
+    $c->close;
+
+    is $txn->q('SELECT sum(v) FROM fupg_copy_test')->val, 1+1+2+2+3+3;
+    $txn->rollback;
+}
+is $conn->q('SELECT sum(v) FROM fupg_copy_test')->val, 1+2+3;
+
+done_testing;

From 9685287523bf2039ee223ed4eea91ea63ac08b62 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 10 Mar 2025 12:43:02 +0100
Subject: [PATCH 10/86] Version 0.3

---
 ChangeLog               | 13 +++++++++++++
 FU.pm                   |  2 +-
 FU/DebugImpl.pm         |  2 +-
 FU/Log.pm               |  2 +-
 FU/MultipartFormData.pm |  2 +-
 FU/Pg.pm                |  2 +-
 FU/SQL.pm               |  2 +-
 FU/Util.pm              |  2 +-
 FU/Validate.pm          |  2 +-
 FU/XMLWriter.pm         |  2 +-
 FU/XS.pm                |  2 +-
 11 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 8a0bb73..7b07d4d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+0.3 - 2025-03-10
+	- FU::Validate: Change API, ->validate() now returns data or throws error on failure
+	- FU::Validate: Rename 'rmwhitespace' to 'trim'
+	- FU::Validate: Support (more) human-readable error messages
+	- FU::Pg: Add support for COPY operations
+	- FU::Pg: Support types with dynamic OIDs
+	- FU: Add support for reading multipart/form-data
+	- FU: Add convenience methods for reading and writing JSON
+	- FU: Fix error in handling a 400
+	- FU::MultipartFormData: New helper module
+	- Fix some tests
+	- Some doc improvements
+
 0.2 - 2025-02-28
 	- FU: Add debug_info web interface
 	- FU: Add fu->denied and fu->notfound methods
diff --git a/FU.pm b/FU.pm
index 1b3f6aa..d6b9268 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 0.2;
+package FU 0.3;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 1fc09cd..2d0ca50 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 0.2;
+package FU::DebugImpl 0.3;
 use v5.36;
 use experimental 'for_list';
 use FU;
diff --git a/FU/Log.pm b/FU/Log.pm
index 17f809c..a0e78ff 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 0.2;
+package FU::Log 0.3;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index da415f8..e19d7bc 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData;
+package FU::MultipartFormData 0.3;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 4bd4b23..ba574d6 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 0.2;
+package FU::Pg 0.3;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index c86a3fb..8f091e5 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 0.2;
+package FU::SQL 0.3;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index 275cac5..cd34fd3 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 0.2;
+package FU::Util 0.3;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 1c7c94b..4b93eb9 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 0.2;
+package FU::Validate 0.3;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 08fc54d..1d01e81 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 0.2;
+package FU::XMLWriter 0.3;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index d2f7d19..1109318 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 0.2;
+package FU::XS 0.3;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');

From 64a105e0131477962cb1eca016c9b7ddb1022ff0 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 14 Mar 2025 06:50:49 +0100
Subject: [PATCH 11/86] Validate: remove a level of indirection

---
 FU/MultipartFormData.pm |   4 +-
 FU/Pg.pm                |   2 +-
 FU/Validate.pm          | 106 ++++++++++++++++++++--------------------
 3 files changed, 56 insertions(+), 56 deletions(-)

diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index e19d7bc..a777823 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -123,7 +123,9 @@ module makes an attempt to avoid any further copies of data values.
 
 =head2 Parsing
 
-=over FU::MultipartFormData->parse($header, $body)
+=over
+
+=item FU::MultipartFormData->parse($header, $body)
 
 Returns an array of field objects from the given C<$header>, which must be a
 valid value for the C<Content-Type> request header, and the given C<$body>,
diff --git a/FU/Pg.pm b/FU/Pg.pm
index ba574d6..5d8c2a7 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -806,7 +806,7 @@ error if something went wrong.
 It is possible to close a read-copy operation before all data has been
 consumed, but that causes all data to still be read and discarded during
 C<close()>. If you really want to interrupt a large read operation, a more
-efficient approach is to call C<< $conn->close >> and discard the entire
+efficient approach is to call C<< $conn->disconnect >> and discard the entire
 connection.
 
 It is not I<necessary> to call this method, simply letting the C<$copy> object
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 4b93eb9..fd180b8 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -94,8 +94,8 @@ our %default_validations = (
 #
 #   name        => $name_or_undef,
 #   validations => [ $recursive_compiled_object, .. ],
-#   schema      => $builtin_validations,
 #   known_keys  => { $key => 1, .. }  # Extracted from 'keys', Used for the 'unknown' validation
+#   %builtin_validations
 #
 sub _compile($schema, $validations, $rec) {
     my(%top, @val);
@@ -125,26 +125,24 @@ sub _compile($schema, $validations, $rec) {
 
     # Inherit some builtin options from validations
     for my $t (@val) {
-        if ($top{type} && $t->{schema}{type} && $top{type} ne $t->{schema}{type}) {
-            confess "Incompatible types, the schema specifies '$top{type}' but validation '$t->{name}' requires '$t->{schema}{type}'" if $schema->{type};
-            confess "Incompatible types, '$t->[0]' requires '$t->{schema}{type}', but another validation requires '$top{type}'";
+        if ($top{type} && $t->{type} && $top{type} ne $t->{type}) {
+            confess "Incompatible types, the schema specifies '$top{type}' but validation '$t->{name}' requires '$t->{type}'" if $schema->{type};
+            confess "Incompatible types, '$t->[0]' requires '$t->{type}', but another validation requires '$top{type}'";
         }
-        exists $t->{schema}{$_} and !exists $top{$_} and $top{$_} = delete $t->{schema}{$_}
+        exists $t->{$_} and !exists $top{$_} and $top{$_} = delete $t->{$_}
             for qw/default onerror trim type scalar unknown missing sort unique/;
 
         push @keys, keys %{ delete $t->{known_keys} };
-        push @keys, keys %{ $t->{schema}{keys} } if $t->{schema}{keys};
+        push @keys, keys %{ $t->{keys} } if $t->{keys};
     }
 
     # Compile sub-schemas
     $top{keys} = { map +($_, __PACKAGE__->compile($top{keys}{$_}, $validations)), keys $top{keys}->%* } if $top{keys};
     $top{values} = __PACKAGE__->compile($top{values}, $validations) if $top{values};
 
-    return {
-        validations => \@val,
-        schema      => \%top,
-        known_keys  => { map +($_,1), @keys },
-    };
+    $top{validations} = \@val;
+    $top{known_keys}  = { map +($_,1), @keys };
+    \%top;
 }
 
 
@@ -153,26 +151,26 @@ sub compile($pkg, $schema, $validations={}) {
 
     my $c = _compile $schema, $validations, 64;
 
-    $c->{schema}{type} //= 'scalar';
-    $c->{schema}{missing} //= 'create';
-    $c->{schema}{trim} //= 1 if $c->{schema}{type} eq 'scalar';
-    $c->{schema}{unknown} //= 'remove' if $c->{schema}{type} eq 'hash';
+    $c->{type} //= 'scalar';
+    $c->{missing} //= 'create';
+    $c->{trim} //= 1 if $c->{type} eq 'scalar';
+    $c->{unknown} //= 'remove' if $c->{type} eq 'hash';
 
-    confess "Invalid value for 'type': $c->{schema}{type}" if !$type_vals{$c->{schema}{type}};
-    confess "Invalid value for 'missing': $c->{schema}{missing}" if !$missing_vals{$c->{schema}{missing}};
-    confess "Invalid value for 'unknown': $c->{schema}{unknown}" if exists $c->{schema}{unknown} && !$unknown_vals{$c->{schema}{unknown}};
+    confess "Invalid value for 'type': $c->{type}" if !$type_vals{$c->{type}};
+    confess "Invalid value for 'missing': $c->{missing}" if !$missing_vals{$c->{missing}};
+    confess "Invalid value for 'unknown': $c->{unknown}" if exists $c->{unknown} && !$unknown_vals{$c->{unknown}};
 
-    delete $c->{schema}{default} if ref $c->{schema}{default} eq 'SCALAR' && ${$c->{schema}{default}} eq 'required';
+    delete $c->{default} if ref $c->{default} eq 'SCALAR' && ${$c->{default}} eq 'required';
 
-    if (exists $c->{schema}{sort}) {
-        my $s = $c->{schema}{sort};
-        $c->{schema}{sort} =
+    if (exists $c->{sort}) {
+        my $s = $c->{sort};
+        $c->{sort} =
             ref $s eq 'CODE' ? $s
             :    $s eq 'str' ? sub($x,$y) { $x cmp $y }
             :    $s eq 'num' ? sub($x,$y) { $x <=> $y }
-            : confess "Unknown value for 'sort': $c->{schema}{sort}";
+            : confess "Unknown value for 'sort': $c->{sort}";
     }
-    $c->{schema}{unique} = sub { $_[0] } if $c->{schema}{unique} && !ref $c->{schema}{unique} && !$c->{schema}{sort};
+    $c->{unique} = sub { $_[0] } if $c->{unique} && !ref $c->{unique} && !$c->{sort};
 
     bless $c, $pkg;
 }
@@ -182,14 +180,14 @@ sub _validate_rec {
     my $c = $_[0];
 
     # hash keys
-    if ($c->{schema}{keys}) {
+    if ($c->{keys}) {
         my @err;
-        for my ($k, $s) ($c->{schema}{keys}->%*) {
+        for my ($k, $s) ($c->{keys}->%*) {
             if (!exists $_[1]{$k}) {
-                next if $s->{schema}{missing} eq 'ignore';
-                return { validation => 'missing', key => $k } if $s->{schema}{missing} eq 'reject';
-                $_[1]{$k} = ref $s->{schema}{default} eq 'CODE' ? $s->{schema}{default}->() : $s->{schema}{default} // undef;
-                next if exists $s->{schema}{default};
+                next if $s->{missing} eq 'ignore';
+                return { validation => 'missing', key => $k } if $s->{missing} eq 'reject';
+                $_[1]{$k} = ref $s->{default} eq 'CODE' ? $s->{default}->() : $s->{default} // undef;
+                next if exists $s->{default};
             }
 
             my $r = _validate($s, $_[1]{$k});
@@ -202,10 +200,10 @@ sub _validate_rec {
     }
 
     # array values
-    if ($c->{schema}{values}) {
+    if ($c->{values}) {
         my @err;
         for my $i (0..$#{$_[1]}) {
-            my $r = _validate($c->{schema}{values}, $_[1][$i]);
+            my $r = _validate($c->{values}, $_[1][$i]);
             if ($r) {
                 $r->{index} = $i;
                 push @err, $r;
@@ -226,8 +224,8 @@ sub _validate_rec {
     }
 
     # func
-    if ($c->{schema}{func}) {
-        my $r = $c->{schema}{func}->($_[1]);
+    if ($c->{func}) {
+        my $r = $c->{func}->($_[1]);
         return { %$r, validation => 'func' } if ref $r eq 'HASH';
         return { validation => 'func', result => $r } if !$r;
     }
@@ -236,24 +234,24 @@ sub _validate_rec {
 
 sub _validate_array {
     my $c = $_[0];
-    return if $c->{schema}{type} ne 'array';
+    return if $c->{type} ne 'array';
 
-    $_[1] = [sort { $c->{schema}{sort}->($a, $b) } $_[1]->@* ] if $c->{schema}{sort};
+    $_[1] = [sort { $c->{sort}->($a, $b) } $_[1]->@* ] if $c->{sort};
 
     # Key-based uniqueness
-    if ($c->{schema}{unique} && ref $c->{schema}{unique} eq 'CODE') {
+    if ($c->{unique} && ref $c->{unique} eq 'CODE') {
         my %h;
         for my $i (0..$#{$_[1]}) {
-            my $k = $c->{schema}{unique}->($_[1][$i]);
+            my $k = $c->{unique}->($_[1][$i]);
             return { validation => 'unique', index_a => $h{$k}, value_a => $_[1][$h{$k}], index_b => $i, value_b => $_[1][$i], key => $k } if exists $h{$k};
             $h{$k} = $i;
         }
 
     # Comparison-based uniqueness
-    } elsif ($c->{schema}{unique}) {
+    } elsif ($c->{unique}) {
         for my $i (0..$#{$_[1]}-1) {
             return { validation => 'unique', index_a => $i, value_a => $_[1][$i], index_b => $i+1, value_b => $_[1][$i+1] }
-                if $c->{schema}{sort}->($_[1][$i], $_[1][$i+1]) == 0
+                if $c->{sort}->($_[1][$i], $_[1][$i+1]) == 0
         }
     }
 }
@@ -263,29 +261,29 @@ sub _validate_input {
     my $c = $_[0];
 
     # trim (needs to be done before the 'default' test)
-    $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $c->{schema}{type} eq 'scalar' && $c->{schema}{trim};
+    $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $c->{type} eq 'scalar' && $c->{trim};
 
     # default
     if (!defined $_[1] || (!ref $_[1] && $_[1] eq '')) {
-        if (exists $c->{schema}{default}) {
-            $_[1] = ref $c->{schema}{default} eq 'CODE' ? $c->{schema}{default}->($_[1]) : $c->{schema}{default};
+        if (exists $c->{default}) {
+            $_[1] = ref $c->{default} eq 'CODE' ? $c->{default}->($_[1]) : $c->{default};
             return;
         }
         return { validation => 'required' };
     }
 
-    if ($c->{schema}{type} eq 'scalar') {
+    if ($c->{type} eq 'scalar') {
         return { validation => 'type', expected => 'scalar', got => lc ref $_[1] } if ref $_[1];
 
-    } elsif ($c->{schema}{type} eq 'hash') {
+    } elsif ($c->{type} eq 'hash') {
         return { validation => 'type', expected => 'hash', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'HASH';
 
         # Each branch below makes a shallow copy of the hash, so that further
         # validations can perform in-place modifications without affecting the
         # input.
-        if ($c->{schema}{unknown} eq 'remove') {
+        if ($c->{unknown} eq 'remove') {
             $_[1] = { map +($_, $_[1]{$_}), grep $c->{known_keys}{$_}, keys $_[1]->%* };
-        } elsif ($c->{schema}{unknown} eq 'reject') {
+        } elsif ($c->{unknown} eq 'reject') {
             my @err = grep !$c->{known_keys}{$_}, keys $_[1]->%*;
             return { validation => 'unknown', keys => \@err, expected => [ sort keys %{$c->{known_keys}} ] } if @err;
             $_[1] = { $_[1]->%* };
@@ -293,16 +291,16 @@ sub _validate_input {
             $_[1] = { $_[1]->%* };
         }
 
-    } elsif ($c->{schema}{type} eq 'array') {
-        $_[1] = [$_[1]] if $c->{schema}{scalar} && !ref $_[1];
-        return { validation => 'type', expected => $c->{schema}{scalar} ? 'array or scalar' : 'array', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'ARRAY';
+    } elsif ($c->{type} eq 'array') {
+        $_[1] = [$_[1]] if $c->{scalar} && !ref $_[1];
+        return { validation => 'type', expected => $c->{scalar} ? 'array or scalar' : 'array', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'ARRAY';
         $_[1] = [$_[1]->@*]; # Create a shallow copy to prevent in-place modification.
 
-    } elsif ($c->{schema}{type} eq 'any') {
+    } elsif ($c->{type} eq 'any') {
         # No need to do anything here.
 
     } else {
-        confess "Unknown type '$c->{schema}{type}'"; # Already checked in compile(), but be extra safe
+        confess "Unknown type '$c->{type}'"; # Already checked in compile(), but be extra safe
     }
 
     &_validate_rec || &_validate_array;
@@ -312,8 +310,8 @@ sub _validate_input {
 sub _validate {
     my $c = $_[0];
     my $r = &_validate_input;
-    ($r, $_[1]) = (undef, ref $c->{schema}{onerror} eq 'CODE' ? $c->{schema}{onerror}->($_[0], bless $r, 'FU::Validate::err') : $c->{schema}{onerror})
-        if $r && exists $c->{schema}{onerror};
+    ($r, $_[1]) = (undef, ref $c->{onerror} eq 'CODE' ? $c->{onerror}->($_[0], bless $r, 'FU::Validate::err') : $c->{onerror})
+        if $r && exists $c->{onerror};
     $r
 }
 

From 1363e112698c186427bf1689bfa861b97cea0357 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 14 Mar 2025 06:57:59 +0100
Subject: [PATCH 12/86] Validate: allow array schemas + defer known_keys hash
 creation

Doesn't allow multiple 'func' options yet, needs work.
---
 FU/Validate.pm | 44 +++++++++++++++++++++++++++++---------------
 t/validate.t   |  5 ++++-
 2 files changed, 33 insertions(+), 16 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index fd180b8..54b3c77 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -76,7 +76,7 @@ our %default_validations = (
     uint  => { _reg $re_uint }, # implies num
     min   => sub($min) { +{ num => 1, func => sub { $_[0] >= $min ? 1 : { expected => $min, got => $_[0] } } } },
     max   => sub($max) { +{ num => 1, func => sub { $_[0] <= $max ? 1 : { expected => $max, got => $_[0] } } } },
-    range => sub { +{ min => $_[0][0], max => $_[0][1] } },
+    range => sub { [ min => $_[0][0], max => $_[0][1] ] },
 
     ascii  => { _reg qr/^[\x20-\x7E]*$/ },
     sl     => { _reg qr/^[^\t\r\n]+$/ },
@@ -99,10 +99,12 @@ our %default_validations = (
 #
 sub _compile($schema, $validations, $rec) {
     my(%top, @val);
-    my @keys = keys $schema->{keys}->%* if $schema->{keys};
 
-    for my($name, $val) (%$schema) {
+    for my($name, $val) (ref $schema eq 'ARRAY' ? @$schema : %$schema) {
         if ($builtin{$name}) {
+            confess "Invalid value for 'type': $val" if $name eq 'type' && !$type_vals{$val};
+            confess "Invalid value for 'missing': $val" if $name eq 'missing' && !$missing_vals{$val};
+            confess "Invalid value for 'unknown': $val" if $name eq 'unknown' && !$unknown_vals{$val};
             $top{$name} = $schema->{$name};
             next;
         }
@@ -117,6 +119,8 @@ sub _compile($schema, $validations, $rec) {
         push @val, $v;
     }
 
+    my @keys = keys $top{keys}->%* if $top{keys};
+
     for my ($n,$t) (qw/keys hash unknown hash  values array sort array unique array/) {
         next if !exists $top{$n};
         confess "Incompatible types, the schema specifies '$top{type}' but the '$n' validation implies '$t'" if $top{type} && $top{type} ne $t;
@@ -132,8 +136,8 @@ sub _compile($schema, $validations, $rec) {
         exists $t->{$_} and !exists $top{$_} and $top{$_} = delete $t->{$_}
             for qw/default onerror trim type scalar unknown missing sort unique/;
 
-        push @keys, keys %{ delete $t->{known_keys} };
-        push @keys, keys %{ $t->{keys} } if $t->{keys};
+        push @keys, delete($t->{known_keys})->@* if $t->{known_keys};
+        push @keys, keys $t->{keys}->%* if $t->{keys};
     }
 
     # Compile sub-schemas
@@ -141,24 +145,20 @@ sub _compile($schema, $validations, $rec) {
     $top{values} = __PACKAGE__->compile($top{values}, $validations) if $top{values};
 
     $top{validations} = \@val;
-    $top{known_keys}  = { map +($_,1), @keys };
+    $top{known_keys}  = \@keys;
     \%top;
 }
 
 
 sub compile($pkg, $schema, $validations={}) {
     return $schema if $schema isa __PACKAGE__;
-
     my $c = _compile $schema, $validations, 64;
 
     $c->{type} //= 'scalar';
     $c->{missing} //= 'create';
     $c->{trim} //= 1 if $c->{type} eq 'scalar';
     $c->{unknown} //= 'remove' if $c->{type} eq 'hash';
-
-    confess "Invalid value for 'type': $c->{type}" if !$type_vals{$c->{type}};
-    confess "Invalid value for 'missing': $c->{missing}" if !$missing_vals{$c->{missing}};
-    confess "Invalid value for 'unknown': $c->{unknown}" if exists $c->{unknown} && !$unknown_vals{$c->{unknown}};
+    $c->{known_keys} = { map +($_,1), $c->{known_keys}->@* } if $c->{known_keys};
 
     delete $c->{default} if ref $c->{default} eq 'SCALAR' && ${$c->{default}} eq 'required';
 
@@ -420,10 +420,11 @@ validation. These are documented in L</SCHEMA DEFINITION> below.
 
 =head1 SCHEMA DEFINITION
 
-A schema is a hashref, each key is the name of a built-in option or of a
-validation to be performed. None of the options or validations are required,
-but some built-ins have default values. This means that the empty schema C<{}>
-is actually equivalent to:
+A schema is an arrayref or hashref, where each key is the name of a built-in
+option or of a validation to be performed and the values are the arguments to
+those validations. None of the options or validations are required, but some
+built-ins have default values. This means that the empty schema C<{}> is
+actually equivalent to:
 
   { type    => 'scalar',
     trim    => 1,
@@ -431,6 +432,19 @@ is actually equivalent to:
     missing => 'create',
   }
 
+Built-in options are always validated in a fixed order, but the order in which
+standard and custom validations are performed is random when the schema is
+given as a hashref. This is rarely a problem, but it can in some cases affect
+the returned error message or whether a later validation will receive data
+normalized by a previous validation. An arrayref can be used to enforce a
+validation order:
+
+  [ enum => [1, 2, 'a'], int => 1 ]
+
+Or to use the same validation multiple times:
+
+  [ regex => qr/^a/, regex => qr/z$/ ]
+
 =head2 Built-in options
 
 =over
diff --git a/t/validate.t b/t/validate.t
index b01352b..2f8d8d9 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -131,6 +131,9 @@ f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validatio
 t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1};
 t { regex => '^a' }, 'abc', 'abc';  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
 f { regex => '^a' }, 'cba', { validation => 'regex', regex => '^a', got => 'cba' }, "failed validation 'regex'";
+t [ regex => '^a', regex => 'z$' ], 'abcxyz', 'abcxyz';
+f [ regex => '^a', regex => 'z$' ], 'bcxyz', { validation => 'regex', regex => '^a', got => 'bcxyz' }, "failed validation 'regex'";
+f [ regex => '^a', regex => 'z$' ], 'abcxy', { validation => 'regex', regex => 'z$', got => 'abcxy' }, "failed validation 'regex'";
 t { enum => [1,2] }, 1, 1;
 t { enum => [1,2] }, 2, 2;
 f { enum => [1,2] }, 3, { validation => 'enum', expected => [1,2], got => 3 }, "failed validation 'enum'";
@@ -205,7 +208,7 @@ t { range => [1,2] }, 1, 1;
 t { range => [1,2] }, 2, 2;
 f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } }, "validation 'range': failed validation 'min'";
 f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } }, "validation 'range': failed validation 'max'";
-#t { range => [1,2] }, 'a', 'a', { validation => 'range', error => { validation => 'max', error => nerr 'a' } }; # XXX: Error validation type depends on evaluation order
+f { range => [1,2] }, 'a', { validation => 'range', error => { validation => 'min', error => (nerr 'a')[0] } }, "validation 'range': validation 'min': failed validation 'num'";
 
 # email template
 use utf8;

From f248a33c1c4cfe0edadf4146681ee2e1b13c0f01 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 14 Mar 2025 09:25:56 +0100
Subject: [PATCH 13/86] Validate: Allow multiple func validations in arrayref
 schema

---
 FU/Validate.pm | 63 ++++++++++++++++++++++++++++----------------------
 t/validate.t   |  6 ++++-
 2 files changed, 41 insertions(+), 28 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 54b3c77..5ea79d6 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -23,10 +23,10 @@ my %unknown_vals = map +($_,1), qw/remove reject pass/;
 my %missing_vals = map +($_,1), qw/create reject ignore/;
 
 sub _length($exp, $min, $max) {
-    +{ func => sub($v) {
+    [ func => sub($v) {
         my $got = ref $v eq 'HASH' ? keys %$v : ref $v eq 'ARRAY' ? @$v : length $v;
         (!defined $min || $got >= $min) && (!defined $max || $got <= $max) ? 1 : { expected => $exp, got => $got };
-    }}
+    }]
 }
 
 # Basically the same as ( regex => $arg ), but hides the regex error
@@ -100,7 +100,15 @@ our %default_validations = (
 sub _compile($schema, $validations, $rec) {
     my(%top, @val);
 
-    for my($name, $val) (ref $schema eq 'ARRAY' ? @$schema : %$schema) {
+    # 'func' is always evaluated last in hashref schemas.
+    my $func = ref $schema eq 'HASH' && $schema->{func};
+
+    for my($name, $val) (ref $schema eq 'HASH' ? %$schema : @$schema) {
+        if ($name eq 'func') {
+            push @val, $val if !$func;
+            next;
+        }
+
         if ($builtin{$name}) {
             confess "Invalid value for 'type': $val" if $name eq 'type' && !$type_vals{$val};
             confess "Invalid value for 'missing': $val" if $name eq 'missing' && !$missing_vals{$val};
@@ -112,13 +120,15 @@ sub _compile($schema, $validations, $rec) {
         my $t = $validations->{$name} || $default_validations{$name};
         confess "Unknown validation: $name" if !$t;
         confess "Recursion limit exceeded while resolving validation '$name'" if $rec < 1;
-        $t = ref $t eq 'HASH' ? $t : $t->($val);
+        $t = ref $t eq 'CODE' ? $t->($val) : $t;
 
         my $v = _compile($t, $validations, $rec-1);
         $v->{name} = $name;
         push @val, $v;
     }
 
+    push @val, $func if $func;
+
     my @keys = keys $top{keys}->%* if $top{keys};
 
     for my ($n,$t) (qw/keys hash unknown hash  values array sort array unique array/) {
@@ -128,7 +138,7 @@ sub _compile($schema, $validations, $rec) {
     }
 
     # Inherit some builtin options from validations
-    for my $t (@val) {
+    for my $t (grep ref $_ eq 'HASH', @val) {
         if ($top{type} && $t->{type} && $top{type} ne $t->{type}) {
             confess "Incompatible types, the schema specifies '$top{type}' but validation '$t->{name}' requires '$t->{type}'" if $schema->{type};
             confess "Incompatible types, '$t->[0]' requires '$t->{type}', but another validation requires '$top{type}'";
@@ -213,21 +223,20 @@ sub _validate_rec {
     }
 
     # validations
-    for ($c->{validations}->@*) {
-        my $r = _validate_rec($_, $_[1]);
-        return {
-            # If the error was a custom 'func' object, then make that the primary cause.
-            # This makes it possible for validations to provide their own error objects.
-            $r->{validation} eq 'func' && (!exists $r->{result} || keys $r->%* > 2) ? $r->%* : (error => $r),
-            validation => $_->{name},
-        } if $r;
-    }
-
-    # func
-    if ($c->{func}) {
-        my $r = $c->{func}->($_[1]);
-        return { %$r, validation => 'func' } if ref $r eq 'HASH';
-        return { validation => 'func', result => $r } if !$r;
+    for my $v ($c->{validations}->@*) {
+        if (ref $v eq 'CODE') {
+            my $r = $v->($_[1]);
+            return { %$r, validation => 'func' } if ref $r eq 'HASH';
+            return { validation => 'func', result => $r } if !$r;
+        } else {
+            my $r = _validate_rec($v, $_[1]);
+            return {
+                # If the error was a custom 'func' object, then make that the primary cause.
+                # This makes it possible for validations to provide their own error objects.
+                $r->{validation} eq 'func' && (!exists $r->{result} || keys $r->%* > 2) ? $r->%* : (error => $r),
+                validation => $v->{name},
+            } if $r;
+        }
     }
 }
 
@@ -669,10 +678,10 @@ assumes the first schema from the previous example.
 =item func => $sub
 
 Run the input through a subroutine to perform additional validation or
-normalization. The subroutine is only called after all other validations have
-succeeded. The subroutine is called with the input as its only argument.
-Normalization of the input can be done by assigning to the first argument or
-modifying its value in-place.
+normalization. When the schema is a hashref, the subroutine is only called
+after all other validations have succeeded. The subroutine is called with the
+input as its only argument.  Normalization of the input can be done by
+assigning to the first argument or modifying its value in-place.
 
 On success, the subroutine should return a true value. On failure, it should
 return either a false value or a hashref. The hashref will have the
@@ -873,9 +882,9 @@ considered "known".
 
 All other built-in options follow inheritance semantics: These options can be
 set in a custom validation, and they are inherited by the top-level schema.  If
-the same option is set in multiple validations a random one will be inherited,
-so that's not a good idea. The top-level schema can always override options set
-by custom validations.
+the same option is set in multiple validations, the final one will be
+inherited. The top-level schema can always override options set by custom
+validations.
 
 
 =head3 Global custom validations
diff --git a/t/validate.t b/t/validate.t
index 2f8d8d9..c04288a 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -17,6 +17,7 @@ my %validations = (
     onerrorsub => { onerror => sub { ref $_[1] } },
     collapsews => { trim => 0, func => sub { $_[0] =~ s/\s+/ /g; 1 } },
     neverfails => { onerror => 'err' },
+    doublefunc => [ func => sub { $_[0] == 0 }, func => sub { $_[0] = 2; 1; } ],
     revnum => { type => 'array', sort => sub($x,$y) { $y <=> $x } },
     uniquelength => { type => 'array', values => { type => 'array' }, unique => sub { scalar @{$_[0]} } },
     person => {
@@ -38,7 +39,7 @@ sub t($schema, $input, $output) {
     my $input_copy = dclone([$input])->[0];
 
     my $res = FU::Validate->compile($schema, \%validations)->validate($input);
-    #diag explain FU::Validate->compile($schema, \%validations) if $line == 139;
+    #diag explain FU::Validate->compile($schema, \%validations) if $line == 98;
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
     is_deeply $res, $output, "data ok $line";
@@ -51,6 +52,7 @@ sub f($schema, $input, $error, @msg) {
     my $input_copy = dclone([$input])->[0];
 
     ok !eval { FU::Validate->compile($schema, \%validations)->validate($input); 1 }, "eval $line";
+    #diag explain FU::Validate->compile($schema, \%validations) if $line == 162;
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
     delete $@->{longmess};
@@ -179,6 +181,8 @@ t { person => 1, unknown => 'remove' }, {name => 'x', sex => 'y', extra => 1}, {
 t { neverfails => 1, int => 1 }, undef, 'err';
 t { neverfails => 1, int => 1 }, 'x', 'err';
 t { neverfails => 1, int => 1, onerror => undef }, 'x', undef; # XXX: no way to 'unset' an inherited onerror clause, hmm.
+t { doublefunc => 1 }, 0, 2;
+f { doublefunc => 1 }, 1, { validation => 'doublefunc', error => { validation => 'func', result => '' } }, "validation 'doublefunc': failed validation 'func'";
 
 # numbers
 sub nerr { ({ validation => 'num', got => $_[0] }, "failed validation 'num'") }

From cea691dd55ed427975aeeb0583b73300bc441fa1 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 14 Mar 2025 10:52:09 +0100
Subject: [PATCH 14/86] Validate: drop creation of default values for built-ins

To better support merging multiple validations, which'll come next.
Probably.
---
 FU/Validate.pm | 30 ++++++++++++------------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 5ea79d6..75af46d 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -164,10 +164,6 @@ sub compile($pkg, $schema, $validations={}) {
     return $schema if $schema isa __PACKAGE__;
     my $c = _compile $schema, $validations, 64;
 
-    $c->{type} //= 'scalar';
-    $c->{missing} //= 'create';
-    $c->{trim} //= 1 if $c->{type} eq 'scalar';
-    $c->{unknown} //= 'remove' if $c->{type} eq 'hash';
     $c->{known_keys} = { map +($_,1), $c->{known_keys}->@* } if $c->{known_keys};
 
     delete $c->{default} if ref $c->{default} eq 'SCALAR' && ${$c->{default}} eq 'required';
@@ -194,8 +190,8 @@ sub _validate_rec {
         my @err;
         for my ($k, $s) ($c->{keys}->%*) {
             if (!exists $_[1]{$k}) {
-                next if $s->{missing} eq 'ignore';
-                return { validation => 'missing', key => $k } if $s->{missing} eq 'reject';
+                next if $s->{missing} && $s->{missing} eq 'ignore';
+                return { validation => 'missing', key => $k } if $s->{missing} && $s->{missing} eq 'reject';
                 $_[1]{$k} = ref $s->{default} eq 'CODE' ? $s->{default}->() : $s->{default} // undef;
                 next if exists $s->{default};
             }
@@ -243,7 +239,6 @@ sub _validate_rec {
 
 sub _validate_array {
     my $c = $_[0];
-    return if $c->{type} ne 'array';
 
     $_[1] = [sort { $c->{sort}->($a, $b) } $_[1]->@* ] if $c->{sort};
 
@@ -269,8 +264,10 @@ sub _validate_array {
 sub _validate_input {
     my $c = $_[0];
 
+    my $type = $c->{type} // 'scalar';
+
     # trim (needs to be done before the 'default' test)
-    $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $c->{type} eq 'scalar' && $c->{trim};
+    $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $type eq 'scalar' && (!exists $c->{trim} || $c->{trim});
 
     # default
     if (!defined $_[1] || (!ref $_[1] && $_[1] eq '')) {
@@ -281,18 +278,18 @@ sub _validate_input {
         return { validation => 'required' };
     }
 
-    if ($c->{type} eq 'scalar') {
+    if ($type eq 'scalar') {
         return { validation => 'type', expected => 'scalar', got => lc ref $_[1] } if ref $_[1];
 
-    } elsif ($c->{type} eq 'hash') {
+    } elsif ($type eq 'hash') {
         return { validation => 'type', expected => 'hash', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'HASH';
 
         # Each branch below makes a shallow copy of the hash, so that further
         # validations can perform in-place modifications without affecting the
         # input.
-        if ($c->{unknown} eq 'remove') {
+        if (!$c->{unknown} || $c->{unknown} eq 'remove') {
             $_[1] = { map +($_, $_[1]{$_}), grep $c->{known_keys}{$_}, keys $_[1]->%* };
-        } elsif ($c->{unknown} eq 'reject') {
+        } elsif ($c->{unknown} && $c->{unknown} eq 'reject') {
             my @err = grep !$c->{known_keys}{$_}, keys $_[1]->%*;
             return { validation => 'unknown', keys => \@err, expected => [ sort keys %{$c->{known_keys}} ] } if @err;
             $_[1] = { $_[1]->%* };
@@ -300,19 +297,16 @@ sub _validate_input {
             $_[1] = { $_[1]->%* };
         }
 
-    } elsif ($c->{type} eq 'array') {
+    } elsif ($type eq 'array') {
         $_[1] = [$_[1]] if $c->{scalar} && !ref $_[1];
         return { validation => 'type', expected => $c->{scalar} ? 'array or scalar' : 'array', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'ARRAY';
         $_[1] = [$_[1]->@*]; # Create a shallow copy to prevent in-place modification.
 
-    } elsif ($c->{type} eq 'any') {
+    } elsif ($type eq 'any') {
         # No need to do anything here.
-
-    } else {
-        confess "Unknown type '$c->{type}'"; # Already checked in compile(), but be extra safe
     }
 
-    &_validate_rec || &_validate_array;
+    &_validate_rec || ($type eq 'array' && &_validate_array);
 }
 
 

From fa24ca53e3e998187400ebb28119f942fa1d8a01 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 14 Mar 2025 14:28:26 +0100
Subject: [PATCH 15/86] Validate: improved arrayref-schema semantics

This allows all built-in options to be duplicated inside a single
schema, the semantics of which are the same as the kind of merging
done as part of inheriting options from custom validations.

This also causes all 'keys' and 'values' validation schemas to be
merged, which changes error messages a bit but is great for
introspection. Probably slightly improves performance as well.
---
 FU/Validate.pm | 217 ++++++++++++++++++++++++-------------------------
 t/validate.t   |  13 ++-
 2 files changed, 114 insertions(+), 116 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 75af46d..feeabb3 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -21,6 +21,11 @@ my %builtin = map +($_,1), qw/
 my %type_vals = map +($_,1), qw/scalar hash array any/;
 my %unknown_vals = map +($_,1), qw/remove reject pass/;
 my %missing_vals = map +($_,1), qw/create reject ignore/;
+my %implied_type = qw/keys hash unknown hash  values array sort array unique array/;
+my %sort_vals = (
+    str => sub($x,$y) { $x cmp $y },
+    num => sub($x,$y) { $x <=> $y },
+);
 
 sub _length($exp, $min, $max) {
     [ func => sub($v) {
@@ -89,136 +94,119 @@ our %default_validations = (
 );
 
 
-# Loads a hashref of validations and a schema definition, and converts it into
-# an object with:
-#
-#   name        => $name_or_undef,
-#   validations => [ $recursive_compiled_object, .. ],
-#   known_keys  => { $key => 1, .. }  # Extracted from 'keys', Used for the 'unknown' validation
-#   %builtin_validations
-#
-sub _compile($schema, $validations, $rec) {
-    my(%top, @val);
+sub _new { bless { validations => [], @_ }, __PACKAGE__ }
 
-    # 'func' is always evaluated last in hashref schemas.
-    my $func = ref $schema eq 'HASH' && $schema->{func};
 
-    for my($name, $val) (ref $schema eq 'HASH' ? %$schema : @$schema) {
+sub _compile($schema, $custom, $rec, $top, $validations=$top->{validations}) {
+    # For hashref schemas, builtins always override other validations
+    $schema = [
+        map +($_, $schema->{$_}),
+            (grep !$builtin{$_}, keys %$schema),
+            (grep $builtin{$_}, keys %$schema),
+    ] if ref $schema eq 'HASH';
+
+    for my($name, $val) (@$schema) {
+        if ($name eq 'type') {
+            confess "Invalid value for 'type': $val" if $name eq 'type' && !$type_vals{$val};
+            confess "Incompatible types, the schema specifies '$val', but another validation requires '$top->{type}'" if $top->{type} && $top->{type} ne $val;;
+            $top->{type} = $val;
+            next;
+        }
+
+        my $type = $implied_type{$name};
+        if ($type) {
+            confess "Incompatible types, the schema specifies '$top->{type}' but the '$name' validation implies '$type'" if $top->{type} && $top->{type} ne $type;
+            $top->{type} = $type;
+        }
+
+        if ($name eq 'values') {
+            $top->{values} ||= _new;
+            _compile($val, $custom, $rec-1, $top->{values});
+            next;
+        }
+
+        if ($name eq 'keys') {
+            $top->{keys} ||= {};
+            for my($n,$v) (%$val) {
+                $top->{keys}{$n} ||= _new;
+                _compile($v, $custom, $rec-1, $top->{keys}{$n});
+            }
+            next;
+        }
+
         if ($name eq 'func') {
-            push @val, $val if !$func;
+            push @$validations, $val;
+            next;
+        }
+
+        if ($name eq 'default') {
+            $top->{default} = $val;
+            delete $top->{default} if ref $val eq 'SCALAR' && $$val eq 'required';
             next;
         }
 
         if ($builtin{$name}) {
-            confess "Invalid value for 'type': $val" if $name eq 'type' && !$type_vals{$val};
             confess "Invalid value for 'missing': $val" if $name eq 'missing' && !$missing_vals{$val};
             confess "Invalid value for 'unknown': $val" if $name eq 'unknown' && !$unknown_vals{$val};
-            $top{$name} = $schema->{$name};
+            $val = $sort_vals{$val} || confess "Unknown value for 'sort': $val" if $name eq 'sort' && ref $val ne 'CODE';
+            $top->{$name} = $val;
             next;
         }
 
-        my $t = $validations->{$name} || $default_validations{$name};
+        my $t = $custom->{$name} || $default_validations{$name};
         confess "Unknown validation: $name" if !$t;
         confess "Recursion limit exceeded while resolving validation '$name'" if $rec < 1;
         $t = ref $t eq 'CODE' ? $t->($val) : $t;
 
-        my $v = _compile($t, $validations, $rec-1);
-        $v->{name} = $name;
-        push @val, $v;
+        my $v = _new name => $name;
+        _compile($t, $custom, $rec-1, $top, $v->{validations});
+        push @$validations, $v if $v->{validations}->@*;
     }
-
-    push @val, $func if $func;
-
-    my @keys = keys $top{keys}->%* if $top{keys};
-
-    for my ($n,$t) (qw/keys hash unknown hash  values array sort array unique array/) {
-        next if !exists $top{$n};
-        confess "Incompatible types, the schema specifies '$top{type}' but the '$n' validation implies '$t'" if $top{type} && $top{type} ne $t;
-        $top{type} = $t;
-    }
-
-    # Inherit some builtin options from validations
-    for my $t (grep ref $_ eq 'HASH', @val) {
-        if ($top{type} && $t->{type} && $top{type} ne $t->{type}) {
-            confess "Incompatible types, the schema specifies '$top{type}' but validation '$t->{name}' requires '$t->{type}'" if $schema->{type};
-            confess "Incompatible types, '$t->[0]' requires '$t->{type}', but another validation requires '$top{type}'";
-        }
-        exists $t->{$_} and !exists $top{$_} and $top{$_} = delete $t->{$_}
-            for qw/default onerror trim type scalar unknown missing sort unique/;
-
-        push @keys, delete($t->{known_keys})->@* if $t->{known_keys};
-        push @keys, keys $t->{keys}->%* if $t->{keys};
-    }
-
-    # Compile sub-schemas
-    $top{keys} = { map +($_, __PACKAGE__->compile($top{keys}{$_}, $validations)), keys $top{keys}->%* } if $top{keys};
-    $top{values} = __PACKAGE__->compile($top{values}, $validations) if $top{values};
-
-    $top{validations} = \@val;
-    $top{known_keys}  = \@keys;
-    \%top;
 }
 
 
-sub compile($pkg, $schema, $validations={}) {
+sub compile($pkg, $schema, $custom={}) {
     return $schema if $schema isa __PACKAGE__;
-    my $c = _compile $schema, $validations, 64;
+    my $c = _new;
+    _compile $schema, $custom, 64, $c;
+    $c
+}
 
-    $c->{known_keys} = { map +($_,1), $c->{known_keys}->@* } if $c->{known_keys};
 
-    delete $c->{default} if ref $c->{default} eq 'SCALAR' && ${$c->{default}} eq 'required';
+sub _validate_keys {
+    my @err;
+    for my ($k, $s) ($_[0]{keys}->%*) {
+        if (!exists $_[1]{$k}) {
+            next if $s->{missing} && $s->{missing} eq 'ignore';
+            return { validation => 'missing', key => $k } if $s->{missing} && $s->{missing} eq 'reject';
+            $_[1]{$k} = ref $s->{default} eq 'CODE' ? $s->{default}->() : $s->{default} // undef;
+            next if exists $s->{default};
+        }
 
-    if (exists $c->{sort}) {
-        my $s = $c->{sort};
-        $c->{sort} =
-            ref $s eq 'CODE' ? $s
-            :    $s eq 'str' ? sub($x,$y) { $x cmp $y }
-            :    $s eq 'num' ? sub($x,$y) { $x <=> $y }
-            : confess "Unknown value for 'sort': $c->{sort}";
+        my $r = _validate($s, $_[1]{$k});
+        if ($r) {
+            $r->{key} = $k;
+            push @err, $r;
+        }
     }
-    $c->{unique} = sub { $_[0] } if $c->{unique} && !ref $c->{unique} && !$c->{sort};
+    return { validation => 'keys', errors => [ sort { $a->{key} cmp $b->{key} } @err ] } if @err;
+}
 
-    bless $c, $pkg;
+sub _validate_values {
+    my @err;
+    for my $i (0..$#{$_[1]}) {
+        my $r = _validate($_[0]{values}, $_[1][$i]);
+        if ($r) {
+            $r->{index} = $i;
+            push @err, $r;
+        }
+    }
+    return { validation => 'values', errors => \@err } if @err;
 }
 
 
 sub _validate_rec {
     my $c = $_[0];
-
-    # hash keys
-    if ($c->{keys}) {
-        my @err;
-        for my ($k, $s) ($c->{keys}->%*) {
-            if (!exists $_[1]{$k}) {
-                next if $s->{missing} && $s->{missing} eq 'ignore';
-                return { validation => 'missing', key => $k } if $s->{missing} && $s->{missing} eq 'reject';
-                $_[1]{$k} = ref $s->{default} eq 'CODE' ? $s->{default}->() : $s->{default} // undef;
-                next if exists $s->{default};
-            }
-
-            my $r = _validate($s, $_[1]{$k});
-            if ($r) {
-                $r->{key} = $k;
-                push @err, $r;
-            }
-        }
-        return { validation => 'keys', errors => \@err } if @err;
-    }
-
-    # array values
-    if ($c->{values}) {
-        my @err;
-        for my $i (0..$#{$_[1]}) {
-            my $r = _validate($c->{values}, $_[1][$i]);
-            if ($r) {
-                $r->{index} = $i;
-                push @err, $r;
-            }
-        }
-        return { validation => 'values', errors => \@err } if @err;
-    }
-
-    # validations
     for my $v ($c->{validations}->@*) {
         if (ref $v eq 'CODE') {
             my $r = $v->($_[1]);
@@ -243,10 +231,10 @@ sub _validate_array {
     $_[1] = [sort { $c->{sort}->($a, $b) } $_[1]->@* ] if $c->{sort};
 
     # Key-based uniqueness
-    if ($c->{unique} && ref $c->{unique} eq 'CODE') {
+    if ($c->{unique} && (!$c->{sort} || ref $c->{unique} eq 'CODE')) {
         my %h;
         for my $i (0..$#{$_[1]}) {
-            my $k = $c->{unique}->($_[1][$i]);
+            my $k = ref $c->{unique} eq 'CODE' ? $c->{unique}->($_[1][$i]) : $_[1][$i];
             return { validation => 'unique', index_a => $h{$k}, value_a => $_[1][$h{$k}], index_b => $i, value_b => $_[1][$i], key => $k } if exists $h{$k};
             $h{$k} = $i;
         }
@@ -288,10 +276,10 @@ sub _validate_input {
         # validations can perform in-place modifications without affecting the
         # input.
         if (!$c->{unknown} || $c->{unknown} eq 'remove') {
-            $_[1] = { map +($_, $_[1]{$_}), grep $c->{known_keys}{$_}, keys $_[1]->%* };
+            $_[1] = { map +($_, $_[1]{$_}), grep $c->{keys}{$_}, keys $_[1]->%* };
         } elsif ($c->{unknown} && $c->{unknown} eq 'reject') {
-            my @err = grep !$c->{known_keys}{$_}, keys $_[1]->%*;
-            return { validation => 'unknown', keys => \@err, expected => [ sort keys %{$c->{known_keys}} ] } if @err;
+            my @err = grep !$c->{keys}{$_}, keys $_[1]->%*;
+            return { validation => 'unknown', keys => \@err, expected => [ sort keys $c->{keys}->%* ] } if @err;
             $_[1] = { $_[1]->%* };
         } else {
             $_[1] = { $_[1]->%* };
@@ -306,7 +294,10 @@ sub _validate_input {
         # No need to do anything here.
     }
 
-    &_validate_rec || ($type eq 'array' && &_validate_array);
+    ($c->{keys} && &_validate_keys) ||
+    ($c->{values} && &_validate_values) ||
+    &_validate_rec ||
+    ($type eq 'array' && &_validate_array)
 }
 
 
@@ -866,13 +857,15 @@ error:
       int => 1
   });
 
-The I<keys>, I<values> and C<func> built-in options are validated separately
-for each custom validation. So if you have multiple custom validations that set
-the I<values> option, then the array elements must validate all the listed
-schemas. The same applies to I<keys>: If the same key is listed in multiple
-custom validations, then the key must conform to all schemas. With respect to
-the I<unknown> option, a key that is mentioned in any of the I<keys> options is
-considered "known".
+The I<func> option is validated separately for each custom validation.
+
+Multiple I<keys> and I<values> validations are merged into a single validation.
+So if you have multiple custom validations that set the I<values> option, a
+single combined schema is created that validates all array elements. The same
+applies to I<keys>: if the same key is listed in multiple custom validations,
+then the key must conform to all schemas. With respect to the I<unknown>
+option, a key that is mentioned in any of the I<keys> options is considered
+"known".
 
 All other built-in options follow inheritance semantics: These options can be
 set in a custom validation, and they are inherited by the top-level schema.  If
diff --git a/t/validate.t b/t/validate.t
index c04288a..7533a02 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -38,8 +38,8 @@ sub t($schema, $input, $output) {
     my $schema_copy = dclone([$schema])->[0];
     my $input_copy = dclone([$input])->[0];
 
+    #diag explain FU::Validate->compile($schema, \%validations) if $line == 95;
     my $res = FU::Validate->compile($schema, \%validations)->validate($input);
-    #diag explain FU::Validate->compile($schema, \%validations) if $line == 98;
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
     is_deeply $res, $output, "data ok $line";
@@ -51,8 +51,8 @@ sub f($schema, $input, $error, @msg) {
     my $schema_copy = dclone([$schema])->[0];
     my $input_copy = dclone([$input])->[0];
 
+    #diag explain FU::Validate->compile($schema, \%validations) if $line == 176;
     ok !eval { FU::Validate->compile($schema, \%validations)->validate($input); 1 }, "eval $line";
-    #diag explain FU::Validate->compile($schema, \%validations) if $line == 162;
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
     delete $@->{longmess};
@@ -67,6 +67,7 @@ f {}, '', { validation => 'required' }, 'required value missing';
 f {}, undef, { validation => 'required' }, 'required value missing';
 t { default => undef }, undef, undef;
 t { default => undef }, '', undef;
+f { default => \'required' }, '', { validation => 'required' }, 'required value missing';
 t { defaultsub1 => 1 }, undef, 2;
 t { defaultsub2 => 1 }, undef, '';
 t { defaultsub2 => 1 }, '', 1;
@@ -115,6 +116,10 @@ t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>
 t { type => 'hash', setundef => 1 }, {}, undef;
 t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef};
 
+t [ keys => { a => {} }, keys => { b => {} } ], {a=>1, b=>2}, {a=>1, b=>2};
+f [ keys => { a => {} }, keys => { b => {} } ], {a=>1}, { validation => 'keys', errors => [{ key => 'b', validation => 'required' }] }, '.b: required value missing';
+f [ keys => { a => {} }, keys => { a => { int => 1 } } ], {a=>'abc'}, { validation => 'keys', errors => [{ key => 'a', validation => 'int', got => 'abc' }] }, ".a: failed validation 'int'";
+
 # default validations
 f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 }, "failed validation 'minlength'";
 t { minlength => 3 }, 'abc', 'abc';
@@ -171,9 +176,9 @@ t { collapsews => 1 }, '   x  ', ' x ';
 t { collapsews => 1, trim => 1 }, '   x  ', 'x';
 f { person => 1 }, 1, { validation => 'type', expected => 'hash', got => 'scalar' }, "invalid type, expected 'hash' but got 'scalar'";
 t { person => 1, default => 1 }, undef, 1;
-f { person => 1 }, { sex => 1 }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } }, "validation 'person'.name: required value missing";
+f { person => 1 }, { sex => 1 }, { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] }, ".name: required value missing";
 t { person => 1 }, { sex => undef, name => 'y' }, { sex => 1, name => 'y' };
-f { person => 1, keys => {age => {default => \'required'}} }, {name => 'x', sex => 'y'}, { validation => 'keys', errors => [{ key => 'age', validation => 'required' }] }, '.age: required value missing';
+f { person => 1, keys => {age => {missing => 'reject'}} }, {name => 'x', sex => 'y'}, { key => 'age', validation => 'missing' }, '.age: required key missing';
 t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => 1},  { name => 'x', sex => 'y', extra => 1 };
 f { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] }, '.extra: required value missing';
 t { person => 1 }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y', extra => 1};

From 3fad7feec3770034dd7af22cc219ea342b289645 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 14 Mar 2025 16:44:08 +0100
Subject: [PATCH 16/86] Validate: Rename "values"->"elems", repurpose "values"
 to validate hash values

I'm breaking stuff left and right while I still can.

Idea: "key_names" validation?

Idea: "tuple" validation that works like "keys" but for arrays.
  (i.e. { tuple => { $index => $schema } }, could make "missing" and
  "unknown" work for arrays, too)
---
 FU/Validate.pm | 120 ++++++++++++++++++++++++++++++-------------------
 t/validate.t   |  13 +++---
 2 files changed, 82 insertions(+), 51 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index feeabb3..8d8db91 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -13,15 +13,18 @@ my %builtin = map +($_,1), qw/
     default
     onerror
     trim
-    values scalar sort unique
-    keys unknown missing
+    elems scalar sort unique
+    keys values unknown missing
     func
 /;
 
 my %type_vals = map +($_,1), qw/scalar hash array any/;
 my %unknown_vals = map +($_,1), qw/remove reject pass/;
 my %missing_vals = map +($_,1), qw/create reject ignore/;
-my %implied_type = qw/keys hash unknown hash  values array sort array unique array/;
+my %implied_type = qw/
+    keys hash values hash unknown hash
+    elems array sort array unique array scalar array
+/;
 my %sort_vals = (
     str => sub($x,$y) { $x cmp $y },
     num => sub($x,$y) { $x <=> $y },
@@ -119,9 +122,9 @@ sub _compile($schema, $custom, $rec, $top, $validations=$top->{validations}) {
             $top->{type} = $type;
         }
 
-        if ($name eq 'values') {
-            $top->{values} ||= _new;
-            _compile($val, $custom, $rec-1, $top->{values});
+        if ($name eq 'elems' || $name eq 'values') {
+            $top->{$name} ||= _new;
+            _compile($val, $custom, $rec-1, $top->{$name});
             next;
         }
 
@@ -173,35 +176,51 @@ sub compile($pkg, $schema, $custom={}) {
 }
 
 
-sub _validate_keys {
-    my @err;
-    for my ($k, $s) ($_[0]{keys}->%*) {
-        if (!exists $_[1]{$k}) {
-            next if $s->{missing} && $s->{missing} eq 'ignore';
-            return { validation => 'missing', key => $k } if $s->{missing} && $s->{missing} eq 'reject';
-            $_[1]{$k} = ref $s->{default} eq 'CODE' ? $s->{default}->() : $s->{default} // undef;
-            next if exists $s->{default};
-        }
+sub _validate_hash {
+    my $c = $_[0];
 
-        my $r = _validate($s, $_[1]{$k});
-        if ($r) {
-            $r->{key} = $k;
-            push @err, $r;
+    if ($c->{keys}) {
+        my @err;
+        for my ($k, $s) ($c->{keys}->%*) {
+            if (!exists $_[1]{$k}) {
+                next if $s->{missing} && $s->{missing} eq 'ignore';
+                return { validation => 'missing', key => $k } if $s->{missing} && $s->{missing} eq 'reject';
+                $_[1]{$k} = ref $s->{default} eq 'CODE' ? $s->{default}->() : $s->{default} // undef;
+                next if exists $s->{default};
+            }
+
+            my $r = _validate($s, $_[1]{$k});
+            if ($r) {
+                $r->{key} = $k;
+                push @err, $r;
+            }
         }
+        return { validation => 'keys', errors => [ sort { $a->{key} cmp $b->{key} } @err ] } if @err;
+    }
+
+    if ($c->{values}) {
+        my @err;
+        for my ($k, $v) ($_[1]->%*) {
+            my $r = _validate($c->{values}, $v);
+            if ($r) {
+                $r->{key} = $k;
+                push @err, $r;
+            }
+        }
+        return { validation => 'values', errors => [ sort { $a->{key} cmp $b->{key} } @err ] } if @err;
     }
-    return { validation => 'keys', errors => [ sort { $a->{key} cmp $b->{key} } @err ] } if @err;
 }
 
-sub _validate_values {
+sub _validate_elems {
     my @err;
     for my $i (0..$#{$_[1]}) {
-        my $r = _validate($_[0]{values}, $_[1][$i]);
+        my $r = _validate($_[0]{elems}, $_[1][$i]);
         if ($r) {
             $r->{index} = $i;
             push @err, $r;
         }
     }
-    return { validation => 'values', errors => \@err } if @err;
+    return { validation => 'elems', errors => \@err } if @err;
 }
 
 
@@ -275,14 +294,14 @@ sub _validate_input {
         # Each branch below makes a shallow copy of the hash, so that further
         # validations can perform in-place modifications without affecting the
         # input.
-        if (!$c->{unknown} || $c->{unknown} eq 'remove') {
+        if (!$c->{keys} || ($c->{unknown} && $c->{unknown} eq 'pass')) {
+            $_[1] = { $_[1]->%* };
+        } elsif (!$c->{unknown} || $c->{unknown} eq 'remove') {
             $_[1] = { map +($_, $_[1]{$_}), grep $c->{keys}{$_}, keys $_[1]->%* };
-        } elsif ($c->{unknown} && $c->{unknown} eq 'reject') {
+        } else {
             my @err = grep !$c->{keys}{$_}, keys $_[1]->%*;
             return { validation => 'unknown', keys => \@err, expected => [ sort keys $c->{keys}->%* ] } if @err;
             $_[1] = { $_[1]->%* };
-        } else {
-            $_[1] = { $_[1]->%* };
         }
 
     } elsif ($type eq 'array') {
@@ -294,8 +313,8 @@ sub _validate_input {
         # No need to do anything here.
     }
 
-    ($c->{keys} && &_validate_keys) ||
-    ($c->{values} && &_validate_values) ||
+    ($type eq 'hash' && &_validate_hash) ||
+    ($c->{elems} && &_validate_elems) ||
     &_validate_rec ||
     ($type eq 'array' && &_validate_array)
 }
@@ -340,8 +359,9 @@ sub errors($e, $prefix='') {
     my $val = $e->{validation};
     my $p = $prefix ? "$prefix: " : '';
     $val eq 'keys'     ? map errors($_, $prefix.'.'._fmtkey($_->{key})), $e->{errors}->@* :
+    $val eq 'values'   ? map errors($_, $prefix.'.'._fmtkey($_->{key})), $e->{errors}->@* :
     $val eq 'missing'  ? $prefix.'.'._fmtkey($e->{key}).': required key missing' :
-    $val eq 'values'   ? map errors($_, $prefix."[$_->{index}]"), $e->{errors}->@* :
+    $val eq 'elems'    ? map errors($_, $prefix."[$_->{index}]"), $e->{errors}->@* :
     $val eq 'unique'   ? $prefix."[$e->{index_b}] value '"._fmtval($e->{value_a})."' duplicated" :
     $val eq 'required' ? "${p}required value missing" :
     $val eq 'type'     ? "${p}invalid type, expected '$e->{expected}' but got '$e->{got}'" :
@@ -374,10 +394,10 @@ validate the format and the structure of the data, but it does not support
 validations that depend on other input values. For example, it is not possible
 to specify that the contents of a I<password> field must be equivalent to that
 of a I<confirm_password> field, but you can specify that both fields need to be
-filled out. Recursive data structures are not supported. There is also no
-built-in support for validating hashes with dynamic keys or arrays where not
-all elements conform to the same schema. These could technically still be
-validated with custom validations, but it won't be as convenient.
+filled out. Recursive data structures are not supported. There is also no good
+support for validating hashes with dynamic keys or arrays where not all
+elements conform to the same schema. These could technically still be validated
+with custom validations, but it won't be as convenient.
 
 This module is designed to validate any kind of program input after it has been
 parsed into a Perl data structure. It should not be used to validate function
@@ -521,14 +541,22 @@ like:
     ]
   }
 
+=item values => $schema
+
+Implies C<< type => 'hash' >>, set a schema that is used to validate every hash
+value. Can be used together with I<keys>, in which case values must validate
+both this C<$schema> and the schema corresponding to the key.
+
 =item unknown => $option
 
 Implies C<< type => 'hash' >>, this option specifies what to do with keys in
 the input data that have not been defined in the I<keys> option. Possible
 values are I<remove> to remove unknown keys from the output data (this is the
 default), I<reject> to return an error if there are unknown keys in the input,
-or I<pass> to pass through any unknown keys to the output data. Note that the
-values for passed-through keys are not validated against any schema!
+or I<pass> to pass through any unknown keys to the output data. Values for
+passed-through keys are only validated when the I<values> option is set,
+otherwise they are passed through as-is. This option has no effect when the
+I<keys> option is never set, in that case all values are always passed through.
 
 In the case of I<reject>, the error object will look like:
 
@@ -549,7 +577,8 @@ undef), I<reject> to return an error if the option is missing or I<ignore> to
 leave the key out of the returned data.
 
 The default is I<create>, but if no I<default> option is set for this key then
-that is effectively the same as I<reject>.
+that is effectively the same as I<reject>. Values created through I<create> are
+still validated through I<values> if that has been set.
 
 In the case of I<reject>, the error object will look like:
 
@@ -557,15 +586,15 @@ In the case of I<reject>, the error object will look like:
     key        => 'field'
   }
 
-=item values => $schema
+=item elems => $schema
 
 Implies C<< type => 'array' >>, this defines the schema that is applied to
-every item in the array.  The schema definition may be a bare hashref or a
+every element in the array.  The schema definition may be a bare hashref or a
 validator returned by C<compile()>.
 
 Failure is reported in a similar fashion to I<keys>:
 
-  { validation => 'values',
+  { validation => 'elems',
     errors => [
       { index => 1, validation => 'required' }
     ]
@@ -626,8 +655,7 @@ All of that may sound complicated, but it's quite easy to use. Here's a few
 examples:
 
   # This describes an array of hashes with keys 'id' and 'name'.
-  { values => {
-      type => 'hash',
+  { elems => {
       keys => {
         id   => { uint => 1 },
         name => {}
@@ -641,7 +669,7 @@ examples:
 
   # Contrived example: An array of strings, and we want
   # each string to start with a different character.
-  { values => { minlength => 1 },
+  { elems => { minlength => 1 },
     unique => sub { substr $_[0], 0, 1 }
   }
 
@@ -845,7 +873,7 @@ used in that schema may get input with whitespace around it.
 
 All validations used in a schema need to agree upon a single I<type> option.
 If a custom validation does not specify a I<type> option (and no type is
-implied by another validation such as I<keys> or I<values>), then the
+implied by another validation such as I<keys> or I<elems>), then the
 validation should work with every type. It is an error to define a schema that
 mixes validations of different types. For example, the following throws an
 error:
@@ -859,8 +887,8 @@ error:
 
 The I<func> option is validated separately for each custom validation.
 
-Multiple I<keys> and I<values> validations are merged into a single validation.
-So if you have multiple custom validations that set the I<values> option, a
+Multiple I<keys> and I<elems> validations are merged into a single validation.
+So if you have multiple custom validations that set the I<elems> option, a
 single combined schema is created that validates all array elements. The same
 applies to I<keys>: if the same key is listed in multiple custom validations,
 then the key must conform to all schemas. With respect to the I<unknown>
diff --git a/t/validate.t b/t/validate.t
index 7533a02..de37264 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -19,7 +19,7 @@ my %validations = (
     neverfails => { onerror => 'err' },
     doublefunc => [ func => sub { $_[0] == 0 }, func => sub { $_[0] = 2; 1; } ],
     revnum => { type => 'array', sort => sub($x,$y) { $y <=> $x } },
-    uniquelength => { type => 'array', values => { type => 'array' }, unique => sub { scalar @{$_[0]} } },
+    uniquelength => { elems => { type => 'array' }, unique => sub { scalar @{$_[0]} } },
     person => {
         type => 'hash',
         unknown => 'pass',
@@ -85,8 +85,8 @@ f { type => 'array' }, 1, { validation => 'type', expected => 'array', got => 's
 t { type => 'array' }, [], [];
 t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}];
 t { type => 'array', scalar => 1 }, 1, [1];
-f { type => 'array', values => {} }, [undef], { validation => 'values', errors => [{ index => 0, validation => 'required' }] }, "[0]: required value missing";
-t { type => 'array', values => {} }, [' a '], ['a'];
+f { type => 'array', elems => {} }, [undef], { validation => 'elems', errors => [{ index => 0, validation => 'required' }] }, "[0]: required value missing";
+t { type => 'array', elems => {} }, [' a '], ['a'];
 t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/];
 t { type => 'array', sort => 'num' }, [qw/20 100 3/], [qw/3 20 100/];
 t { revnum => 1 },                    [qw/20 100 3/], [qw/100 20 3/];
@@ -97,12 +97,12 @@ f { type => 'array', unique => 1 }, [qw/3 1 3/], { validation => 'unique', index
 t { uniquelength => 1 }, [[],[1],[1,2]], [[],[1],[1,2]];
 f { uniquelength => 1 }, [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 }, q{[2] value '[1]' duplicated};
 t { type => 'array', setundef => 1 }, [], undef;
-t { type => 'array', values => { type => 'any', setundef => 1 } }, [[]], [undef];
+t { type => 'array', elems => { type => 'any', setundef => 1 } }, [[]], [undef];
 
 # hashes
 f { type => 'hash' }, [], { validation => 'type', expected => 'hash', got => 'array' }, "invalid type, expected 'hash' but got 'array'";
 f { type => 'hash' }, 'a', { validation => 'type', expected => 'hash', got => 'scalar' }, "invalid type, expected 'hash' but got 'scalar'";
-t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {};
+t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {a=>[],b=>undef,c=>{}};
 f { type => 'hash', keys => { a=>{} } }, {}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] }, '.a: required value missing';
 t { type => 'hash', keys => { a=>{missing=>'ignore'} } }, {}, {};
 t { type => 'hash', keys => { a=>{default=>undef} } }, {}, {a=>undef};
@@ -120,6 +120,9 @@ t [ keys => { a => {} }, keys => { b => {} } ], {a=>1, b=>2}, {a=>1, b=>2};
 f [ keys => { a => {} }, keys => { b => {} } ], {a=>1}, { validation => 'keys', errors => [{ key => 'b', validation => 'required' }] }, '.b: required value missing';
 f [ keys => { a => {} }, keys => { a => { int => 1 } } ], {a=>'abc'}, { validation => 'keys', errors => [{ key => 'a', validation => 'int', got => 'abc' }] }, ".a: failed validation 'int'";
 
+t { values => { int => 1 } }, { a => -1, b => 1 }, { a => -1, b => 1 };
+f { values => { int => 1 } }, { a => undef }, { validation => 'values', errors => [{ key => 'a', validation => 'required' }] }, '.a: required value missing';
+
 # default validations
 f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 }, "failed validation 'minlength'";
 t { minlength => 3 }, 'abc', 'abc';

From f8fe53cba90f5b39ae4666aee97d053a421c61ef Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 16 Mar 2025 15:03:32 +0100
Subject: [PATCH 17/86] json_format: Add html_safe option

---
 FU/Util.pm      | 27 ++++++++++++++++++++-------
 c/jsonfmt.c     | 29 ++++++++++++++++-------------
 t/json_format.t |  1 +
 3 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/FU/Util.pm b/FU/Util.pm
index cd34fd3..a9cca1d 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -212,13 +212,6 @@ roughly similar to:
 
   JSON::PP->new->allow_nonref->core_bools->convert_blessed->encode($scalar);
 
-Some modules escape the slash character in encoded strings to prevent a
-potential XSS vulnerability when embedding JSON inside C<< <script> ..
-</script> >> tags.  This function does I<not> do that because it might not even
-be sufficient. The following is probably an improvement:
-
-  json_format($data) =~ s{</}{<\\/}rg =~ s/<!--/<\\u0021--/rg;
-
 This function generates invalid JSON if you pass it a string with invalid
 Unicode characters; I don't see how you'd ever accidentally end up with such a
 string, anyway.
@@ -244,6 +237,26 @@ versions.
 
 Boolean, returns a UTF-8 encoded byte string instead of a Perl Unicode string.
 
+=item html_safe
+
+Boolean. When set, the encoded JSON is safe for (unescaped) inclusion into HTML
+or XML content. This encodes C<< < >>, C<< > >> and C<< & >> as Unicode escapes.
+Commonly used to embed data inside a HTML page:
+
+  $html = '<script id="site_data" type="application/json">'
+        . json_format($data, html_safe => 1)
+        . '</script>';
+
+This option does NOT make it safe to include the encoded JSON as an attribute
+value. There is no way to do that without violating JSON specs, so you should
+use entity escaping instead.
+
+Some JSON modules escape the forward slash (C</>) character instead, but that
+is, at best, B<only> sufficient for embedding inside a C<< <script> >> tag (I'm
+not sure how C<< <!-- >> and C<< <![CDATA[ >> are treated in that context). In
+any other context, you'll need the more thourough escaping provided by this
+C<html_safe> option.
+
 =item max_size
 
 Maximum permitted size, in bytes, of the generated JSON string. Defaults to 1 GiB.
diff --git a/c/jsonfmt.c b/c/jsonfmt.c
index e6aab96..f605e8d 100644
--- a/c/jsonfmt.c
+++ b/c/jsonfmt.c
@@ -3,6 +3,7 @@ typedef struct {
     UV depth;
     int canon;
     int pretty; /* <0 when disabled, current nesting level otherwise */
+    int htmlsafe;
 } fujson_fmt_ctx;
 
 static void fujson_fmt(pTHX_ fujson_fmt_ctx *, SV *);
@@ -27,21 +28,22 @@ static void fujson_fmt_str(pTHX_ fujson_fmt_ctx *ctx, const char *stri, size_t l
     while (off < len) {
         /* Fast path: no escaping needed */
         loff = off;
+
+#define SKIPUNTIL(cond) \
+                while (off < len) { \
+                    x = str[off]; \
+                    if (x <= 0x1f || x == '"' || x == '\\' || cond) break; \
+                    off++;\
+                }
         if (utf8) {
-            /* assume >=0x80 is valid utf8 */
-            while (off < len) {
-                x = str[off];
-                if (x <= 0x1f || x == '"' || x == '\\' || x == 0x7f) break;
-                off++;
-            }
+            if (!ctx->htmlsafe) { SKIPUNTIL(x == 0x7f) }
+            else                { SKIPUNTIL(x == 0x7f || x == '<' || x == '>' || x == '&') }
         } else {
-            /* binary strings need special handling for >=0x80 */
-            while (off < len) {
-                x = str[off];
-                if (x <= 0x1f || x == '"' || x == '\\' || x >= 0x7f) break;
-                off++;
-            }
+            if (!ctx->htmlsafe) { SKIPUNTIL(x >= 0x7f) }
+            else                { SKIPUNTIL(x >= 0x7f || x == '<' || x == '>' || x == '&') }
         }
+#undef SKIPUNTIL
+
         fustr_write(ctx->out, (char *)str+loff, off-loff);
 
         if (off < len) { /* early break, which means current byte needs special processing */
@@ -279,7 +281,7 @@ static SV *fujson_fmt_xs(pTHX_ I32 ax, I32 argc, SV *val) {
     ctx.out = &out;
     ctx.depth = 0;
     ctx.pretty = INT_MIN;
-    ctx.canon = 0;
+    ctx.canon = ctx.htmlsafe = 0;
     while (i < argc) {
         arg = SvPV_nolen(ST(i));
         i++;
@@ -289,6 +291,7 @@ static SV *fujson_fmt_xs(pTHX_ I32 ax, I32 argc, SV *val) {
 
         if (strcmp(arg, "canonical") == 0) ctx.canon = SvPVXtrue(r);
         else if (strcmp(arg, "pretty") == 0) ctx.pretty = SvPVXtrue(r) ? 0 : INT_MIN;
+        else if (strcmp(arg, "html_safe") == 0) ctx.htmlsafe = !!SvPVXtrue(r);
         else if (strcmp(arg, "utf8") == 0) encutf8 = SvPVXtrue(r);
         else if (strcmp(arg, "max_size") == 0) out.maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
diff --git a/t/json_format.t b/t/json_format.t
index 5288e1a..89902e7 100644
--- a/t/json_format.t
+++ b/t/json_format.t
@@ -110,6 +110,7 @@ is json_format(
 }
 _
 
+is json_format('<hello & world>', html_safe => 1), '"\u003chello \u0026 world\u003e"';
 
 eval { json_format [[]], max_depth => 2 };
 like $@, qr/max_depth exceeded while formatting JSON/;

From 3382deba9acb1c77e3cca464dc8f59801b62b1c5 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 16 Mar 2025 15:39:10 +0100
Subject: [PATCH 18/86] Validate: Rename "scalar" to "accept_scalar" and add
 "accept_array"

---
 FU/Validate.pm | 33 ++++++++++++++++++++++++---------
 t/validate.t   | 10 +++++++++-
 2 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 8d8db91..6016b28 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -13,7 +13,8 @@ my %builtin = map +($_,1), qw/
     default
     onerror
     trim
-    elems scalar sort unique
+    elems sort unique
+    accept_scalar accept_array
     keys values unknown missing
     func
 /;
@@ -22,8 +23,9 @@ my %type_vals = map +($_,1), qw/scalar hash array any/;
 my %unknown_vals = map +($_,1), qw/remove reject pass/;
 my %missing_vals = map +($_,1), qw/create reject ignore/;
 my %implied_type = qw/
+    accept_array scalar
     keys hash values hash unknown hash
-    elems array sort array unique array scalar array
+    elems array sort array unique array accept_scalar array
 /;
 my %sort_vals = (
     str => sub($x,$y) { $x cmp $y },
@@ -151,6 +153,7 @@ sub _compile($schema, $custom, $rec, $top, $validations=$top->{validations}) {
         if ($builtin{$name}) {
             confess "Invalid value for 'missing': $val" if $name eq 'missing' && !$missing_vals{$val};
             confess "Invalid value for 'unknown': $val" if $name eq 'unknown' && !$unknown_vals{$val};
+            confess "Invalid value for 'accept_array': $val" if $name eq 'accept_array' && $val && $val ne 'first' && $val ne 'last';
             $val = $sort_vals{$val} || confess "Unknown value for 'sort': $val" if $name eq 'sort' && ref $val ne 'CODE';
             $top->{$name} = $val;
             next;
@@ -273,6 +276,10 @@ sub _validate_input {
 
     my $type = $c->{type} // 'scalar';
 
+    # accept_array (needs to be done before 'trim')
+    $_[1] = $_[1]->@* == 0 ? undef : $c->{accept_array} eq 'first' ? $_[1][0] : $_[1][ $#{$_[1]} ]
+        if $c->{accept_array} && ref $_[1] eq 'ARRAY';
+
     # trim (needs to be done before the 'default' test)
     $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $type eq 'scalar' && (!exists $c->{trim} || $c->{trim});
 
@@ -305,8 +312,8 @@ sub _validate_input {
         }
 
     } elsif ($type eq 'array') {
-        $_[1] = [$_[1]] if $c->{scalar} && !ref $_[1];
-        return { validation => 'type', expected => $c->{scalar} ? 'array or scalar' : 'array', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'ARRAY';
+        $_[1] = [$_[1]] if $c->{accept_scalar} && !ref $_[1];
+        return { validation => 'type', expected => $c->{accept_scalar} ? 'array or scalar' : 'array', got => lc ref $_[1] || 'scalar' } if ref $_[1] ne 'ARRAY';
         $_[1] = [$_[1]->@*]; # Create a shallow copy to prevent in-place modification.
 
     } elsif ($type eq 'any') {
@@ -600,10 +607,10 @@ Failure is reported in a similar fashion to I<keys>:
     ]
   }
 
-=item scalar => 0/1
+=item accept_scalar => 0/1
 
 Implies C<< type => 'array' >>, this option will also permit the input to be a
-scalar. In this case, the input is interpreted and returned as an array with
+scalar. In that case, the input is interpreted and returned as an array with
 only one element. This option exists to make it easy to validate multi-value
 form inputs. For example, consider C<query_decode()> in L<FU::Util>: a
 parameter in a query string is decoded into an array if it is listed multiple
@@ -613,16 +620,24 @@ times, a scalar if it only occcurs once. So we could either end up with:
   # OR:
   { a => [1, 3], b => 1 }
 
-With the I<scalar> option, we can accept both forms for C<a> and normalize into
-an array. The following schema definition can validate the above examples:
+With the I<accept_scalar> option, we can accept both forms for C<a> and
+normalize into an array. The following schema definition can validate the above
+examples:
 
   { type => 'hash',
     keys => {
-      a => { type => 'array', scalar => 1 },
+      a => { type => 'array', accept_scalar => 1 },
       b => { }
     }
   }
 
+=item accept_array => false/'first'/'last'
+
+Implies C<< type => 'scalar' >>. Similar to I<accept_scalar> but normalizes in
+the other direction: when the input is an array, only the first or last item is
+extracted and the other elements are ignored. If the input is an empty array,
+the value is taken to be C<undef>.
+
 =item sort => $option
 
 Implies C<< type => 'array' >>, sort the array after validating its elements.
diff --git a/t/validate.t b/t/validate.t
index de37264..1481193 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -79,12 +79,20 @@ t { trim => 0 }, " Va\rl id \n ", " Va\rl id \n ";
 f {}, '  ', { validation => 'required' }, 'required value missing';
 t { trim => 0 }, '  ', '  ';
 
+# accept_array
+t { default => undef, accept_array => 'first' }, [], undef;
+t { default => undef, accept_array => 'first' }, [' x '], 'x';
+t { accept_array => 'first' }, [1,2,3], 1;
+t { accept_array => 'last' }, [1,2,3], 3;
+f { accept_array => 'first' }, ['  ', 1], { validation => 'required' }, 'required value missing';
+f { accept_array => 'first' }, [], { validation => 'required' }, 'required value missing';
+
 # arrays
 f {}, [], { validation => 'type', expected => 'scalar', got => 'array' }, "invalid type, expected 'scalar' but got 'array'";
 f { type => 'array' }, 1, { validation => 'type', expected => 'array', got => 'scalar' }, "invalid type, expected 'array' but got 'scalar'";
 t { type => 'array' }, [], [];
 t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}];
-t { type => 'array', scalar => 1 }, 1, [1];
+t { type => 'array', accept_scalar => 1 }, 1, [1];
 f { type => 'array', elems => {} }, [undef], { validation => 'elems', errors => [{ index => 0, validation => 'required' }] }, "[0]: required value missing";
 t { type => 'array', elems => {} }, [' a '], ['a'];
 t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/];

From 65cf842500ff05c3610b5a40835fbe58acaaed8a Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 16 Mar 2025 18:26:57 +0100
Subject: [PATCH 19/86] FU: Some fixes and consistency in fu->json and
 fu->formdata

---
 FU.pm | 35 ++++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/FU.pm b/FU.pm
index d6b9268..d5978e2 100644
--- a/FU.pm
+++ b/FU.pm
@@ -393,13 +393,13 @@ sub _do_req($c) {
     }
 
     my $proc_ms = ($REQ->{trace_end} - $REQ->{trace_start}) * 1000;
-    log_write(sprintf "%.0fms%s %s-%s %s-%d\n", $proc_ms,
+    log_write(sprintf "%.0fms%s %s-%s %d-%s\n", $proc_ms,
         $REQ->{trace_nsql} ?
             sprintf ' (sql %.0f+%.0fms, %d/%d/%d)',
             ($REQ->{trace_sqlexec}||0)*1000, ($REQ->{trace_sqlprep}||0)*1000,
             $REQ->{trace_nsqldirect}||0, $REQ->{trace_nsqlprep}||0, $REQ->{trace_nsql} : '',
         $REQ->{status}, ($REQ->{reshdr}{'content-type'}//'-') =~ s/;.+$//r,
-        $REQ->{reshdr}{'content-encoding'}//'bytes', length($REQ->{resbody}),
+        length($REQ->{resbody}), substr($REQ->{reshdr}{'content-encoding'}//'bytes', 0, 1)
     ) if FU::debug || $proc_ms > (FU::log_slow_reqs||1e10);
 }
 
@@ -657,8 +657,10 @@ sub query {
 
 sub json {
     shift;
+    fu->error(400, "Invalid content type for json") if (fu->header('content-type')||'') ne 'application/json';
+    return FU::Util::utf8_decode(my $x = $FU::REQ->{body}) if !@_;
     $FU::REQ->{json} ||= eval {
-        FU::Util::json_parse($FU::REQ->{data}, utf8 => 1)
+        FU::Util::json_parse($FU::REQ->{body}, utf8 => 1)
     } || fu->error(400, "JSON parse error: $@");
     return $FU::REQ->{json} if !@_;
     _getfield $FU::REQ->{json}, @_;
@@ -666,10 +668,10 @@ sub json {
 
 sub formdata {
     shift;
+    fu->error(400, "Invalid content type for form data") if (fu->header('content-type')||'') ne 'application/x-www-form-urlencoded';
+    return FU::Util::utf8_decode(my $x = $FU::REQ->{body}) if !@_;
     $FU::REQ->{formdata} ||= eval {
-        confess "Invalid content type for form data"
-            if (fu->header('content-type')||'') ne 'application/x-www-form-urlencoded';
-        FU::Util::query_decode($FU::REQ->{data});
+        FU::Util::query_decode($FU::REQ->{body});
     } || fu->error(400, $@);
     _getfield $FU::REQ->{formdata}, @_;
 }
@@ -1235,9 +1237,9 @@ C<https://example.com/some/path?query> this returns C<query>.
 =item fu->query($name)
 
 Parses the raw query string with C<query_decode> in L<FU::Util> and returns the
-value with the given $name. Beware: multiple values are returned as an array.
-Prefer to use the C<$schema>-based validation methods below to reliably handle
-all sorts of query strings.
+value with the given $name. Beware: an array is returned if the given key is
+repeated in the query string.  Prefer to use the C<$schema>-based validation
+methods below to reliably handle all sorts of query strings.
 
 =item fu->query($name => $schema)
 
@@ -1260,12 +1262,19 @@ Parse, validate and return multiple query parameters.
   # Or, more concisely:
   my $data = fu->query(a => {anybool => 1}, b => {});
 
-=item fu->json(@args)
+To fetch all query paramaters as decoded by C<query_decode()>, use:
 
-Like C<< fu->query() >> but parses the request body as JSON. Returns the
-decoded JSON value if C<@args> is empty.
+  my $data = fu->query({type=>'any'});
 
-=item fu->formdata(@args)
+=item fu->json(...)
+
+Like C<< fu->query() >> but parses the request body as JSON. Returns the raw
+(unvalidated!) JSON Unicode string if no arguments are given. To retrieve the
+decoded JSON data without performing further validation, use:
+
+  my $data = fu->json({type=>'any'});
+
+=item fu->formdata(...)
 
 Like C<< fu->query() >> but returns data from the POST request body. This
 method only supports form data encoded as C<application/x-www-form-urlencoded>,

From a7bfe146b1877ea2924782f7bdcf4dbdeb0e11aa Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 17 Mar 2025 12:50:21 +0100
Subject: [PATCH 20/86] Validate: Fix referencing & merging already compiled
 schemas

---
 FU/Validate.pm | 9 ++++++++-
 t/validate.t   | 8 ++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 6016b28..112d8cc 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -103,12 +103,14 @@ sub _new { bless { validations => [], @_ }, __PACKAGE__ }
 
 
 sub _compile($schema, $custom, $rec, $top, $validations=$top->{validations}) {
+    my $iscompiled = $schema isa __PACKAGE__;
+
     # For hashref schemas, builtins always override other validations
     $schema = [
         map +($_, $schema->{$_}),
             (grep !$builtin{$_}, keys %$schema),
             (grep $builtin{$_}, keys %$schema),
-    ] if ref $schema eq 'HASH';
+    ] if ref $schema ne 'ARRAY';
 
     for my($name, $val) (@$schema) {
         if ($name eq 'type') {
@@ -159,6 +161,11 @@ sub _compile($schema, $custom, $rec, $top, $validations=$top->{validations}) {
             next;
         }
 
+        if ($iscompiled && $name eq 'validations') {
+            push @$validations, @$val;
+            next;
+        }
+
         my $t = $custom->{$name} || $default_validations{$name};
         confess "Unknown validation: $name" if !$t;
         confess "Recursion limit exceeded while resolving validation '$name'" if $rec < 1;
diff --git a/t/validate.t b/t/validate.t
index 1481193..626dbbd 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -275,6 +275,14 @@ t { weburl => 1}, $_, $_ for (
   'https://blicky.net/?#Who\'d%20ever%22makeaurl_like-this/!idont.know',
 );
 
+# Merging nested schemas
+my $pa = FU::Validate->compile({ regex => '^a' });
+my $pz = FU::Validate->compile({ regex => 'z$' });
+my $com = FU::Validate->compile([ elems => $pa, elems => $pz ]);
+is_deeply $com->validate(['axz']), ['axz'];
+ok !eval { $com->validate(['bz', 'axz', 'ax']) };
+is [$@->errors]->[0], "[0]: failed validation 'regex'";
+is [$@->errors]->[1], "[2]: failed validation 'regex'";
 
 # Things that should fail
 ok !eval { FU::Validate->compile({ recursive => 1 }, { recursive => { recursive => 1 } }); 1 }, 'recursive';

From d8ecc71abb4b6c63b6d063e3303aa552de328e2a Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 17 Mar 2025 13:46:03 +0100
Subject: [PATCH 21/86] FU: Add fu->set_cookie() (+ uri_unescape '+' fix)

---
 FU.pm      | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
 FU/Util.pm |  1 +
 2 files changed, 62 insertions(+), 2 deletions(-)

diff --git a/FU.pm b/FU.pm
index d5978e2..23aac6c 100644
--- a/FU.pm
+++ b/FU.pm
@@ -6,6 +6,7 @@ use POSIX ();
 use Time::HiRes 'time';
 use FU::Log 'log_write';
 use FU::Util;
+use FU::Validate;
 
 
 sub import($pkg, @opt) {
@@ -641,7 +642,6 @@ sub _getfield($data, @a) {
         fu->error(400, "Expected top-level to be a hash") if ref $data ne 'HASH';
         return $data->{$a[0]};
     }
-    require FU::Validate;
     my $schema = FU::Validate->compile(@a > 1 ? { keys => {@a} } : $a[0]);
     my $res = eval { $schema->validate($data) };
     fu->error(400, "Input validation failed: $@") if $@;
@@ -729,6 +729,39 @@ sub set_header($, $hdr, $val=undef) {
     $FU::REQ->{reshdr}{ lc $hdr } = $val;
 }
 
+sub set_cookie($, $name, $val=undef, %opt) {
+    confess "Invalid cookie name '$name'" if $name !~ /^$FU::hdrname_re$/;
+    return delete $FU::REQ->{rescookie}{$name} if !defined $val;
+    confess "Invalid cookie value: $val" if $val =~ /[\0-\x1f\x7f-\x{10ffff}\s\r\n\t",;\\]/;
+    my $c = "$name=$val";
+    for my ($k,$v) (%opt) {
+        $k = lc $k; # attributes are case-insensitive
+        if ($k eq 'domain') {
+            confess "Invalid cookie domain: $v" if $v !~ $FU::Validate::re_domain;
+        } elsif ($k eq 'expires') {
+            confess "Cookie 'Expires' attribute should be a UNIX timestamp" if defined $v && $v !~ /^[0-9]+$/;
+            $v = FU::Util::httpdate_format($v || 0);
+        } elsif ($k eq 'httponly') {
+            $c .= "; $k" if $v;
+            next;
+        } elsif ($k eq 'max-age') {
+            confess "Invalid 'Max-Age' cookie attribute: $v" if $v !~ /^[0-9]+$/;
+        } elsif ($k eq 'partitioned') {
+            $c .= "; $k" if $v;
+            next;
+        } elsif ($k eq 'path') {
+            confess "Invalid 'Path' cookie attribute: $v" if $v =~ /[\0-\x1f\x7f-\x{10ffff}\s\r\n\t",;\\]/;
+        } elsif ($k eq 'secure') {
+            $c .= "; $k" if $v;
+            next;
+        } elsif ($k eq 'samesite') {
+            confess "Invalid 'SameSite' cookie attribute: $v" if $v !~ /^(?:Strict|Lax|None)$/;
+        }
+        $c .= "; $k=$v";
+    }
+    $FU::REQ->{rescookie}{$name} = $c;
+}
+
 sub send_json($, $data) {
     fu->set_header('content-type', 'application/json');
     fu->set_body(FU::Util::json_format($data, canonical => 1, utf8 => 1));
@@ -804,6 +837,8 @@ sub _finalize {
     state $haszlib = eval { require Compress::Raw::Zlib; 1 };
     my $r = $FU::REQ;
 
+    fu->add_header('set-cookie', $_) for $r->{rescookie} ? sort values $r->{rescookie}->%* : ();
+
     if ($r->{status} == 204 || $r->{status} == 304) {
         delete $r->{reshdr}{'content-length'};
         delete $r->{reshdr}{'content-encoding'};
@@ -1335,6 +1370,31 @@ Add a response header, can be used to add multiple headers with the same name.
 Add a response header or overwrite the header with a new value if it already
 exists. Set C<$value> to undef to remove a previously set header.
 
+=item fu->set_cookie($name, $value, %attributes)
+
+Set or overwrite a cookie. Set C<$value> to undef to remove a previously set
+cookie. To fully remove a cookie from the user's browser, set the cookie with
+an empty value and zero C<Max-Age>:
+
+  fu->set_cookie(my_cookie => '', 'Max-Age' => 0);
+
+C<%attributes> can be any of the supported L<cookie
+attributes|https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Set-Cookie>.
+The C<Expires> attribute, when given, must be a UNIX timestamp. Boolean
+attributes are interpreted according to Perl's idea of truthiness. For example:
+
+  fu->set_cookie(auth => $auth_token,
+      Expires => time()+30*24*3600,
+      Domain => 'example.com',
+      Secure => 1,
+      SameSite => 'Lax'
+  );
+
+This method does not encode or escape the cookie value in any way. If you want
+to set a non-ASCII value or a value containing characters that are not
+permitted in the C<Set-Cookie> header, use C<uri_escape()> in L<FU::Util> or
+your favorite alternative cookie-safe encoding.
+
 =item fu->set_body($data)
 
 Set the (raw, binary) body of the response to C<$data>. This method is not very
@@ -1408,7 +1468,6 @@ one of the following status codes or an alias:
 
 =back
 
-I<TODO:> Setting cookies.
 
 
 =head2 Running the Site
diff --git a/FU/Util.pm b/FU/Util.pm
index a9cca1d..a94aa9e 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -32,6 +32,7 @@ sub uri_escape :prototype($) ($s) {
 sub uri_unescape :prototype($) ($s) {
     return if !defined $s;
     utf8::encode($s);
+    $s =~ tr/+/ /;
     $s =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg;
     utf8_decode $s;
 }

From c2e0f158aca7c06bdbe9905dabe8468141683c71 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 17 Mar 2025 16:18:12 +0100
Subject: [PATCH 22/86] FU: Add fu->cookie()

---
 FU.pm | 34 ++++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/FU.pm b/FU.pm
index 23aac6c..4d7a28b 100644
--- a/FU.pm
+++ b/FU.pm
@@ -655,6 +655,22 @@ sub query {
     _getfield $FU::REQ->{qs_parsed}, @_;
 }
 
+sub cookie {
+    shift;
+    return fu->header('cookie') if !@_;
+    $FU::REQ->{cookie} ||= do {
+        my %c;
+        for my $c (split /; /, fu->header('cookie')||'') {
+            my($n, $v) = split /=/, $c, 2;
+            if (!exists $c{$n}) { $c{$n} = $v }
+            elsif (ref $c{$n}) { push $c{$n}->@*, $v }
+            else { $c{$n} = [ $c{$n}, $v ] }
+        }
+        \%c
+    };
+    _getfield $FU::REQ->{cookie}, @_;
+}
+
 sub json {
     shift;
     fu->error(400, "Invalid content type for json") if (fu->header('content-type')||'') ne 'application/json';
@@ -1301,6 +1317,22 @@ To fetch all query paramaters as decoded by C<query_decode()>, use:
 
   my $data = fu->query({type=>'any'});
 
+To fetch a query parameter that may have multiple values, use:
+
+  my $arrayref = fu->query(q => {accept_scalar => 1});
+
+  # OR:
+  my $first_value = fu->query(q => {accept_array => 'first'});
+
+  # OR:
+  my $last_value = fu->query(q => {accept_array => 'last'});
+
+=item fu->cookie(...)
+
+Like C<< fu->query() >> but parses the C<Cookie> request header. Beware that,
+exactly like with query parameters, it's possible for a cookie to have multiple
+values and thus get represented as an array.
+
 =item fu->json(...)
 
 Like C<< fu->query() >> but parses the request body as JSON. Returns the raw
@@ -1323,8 +1355,6 @@ objects.  Refer to L<FU::MultipartFormData> for more information.
 
 =back
 
-I<TODO:> Cookie parsing.
-
 
 =head2 Generating Responses
 

From bc33fe53f033003080d7babb56570792bd09cf55 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Tue, 18 Mar 2025 16:58:31 +0100
Subject: [PATCH 23/86] FU::Util: Add gzip_compress() wrapper for
 libdeflate/zlib-ng/zlib

And use it for automatic output compression in FU, as (potentially)
faster alternative to Compress::Raw::Zlib.

Was also planning to maybe add support for Zstd or Brotli, but given the
performance of libdeflate, I'm not sure that's really necessary. Brotli
does tend to do a better job at compressing HTML, though.
---
 FU.pm        |  14 +++---
 FU.xs        |  15 +++++-
 FU/Util.pm   |  43 +++++++++++++++++
 c/compress.c | 128 +++++++++++++++++++++++++++++++++++++++++++++++++++
 t/compress.t |  47 +++++++++++++++++++
 5 files changed, 237 insertions(+), 10 deletions(-)
 create mode 100644 c/compress.c
 create mode 100644 t/compress.t

diff --git a/FU.pm b/FU.pm
index 4d7a28b..999dda2 100644
--- a/FU.pm
+++ b/FU.pm
@@ -850,7 +850,6 @@ sub _error_page($, $code, $title, $msg) {
 }
 
 sub _finalize {
-    state $haszlib = eval { require Compress::Raw::Zlib; 1 };
     my $r = $FU::REQ;
 
     fu->add_header('set-cookie', $_) for $r->{rescookie} ? sort values $r->{rescookie}->%* : ();
@@ -862,18 +861,14 @@ sub _finalize {
         $r->{resbody} = '';
 
     } else {
-        if ($haszlib && length($r->{resbody}) > 256
+        if (FU::Util::gzip_lib() && length($r->{resbody}) > 256
                 && !defined $r->{reshdr}{'content-encoding'} && FU::compress_mimes->{$r->{reshdr}{'content-type'}}) {
 
             $r->{reshdr}{'vary'} = ($r->{reshdr}{'vary'} ? $r->{reshdr}{'vary'}.', ' : '').'accept-encoding'
                 if ($r->{reshdr}{'vary'}||'') !~ /accept-encoding/i;
 
-            if ($haszlib && ($r->{hdr}{'accept-encoding'}||'') =~ /gzip/) {
-                # Use lower-level API because the higher-level Compress::Zlib loads a whole bunch of other modules.
-                my $z = Compress::Raw::Zlib::Deflate->new(-WindowBits => Compress::Raw::Zlib::WANT_GZIP(), -Level => 3, -AppendOutput => 1);
-                $z->deflate($r->{resbody}, my $buf);
-                $z->flush($buf);
-                $r->{resbody} = $buf;
+            if (($r->{hdr}{'accept-encoding'}||'') =~ /gzip/) {
+                $r->{resbody} = FU::Util::gzip_compress(6, $r->{resbody});
                 $r->{reshdr}{'content-encoding'} = 'gzip';
             }
         }
@@ -993,6 +988,9 @@ is). There are a few additional optional dependencies:
 =item * C<libpq.so> - required for L<FU::Pg>, dynamically loaded through
 C<dlopen()>.
 
+=item * C<libdeflate.so> or C<libz-ng.so> or C<libz.so> - required for
+C<gzip_compress()> in L<FU::Util> and used for HTTP output compression.
+
 =back
 
 
diff --git a/FU.xs b/FU.xs
index 1d63ffe..b775f3c 100644
--- a/FU.xs
+++ b/FU.xs
@@ -27,10 +27,12 @@
 
 #include "c/khashl.h"
 #include "c/common.c"
+
+#include "c/compress.c"
+#include "c/fcgi.c"
+#include "c/fdpass.c"
 #include "c/jsonfmt.c"
 #include "c/jsonparse.c"
-#include "c/fdpass.c"
-#include "c/fcgi.c"
 #include "c/xmlwr.c"
 
 #include "c/libpq.h"
@@ -115,6 +117,15 @@ void json_parse(SV *val, ...)
   CODE:
     ST(0) = fujson_parse_xs(aTHX_ ax, items, val);
 
+void gzip_lib()
+  PROTOTYPE:
+  CODE:
+    ST(0) = sv_2mortal(newSVpv(fugz_lib(), 0));
+
+void gzip_compress(IV level, SV *in)
+  CODE:
+    ST(0) = fugz_compress(aTHX_ level, in);
+
 void fdpass_send(int socket, int fd, SV *data)
   CODE:
     STRLEN buflen;
diff --git a/FU/Util.pm b/FU/Util.pm
index a94aa9e..4ff1c2b 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -13,6 +13,7 @@ our @EXPORT_OK = qw/
     utf8_decode uri_escape uri_unescape
     query_decode query_encode
     httpdate_format httpdate_parse
+    gzip_lib gzip_compress
     fdpass_send fdpass_recv
 /;
 
@@ -367,6 +368,48 @@ This will not happen if your local timezone is UTC.
 =back
 
 
+=head2 Gzip Compression
+
+Gzip compression can be done with a few different libraries. The canonical one
+is I<zlib>, which is old and not well optimized for modern systems. There's
+also I<zlib-ng>, a (much) more performant reimplementation that remains
+API-compatible with I<zlib>. And there's I<libdeflate>, which offers a
+different API that does not support streaming compression but is, in exchange,
+even faster than I<zlib-ng>.
+
+There are more implementations, of course, but this module only supports those
+three and (attempts to) pick the best one that's available on your system.
+
+=over
+
+=item gzip_lib()
+
+Returns an empty string if no supported gzip library was found on your system
+(unlikely but possible), otherwise returns the selected implementation: either
+C<"libdeflate">, C<"zlib-ng"> or C<"zlib">.
+
+This function does not try very hard to differentiate between I<zlib> and
+I<zlib-ng>, so it may report that I<zlib> is being used on systems where
+C<libz.so> is, in fact, I<zlib-ng>.
+
+=item gzip_compress($level, $data)
+
+Returns a byte string with the gzip-compressed version of C<$data> at the given
+gzip C<$level>, which is a number between 0 (no compression) and 12 (strongest
+compression). Only I<libdeflate> supports levels higher than 9, for
+I<zlib(-ng)> the level is capped at 9. 6 is typically used as a default.
+
+Throws an error if no suitable library was found.
+
+=back
+
+This module does not currently implement decompression. If you need that, or
+streaming, or other functionality not provided here, there's
+L<Compress::Raw::Zlib> and L<Compress::Zlib> in the core Perl distribution and
+L<Gzip::Deflate> on CPAN.
+
+
+
 =head2 File Descriptor Passing
 
 UNIX sockets (see L<IO::Socket::UNIX>) have the fancy property of letting you
diff --git a/c/compress.c b/c/compress.c
new file mode 100644
index 0000000..9c4a1a9
--- /dev/null
+++ b/c/compress.c
@@ -0,0 +1,128 @@
+static const char *fugz_imps[] = {"", "libdeflate", "zlib-ng", "zlib"};
+static int fugz_imp = -1;
+
+
+/* zlib & zlib-ng */
+
+typedef struct {
+    const char *next_in;
+    unsigned int avail_in;
+    unsigned long total_in;
+    char *next_out;
+    unsigned int avail_out;
+    unsigned long total_out;
+    const char *msg;
+    struct internal_state *state;
+    void *zalloc;
+    void *zfree;
+    void *opaque;
+    int data_type;
+    unsigned long adler;
+    unsigned long reserved;
+} z_stream;
+
+static int (*deflate)(z_stream *, int);
+static int (*deflateEnd)(z_stream *);
+static int (*deflateInit2)(z_stream *, int, int, int, int, int);
+static int (*deflateInit2_)(z_stream *, int, int, int, int, int, const char *, int);
+static unsigned long (*compressBound)(unsigned long);
+
+
+/* libdeflate */
+
+static struct libdeflate_compressor *fugz_ld_ctx;
+static int fugz_ld_comp = -1;
+
+static struct libdeflate_compressor *(*libdeflate_alloc_compressor)(int);
+static void (*libdeflate_free_compressor)(struct libdeflate_compressor *);
+static size_t (*libdeflate_gzip_compress_bound)(struct libdeflate_compressor *, size_t);
+static size_t (*libdeflate_gzip_compress)(struct libdeflate_compressor *, const void *, size_t, void *, size_t);
+
+
+
+static const char *fugz_lib() {
+    if (fugz_imp >= 0) goto done;
+
+    void *handle;
+    if ((handle = dlopen("libdeflate.so", RTLD_LAZY))) {
+        if ((libdeflate_alloc_compressor = dlsym(handle, "libdeflate_alloc_compressor"))
+           && (libdeflate_free_compressor = dlsym(handle, "libdeflate_free_compressor"))
+           && (libdeflate_gzip_compress_bound = dlsym(handle, "libdeflate_gzip_compress_bound"))
+           && (libdeflate_gzip_compress = dlsym(handle, "libdeflate_gzip_compress"))) {
+            fugz_imp = 1;
+            goto done;
+        }
+    }
+
+    int i;
+    for (i=2; i<=3; i++) {
+        if ((handle = dlopen(i == 2 ? "libz-ng.so" : "libz.so", RTLD_LAZY))) {
+            if (((deflate = dlsym(handle, "zng_deflate")) || (deflate = dlsym(handle, "deflate")))
+               && ((deflateEnd = dlsym(handle, "zng_deflateEnd")) || (deflateEnd = dlsym(handle, "deflateEnd")))
+               && ((deflateInit2 = dlsym(handle, "zng_deflateInit2")) || (deflateInit2_ = dlsym(handle, "deflateInit2_")))
+               && ((compressBound = dlsym(handle, "zng_compressBound")) || (compressBound = dlsym(handle, "compressBound")))) {
+                fugz_imp = i;
+                goto done;
+            }
+        }
+    }
+    fugz_imp = 0;
+
+done:
+    return fugz_imps[fugz_imp];
+}
+
+
+static SV *fugz_compress_ld(pTHX_ int level, const char *bytes, size_t inlen) {
+    if (fugz_ld_comp != level) {
+        if (fugz_ld_ctx) libdeflate_free_compressor(fugz_ld_ctx);
+        fugz_ld_ctx = NULL;
+        fugz_ld_comp = level;
+    }
+    if (!fugz_ld_ctx) fugz_ld_ctx = libdeflate_alloc_compressor(level);
+
+    size_t outlen = libdeflate_gzip_compress_bound(fugz_ld_ctx, inlen);
+    SV *out = sv_2mortal(newSV(outlen));
+    SvPOK_only(out);
+    size_t len = libdeflate_gzip_compress(fugz_ld_ctx, bytes, inlen, SvPVX(out), outlen);
+    if (!len) fu_confess("Libdeflate compression failed"); /* Shouldn't happen */
+    SvCUR_set(out, len);
+    return out;
+}
+
+
+static SV *fugz_compress_zlib(pTHX_ int level, const char *bytes, size_t inlen) {
+    z_stream stream;
+    memset(&stream, 0, sizeof(stream));
+
+    int r = deflateInit2
+        ? deflateInit2(&stream, level > 9 ? 9 : level, 8, 16+15, 9, 0)
+        : deflateInit2_(&stream, level > 9 ? 9 : level, 8, 16+15, 9, 0, "1.3.1", (int)sizeof(stream));
+    if (r) fu_confess("Zlib compression failed (%d)", r);
+
+    stream.avail_out = compressBound(inlen) + 64; /* compressBound() does not include the gzip header */
+    SV *out = sv_2mortal(newSV(stream.avail_out));
+    SvPOK_only(out);
+    stream.next_out = SvPVX(out);
+    stream.next_in = bytes;
+    stream.avail_in = inlen;
+
+    if ((r = deflate(&stream, 4)) != 1) fu_confess("Zlib compression failed (%d)", r);
+
+    SvCUR_set(out, stream.total_out);
+    deflateEnd(&stream);
+    return out;
+}
+
+
+static SV *fugz_compress(pTHX_ IV level, SV *in) {
+    if (level < 0 || level > 12) fu_confess("Invalid compression level: %"IVdf, level);
+    if (!*fugz_lib()) fu_confess("Unable to load a suitable compression library");
+
+    STRLEN inlen;
+    const char *bytes = SvPVbyte(in, inlen);
+
+    if (fugz_imp == 1) return fugz_compress_ld(aTHX_ level, bytes, inlen);
+    else return fugz_compress_zlib(aTHX_ level, bytes, inlen);
+    return &PL_sv_undef;
+}
diff --git a/t/compress.t b/t/compress.t
new file mode 100644
index 0000000..ef17c77
--- /dev/null
+++ b/t/compress.t
@@ -0,0 +1,47 @@
+use v5.36;
+use Test::More;
+use FU::Util qw/gzip_lib gzip_compress/;
+
+like gzip_lib, qr/^(|libdeflate|zlib-ng|zlib)$/, gzip_lib;
+
+plan skip_all => 'No suitable gzip library found' if !gzip_lib;
+plan skip_all => 'Compress::Zlib not found' if !eval { require Compress::Zlib };
+
+my $incompressible = Compress::Zlib::memGzip(join '', map chr(rand 256), 0..93123);
+
+for my $str ('', 'Hello world!', 'x'x4096, $incompressible) {
+    is Compress::Zlib::memGunzip(gzip_compress(0, $str)), $str;
+    is Compress::Zlib::memGunzip(gzip_compress(12, $str)), $str;
+}
+
+
+done_testing;
+
+
+__END__
+
+# Test for leaks:
+
+use Test::LeakTrace;
+diag count_sv;
+for (0..1000) {
+    for my $str ('', 'Hello world!', 'x'x4096, $incompressible) {
+        local $_ = gzip_lib;
+        $_ = gzip_compress(0, $str);
+        $_ = gzip_compress(12, $str);
+    }
+}
+diag count_sv;
+
+
+# Compare performance:
+
+use Benchmark 'cmpthese';
+open my $F, '<', 'FU.pm';
+local $/ = undef;
+my $data = <$F>;
+
+cmpthese -3, {
+    memGzip => 'Compress::Zlib::memGzip($data)',
+    gzip_compress => 'gzip_compress(6, $data)',
+};

From 6159b33950e0bfded87594a5af9b5b0e5044596d Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 19 Mar 2025 10:12:06 +0100
Subject: [PATCH 24/86] FU::Util: Add brotli_compress() and use it for FU
 output compression

Seems to compresses and perform better than libdeflate at level 6, so
certainly worth using.
---
 FU.pm        | 19 +++++++++++++++----
 FU.xs        |  4 ++++
 FU/Util.pm   | 20 +++++++++++++++++++-
 c/compress.c | 35 ++++++++++++++++++++++++++++++++++-
 t/compress.t | 35 +++++++++++++++++++++++++++--------
 5 files changed, 99 insertions(+), 14 deletions(-)

diff --git a/FU.pm b/FU.pm
index 999dda2..40dd19e 100644
--- a/FU.pm
+++ b/FU.pm
@@ -400,7 +400,7 @@ sub _do_req($c) {
             ($REQ->{trace_sqlexec}||0)*1000, ($REQ->{trace_sqlprep}||0)*1000,
             $REQ->{trace_nsqldirect}||0, $REQ->{trace_nsqlprep}||0, $REQ->{trace_nsql} : '',
         $REQ->{status}, ($REQ->{reshdr}{'content-type'}//'-') =~ s/;.+$//r,
-        length($REQ->{resbody}), substr($REQ->{reshdr}{'content-encoding'}//'bytes', 0, 1)
+        length($REQ->{resbody}), substr($REQ->{reshdr}{'content-encoding'}//'r', 0, 1)
     ) if FU::debug || $proc_ms > (FU::log_slow_reqs||1e10);
 }
 
@@ -850,6 +850,8 @@ sub _error_page($, $code, $title, $msg) {
 }
 
 sub _finalize {
+    state $hasgzip = FU::Util::gzip_lib();
+    state $hasbrotli = eval { FU::Util::brotli_compress(6, ''); 1 };
     my $r = $FU::REQ;
 
     fu->add_header('set-cookie', $_) for $r->{rescookie} ? sort values $r->{rescookie}->%* : ();
@@ -861,13 +863,19 @@ sub _finalize {
         $r->{resbody} = '';
 
     } else {
-        if (FU::Util::gzip_lib() && length($r->{resbody}) > 256
-                && !defined $r->{reshdr}{'content-encoding'} && FU::compress_mimes->{$r->{reshdr}{'content-type'}}) {
+        if (($hasgzip || $hasbrotli) && length($r->{resbody}) > 256
+                && !defined $r->{reshdr}{'content-encoding'}
+                && FU::compress_mimes->{$r->{reshdr}{'content-type'}}
+        ) {
 
             $r->{reshdr}{'vary'} = ($r->{reshdr}{'vary'} ? $r->{reshdr}{'vary'}.', ' : '').'accept-encoding'
                 if ($r->{reshdr}{'vary'}||'') !~ /accept-encoding/i;
 
-            if (($r->{hdr}{'accept-encoding'}||'') =~ /gzip/) {
+            if ($hasbrotli && ($r->{hdr}{'accept-encoding'}||'') =~ /\bbr\b/) {
+                $r->{resbody} = FU::Util::brotli_compress(6, $r->{resbody});
+                $r->{reshdr}{'content-encoding'} = 'br';
+
+            } elsif ($hasgzip && ($r->{hdr}{'accept-encoding'}||'') =~ /\bgzip\b/) {
                 $r->{resbody} = FU::Util::gzip_compress(6, $r->{resbody});
                 $r->{reshdr}{'content-encoding'} = 'gzip';
             }
@@ -991,6 +999,9 @@ C<dlopen()>.
 =item * C<libdeflate.so> or C<libz-ng.so> or C<libz.so> - required for
 C<gzip_compress()> in L<FU::Util> and used for HTTP output compression.
 
+=item * C<libbrotlienc.so> - required for C<brotli_compress()> in L<FU::Util>
+and used for HTTP output compression.
+
 =back
 
 
diff --git a/FU.xs b/FU.xs
index b775f3c..3e7a0b2 100644
--- a/FU.xs
+++ b/FU.xs
@@ -126,6 +126,10 @@ void gzip_compress(IV level, SV *in)
   CODE:
     ST(0) = fugz_compress(aTHX_ level, in);
 
+void brotli_compress(IV level, SV *in)
+  CODE:
+    ST(0) = fubr_compress(aTHX_ level, in);
+
 void fdpass_send(int socket, int fd, SV *data)
   CODE:
     STRLEN buflen;
diff --git a/FU/Util.pm b/FU/Util.pm
index 4ff1c2b..27a0b62 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -13,7 +13,7 @@ our @EXPORT_OK = qw/
     utf8_decode uri_escape uri_unescape
     query_decode query_encode
     httpdate_format httpdate_parse
-    gzip_lib gzip_compress
+    gzip_lib gzip_compress brotli_compress
     fdpass_send fdpass_recv
 /;
 
@@ -401,6 +401,8 @@ I<zlib(-ng)> the level is capped at 9. 6 is typically used as a default.
 
 Throws an error if no suitable library was found.
 
+This function is B<NOT> safe to use from multiple threads!
+
 =back
 
 This module does not currently implement decompression. If you need that, or
@@ -409,6 +411,22 @@ L<Compress::Raw::Zlib> and L<Compress::Zlib> in the core Perl distribution and
 L<Gzip::Deflate> on CPAN.
 
 
+=head2 Brotli Compression
+
+Just a small wrapper around C<libbrotlienc.so>'s one-shot compression
+interface.
+
+=over
+
+=item brotli_compress($level, $data)
+
+Returns a byte string with the brotli-compressed version of C<$data> at the
+given quality C<$level> (between 0 and 11).
+
+Throws an error if C<libbrotlienc.so> could not be found or loaded.
+
+=back
+
 
 =head2 File Descriptor Passing
 
diff --git a/c/compress.c b/c/compress.c
index 9c4a1a9..20a59be 100644
--- a/c/compress.c
+++ b/c/compress.c
@@ -124,5 +124,38 @@ static SV *fugz_compress(pTHX_ IV level, SV *in) {
 
     if (fugz_imp == 1) return fugz_compress_ld(aTHX_ level, bytes, inlen);
     else return fugz_compress_zlib(aTHX_ level, bytes, inlen);
-    return &PL_sv_undef;
+}
+
+
+
+/* Brotli */
+
+typedef enum { BROTLI_MODE_GENERIC = 0, BROTLI_MODE_TEXT = 1, BROTLI_MODE_FONT = 2 } BrotliEncoderMode;
+
+static size_t (*BrotliEncoderMaxCompressedSize)(size_t);
+static int (*BrotliEncoderCompress)(int, int, BrotliEncoderMode, size_t, const char *, size_t *, char *);
+
+static SV *fubr_compress(pTHX_ IV level, SV *in) {
+    if (!BrotliEncoderCompress) {
+        void *handle;
+        if (!(handle = dlopen("libbrotlienc.so", RTLD_LAZY))
+         || !(BrotliEncoderMaxCompressedSize = dlsym(handle, "BrotliEncoderMaxCompressedSize"))
+         || !(BrotliEncoderCompress = dlsym(handle, "BrotliEncoderCompress")))
+            fu_confess("Unable to load libbrotlienc.so: %s", dlerror());
+    }
+    if (level < 0 || level > 11) fu_confess("Invalid compression level: %"IVdf, level);
+
+    STRLEN inlen;
+    const char *bytes = SvPVbyte(in, inlen);
+
+    size_t outlen = BrotliEncoderMaxCompressedSize(inlen);
+    /* "Result is only valid if quality is at least 2", so let's use a (more conservative?) fallback */
+    if (level < 2 && outlen < inlen + 256) outlen = inlen + 256;
+
+    SV *out = sv_2mortal(newSV(outlen));
+    SvPOK_only(out);
+    if (!BrotliEncoderCompress(level, 22, BROTLI_MODE_GENERIC, inlen, bytes, &outlen, SvPVX(out)))
+        fu_confess("Brotli compression failed");
+    SvCUR_set(out, outlen);
+    return out;
 }
diff --git a/t/compress.t b/t/compress.t
index ef17c77..ae236ae 100644
--- a/t/compress.t
+++ b/t/compress.t
@@ -1,18 +1,34 @@
 use v5.36;
 use Test::More;
-use FU::Util qw/gzip_lib gzip_compress/;
+use FU::Util qw/gzip_lib gzip_compress brotli_compress/;
 
 like gzip_lib, qr/^(|libdeflate|zlib-ng|zlib)$/, gzip_lib;
 
-plan skip_all => 'No suitable gzip library found' if !gzip_lib;
-plan skip_all => 'Compress::Zlib not found' if !eval { require Compress::Zlib };
+my $incompressible;
 
-my $incompressible = Compress::Zlib::memGzip(join '', map chr(rand 256), 0..93123);
+subtest 'gzip_compress', sub {
+    plan skip_all => 'No suitable gzip library found' if !gzip_lib;
+    plan skip_all => 'Compress::Zlib not found' if !eval { require Compress::Zlib };
 
-for my $str ('', 'Hello world!', 'x'x4096, $incompressible) {
-    is Compress::Zlib::memGunzip(gzip_compress(0, $str)), $str;
-    is Compress::Zlib::memGunzip(gzip_compress(12, $str)), $str;
-}
+    $incompressible = Compress::Zlib::memGzip(join '', map chr(rand 256), 0..93123);
+
+    for my $str ('', 'Hello world!', 'x'x4096, $incompressible) {
+        is Compress::Zlib::memGunzip(gzip_compress(0, $str)), $str;
+        is Compress::Zlib::memGunzip(gzip_compress(12, $str)), $str;
+    }
+};
+
+
+subtest 'brotli_compress', sub {
+    plan skip_all => 'libbrotlienc not available'
+        if !eval { brotli_compress 6, '' } && $@ =~ /Unable to load/;
+
+    ok length(brotli_compress 0, '') > 0;
+    ok length(brotli_compress 11, '') > 0;
+    # '0' does not disable compression...
+    ok length(brotli_compress 0, 'Hello world!'x100) < 200;
+    ok length(brotli_compress 11, 'Hello world!'x100) < 100;
+};
 
 
 done_testing;
@@ -29,6 +45,8 @@ for (0..1000) {
         local $_ = gzip_lib;
         $_ = gzip_compress(0, $str);
         $_ = gzip_compress(12, $str);
+        $_ = brotli_compress(0, $str);
+        $_ = brotli_compress(11, $str);
     }
 }
 diag count_sv;
@@ -44,4 +62,5 @@ my $data = <$F>;
 cmpthese -3, {
     memGzip => 'Compress::Zlib::memGzip($data)',
     gzip_compress => 'gzip_compress(6, $data)',
+    brotli_compress => 'brotli_compress(6, $data)',
 };

From 7c765f33bb5a66ad2fc7a6afd3b9e30fa2705321 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 19 Mar 2025 10:54:23 +0100
Subject: [PATCH 25/86] Version 0.4

---
 ChangeLog               | 15 +++++++++++++++
 FU.pm                   |  2 +-
 FU/DebugImpl.pm         |  2 +-
 FU/Log.pm               |  2 +-
 FU/MultipartFormData.pm |  2 +-
 FU/Pg.pm                |  2 +-
 FU/SQL.pm               |  2 +-
 FU/Util.pm              |  2 +-
 FU/Validate.pm          |  2 +-
 FU/XMLWriter.pm         |  2 +-
 FU/XS.pm                |  2 +-
 11 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 7b07d4d..f9c7266 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+0.4 - 2025-03-19
+	- FU::Validate: Support arrayref schemas
+	- FU::Validate: Rename 'values' option to 'elems'
+	- FU::Validate: Repurpose 'values' option to validate hash values
+	- FU::Validate: Merge nested 'elems', 'keys' and 'values' schemas during compile()
+	- FU::Validate: Rename 'scalar' to 'accept_scalar'
+	- FU::Validate: Add 'accept_array' option
+	- FU::Util: Add 'html_safe' option to json_format()
+	- FU::Util: Add gzip_compress() wrapper for libdeflate.so, zlib-ng.so or zlib.so
+	- FU::Util: Add brotli_compress() wrapper for libbrotlienc.so
+	- FU: Consistency fixes for fu->json() and fu->formdata()
+	- FU: Add fu->cookie() and fu->set_cookie()
+	- FU: Add support for brotli output compression
+	- FU: Use gzip_compress() for faster gzip output compression
+
 0.3 - 2025-03-10
 	- FU::Validate: Change API, ->validate() now returns data or throws error on failure
 	- FU::Validate: Rename 'rmwhitespace' to 'trim'
diff --git a/FU.pm b/FU.pm
index 40dd19e..cf13473 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 0.3;
+package FU 0.4;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 2d0ca50..08252ad 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 0.3;
+package FU::DebugImpl 0.4;
 use v5.36;
 use experimental 'for_list';
 use FU;
diff --git a/FU/Log.pm b/FU/Log.pm
index a0e78ff..612ecd1 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 0.3;
+package FU::Log 0.4;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index a777823..8c2542a 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 0.3;
+package FU::MultipartFormData 0.4;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 5d8c2a7..0d5579f 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 0.3;
+package FU::Pg 0.4;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 8f091e5..6a30332 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 0.3;
+package FU::SQL 0.4;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index 27a0b62..41d01f3 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 0.3;
+package FU::Util 0.4;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 112d8cc..bc63845 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 0.3;
+package FU::Validate 0.4;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 1d01e81..3fc8626 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 0.3;
+package FU::XMLWriter 0.4;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index 1109318..f9660c8 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 0.3;
+package FU::XS 0.4;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');

From 0925ae79a1b49ee3d63e542dd74acdb0c580c6c9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 19 Mar 2025 17:33:10 +0100
Subject: [PATCH 26/86] XS: Ensure created Perl strings are nul-terminated

Most of these are binary strings and shouldn't be interpreted as C
strings in the first place, but better be safe in case they are, anyway.
The lack of nul-termination of FU::Pg `$hex` strings was more likely to
be problematic.
---
 c/compress.c | 3 +++
 c/fdpass.c   | 1 +
 c/pgtypes.c  | 1 +
 3 files changed, 5 insertions(+)

diff --git a/c/compress.c b/c/compress.c
index 20a59be..da986a3 100644
--- a/c/compress.c
+++ b/c/compress.c
@@ -87,6 +87,7 @@ static SV *fugz_compress_ld(pTHX_ int level, const char *bytes, size_t inlen) {
     size_t len = libdeflate_gzip_compress(fugz_ld_ctx, bytes, inlen, SvPVX(out), outlen);
     if (!len) fu_confess("Libdeflate compression failed"); /* Shouldn't happen */
     SvCUR_set(out, len);
+    SvPVX(out)[len] = 0;
     return out;
 }
 
@@ -110,6 +111,7 @@ static SV *fugz_compress_zlib(pTHX_ int level, const char *bytes, size_t inlen)
     if ((r = deflate(&stream, 4)) != 1) fu_confess("Zlib compression failed (%d)", r);
 
     SvCUR_set(out, stream.total_out);
+    SvPVX(out)[stream.total_out] = 0;
     deflateEnd(&stream);
     return out;
 }
@@ -157,5 +159,6 @@ static SV *fubr_compress(pTHX_ IV level, SV *in) {
     if (!BrotliEncoderCompress(level, 22, BROTLI_MODE_GENERIC, inlen, bytes, &outlen, SvPVX(out)))
         fu_confess("Brotli compression failed");
     SvCUR_set(out, outlen);
+    SvPVX(out)[outlen] = 0;
     return out;
 }
diff --git a/c/fdpass.c b/c/fdpass.c
index 74a1229..ae4b141 100644
--- a/c/fdpass.c
+++ b/c/fdpass.c
@@ -71,6 +71,7 @@ static int fufdpass_recv(pTHX_ I32 ax, int socket, size_t len) {
     }
 
     SvCUR_set(buf, r);
+    SvPVX(buf)[r] = 0;
     ST(1) = buf;
     return 2;
 }
diff --git a/c/pgtypes.c b/c/pgtypes.c
index 6fb8835..b307cc0 100644
--- a/c/pgtypes.c
+++ b/c/pgtypes.c
@@ -166,6 +166,7 @@ RECVFN(hex) {
         *out++ = PL_hexdigit[(in[i] >> 4) & 0x0f];
         *out++ = PL_hexdigit[in[i] & 0x0f];
     }
+    *out = 0;
     SvCUR_set(r, len * 2);
     return r;
 }

From 90881924d4f8331b7d4c67446dc9b9024bc2cf26 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 21 Mar 2025 11:29:52 +0100
Subject: [PATCH 27/86] Doc fixes

---
 FU.pm                   | 32 ++++++++++++++++----------------
 FU/Log.pm               |  4 ++--
 FU/MultipartFormData.pm |  4 ++--
 FU/Pg.pm                | 12 ++++++------
 FU/SQL.pm               |  4 ++--
 FU/Util.pm              | 25 +++++++++++++------------
 FU/Validate.pm          | 34 +++++++++++++++++-----------------
 FU/XMLWriter.pm         |  8 ++++----
 8 files changed, 62 insertions(+), 61 deletions(-)

diff --git a/FU.pm b/FU.pm
index cf13473..02b1b8b 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1046,7 +1046,7 @@ Unless specifically mentioned otherwise, all methods and functions taking or
 returning strings deal with perl Unicode strings, not raw bytes.
 
 
-=head2 Framework Configuration
+=head1 Framework Configuration
 
 =over
 
@@ -1143,7 +1143,7 @@ restart loop.
 =back
 
 
-=head2 Handlers & Routing
+=head1 Handlers & Routing
 
 =over
 
@@ -1211,7 +1211,7 @@ for a certain error code, C<500> is used as fallback.
 =back
 
 
-=head2 The 'fu' Object
+=head1 The 'fu' Object
 
 While the C<FU::> namespace is used for global configuration and utility
 functions, the C<fu> object is intended for methods that deal with request
@@ -1261,7 +1261,7 @@ Convenient short-hand for C<< fu->db->Q(@args) >>.
 
 =back
 
-=head2 Request Information
+=head1 Request Information
 
 =over
 
@@ -1307,6 +1307,16 @@ Parse, validate and return the query parameter identified by C<$name> with the
 given L<FU::Validate> schema. Calls C<< fu->error(400) >> with a useful error
 message if validation fails.
 
+To fetch a query parameter that may have multiple values, use:
+
+  my $arrayref = fu->query(q => {accept_scalar => 1});
+
+  # OR:
+  my $first_value = fu->query(q => {accept_array => 'first'});
+
+  # OR:
+  my $last_value = fu->query(q => {accept_array => 'last'});
+
 =item fu->query($schema)
 
 =item fu->query($name1 => $schema1, $name2 => $schema2, ..)
@@ -1326,16 +1336,6 @@ To fetch all query paramaters as decoded by C<query_decode()>, use:
 
   my $data = fu->query({type=>'any'});
 
-To fetch a query parameter that may have multiple values, use:
-
-  my $arrayref = fu->query(q => {accept_scalar => 1});
-
-  # OR:
-  my $first_value = fu->query(q => {accept_array => 'first'});
-
-  # OR:
-  my $last_value = fu->query(q => {accept_array => 'last'});
-
 =item fu->cookie(...)
 
 Like C<< fu->query() >> but parses the C<Cookie> request header. Beware that,
@@ -1365,7 +1365,7 @@ objects.  Refer to L<FU::MultipartFormData> for more information.
 =back
 
 
-=head2 Generating Responses
+=head1 Response Generation
 
 =over
 
@@ -1509,7 +1509,7 @@ one of the following status codes or an alias:
 
 
 
-=head2 Running the Site
+=head1 Running the Site
 
 When your script is done setting L</"Framework Configuration"> and registering
 L</"Handlers & Routing">, it should call C<FU::run> to actually start serving
diff --git a/FU/Log.pm b/FU/Log.pm
index 612ecd1..24dafbf 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -89,7 +89,7 @@ interface either; the entire point of this module is that it only handles
 process-global logging. This module mainly exists for users of the L<FU>
 framework.
 
-=head2 Configuration
+=head1 Configuration
 
 =over
 
@@ -119,7 +119,7 @@ is then used instead. This is to avoid recursion.
 
 =back
 
-=head2 Exportable function
+=head1 Exportable function
 
 =over
 
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 8c2542a..8441740 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -121,7 +121,7 @@ C<multipart/form-data>, which is typically used to handle file uploads.
 The entire request body is assumed to be in memory as a Perl string, but this
 module makes an attempt to avoid any further copies of data values.
 
-=head2 Parsing
+=head1 Parsing
 
 =over
 
@@ -138,7 +138,7 @@ tried.
 
 =back
 
-=head2 Field Object
+=head1 Field Object
 
 Each field is parsed into a field object that supports the following methods:
 
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 0d5579f..83b2111 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -61,7 +61,7 @@ FU::Pg is a client module for PostgreSQL with a convenient high-level API and
 support for flexible and complex type conversions. This module interfaces
 directly with C<libpq>.
 
-=head2 Connection setup
+=head1 Connection setup
 
 =over
 
@@ -173,7 +173,7 @@ attempts to use C<$conn> throw an error.
 
 =back
 
-=head2 Querying
+=head1 Querying
 
 =over
 
@@ -420,7 +420,7 @@ Returns the respective configuration parameters.
 
 
 
-=head2 Transactions
+=head1 Transactions
 
 This module provides a convenient and safe API for I<scoped transactions> and
 I<subtransactions>. A new transaction can be started with C<< $conn->txn >>,
@@ -549,7 +549,7 @@ Just don't try to use transaction objects and manual transaction commands at
 the same time, that won't end well.
 
 
-=head2 Formats and Types
+=head1 Formats and Types
 
 The PostgreSQL wire protocol supports sending bind parameters and receiving
 query results in two different formats: text and binary. While the exact wire
@@ -752,7 +752,7 @@ I<TODO:> Methods to convert between the various formats.
 I<TODO:> Methods to query type info.
 
 
-=head2 COPY support
+=head1 COPY support
 
 You can use L<COPY
 statements|https://www.postgresql.org/docs/current/sql-copy.html> for efficient
@@ -816,7 +816,7 @@ silently discarded. An explicit C<close()> is recommended to catch errors.
 =back
 
 
-=head2 Errors
+=head1 Errors
 
 All methods can throw an exception on error. When possible, the error message
 is constructed using L<Carp>'s C<confess()>, including a full stack trace.
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 6a30332..15adbb1 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -124,7 +124,7 @@ changes, see the main L<FU> module for details.
 
 =head1 DESCRIPTION
 
-=head2 Compiling SQL
+=head1 Compiling SQL
 
 All functions listed under L</"Constructing SQL"> return an object that can be
 passed to other construction functions or compiled into SQL and bind
@@ -165,7 +165,7 @@ for details.
 
 =back
 
-=head2 Constructing SQL
+=head1 Constructing SQL
 
 All of the functions below return an object with a C<compile()> method. All
 functions are exported by default.
diff --git a/FU/Util.pm b/FU/Util.pm
index 41d01f3..7b8c6ad 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -95,10 +95,7 @@ __END__
 
 =head1 NAME
 
-FU::Util - Miscellaneous utility functions that really should have been part of
-a core Perl installation but aren't for some reason because the Perl community
-doesn't believe in the concept of a "batteries included" standard library.
-</rant>
+FU::Util - Miscellaneous Utility Functions
 
 =head1 EXPERIMENTAL
 
@@ -113,7 +110,11 @@ changes, see the main L<FU> module for details.
 
 =head1 DESCRIPTION
 
-=head2 Boolean Stuff
+A bunch of functions that are too small (or I'm too lazy) to split out into
+separate modules. Some of these functions really ought to be part of Perl core.
+
+
+=head1 Boolean Stuff
 
 Perl has had a builtin boolean type since version 5.36 and FU uses that where
 appropriate, but there's still a lot of older code out there using different
@@ -137,7 +138,7 @@ value for C<$val>, due to C<\0> and C<\1> being considered booleans.
 
 =back
 
-=head2 JSON parsing & formatting
+=head1 JSON Parsing & Formatting
 
 This module comes with a custom C-based JSON parser and formatter. These
 functions conform strictly to L<RFC-8259|https://tools.ietf.org/html/rfc8259>,
@@ -281,7 +282,7 @@ functions, L<JSON::PP> and L<Cpanel::JSON::XS> are perfectly fine alternatives.
 L<JSON::SIMD> and L<JSON::Tiny> also look like good and maintained candidates.)
 
 
-=head2 URI-Related Functions
+=head1 URI-Related Functions
 
 While URIs are capable of encoding arbitrary binary data, the functions below
 assume you're only dealing with text. This makes them more robust against weird
@@ -343,7 +344,7 @@ then encoded.
 =back
 
 
-=head2 HTTP Date Formatting
+=head1 HTTP Date Formatting
 
 The HTTP date format is utter garbage, but with the right tools it doesn't
 require I<too> much code to work with.
@@ -368,7 +369,7 @@ This will not happen if your local timezone is UTC.
 =back
 
 
-=head2 Gzip Compression
+=head1 Gzip Compression
 
 Gzip compression can be done with a few different libraries. The canonical one
 is I<zlib>, which is old and not well optimized for modern systems. There's
@@ -408,10 +409,10 @@ This function is B<NOT> safe to use from multiple threads!
 This module does not currently implement decompression. If you need that, or
 streaming, or other functionality not provided here, there's
 L<Compress::Raw::Zlib> and L<Compress::Zlib> in the core Perl distribution and
-L<Gzip::Deflate> on CPAN.
+L<Gzip::Faster>, L<Gzip::Zopfli> and L<Gzip::Libdeflate> on CPAN.
 
 
-=head2 Brotli Compression
+=head1 Brotli Compression
 
 Just a small wrapper around C<libbrotlienc.so>'s one-shot compression
 interface.
@@ -428,7 +429,7 @@ Throws an error if C<libbrotlienc.so> could not be found or loaded.
 =back
 
 
-=head2 File Descriptor Passing
+=head1 File Descriptor Passing
 
 UNIX sockets (see L<IO::Socket::UNIX>) have the fancy property of letting you
 send file descriptors over them, allowing you to pass, for example, a socket
diff --git a/FU/Validate.pm b/FU/Validate.pm
index bc63845..da29a2e 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -419,15 +419,15 @@ parameters within Perl code. In fact, the correct answer to "how do I validate
 function parameters?" is "don't, document your assumptions instead".
 
 
-=head2 Validation API
+=head1 Validation API
 
 To validate some input, you first need a schema. A schema can be compiled as
 follows:
 
   my $validator = FU::Validate->compile($schema, $validations);
 
-C<$schema> is the schema that describes the data to be validated (see L</SCHEMA
-DEFINITION> below) and C<$validations> is an optional hashref containing
+C<$schema> is the schema that describes the data to be validated (see L</Schema
+Definition> below) and C<$validations> is an optional hashref containing
 L<custom validations|/Custom validations> that C<$schema> can refer to.  An
 error is thrown if the C<$validations> or C<$schema> are invalid.
 
@@ -443,10 +443,10 @@ An error is thrown if the input does not validate. The error object is a
 C<FU::Validate::err>-blessed hashref containing at least one key:
 I<validation>, which indicates the name of the validation that failed.
 Additional keys with more detailed information may be present, depending on the
-validation. These are documented in L</SCHEMA DEFINITION> below.
+validation. These are documented in L</Schema Definition> below.
 
 
-=head1 SCHEMA DEFINITION
+=head1 Schema Definition
 
 A schema is an arrayref or hashref, where each key is the name of a built-in
 option or of a validation to be performed and the values are the arguments to
@@ -473,7 +473,7 @@ Or to use the same validation multiple times:
 
   [ regex => qr/^a/, regex => qr/z$/ ]
 
-=head2 Built-in options
+=head1 Built-in options
 
 =over
 
@@ -730,7 +730,7 @@ reporting.
 
 =back
 
-=head2 Standard validations
+=head1 Standard validations
 
 Standard validations are provided by the module. It is possible to override,
 re-implement and supplement these with custom validations. Internally, these
@@ -855,7 +855,7 @@ given the year and month.
 =back
 
 
-=head2 Custom validations
+=head1 Custom validations
 
 Custom validations can be passed to C<compile()> as the C<$validations> hashref
 argument.  A custom validation is, in simple terms, either a schema or a
@@ -885,7 +885,7 @@ schema that contains the I<func> built-in option to do the actual validation.
   my $schema = { prefix => 'Hello, ' };
   my $result = FU::Validate->compile($schema, $validations)->validate('Hello, World!');
 
-=head3 Custom validations and built-in options
+=head2 Custom validations and built-in options
 
 Custom validations can also set built-in options, but the semantics differ a
 little depending on the option. First, be aware that many of the built-in
@@ -909,13 +909,13 @@ error:
 
 The I<func> option is validated separately for each custom validation.
 
-Multiple I<keys> and I<elems> validations are merged into a single validation.
-So if you have multiple custom validations that set the I<elems> option, a
-single combined schema is created that validates all array elements. The same
-applies to I<keys>: if the same key is listed in multiple custom validations,
-then the key must conform to all schemas. With respect to the I<unknown>
-option, a key that is mentioned in any of the I<keys> options is considered
-"known".
+Multiple I<keys>, I<values> and I<elems> validations are merged into a single
+validation.  So if you have multiple custom validations that set the I<elems>
+option, a single combined schema is created that validates all array elements.
+The same applies to I<keys>: if the same key is listed in multiple custom
+validations, then the key must conform to all schemas. With respect to the
+I<unknown> option, a key that is mentioned in any of the I<keys> options is
+considered "known".
 
 All other built-in options follow inheritance semantics: These options can be
 set in a custom validation, and they are inherited by the top-level schema.  If
@@ -924,7 +924,7 @@ inherited. The top-level schema can always override options set by custom
 validations.
 
 
-=head3 Global custom validations
+=head2 Global custom validations
 
 Instead of passing a C<$validations> argument every time you call C<compile()>,
 you can also add custom validations to the global list of built-in validations:
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 3fc8626..35e0702 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -138,7 +138,7 @@ other XML writing modules on CPAN that I tried, but whether this approach is
 faster than typical templating solutions... I've no idea. Check out
 L<FU::Benchmarks> for some benchmarks.
 
-=head2 Top-level functions
+=head1 Top-level functions
 
 These functions all return a byte string with (UTF-8) encoded XML.
 
@@ -170,7 +170,7 @@ passed to the C<tag_()> call for the top-level C<< <html> >> element.
 
 =back
 
-=head2 Output functions
+=head1 Output functions
 
 =over
 
@@ -252,7 +252,7 @@ provided it defaults to C<undef>.
 
 =back
 
-=head2 Utility function
+=head1 Utility function
 
 =over
 
@@ -263,7 +263,7 @@ and C<"> are replaced with their XML entity.
 
 =back
 
-=head2 Import options
+=head1 Import options
 
 All of the functions mentioned in this document can be imported individually.
 There are also two import groups:

From 17584f2b8ca71e2aeb8899e60f475df78e1008bc Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 22 Mar 2025 14:58:38 +0100
Subject: [PATCH 28/86] FU: Fix DB reconnect + not setting debug_info()

---
 FU.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index 02b1b8b..c178072 100644
--- a/FU.pm
+++ b/FU.pm
@@ -123,6 +123,7 @@ sub query_trace($st,@) {
 sub _connect_db {
     $DB = ref $INIT_DB eq 'CODE' ? $INIT_DB->() : FU::Pg->connect($INIT_DB);
     $DB->query_trace(\&query_trace);
+    $DB
 }
 sub init_db($info) {
     require FU::Pg;
@@ -214,7 +215,7 @@ sub _monitor {
 }
 
 
-our $debug_info = [];
+our $debug_info = {};
 sub debug_info($path, $storage=undef, $history=100) {
     $debug_info = { path => $path, storage => $storage, history => $history }
 }

From 9e1be5bc719c8fc84ba2be05d576f72d25353cc2 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 22 Mar 2025 15:10:59 +0100
Subject: [PATCH 29/86] FU: Log errors thrown from error handler

---
 FU.pm | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/FU.pm b/FU.pm
index c178072..81bcf94 100644
--- a/FU.pm
+++ b/FU.pm
@@ -380,10 +380,11 @@ sub _do_req($c) {
         my($code, $msg) = ref $err eq 'FU::err' ? $err->@* : (500, $err);
         fu->reset;
         fu->status($code);
-        eval {
-            ($onerr{$code} || $onerr{500})->($code, $msg);
-            1;
-        } || _err_500();
+        my $ok = eval { ($onerr{$code} || $onerr{500})->($code, $msg) };
+        if (!$ok && !_is_done($@)) {
+            _log_err $@;
+            _err_500();
+        }
     }
 
     $REQ->{trace_end} = time;

From 2f50736782894519a81136691363c3f1227efba5 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 24 Mar 2025 11:07:36 +0100
Subject: [PATCH 30/86] fdpass_recv: Set O_CLOEXEC on received fds

Turns out this is necessary even if the fd is going to be passed through
exec() soon, because the supervisor might receive multiple fds before
spawning another process, in which case all of them are going to be
passed to the new process instead of just one.
---
 FU.pm      | 13 +++++--------
 FU/Util.pm |  7 +++----
 c/fdpass.c |  2 +-
 3 files changed, 9 insertions(+), 13 deletions(-)

diff --git a/FU.pm b/FU.pm
index 81bcf94..1451c74 100644
--- a/FU.pm
+++ b/FU.pm
@@ -475,26 +475,23 @@ sub _supervisor($c) {
         }
 
         # Don't bother spawning more than 1 at a time while in error state
-        my $spawn = !$err ? $c->{proc} - keys %childs : (grep $_ == 1, values %childs) ? 0 : 1;
+        my $spawn = !$err ? $c->{proc} - keys %childs : !@client_fd && (grep $_ == 1, values %childs) ? 0 : 1;
         for (1..$spawn) {
-            my $client = shift @client_fd;
+            my $client = @client_fd ? IO::Socket->new_from_fd(shift(@client_fd), 'r') : undef;
             my $pid = fork;
             die $! if !defined $pid;
             if (!$pid) { # child
                 $SIG{CHLD} = $SIG{HUP} = $SIG{INT} = $SIG{TERM} = undef;
+                # In error state, wait with loading the script until we've received a request.
+                # Otherwise we'll end up in an infinite spawning loop if the script doesn't start properly.
+                $client = $c->{listen_sock}->accept() or die $! if !$client && $err;
                 if ($client) {
-                    $ENV{FU_CLIENT_FD} = $client;
-                } elsif ($err) {
-                    # In error state, wait with loading the script until we've received a request.
-                    # Otherwise we'll end up in an infinite spawning loop if the script doesn't start properly.
-                    $client = $c->{listen_sock}->accept() or die $!;
                     fcntl $client, Fcntl::F_SETFD, 0;
                     $ENV{FU_CLIENT_FD} = fileno $client;
                 }
                 exec $^X, (map "-I$_", @INC), $0;
                 exit 1;
             }
-            $client && IO::Socket->new_from_fd($client, 'r'); # close() the fd if we have one
             $childs{$pid} = 1;
         }
 
diff --git a/FU/Util.pm b/FU/Util.pm
index 7b8c6ad..e823d79 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -458,10 +458,9 @@ Like regular socket I/O, a single C<fdpass_send()> message may be split across
 multiple C<fdpass_recv()> calls; in that case the C<$fd> is only received on
 the first call.
 
-Don't use this function if the sender may include multiple file descriptors in
-a single message, weird things can happen. File descriptors received this way
-do not have the C<CLOEXEC> flag and will thus survive a call to C<exec()>.
-Refer to L<this wonderful
+The C<O_CLOEXEC> flag is set on received file descriptors. Don't use this
+function if the sender may include multiple file descriptors in a single
+message, weird things can happen. Refer to L<this wonderful
 discussion|https://gist.github.com/kentonv/bc7592af98c68ba2738f4436920868dc>
 for more weirdness and edge cases.
 
diff --git a/c/fdpass.c b/c/fdpass.c
index ae4b141..b54df7d 100644
--- a/c/fdpass.c
+++ b/c/fdpass.c
@@ -53,7 +53,7 @@ static int fufdpass_recv(pTHX_ I32 ax, int socket, size_t len) {
     msg.msg_controllen = sizeof(cmsgbuf.buf);
     msg.msg_flags = 0;
 
-    ssize_t r = recvmsg(socket, &msg, 0);
+    ssize_t r = recvmsg(socket, &msg, MSG_CMSG_CLOEXEC);
     if (r < 0) {
         ST(0) = &PL_sv_undef;
         ST(1) = &PL_sv_undef;

From 13661b46f900b0ef7dcca6404c9b9fd276624f40 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 30 Mar 2025 13:51:14 +0200
Subject: [PATCH 31/86] Validate: Normalize num/int/uint to Perl numeric types
 + add 64bit limit to int/uint

Normalization may be undone by later validations, but this should work
in most cases.
---
 FU/Validate.pm | 15 ++++++++-------
 t/validate.t   |  4 ++--
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index da29a2e..7352053 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -81,9 +81,9 @@ our %default_validations = (
     bool      => { type => 'any', func => sub { my $r = to_bool $_[0]; return {} if !defined $r; $_[0] = $r; 1 } },
     anybool   => { type => 'any', default => false, func => sub { $_[0] = $_[0] ? true : false; 1 } },
 
-    num   => { _reg $re_num },
-    int   => { _reg $re_int }, # implies num
-    uint  => { _reg $re_uint }, # implies num
+    num   => [ _reg($re_num), func => sub { $_[0] = $_[0]*1; 1 } ],
+    int   => [ _reg($re_int), func => sub { return { message => 'integer out of range' } if $_[0] < -9223372036854775808 || $_[0] > 9223372036854775807; $_[0] = int $_[0]; 1 } ],
+    uint  => [ _reg($re_uint), func => sub { return { message => 'integer out of range' } if $_[0] > 18446744073709551615; $_[0] = int $_[0]; 1 } ],
     min   => sub($min) { +{ num => 1, func => sub { $_[0] >= $min ? 1 : { expected => $min, got => $_[0] } } } },
     max   => sub($max) { +{ num => 1, func => sub { $_[0] <= $max ? 1 : { expected => $max, got => $_[0] } } } },
     range => sub { [ min => $_[0][0], max => $_[0][1] ] },
@@ -780,17 +780,18 @@ Require the input to be a boolean type as per C<to_bool()> in L<FU::Util>.
 Implies C<< type => 'scalar' >>. Require the input to be a number formatted
 using the format permitted by JSON. Note that this is slightly more restrictive
 from Perl's number formatting, in that 'NaN', 'Inf' and thousand separators are
-not permitted.
+not permitted. The value is normalized to a Perl integer or floating point
+value, which means precision for large numbers may be lost.
 
 =item int => 1
 
-Implies C<< type => 'scalar' >>. Require the input to be an (arbitrarily large)
+Implies C<< type => 'scalar' >>. Require the input to be an (at most) 64-bit
 integer.
 
 =item uint => 1
 
-Implies C<< type => 'scalar' >>. Require the input to be an (arbitrarily large)
-positive integer.
+Implies C<< type => 'scalar' >>. Require the input to be an (at most) 64-bit
+unsigned integer.
 
 =item min => $num
 
diff --git a/t/validate.t b/t/validate.t
index 626dbbd..a29dc4c 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -209,8 +209,8 @@ t { num => 1 }, '1', '1';
 f { num => 1 }, '1.1.', nerr '1.1.';
 f { num => 1 }, '1.-1', nerr '1.-1';
 f { num => 1 }, '.1', nerr '.1';
-t { num => 1 }, '0.1e5', '0.1e5';
-t { num => 1 }, '0.1e+5', '0.1e+5';
+t { num => 1 }, '0.1e5', 10000;
+t { num => 1 }, '0.1e+5', 10000;
 f { num => 1 }, '0.1e5.1', nerr '0.1e5.1';
 t { int => 1 }, 0, 0;
 t { int => 1 }, -123, -123;

From 3bf98e4d8fd5dbd4173a50b826eec608fe028ec8 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 3 Apr 2025 15:58:38 +0200
Subject: [PATCH 32/86] FU: Fix fu->reset also resetting cookies

---
 FU.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/FU.pm b/FU.pm
index 1451c74..d59f821 100644
--- a/FU.pm
+++ b/FU.pm
@@ -722,6 +722,7 @@ sub reset {
     $FU::REQ->{reshdr} = {
         'content-type', 'text/html',
     };
+    delete $FU::REQ->{rescookie};
 }
 
 

From b3281924d102f01d203df4cefb58a0c17d09990f Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 7 Apr 2025 13:45:24 +0200
Subject: [PATCH 33/86] Pg: Add escape_literal() and escape_identifier()

Didn't expect I'd ever need these, but they're useful for generating SQL
scripts.
---
 FU.xs         | 18 ++++++++++++++++++
 FU/Pg.pm      | 11 +++++++++++
 c/libpq.h     |  2 ++
 t/pgconnect.t | 10 ++++++++++
 4 files changed, 41 insertions(+)

diff --git a/FU.xs b/FU.xs
index 3e7a0b2..ec12a7c 100644
--- a/FU.xs
+++ b/FU.xs
@@ -216,6 +216,24 @@ void status(fupg_conn *c)
   CODE:
     ST(0) = sv_2mortal(newSVpv(fupg_conn_status(c), 0));
 
+void escape_literal(fupg_conn *c, SV *v)
+  CODE:
+    STRLEN len;
+    const char *str = SvPVutf8(v, len);
+    char *r = PQescapeLiteral(c->conn, str, len);
+    if (!r) fupg_conn_croak(c, "escapeLiteral");
+    ST(0) = newSVpvn_flags(r, strlen(r), SVf_UTF8|SVs_TEMP);
+    PQfreemem(r);
+
+void escape_identifier(fupg_conn *c, SV *v)
+  CODE:
+    STRLEN len;
+    const char *str = SvPVutf8(v, len);
+    char *r = PQescapeIdentifier(c->conn, str, len);
+    if (!r) fupg_conn_croak(c, "escapeIdentifier");
+    ST(0) = newSVpvn_flags(r, strlen(r), SVf_UTF8|SVs_TEMP);
+    PQfreemem(r);
+
 void cache(fupg_conn *x, ...)
   ALIAS:
     FU::Pg::conn::text_params  = FUPG_TEXT_PARAMS
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 83b2111..e43776e 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -123,6 +123,17 @@ Connection is dead or otherwise unusable.
 
 =back
 
+=item $conn->escape_literal($str)
+
+Return an escaped version of C<$str> suitable for use as a string literal in an
+SQL statement. You'll rarely need this, it's often better to pass data as bind
+parameters instead.
+
+=item $conn->escape_identifier($str)
+
+Return an escaped version of C<$str> suitable for use as an identifier (name of
+a table, column, function, etc) in an SQL statement.
+
 =item $conn->cache($enable)
 
 =item $conn->text_params($enable)
diff --git a/c/libpq.h b/c/libpq.h
index 94d817e..d47dd16 100644
--- a/c/libpq.h
+++ b/c/libpq.h
@@ -43,6 +43,8 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
     X(PQconnectdb, PGconn *, const char *) \
     X(PQenterPipelineMode, int, PGconn *) \
     X(PQerrorMessage, char *, const PGconn *) \
+    X(PQescapeIdentifier, char *, PGconn *, const char *, size_t) \
+    X(PQescapeLiteral, char *, PGconn *, const char *, size_t) \
     X(PQexec, PGresult *, PGconn *, const char *) \
     X(PQexecParams, PGresult *, PGconn *, const char *, int, const Oid *, const char * const *, const int *, const int *, int) \
     X(PQexecPrepared, PGresult *, PGconn *, const char *, int, const char * const *, const int *, const int *, int) \
diff --git a/t/pgconnect.t b/t/pgconnect.t
index b7d1996..d2dfbf5 100644
--- a/t/pgconnect.t
+++ b/t/pgconnect.t
@@ -353,6 +353,16 @@ subtest 'txn', sub {
     is_deeply $st->val, [1,3], 'not deep copy';
 }
 
+
+{
+    # Exact format returned by escape_literal() can differ between Postgres versions and configurations.
+    my $x = q{"' \" \\};
+    is $conn->q('SELECT '.$conn->escape_literal($x))->val, $x;
+
+    # Format can also change, but unsure how to test this otherwise.
+    is $conn->escape_identifier('hel\l"o'), '"hel\l""o"';
+}
+
 subtest 'Prepared statement cache', sub {
     my $txn = $conn->cache_size(2)->txn->cache;
     my sub numexec($sql) {

From e7a9f165deffdf7075014f7c37da9f0f65b0f218 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 7 Apr 2025 14:29:21 +0200
Subject: [PATCH 34/86] Fix use of SvPVXtrue() where SvTRUEx() was intended

---
 FU.xs         | 2 +-
 c/jsonfmt.c   | 8 ++++----
 c/jsonparse.c | 2 +-
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/FU.xs b/FU.xs
index ec12a7c..ef8d60b 100644
--- a/FU.xs
+++ b/FU.xs
@@ -18,7 +18,7 @@
 #define av_push_simple av_push
 #endif
 #ifndef BOOL_INTERNALS_sv_isbool_true
-#define BOOL_INTERNALS_sv_isbool_true(x) SvPVXtrue(x)
+#define BOOL_INTERNALS_sv_isbool_true(x) SvTRUEx(x)
 #endif
 
 /* Disable key/value struct packing in khashl, so we can safely take a pointer
diff --git a/c/jsonfmt.c b/c/jsonfmt.c
index f605e8d..fff3e4f 100644
--- a/c/jsonfmt.c
+++ b/c/jsonfmt.c
@@ -289,10 +289,10 @@ static SV *fujson_fmt_xs(pTHX_ I32 ax, I32 argc, SV *val) {
         r = ST(i);
         i++;
 
-        if (strcmp(arg, "canonical") == 0) ctx.canon = SvPVXtrue(r);
-        else if (strcmp(arg, "pretty") == 0) ctx.pretty = SvPVXtrue(r) ? 0 : INT_MIN;
-        else if (strcmp(arg, "html_safe") == 0) ctx.htmlsafe = !!SvPVXtrue(r);
-        else if (strcmp(arg, "utf8") == 0) encutf8 = SvPVXtrue(r);
+        if (strcmp(arg, "canonical") == 0) ctx.canon = SvTRUEx(r);
+        else if (strcmp(arg, "pretty") == 0) ctx.pretty = SvTRUEx(r) ? 0 : INT_MIN;
+        else if (strcmp(arg, "html_safe") == 0) ctx.htmlsafe = !!SvTRUEx(r);
+        else if (strcmp(arg, "utf8") == 0) encutf8 = SvTRUEx(r);
         else if (strcmp(arg, "max_size") == 0) out.maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
         else croak("Unknown flag: '%s'", arg);
diff --git a/c/jsonparse.c b/c/jsonparse.c
index 6c742be..d24b1f9 100644
--- a/c/jsonparse.c
+++ b/c/jsonparse.c
@@ -272,7 +272,7 @@ static SV *fujson_parse_xs(pTHX_ I32 ax, I32 argc, SV *val) {
         r = ST(i);
         i++;
 
-        if (strcmp(arg, "utf8") == 0) decutf8 = SvPVXtrue(r);
+        if (strcmp(arg, "utf8") == 0) decutf8 = SvTRUEx(r);
         else if (strcmp(arg, "max_size") == 0) maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
         else if (strcmp(arg, "offset") == 0) offset = r;

From 196b1cc3cee1fc90c4da43621af2068721722df7 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 7 Apr 2025 16:41:29 +0200
Subject: [PATCH 35/86] FU: Use CLOCK_MONOTONIC for timing

---
 FU.pm           | 8 ++++----
 FU/DebugImpl.pm | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/FU.pm b/FU.pm
index d59f821..c21bb7f 100644
--- a/FU.pm
+++ b/FU.pm
@@ -3,7 +3,7 @@ use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
 use POSIX ();
-use Time::HiRes 'time';
+use Time::HiRes 'clock_gettime', 'CLOCK_MONOTONIC';
 use FU::Log 'log_write';
 use FU::Util;
 use FU::Validate;
@@ -320,7 +320,7 @@ sub _log_err($e) {
 }
 
 sub _do_req($c) {
-    local $REQ = { hdr => {}, trace_start => time, trace_id => sprintf('%010x%08x%04x', int time, $$, int rand 1<<16) };
+    local $REQ = { hdr => {}, trace_start => clock_gettime(CLOCK_MONOTONIC), trace_id => sprintf('%010x%08x%04x', int time, $$, int rand 1<<16) };
     local $fu = bless {}, 'FU::obj';
 
     $REQ->{ip} = $c->{client_sock} isa 'IO::Socket::INET' ? $c->{client_sock}->peerhost : '127.0.0.1';
@@ -328,7 +328,7 @@ sub _do_req($c) {
 
     my $ok = eval {
         _read_req $c;
-        $REQ->{trace_start} = time;
+        $REQ->{trace_start} = clock_gettime(CLOCK_MONOTONIC);
 
         my $path = fu->path;
         my $method = fu->method eq 'HEAD' ? 'GET' : fu->method;
@@ -387,7 +387,7 @@ sub _do_req($c) {
         }
     }
 
-    $REQ->{trace_end} = time;
+    $REQ->{trace_end} = clock_gettime(CLOCK_MONOTONIC);
     fu->_flush($c->{fcgi_obj} || $c->{client_sock});
 
     if (debug && $REQ->{trace_id} && $debug_info->{history} && $debug_info->{storage}) {
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 08252ad..eda2939 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -342,7 +342,7 @@ sub save {
         return;
     };
     my $line = sprintf "%d %f %s %s %s\n",
-        time, time - $FU::REQ->{trace_start}, $FU::REQ->{status},
+        time, $FU::REQ->{trace_end} - $FU::REQ->{trace_start}, $FU::REQ->{status},
         fu->method, fu->path.(fu->query?'?'.fu->query:'');
     utf8::encode($line);
     print $fh $line;

From efa63ca96abdd6582a869cfc5cd8d047647d7ca9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 7 Apr 2025 16:52:45 +0200
Subject: [PATCH 36/86] Pg: Discard temporary hash keys earlier in $st->kv?
 methods

Saves some memory for large query results, didn't notice much of a
performance difference.
---
 c/pgst.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/c/pgst.c b/c/pgst.c
index 1e01392..e5a2a3d 100644
--- a/c/pgst.c
+++ b/c/pgst.c
@@ -460,9 +460,11 @@ static SV *fupg_st_kvv(pTHX_ fupg_st *st) {
     HV *hv = newHV();
     SV *sv = sv_2mortal(newRV_noinc((SV *)hv));
     for (i=0; i<nrows; i++) {
+        SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kvv() query results", SvPV_nolen(key));
         hv_store_ent(hv, key, st->nfields == 1 ? &PL_sv_yes : fupg_st_getval(aTHX_ st, i, 1), 0);
+        FREETMPS;
     }
     return sv;
 }
@@ -474,10 +476,12 @@ static SV *fupg_st_kva(pTHX_ fupg_st *st) {
     HV *hv = newHV();
     SV *sv = sv_2mortal(newRV_noinc((SV *)hv));
     for (i=0; i<nrows; i++) {
+        SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kva() query results", SvPV_nolen(key));
         AV *row = st->nfields == 1 ? newAV() : newAV_alloc_x(st->nfields-1);
         hv_store_ent(hv, key, newRV_noinc((SV *)row), 0);
+        FREETMPS;
         for (j=1; j<st->nfields; j++)
             av_push_simple(row, fupg_st_getval(aTHX_ st, i, j));
     }
@@ -492,10 +496,12 @@ static SV *fupg_st_kvh(pTHX_ fupg_st *st) {
     HV *hv = newHV();
     SV *sv = sv_2mortal(newRV_noinc((SV *)hv));
     for (i=0; i<nrows; i++) {
+        SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kvh() query results", SvPV_nolen(key));
         HV *row = newHV();
         hv_store_ent(hv, key, newRV_noinc((SV *)row), 0);
+        FREETMPS;
         for (j=1; j<st->nfields; j++) {
             const char *key = PQfname(st->result, j);
             hv_store(row, key, -strlen(key), fupg_st_getval(aTHX_ st, i, j), 0);

From 8b807e6dcfff16d917479aa8482dae0bac4a69b0 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 19 Apr 2025 12:36:19 +0200
Subject: [PATCH 37/86] Validate: Add empty() and coerce() methods

Implementing the undocumented coerce_for_json() method from TUWF and
elm_empty() from VNDB.
---
 FU/Validate.pm    | 100 +++++++++++++++++++++++++++++++++++++++++++---
 t/validate-util.t |  36 +++++++++++++++++
 2 files changed, 131 insertions(+), 5 deletions(-)
 create mode 100644 t/validate-util.t

diff --git a/FU/Validate.pm b/FU/Validate.pm
index 7352053..dd767c7 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -61,6 +61,10 @@ our $re_weburl    = qr/^https?:\/\/$re_domain(?::[1-9][0-9]{0,5})?(?:\/[^\s<>"]*
 our $re_date      = qr/^(?:19[0-9][0-9]|20[0-9][0-9])-(?:0[1-9]|1[0-2])-(?:0[1-9]|[12][0-9]|3[01])$/;
 
 
+# There's a special '_scalartype' option used for coerce() and empty(), with the following values:
+#   0/undef/missing: string, 1:num, 2:int, 3:bool
+# The highest number, i.e. most restrictive type, is chosen when multiple validations exist.
+
 our %default_validations = (
     regex => sub($reg) {
         # Error objects should be plain data structures so that they can easily
@@ -78,12 +82,12 @@ our %default_validations = (
     maxlength => sub($v) { _length $v, undef, $v },
     length    => sub($v) { _length $v, ref $v eq 'ARRAY' ? @$v : ($v, $v) },
 
-    bool      => { type => 'any', func => sub { my $r = to_bool $_[0]; return {} if !defined $r; $_[0] = $r; 1 } },
-    anybool   => { type => 'any', default => false, func => sub { $_[0] = $_[0] ? true : false; 1 } },
+    bool      => { _scalartype => 3, type => 'any', func => sub { my $r = to_bool $_[0]; return {} if !defined $r; $_[0] = $r; 1 } },
+    anybool   => { _scalartype => 3, type => 'any', default => false, func => sub { $_[0] = $_[0] ? true : false; 1 } },
 
-    num   => [ _reg($re_num), func => sub { $_[0] = $_[0]*1; 1 } ],
-    int   => [ _reg($re_int), func => sub { return { message => 'integer out of range' } if $_[0] < -9223372036854775808 || $_[0] > 9223372036854775807; $_[0] = int $_[0]; 1 } ],
-    uint  => [ _reg($re_uint), func => sub { return { message => 'integer out of range' } if $_[0] > 18446744073709551615; $_[0] = int $_[0]; 1 } ],
+    num   => [ _scalartype => 1, _reg($re_num), func => sub { $_[0] = $_[0]*1; 1 } ],
+    int   => [ _scalartype => 2, _reg($re_int), func => sub { return { message => 'integer out of range' } if $_[0] < -9223372036854775808 || $_[0] > 9223372036854775807; $_[0] = int $_[0]; 1 } ],
+    uint  => [ _scalartype => 2, _reg($re_uint), func => sub { return { message => 'integer out of range' } if $_[0] > 18446744073709551615; $_[0] = int $_[0]; 1 } ],
     min   => sub($min) { +{ num => 1, func => sub { $_[0] >= $min ? 1 : { expected => $min, got => $_[0] } } } },
     max   => sub($max) { +{ num => 1, func => sub { $_[0] <= $max ? 1 : { expected => $max, got => $_[0] } } } },
     range => sub { [ min => $_[0][0], max => $_[0][1] ] },
@@ -152,6 +156,11 @@ sub _compile($schema, $custom, $rec, $top, $validations=$top->{validations}) {
             next;
         }
 
+        if ($name eq '_scalartype') {
+            $top->{_scalartype} = $val if ($top->{_scalartype}||0) < $val;
+            next;
+        }
+
         if ($builtin{$name}) {
             confess "Invalid value for 'missing': $val" if $name eq 'missing' && !$missing_vals{$val};
             confess "Invalid value for 'unknown': $val" if $name eq 'unknown' && !$unknown_vals{$val};
@@ -353,6 +362,37 @@ sub validate($c, $input) {
 }
 
 
+sub coerce {
+    my $c = $_[0];
+    my %opt = @_[2..$#_];
+    if (!defined $_[1]) {
+        $_[1] = undef;
+    } elsif ($c->{_scalartype}) {
+        $_[1] = $c->{_scalartype} == 3 ? !!$_[1] : $c->{_scalartype} == 2 ? int $_[1] : $_[1]+0;
+    } elsif (!$c->{type} || $c->{type} eq 'scalar') {
+        $_[1] = "$_[1]";
+    } elsif ($c->{type} eq 'array' && $c->{elems} && ref $_[1] eq 'ARRAY') {
+        coerce($c->{elems}, $_, %opt) for $_[1]->@*;
+    } elsif ($c->{type} eq 'hash' && $c->{keys} && ref $_[1] eq 'HASH') {
+        $opt{unknown} ||= $c->{unknown};
+        delete @{$_[1]}{ grep !$c->{keys}{$_}, keys $_[1]->%* }
+            if $opt{unknown} && $opt{unknown} ne 'pass';
+        $_[1]{$_} = exists $_[1]{$_} ? coerce($c->{keys}{$_}, $_[1]{$_}, %opt) : empty($c->{keys}{$_})
+            for keys $c->{keys}->%*;
+    }
+    return $_[1];
+}
+
+
+sub empty($c) {
+    return ref $c->{default} eq 'CODE' ? $c->{default}->(undef) : $c->{default} if exists $c->{default};
+    return [] if $c->{type} && $c->{type} eq 'array';
+    return $c->{keys} ? +{ map +($_, empty($c->{keys}{$_})), keys $c->{keys}->%* } : {} if $c->{type} && $c->{type} eq 'hash';
+    return undef if $c->{type} && $c->{type} eq 'any';
+    # Only scalar types remain
+    return !$c->{_scalartype} ? '' : $c->{_scalartype} == 3 ? !1 : 0;
+}
+
 
 
 package FU::Validate::err;
@@ -445,6 +485,56 @@ I<validation>, which indicates the name of the validation that failed.
 Additional keys with more detailed information may be present, depending on the
 validation. These are documented in L</Schema Definition> below.
 
+Additional utility methods:
+
+=over
+
+=item $validator->empty
+
+Returns an "empty" value that roughly follows the data structure described by
+the schema. The returned value does not necessarily validate but can still be
+useful as a template. Works roughly as follows:
+
+=over
+
+=item * If the schema has a I<default>, then that is returned.
+
+=item * If the schema describes a hash, then a hash is returned with each key
+in I<keys> initialized to an empty value.
+
+=item * If the schema describes an array, an empty array is returned.
+
+=item * If the schema describes a bool, return C<false>.
+
+=item * If the schema describes a number, return C<0>.
+
+=item * If the schema describes a string, return C<''>.
+
+=item * Otherwise, return C<undef>.
+
+=back
+
+=item $validator->coerce($input, %opt)
+
+Perform in-place coercion of C<$input> to the data types described by the
+schema. Also returns the modified C<$input> for convenience. This method assumes
+that C<$input> already has the general structure described by the schema and is
+mainly useful to ensure that encoding the value as JSON will end up with the
+correct data types. i.e. booleans are encoded as booleans, integers as integers
+(truncating if necessary), numbers as numbers, etc.
+
+If an input hash is missing keys described in the schema, then those are
+created with C<< ->empty >>. If the schema has I<unknown> set to either
+I<reject> or I<remove>, unknown keys are removed. This behavior can be
+overriden by passing different I<unknown> value in C<%opt>.
+
+This method does NOT perform any sort of validation and will happily pass
+through garbage if the given C<$input> does not follow the structure of the
+schema. It's basically a faster and lousier normalization-only alternative to
+C<< ->validate() >>.
+
+=back
+
 
 =head1 Schema Definition
 
diff --git a/t/validate-util.t b/t/validate-util.t
new file mode 100644
index 0000000..a0ad106
--- /dev/null
+++ b/t/validate-util.t
@@ -0,0 +1,36 @@
+use v5.36;
+use Test::More;
+use FU::Validate;
+use FU::Util 'json_format';
+
+my $schema = FU::Validate->compile({ keys => {
+    bool => { anybool => 1 },
+    num => { num => 1 },
+    int => { int => 1 },
+    str => { default => 'x' },
+    intarray => { elems => { int => 1 } },
+    any => { type => 'any' },
+}});
+
+
+is json_format($schema->coerce(undef)), 'null';
+is json_format($schema->coerce("str")), '"str"';
+
+is json_format($schema->coerce({
+        bool => 'abc',
+        num => " 1.5 ",
+        int => 9.7,
+        str => !1,
+        intarray => [ 1.5, -10, undef, ' 0E0 ' ],
+        any => {},
+        whatsthis => undef,
+    }, unknown => 'remove'), canonical => 1),
+    '{"any":{},"bool":true,"int":9,"intarray":[1,-10,null,0],"num":1.5,"str":""}';
+
+is json_format($schema->coerce({uhm => 1}), canonical => 1),
+    '{"any":null,"bool":false,"int":0,"intarray":[],"num":0,"str":"x","uhm":1}';
+
+is json_format($schema->empty, canonical => 1),
+    '{"any":null,"bool":false,"int":0,"intarray":[],"num":0,"str":"x"}';
+
+done_testing;

From f2294a709a7d683fb95fa94ce427384a3d97e60b Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 20 Apr 2025 11:31:15 +0200
Subject: [PATCH 38/86] FU: Fix warning when calling fu->set_header() with
 undef value

---
 FU.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index c21bb7f..37f90db 100644
--- a/FU.pm
+++ b/FU.pm
@@ -728,7 +728,7 @@ sub reset {
 
 sub _validate_header($hdr, $val) {
     confess "Invalid response header '$hdr'" if $hdr !~ /^$FU::hdrname_re$/;
-    confess "Invalid attempt to set response header containing a newline" if $val =~ /[\r\n]/;
+    confess "Invalid attempt to set response header containing a newline" if defined $val && $val =~ /[\r\n]/;
 }
 
 sub add_header($, $hdr, $val) {

From 15940067395b2185fc1120122a0b85ff696103a2 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 20 Apr 2025 11:40:27 +0200
Subject: [PATCH 39/86] FU: Improve merging of "Vary" response headers + debug
 header listing

---
 FU.pm           |  7 +++----
 FU/DebugImpl.pm | 11 +++++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/FU.pm b/FU.pm
index 37f90db..6a9ec47 100644
--- a/FU.pm
+++ b/FU.pm
@@ -863,14 +863,12 @@ sub _finalize {
         $r->{resbody} = '';
 
     } else {
+        my @vary = ref $r->{reshdr}{vary} eq 'ARRAY' ? $r->{reshdr}{vary}->@* : defined $r->{reshdr}{vary} ? ($r->{reshdr}{vary}) : ();
         if (($hasgzip || $hasbrotli) && length($r->{resbody}) > 256
                 && !defined $r->{reshdr}{'content-encoding'}
                 && FU::compress_mimes->{$r->{reshdr}{'content-type'}}
         ) {
-
-            $r->{reshdr}{'vary'} = ($r->{reshdr}{'vary'} ? $r->{reshdr}{'vary'}.', ' : '').'accept-encoding'
-                if ($r->{reshdr}{'vary'}||'') !~ /accept-encoding/i;
-
+            push @vary, 'accept-encoding';
             if ($hasbrotli && ($r->{hdr}{'accept-encoding'}||'') =~ /\bbr\b/) {
                 $r->{resbody} = FU::Util::brotli_compress(6, $r->{resbody});
                 $r->{reshdr}{'content-encoding'} = 'br';
@@ -880,6 +878,7 @@ sub _finalize {
                 $r->{reshdr}{'content-encoding'} = 'gzip';
             }
         }
+        $r->{reshdr}{vary} = @vary ? join ', ', @vary : undef;
         $r->{reshdr}{'content-length'} = length $r->{resbody};
         $r->{resbody} = '' if (fu->method//'') eq 'HEAD';
     }
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index eda2939..da5fc47 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -75,10 +75,13 @@ my @tabs = (
         };
         h2_ 'Headers';
         table_ sub {
-            tr_ sub {
-                td_ $_;
-                td_ $r->{reshdr}{$_};
-            } for sort keys $r->{reshdr}->%*;
+            for my $k (sort keys $r->{reshdr}->%*) {
+                my $v = $r->{reshdr}{$k};
+                tr_ sub {
+                    td_ $k;
+                    td_ $_;
+                } for !defined $v ? () : ref $v ? @$v : ($v);
+            }
         };
         ('Response')
     },

From ea8ad9e4838e75b149a357388f94ca34e7a0eca9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 20 Apr 2025 18:37:54 +0200
Subject: [PATCH 40/86] FU: Throw and handle FU::Validate errors directly

Instead of wrapping them in a FU::err that isn't easily inspected.
---
 FU.pm | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/FU.pm b/FU.pm
index 6a9ec47..0161f01 100644
--- a/FU.pm
+++ b/FU.pm
@@ -300,13 +300,13 @@ sub _read_req($c) {
 }
 
 
-sub _is_done($e) { ref $@ eq 'FU::err' && $@->[0] == 200 }
+sub _is_done($e) { ref $e eq 'FU::err' && $e->[0] == 200 }
 
 sub _log_err($e) {
     return if !$e;
-    return if !debug && ref $@ eq 'FU::err' && $@->[0] != 500;
-    if (!$REQ->{full_err} && (ref $@ ne 'FU::err' || $@->[0] == 500)) {
-        $REQ->{full_err}++;
+    my $crit = $e isa 'FU::err' ? $e->[0] == 500 : !($e isa 'FU::Validate::err');
+    return if !debug && !$crit;
+    if ($crit && !$REQ->{full_err}++) {
         $e =~ s/^\s+//;
         $e =~ s/\s+$//;
         log_write join "\n",
@@ -377,7 +377,7 @@ sub _do_req($c) {
     }
 
     if ($err) {
-        my($code, $msg) = ref $err eq 'FU::err' ? $err->@* : (500, $err);
+        my($code, $msg) = $err isa 'FU::err' ? @$err : $err isa 'FU::Validate::err' ? (400, $err) : (500, $err);
         fu->reset;
         fu->status($code);
         my $ok = eval { ($onerr{$code} || $onerr{500})->($code, $msg) };
@@ -642,8 +642,7 @@ sub _getfield($data, @a) {
         return $data->{$a[0]};
     }
     my $schema = FU::Validate->compile(@a > 1 ? { keys => {@a} } : $a[0]);
-    my $res = eval { $schema->validate($data) };
-    fu->error(400, "Input validation failed: $@") if $@;
+    my $res = $schema->validate($data);
     return @a == 2 ? $res->{$a[0]} : $res;
 }
 
@@ -1209,6 +1208,12 @@ for a certain error code, C<500> is used as fallback.
 
 =back
 
+All of the above C<$sub> callbacks are allowed to throw an error. Special
+handling is given to exceptions generated by C<< fu->error() >>, which are
+relegated to the appropriate C<on_error> handler, and errors thrown by the
+C<validate()> method of L<FU::Validate>, which result in the C<400> error
+handler being run. Any other exception is passed to the C<500> error handler.
+
 
 =head1 The 'fu' Object
 
@@ -1303,8 +1308,7 @@ methods below to reliably handle all sorts of query strings.
 =item fu->query($name => $schema)
 
 Parse, validate and return the query parameter identified by C<$name> with the
-given L<FU::Validate> schema. Calls C<< fu->error(400) >> with a useful error
-message if validation fails.
+given L<FU::Validate> schema.
 
 To fetch a query parameter that may have multiple values, use:
 

From 91b2421a84329baca5590bd6080579bba375b653 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Tue, 22 Apr 2025 09:32:56 +0200
Subject: [PATCH 41/86] FU: Add -procname import option and setting $0 to
 something useful

---
 FU.pm | 32 ++++++++++++++++++++++++++++----
 1 file changed, 28 insertions(+), 4 deletions(-)

diff --git a/FU.pm b/FU.pm
index 0161f01..abdde9b 100644
--- a/FU.pm
+++ b/FU.pm
@@ -8,15 +8,22 @@ use FU::Log 'log_write';
 use FU::Util;
 use FU::Validate;
 
+my $procname;
+my $scriptpath = $0;
 
 sub import($pkg, @opt) {
     my $c = caller;
     no strict 'refs';
     *{$c.'::fu'} = \&fu;
+    my $spawn;
     for (@opt) {
-        if ($_ eq '-spawn') { _spawn() }
+        if (ref $procname eq 'FU::ARG') { $procname = $_ }
+        elsif ($_ eq '-procname') { $procname = bless {}, 'FU::ARG' }
+        elsif ($_ eq '-spawn') { $spawn = 1; }
         else { croak "Unknown import option: '$_'" }
     }
+    croak "Missing argument for -procname option" if ref $procname eq 'FU::ARG';
+    _spawn() if $spawn;
 }
 
 
@@ -209,7 +216,7 @@ sub _monitor {
                 die if $m > $data{$_};
             },
             no_chdir => 1
-        }, $0, values %INC, @monitor_paths);
+        }, $scriptpath, values %INC, @monitor_paths);
         0
     } // 1;
 }
@@ -409,6 +416,7 @@ sub _do_req($c) {
 
 sub _run_loop($c) {
     my $stop = 0;
+    my $count = 0;
     local $SIG{HUP} = 'IGNORE';
     local $SIG{TERM} = $SIG{INT} = sub { $stop = 1 };
 
@@ -418,7 +426,13 @@ sub _run_loop($c) {
         exit;
     }
 
+    my sub setstate($state) {
+        $0 = sprintf "%s: %s [#%d%s]", $procname, $state, $count, $c->{max_reqs} ? "/$c->{max_reqs}" : '' if $procname;
+    }
+
     while (!$stop) {
+        setstate 'idle';
+
         $c->{client_sock} ||= $c->{listen_sock}->accept || next;
         $c->{fcgi_obj} ||= $c->{listen_proto} eq 'fcgi' && FU::fcgi::new(fileno $c->{client_sock}, $c->{proc});
 
@@ -427,11 +441,13 @@ sub _run_loop($c) {
             passclient;
         }
 
+        setstate 'working';
         _do_req $c;
 
         $c->{client_sock} = $c->{fcgi_obj} = undef if !($c->{fcgi_obj} && $c->{fcgi_obj}->keepalive);
 
-        passclient if $c->{max_reqs} && !--$c->{max_reqs};
+        $count++;
+        passclient if $c->{max_reqs} && $count >= $c->{max_reqs};
     }
 }
 
@@ -489,12 +505,14 @@ sub _supervisor($c) {
                     fcntl $client, Fcntl::F_SETFD, 0;
                     $ENV{FU_CLIENT_FD} = fileno $client;
                 }
-                exec $^X, (map "-I$_", @INC), $0;
+                exec $^X, (map "-I$_", @INC), $scriptpath;
                 exit 1;
             }
             $childs{$pid} = 1;
         }
 
+        $0 = sprintf "%s: supervisor [%d/%d]", $procname, scalar keys %childs, $c->{proc} if $procname;
+
         my ($fd, $msgadd) = FU::Util::fdpass_recv(fileno($rsock), 500);
         push @client_fd, $fd if $fd;
         next if !defined $msgadd;
@@ -1048,6 +1066,12 @@ returning strings deal with perl Unicode strings, not raw bytes.
 
 =over
 
+=item use FU -procname => $name
+
+When the C<-procname> import option is set, FU automatically updates the
+process name (as displayed in L<top(1)> and L<ps(1)>, see `$0`) with
+information about the current process, prefixed with the given C<$name>.
+
 =item FU::init_db($info)
 
 Set database configuration. C<$info> can either be a connection string for C<<

From 8096de749741f6365887991e6d555dc6b115e6fc Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Tue, 22 Apr 2025 18:54:04 +0200
Subject: [PATCH 42/86] MultipartFormData: Fix ->save() on zero-length values

---
 FU/MultipartFormData.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 8441740..7c4f467 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -77,7 +77,7 @@ sub syswrite($o, $fh) {
 
 sub save($o, $fn) {
     open my $F, '>', $fn or confess "Error opening '$fn': $!";
-    $o->syswrite($F) or confess "Error writing to '$fn': $!";
+    defined $o->syswrite($F) or confess "Error writing to '$fn': $!";
 }
 
 sub describe($o) {

From 13271fa413daab3bd91ac33392605b3b6c88ea9a Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 23 Apr 2025 16:12:39 +0200
Subject: [PATCH 43/86] Minor changes & fixes

---
 FU.pm           | 5 +----
 FU/DebugImpl.pm | 7 ++++---
 README.md       | 7 +++----
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/FU.pm b/FU.pm
index abdde9b..ffdc9bd 100644
--- a/FU.pm
+++ b/FU.pm
@@ -694,7 +694,6 @@ sub json {
     $FU::REQ->{json} ||= eval {
         FU::Util::json_parse($FU::REQ->{body}, utf8 => 1)
     } || fu->error(400, "JSON parse error: $@");
-    return $FU::REQ->{json} if !@_;
     _getfield $FU::REQ->{json}, @_;
 }
 
@@ -945,7 +944,7 @@ __END__
 
 =head1 NAME
 
-FU - Framework Ultimatum: A Lean and Efficient Zero-Dependency Web Framework.
+FU - A Lean and Efficient Zero-Dependency Web Framework.
 
 =head1 EXPERIMENTAL
 
@@ -1481,8 +1480,6 @@ Encode C<$data> as JSON (using C<json_format> in L<FU::Util>), set an
 appropriate C<Content-Type> header and send it to the client. Calls C<<
 fu->done >>.
 
-I<TODO:> Support schema-based normalization.
-
 =item fu->send_file($root, $path)
 
 If a file identified by C<"$root/$path"> exists, set that as response and call
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index da5fc47..d7f0334 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -4,7 +4,7 @@ use v5.36;
 use experimental 'for_list';
 use FU;
 use FU::XMLWriter ':html5_', 'fragment', 'xml_escape';
-use Time::HiRes 'time';
+use Time::HiRes 'time', 'clock_gettime', 'CLOCK_MONOTONIC';
 use POSIX 'strftime';
 
 sub fmtts { strftime '%Y-%m-%d %H:%M:%S UTC', gmtime shift }
@@ -15,7 +15,8 @@ sub loc_($loc) {
         br_ if $_;
         my $l = $loc->[$_];
         my $f = $_ == $#$loc ? '(main)' : $loc->[$_+1][3];
-        txt_ "$l->[1]:$l->[2] $f";
+        $f = "$l->[0]::$f" if $f !~ /^\Q$l->[0]/;
+        txt_ "$f @ $l->[1]:$l->[2]";
     }
 }
 
@@ -35,7 +36,7 @@ my @tabs = (
             tr_ sub { td_ 'Path'; td_ fu->path };
             tr_ sub { td_ 'Query'; td_ fu->query };
             tr_ sub { td_ 'Client IP'; td_ fu->ip };
-            tr_ sub { td_ 'Received'; td_ fmtts $FU::REQ->{trace_start} };
+            tr_ sub { td_ 'Received'; td_ fmtts(time - (($FU::REQ->{trace_end}||clock_gettime(CLOCK_MONOTONIC)) - $FU::REQ->{trace_start})) };
         };
         h2_ 'Headers';
         table_ sub {
diff --git a/README.md b/README.md
index 9cf7cb8..bf545b6 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
-# FU - Framework Ultimatum: A Lean and Efficient Zero-Dependency Web Framework
+# FU - A Lean and Efficient Zero-Dependency Web Framework
 
-FU is a web development framework for Perl and a collection of handy utility
-modules.
+FU (Framework Ultimatum) is a web development framework for Perl and a
+collection of handy utility modules.
 
 *Contributing:* Refer to my [contribution guidelines](https://dev.yorhel.nl/contributing).
 
@@ -23,7 +23,6 @@ Things that may or may not happen:
 
 - FU::JSON - JSON::{XS,PP,etc}-compatible wrapper around FU::Util's JSON functions? I prolly won't need this myself, but could be handy.
 - FU::DBI - DBI wrapper with a FU::Pg-like API, for easier integration into FU.
-- FU::Mailer - Simple sendmail wrapper
 
 # License
 

From ab168bd952042a9e2fbd43173b0805204cf57eef Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 24 Apr 2025 07:23:00 +0200
Subject: [PATCH 44/86] Pg: Don't return self on / cache() or text_*() methods

This is much too easy of a footgun:

  $db->cache->q(...);

Enabled cache on the $db object, not just for the given query.
---
 FU.xs         | 2 +-
 t/pgconnect.t | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/FU.xs b/FU.xs
index ef8d60b..221740b 100644
--- a/FU.xs
+++ b/FU.xs
@@ -55,7 +55,6 @@
         if (!ix) ix = FUPG_CACHE;\
         if (items == 1 || SvTRUE(ST(1))) x->stflags |= ix; \
         else x->stflags &= ~ix; \
-        XSRETURN(1); \
     } while(0)
 
 MODULE = FU
@@ -343,6 +342,7 @@ void cache(fupg_st *x, ...)
   CODE:
     if (ix == 0 && x->prepared) fu_confess("Invalid attempt to change statement configuration after it has already been prepared or executed");
     FUPG_STFLAGS;
+    XSRETURN(1);
 
 void exec(fupg_st *st)
   CODE:
diff --git a/t/pgconnect.t b/t/pgconnect.t
index d2dfbf5..798734c 100644
--- a/t/pgconnect.t
+++ b/t/pgconnect.t
@@ -17,7 +17,9 @@ okerr FATAL => connect => qr/missing "=" after "invalid"/;
 
 ok FU::Pg::lib_version() > 100000;
 
-my $conn = FU::Pg->connect($ENV{FU_TEST_DB})->text->cache(0);
+my $conn = FU::Pg->connect($ENV{FU_TEST_DB});
+$conn->text;
+$conn->cache(0);
 $conn->_debug_trace(0);
 
 is ref $conn, 'FU::Pg::conn';
@@ -364,7 +366,9 @@ subtest 'txn', sub {
 }
 
 subtest 'Prepared statement cache', sub {
-    my $txn = $conn->cache_size(2)->txn->cache;
+    $conn->cache_size(2);
+    my $txn = $conn->txn;
+    $txn->cache;
     my sub numexec($sql) {
         $txn->q('SELECT generic_plans + custom_plans FROM pg_prepared_statements WHERE statement = $1', $sql)->cache(0)->val
     }

From 48334568984eafd20fe82f54106e93eec49349d7 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 24 Apr 2025 10:50:50 +0200
Subject: [PATCH 45/86] FU: Accept charset argument in JSON request
 content-type header

---
 FU.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index ffdc9bd..423c6e1 100644
--- a/FU.pm
+++ b/FU.pm
@@ -689,7 +689,7 @@ sub cookie {
 
 sub json {
     shift;
-    fu->error(400, "Invalid content type for json") if (fu->header('content-type')||'') ne 'application/json';
+    fu->error(400, "Invalid content type for json") if (fu->header('content-type')||'') !~ m{^application/json(?:;\s*charset=utf-?8)?$}i;
     return FU::Util::utf8_decode(my $x = $FU::REQ->{body}) if !@_;
     $FU::REQ->{json} ||= eval {
         FU::Util::json_parse($FU::REQ->{body}, utf8 => 1)

From e88ad65232ad9e6fb4246993cbb1a172a04641ac Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 24 Apr 2025 14:16:17 +0200
Subject: [PATCH 46/86] Version 0.5

---
 ChangeLog               | 17 +++++++++++++++++
 FU.pm                   |  4 ++--
 FU/DebugImpl.pm         |  2 +-
 FU/Log.pm               |  2 +-
 FU/MultipartFormData.pm |  2 +-
 FU/Pg.pm                |  2 +-
 FU/SQL.pm               |  2 +-
 FU/Util.pm              |  2 +-
 FU/Validate.pm          |  2 +-
 FU/XMLWriter.pm         |  2 +-
 FU/XS.pm                |  2 +-
 Makefile.PL             |  1 +
 README.md               |  2 ++
 13 files changed, 31 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f9c7266..3f18b6b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+0.5 - 2025-04-24
+	- FU::Util: Set O_CLOEXEC on fds received through `fdpass_recv()`
+	- FU::Util: Fix interpretation of false options in `json_format()` and
+	  `json_parse()`
+	- FU::Validate: Add `coerce()` and `empty()` utility methods
+	- FU::Validate: Limit values of int/uint input to 64 bits
+	- FU::Validate: Normalize num/int/uint inputs to Perl numeric types
+	- FU::Pg: Add `escape_literal()` and `escape_identifier()` methods
+	- FU::Pg: Use less memory for `kvv()`, `kva()` and `kvh()` methods
+	- FU::Pg: Disallow chaining of `cache()`, `text()`, `text_params()` and
+	  `text_results()` methods on connection and transaction objects
+	- FU: Throw and catch FU::Validate errors without wrapping in `fu->error()`
+	- FU: Add `-progname` option and add diagnostics to process names
+	- FU: Whole bunch of misc fixes
+	- Doc fixes
+	- Fix nul-termination of some XS-created strings
+
 0.4 - 2025-03-19
 	- FU::Validate: Support arrayref schemas
 	- FU::Validate: Rename 'values' option to 'elems'
diff --git a/FU.pm b/FU.pm
index 423c6e1..1f71a91 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 0.4;
+package FU 0.5;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
@@ -1055,7 +1055,7 @@ certainly not great if you plan to transfer large files.
 =back
 
 The rest of this document is reference documentation; there's no easy
-introductionary cookbook-style docs yet, sorry about that.
+introductory cookbook-style docs yet, sorry about that.
 
 Unless specifically mentioned otherwise, all methods and functions taking or
 returning strings deal with perl Unicode strings, not raw bytes.
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index d7f0334..a0de7ea 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 0.4;
+package FU::DebugImpl 0.5;
 use v5.36;
 use experimental 'for_list';
 use FU;
diff --git a/FU/Log.pm b/FU/Log.pm
index 24dafbf..e2da4a2 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 0.4;
+package FU::Log 0.5;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 7c4f467..a92e4d7 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 0.4;
+package FU::MultipartFormData 0.5;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index e43776e..f43c7f8 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 0.4;
+package FU::Pg 0.5;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 15adbb1..2f8566d 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 0.4;
+package FU::SQL 0.5;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index e823d79..5b262fb 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 0.4;
+package FU::Util 0.5;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index dd767c7..b315784 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 0.4;
+package FU::Validate 0.5;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 35e0702..1e9bb90 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 0.4;
+package FU::XMLWriter 0.5;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index f9660c8..52cc757 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 0.4;
+package FU::XS 0.5;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');
diff --git a/Makefile.PL b/Makefile.PL
index 326fdb4..c107fa1 100644
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -17,6 +17,7 @@ WriteMakefile(
         dynamic_config => 0,
         'meta-spec' => { version => 2 },
         resources => {
+            homepage => 'https://dev.yorhel.nl/fu',
             repository => {
                 web => 'https://code.blicky.net/yorhel/fu',
                 type => 'git',
diff --git a/README.md b/README.md
index bf545b6..d29e00c 100644
--- a/README.md
+++ b/README.md
@@ -3,6 +3,8 @@
 FU (Framework Ultimatum) is a web development framework for Perl and a
 collection of handy utility modules.
 
+*Website:* More information @ [dev.yorhel.nl/fu](https://dev.yorhel.nl/fu).
+
 *Contributing:* Refer to my [contribution guidelines](https://dev.yorhel.nl/contributing).
 
 ## Project Status

From 0cd947c545b13ce316d36bda21476ace5b2b3483 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 25 Apr 2025 09:31:43 +0200
Subject: [PATCH 47/86] FastCGI: Ignore HTTP_CONTENT_(TYPE|LENGTH)

The non-HTTP_ versions of these are authoritative, Also fixes a memory
leak when both the HTTP_ and non-HTTP_ versions are included.
---
 c/fcgi.c | 7 +++++--
 t/fcgi.t | 7 +++++++
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/c/fcgi.c b/c/fcgi.c
index fdf8cae..4f886dd 100644
--- a/c/fcgi.c
+++ b/c/fcgi.c
@@ -319,8 +319,11 @@ static int fufcgi_read_params(pTHX_ fufcgi *ctx, fufcgi_rec *rec) {
                 p.name += 5;
                 for (r=0; r<p.namelen; r++)
                     p.name[r] = p.name[r] == '_' ? '-' : p.name[r] >= 'A' && p.name[r] <= 'Z' ? p.name[r] | 0x20 : p.name[r];
-                valsv = newSV(p.vallen+1);
-                hv_store(ctx->headers, p.name, p.namelen, valsv, 0);
+                if (!(p.namelen == 14 && memcmp(p.name, "content-length", 14) == 0)
+                        && !(p.namelen == 12 && memcmp(p.name, "content-type", 12) == 0)) {
+                    valsv = newSV(p.vallen+1);
+                    hv_store(ctx->headers, p.name, p.namelen, valsv, 0);
+                }
 
             } else if (p.namelen == 14 && memcmp(p.name, "CONTENT_LENGTH", 14) == 0) {
                 valsv = newSV(p.vallen+1);
diff --git a/t/fcgi.t b/t/fcgi.t
index 0711d6a..d7860dc 100644
--- a/t/fcgi.t
+++ b/t/fcgi.t
@@ -167,6 +167,13 @@ record 1, 4, "\x0c\x05CONTENT_TYPEsomet";
 record 1, 2, "";
 isrec {'content-type','somet'}, {body => ''}, -6;
 
+start;
+begin;
+record 1, 4, "\x13\x01HTTP_CONTENT_LENGTH3\x0e\x01CONTENT_LENGTH0\x13\x01HTTP_CONTENT_LENGTH5";
+record 1, 4, "";
+record 1, 5, "";
+isrec {'content-length','0'}, {body => ''};
+
 start;
 begin;
 record 1, 4, "\x0e\x05CONTENT_LENGTH65536";

From 5f8809d0523d547ae7be03fac568ec7a36b85e53 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 25 Apr 2025 17:07:56 +0200
Subject: [PATCH 48/86] FU::Util::query_decode(): Properly handle empty
 "&"-parts

---
 FU/Util.pm | 1 +
 t/query.t  | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/FU/Util.pm b/FU/Util.pm
index 5b262fb..7d585d9 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -41,6 +41,7 @@ sub uri_unescape :prototype($) ($s) {
 sub query_decode :prototype($) ($s) {
     my %o;
     for (split /&/, $s//'') {
+        next if !length;
         my($k,$v) = map uri_unescape($_), split /=/, $_, 2;
         $v //= builtin::true;
         if (ref $o{$k}) { push $o{$k}->@*, $v }
diff --git a/t/query.t b/t/query.t
index ebeff80..80f2b00 100644
--- a/t/query.t
+++ b/t/query.t
@@ -10,6 +10,8 @@ is_deeply
 ok !eval { query_decode('%10'); 1 };
 like $@, qr/Invalid control character/;
 
+is_deeply query_decode('&&&a=b'), { a => 'b' };
+
 is query_encode
     { a => builtin::true, b => undef, c => builtin::false, d => 'string', e => "&=\xfe" },
     'a&d=string&e=%26%3d%c3%be';

From 461ed6f39d5522136da212ba2477a0bfbf25613c Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 26 Apr 2025 08:05:09 +0200
Subject: [PATCH 49/86] FU: Suppress warnings about non-existent files in
 FU::monitor_path checking

---
 FU.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index 1f71a91..04809b5 100644
--- a/FU.pm
+++ b/FU.pm
@@ -216,7 +216,7 @@ sub _monitor {
                 die if $m > $data{$_};
             },
             no_chdir => 1
-        }, $scriptpath, values %INC, @monitor_paths);
+        }, grep -e, $scriptpath, values %INC, @monitor_paths);
         0
     } // 1;
 }

From 753cac615a645bc875d4e39ad3b81d80fc01542f Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 26 Apr 2025 15:41:26 +0200
Subject: [PATCH 50/86] Validate: Improved + extendable error message
 formatting

Very much needed for VNDB's advanced search validation.
Also completely undocumented.
---
 FU/Validate.pm | 47 +++++++++++++++++++++++++++++++----------------
 t/validate.t   | 34 +++++++++++++++++-----------------
 2 files changed, 48 insertions(+), 33 deletions(-)

diff --git a/FU/Validate.pm b/FU/Validate.pm
index b315784..2741ee6 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -395,32 +395,47 @@ sub empty($c) {
 
 
 
+sub _fmtkey($k) { $k =~ /^[a-zA-Z0-9_-]+$/ ? $k : FU::Util::json_format($k); }
+sub _fmtval($v) { eval { $v = FU::Util::json_format($v) }; "$v" }
+sub _inval($t,$v) { sprintf 'invalid %s: %s', $t, _fmtval $v }
+
+# validation name => formatting sub
+# TODO: document.
+our %error_format = (
+    required  => sub { 'required value missing' },
+    type      => sub($e) { "invalid type, expected '$e->{expected}' but got '$e->{got}'" },
+    unknown   => sub($e) { sprintf 'unknown key%s: %s', $e->{keys}->@* == 1 ? '' : 's', join ', ', map _fmtkey($_), $e->{keys}->@* },
+    minlength => sub($e) { sprintf "input too short, expected minimum of %d but got %d", $e->{expected}, $e->{got} },
+    maxlength => sub($e) { sprintf "input too long, expected maximum of %d but got %d", $e->{expected}, $e->{got} },
+    length    => sub($e) {
+        !ref $e->{expected}
+        ? sprintf 'invalid input length, expected %d but got %d', $e->{expected}, $e->{got}
+        : sprintf 'invalid input length, expected between %d and %d but got %d', $e->{expected}->@*, $e->{got}
+    },
+    num       => sub($e) { _inval 'number', $e->{got} },
+    min       => sub($e) { $e->{error} ? _inval 'number', $e->{error}{got} : sprintf 'expected minimum %s but got %s', $e->{expected}, $e->{got} },
+    max       => sub($e) { $e->{error} ? _inval 'number', $e->{error}{got} : sprintf 'expected maximum %s but got %s', $e->{expected}, $e->{got} },
+    range     => sub($e) { FU::Validate::err::errors($e->{error}) },
+);
+
+
 package FU::Validate::err;
 use v5.36;
-use FU::Util;
 
 use overload '""' => sub { $_[0]{longmess} || join "\n", $_[0]->errors };
 
-sub _fmtkey($k) {
-    $k =~ /^[a-zA-Z0-9_-]+$/ ? $k : FU::Util::json_format($k);
-}
-
-sub _fmtval($v) {
-    eval { $v = FU::Util::json_format($v) }; "$v"
-}
-
+# TODO: document.
 sub errors($e, $prefix='') {
     my $val = $e->{validation};
     my $p = $prefix ? "$prefix: " : '';
-    $val eq 'keys'     ? map errors($_, $prefix.'.'._fmtkey($_->{key})), $e->{errors}->@* :
-    $val eq 'values'   ? map errors($_, $prefix.'.'._fmtkey($_->{key})), $e->{errors}->@* :
-    $val eq 'missing'  ? $prefix.'.'._fmtkey($e->{key}).': required key missing' :
+    $FU::Validate::error_format{$val} ? map "$p$_", $FU::Validate::error_format{$val}->($e) :
+    $val eq 'keys'     ? map errors($_, $prefix.'.'.FU::Validate::_fmtkey($_->{key})), $e->{errors}->@* :
+    $val eq 'values'   ? map errors($_, $prefix.'.'.FU::Validate::_fmtkey($_->{key})), $e->{errors}->@* :
+    $val eq 'missing'  ? $prefix.'.'.FU::Validate::_fmtkey($e->{key}).': required key missing' :
     $val eq 'elems'    ? map errors($_, $prefix."[$_->{index}]"), $e->{errors}->@* :
-    $val eq 'unique'   ? $prefix."[$e->{index_b}] value '"._fmtval($e->{value_a})."' duplicated" :
-    $val eq 'required' ? "${p}required value missing" :
-    $val eq 'type'     ? "${p}invalid type, expected '$e->{expected}' but got '$e->{got}'" :
-    $val eq 'unknown'  ? ($e->{keys}->@* > 1 ? "${p}unknown keys: ".join(', ', _fmtkey($e->{keys})) : "${p}unknown key '"._fmtkey($e->{keys}[0])."'") :
+    $val eq 'unique'   ? $prefix."[$e->{index_b}] value '".FU::Validate::_fmtval($e->{value_a})."' duplicated" :
     $e->{error}        ? errors($e->{error}, "${p}validation '$val'") :
+    $e->{message}      ? "${p}validation '$val': $e->{message}" :
                          "${p}failed validation '$val'";
 }
 
diff --git a/t/validate.t b/t/validate.t
index a29dc4c..26704cd 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -119,7 +119,7 @@ f { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {key => 'a', valid
 
 t { type => 'hash', keys => { a=>{} } }, {a=>' a '}, {a=>'a'}; # Test against in-place modification
 t { type => 'hash', keys => { a=>{} }, unknown => 'remove' }, { a=>1,b=>1 }, { a=>1 };
-f { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] }, "unknown key 'b'";
+f { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] }, "unknown key: b";
 t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>1,b=>1 };
 t { type => 'hash', setundef => 1 }, {}, undef;
 t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef};
@@ -132,20 +132,20 @@ t { values => { int => 1 } }, { a => -1, b => 1 }, { a => -1, b => 1 };
 f { values => { int => 1 } }, { a => undef }, { validation => 'values', errors => [{ key => 'a', validation => 'required' }] }, '.a: required value missing';
 
 # default validations
-f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 }, "failed validation 'minlength'";
+f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 }, "input too short, expected minimum of 3 but got 2";
 t { minlength => 3 }, 'abc', 'abc';
-f { maxlength => 3 }, 'abcd', { validation => 'maxlength', expected => 3, got => 4 }, "failed validation 'maxlength'";
+f { maxlength => 3 }, 'abcd', { validation => 'maxlength', expected => 3, got => 4 }, "input too long, expected maximum of 3 but got 4";
 t { maxlength => 3 }, 'abc', 'abc';
 t { minlength => 3, maxlength => 3 }, 'abc', 'abc';
-f { length => 3 }, 'ab',   { validation => 'length', expected => 3, got => 2 }, "failed validation 'length'";
-f { length => 3 }, 'abcd', { validation => 'length', expected => 3, got => 4 }, "failed validation 'length'";
+f { length => 3 }, 'ab',   { validation => 'length', expected => 3, got => 2 }, 'invalid input length, expected 3 but got 2';
+f { length => 3 }, 'abcd', { validation => 'length', expected => 3, got => 4 }, 'invalid input length, expected 3 but got 4';
 t { length => 3 }, 'abc',  'abc';
 t { length => [1,3] }, 'abc',  'abc';
-f { length => [1,3] }, 'abcd', { validation => 'length', expected => [1,3], got => 4 }, "failed validation 'length'";
+f { length => [1,3] }, 'abcd', { validation => 'length', expected => [1,3], got => 4 }, "invalid input length, expected between 1 and 3 but got 4";
 t { type => 'array', length => 0 }, [], [];
-f { type => 'array', length => 1 }, [1,2], { validation => 'length', expected => 1, got => 2 }, "failed validation 'length'";
+f { type => 'array', length => 1 }, [1,2], { validation => 'length', expected => 1, got => 2 }, "invalid input length, expected 1 but got 2";
 t { type => 'hash', length => 0 }, {}, {};
-f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 }, "failed validation 'length'";
+f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 }, "invalid input length, expected 1 but got 2";
 t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1};
 t { regex => '^a' }, 'abc', 'abc';  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
 f { regex => '^a' }, 'cba', { validation => 'regex', regex => '^a', got => 'cba' }, "failed validation 'regex'";
@@ -201,7 +201,7 @@ t { doublefunc => 1 }, 0, 2;
 f { doublefunc => 1 }, 1, { validation => 'doublefunc', error => { validation => 'func', result => '' } }, "validation 'doublefunc': failed validation 'func'";
 
 # numbers
-sub nerr { ({ validation => 'num', got => $_[0] }, "failed validation 'num'") }
+sub nerr { ({ validation => 'num', got => $_[0] }, "invalid number: \"$_[0]\"") }
 t { num => 1 }, 0, 0;
 f { num => 1 }, '-', nerr '-';
 f { num => 1 }, '00', nerr '00';
@@ -219,16 +219,16 @@ t { uint => 1 }, 0, 0;
 t { uint => 1 }, 123, 123;
 f { uint => 1 }, -123, { validation => 'uint', got => -123 }, "failed validation 'uint'";
 t { min => 1 }, 1, 1;
-f { min => 1 }, 0.9, { validation => 'min', expected => 1, got => 0.9 }, "failed validation 'min'";
-f { min => 1 }, 'a', { validation => 'min', error => (nerr 'a')[0] }, "validation 'min': failed validation 'num'";
+f { min => 1 }, 0.9, { validation => 'min', expected => 1, got => 0.9 }, "expected minimum 1 but got 0.9";
+f { min => 1 }, 'a', { validation => 'min', error => (nerr 'a')[0] }, 'invalid number: "a"';
 t { max => 1 }, 1, 1;
-f { max => 1 }, 1.1, { validation => 'max', expected => 1, got => 1.1 }, "failed validation 'max'";
-f { max => 1 }, 'a', { validation => 'max', error => (nerr 'a')[0] }, "validation 'max': failed validation 'num'";
+f { max => 1 }, 1.1, { validation => 'max', expected => 1, got => 1.1 }, "expected maximum 1 but got 1.1";
+f { max => 1 }, 'a', { validation => 'max', error => (nerr 'a')[0] }, 'invalid number: "a"';
 t { range => [1,2] }, 1, 1;
 t { range => [1,2] }, 2, 2;
-f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } }, "validation 'range': failed validation 'min'";
-f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } }, "validation 'range': failed validation 'max'";
-f { range => [1,2] }, 'a', { validation => 'range', error => { validation => 'min', error => (nerr 'a')[0] } }, "validation 'range': validation 'min': failed validation 'num'";
+f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } }, 'expected minimum 1 but got 0.9';
+f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } }, 'expected maximum 2 but got 2.1';
+f { range => [1,2] }, 'a', { validation => 'range', error => { validation => 'min', error => (nerr 'a')[0] } }, 'invalid number: "a"';
 
 # email template
 use utf8;
@@ -253,7 +253,7 @@ t { email => 1 }, $_, $_ for (
   'abc@x-y_z.example',
 );
 my $long = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxxxxxxxxxx.xxxxx';
-f { email => 1 }, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } }, "validation 'email': failed validation 'maxlength'";
+f { email => 1 }, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } }, "validation 'email': input too long, expected maximum of 254 but got 255";
 
 # weburl template
 f { weburl => 1 }, $_, { validation => 'weburl', got => $_ }, "failed validation 'weburl'" for (

From 817fa600d0ccc66bdc2e6d0e501e557a01be69d9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 27 Apr 2025 11:17:54 +0200
Subject: [PATCH 51/86] FU: Add fu->log_verbose() + include request body in
 error logs

---
 FU.pm | 44 ++++++++++++++++++++++++++++++++------------
 1 file changed, 32 insertions(+), 12 deletions(-)

diff --git a/FU.pm b/FU.pm
index 04809b5..11c58ff 100644
--- a/FU.pm
+++ b/FU.pm
@@ -303,7 +303,7 @@ sub _read_req($c) {
 
     ($REQ->{path}, my $qs) = split /\?/, $REQ->{path}//'', 2;
     $REQ->{qs} //= $qs;
-    $REQ->{path} = FU::Util::uri_unescape($REQ->{path});
+    eval { $REQ->{path} = FU::Util::uri_unescape($REQ->{path}); 1; } || fu->error(400, $@);
 }
 
 
@@ -313,17 +313,8 @@ sub _log_err($e) {
     return if !$e;
     my $crit = $e isa 'FU::err' ? $e->[0] == 500 : !($e isa 'FU::Validate::err');
     return if !debug && !$crit;
-    if ($crit && !$REQ->{full_err}++) {
-        $e =~ s/^\s+//;
-        $e =~ s/\s+$//;
-        log_write join "\n",
-            'IP: '.($REQ->{ip}||'-'),
-            'Headers:', (map "  $_: $REQ->{hdr}{$_}", sort keys $REQ->{hdr}->%*),
-            'ERROR:'.($e =~ s/(^|\n)/\n  /rg);
-            # TODO: decoded body, if we have that.
-    } else {
-        log_write $e;
-    }
+    return fu->log_verbose($e) if $crit;
+    log_write $e;
 }
 
 sub _do_req($c) {
@@ -643,6 +634,27 @@ sub db {
 sub sql { shift->db->q(@_) }
 sub SQL { shift->db->Q(@_) }
 
+sub _fmt_section($s) { $s =~ s/^\s*/  /r =~ s/\s+$//r =~ s/\n/\n  /rg }
+
+sub log_verbose($,$msg) {
+    my $r = $FU::REQ;
+    return FU::Log::log_write($msg) if $r->{log_verbose}++;
+    FU::Log::log_write(join "\n",
+        'IP: '.($r->{ip}||'-'),
+        'Headers:', (map "  $_: $r->{hdr}{$_}", sort keys $r->{hdr}->%*),
+        $r->{multipart} ? ('Body (multipart):', _fmt_section join "\n", map $_->describe, $r->{multipart}->@*) :
+        $r->{json} ? ('Body (JSON):', _fmt_section FU::Util::json_format($r->{json}, pretty => 1, canonical => 1)) :
+        $r->{formdata} ? ('Body (formdata):', _fmt_section FU::Util::json_format($r->{json}, pretty => 1, canonical => 1)) :
+        length $r->{body} ? do {
+            my $b = substr $r->{body}, 0, 4096;
+            my $trunc = length $r->{body} > 4096 ? ', truncated' : '';
+            utf8::decode($b) ? ("Body (utf8$trunc)", _fmt_section($b =~ s/\r//rg =~ s/\n{4,}/\n[..]\n/rg))
+                             : ("Body (hex$trunc)", _fmt_section(unpack('H*', $b) =~ s/(.{128})/$1\n/rg))
+        } : (),
+        'Message:', _fmt_section $msg
+    );
+}
+
 
 
 
@@ -1286,6 +1298,14 @@ Convenient short-hand for C<< fu->db->q($query, @params) >>.
 
 Convenient short-hand for C<< fu->db->Q(@args) >>.
 
+=item fu->log_verbose($message)
+
+Write a verbose multi-line message to the log, including a full dump of
+information about the request: IP, headers and (potentially reformatted and/or
+truncated) body. This extra info is only written once per request, further
+calls to C<log_verbose()> just go directly to L<FU::Log>'s C<log_write()>
+instead.
+
 =back
 
 =head1 Request Information

From d0c5397e2dda0387c1678a279bc13187c8dd455d Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 28 Apr 2025 10:20:53 +0200
Subject: [PATCH 52/86] json_parse()/pgtypes: Fix accidental creation of
 read-only array/hash values

&PL_sv_* shouldn't be used when constructing arrays or hashes in this
context.
---
 FU.xs               |  6 ++++++
 c/jsonparse.c       |  4 ++--
 c/pgst.c            |  2 +-
 c/pgtypes.c         | 12 ++++++++----
 t/json_parse.t      |  8 +++++++-
 t/pgconnect.t       |  1 +
 t/pgtypes-dynamic.t |  7 ++++++-
 t/pgtypes.t         | 16 ++++++++++++++--
 8 files changed, 45 insertions(+), 11 deletions(-)

diff --git a/FU.xs b/FU.xs
index 221740b..dc19870 100644
--- a/FU.xs
+++ b/FU.xs
@@ -20,6 +20,12 @@
 #ifndef BOOL_INTERNALS_sv_isbool_true
 #define BOOL_INTERNALS_sv_isbool_true(x) SvTRUEx(x)
 #endif
+#ifndef newSV_true
+#define newSV_true() newSVsv(&PL_sv_yes)
+#endif
+#ifndef newSV_false
+#define newSV_false() newSVsv(&PL_sv_no)
+#endif
 
 /* Disable key/value struct packing in khashl, so we can safely take a pointer
  * to values inside the hash table. */
diff --git a/c/jsonparse.c b/c/jsonparse.c
index d24b1f9..6dfee91 100644
--- a/c/jsonparse.c
+++ b/c/jsonparse.c
@@ -236,12 +236,12 @@ static SV *fujson_parse(pTHX_ fujson_parse_ctx *ctx) {
             if (ctx->end - ctx->buf < 4) return NULL;
             if (memcmp(ctx->buf, "true", 4) != 0) return NULL;
             ctx->buf += 4;
-            return &PL_sv_yes;
+            return newSV_true();
         case 'f':
             if (ctx->end - ctx->buf < 5) return NULL;
             if (memcmp(ctx->buf, "false", 5) != 0) return NULL;
             ctx->buf += 5;
-            return &PL_sv_no;
+            return newSV_false();
         case 'n':
             if (ctx->end - ctx->buf < 4) return NULL;
             if (memcmp(ctx->buf, "null", 4) != 0) return NULL;
diff --git a/c/pgst.c b/c/pgst.c
index e5a2a3d..e943450 100644
--- a/c/pgst.c
+++ b/c/pgst.c
@@ -463,7 +463,7 @@ static SV *fupg_st_kvv(pTHX_ fupg_st *st) {
         SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kvv() query results", SvPV_nolen(key));
-        hv_store_ent(hv, key, st->nfields == 1 ? &PL_sv_yes : fupg_st_getval(aTHX_ st, i, 1), 0);
+        hv_store_ent(hv, key, st->nfields == 1 ? newSV_true() : fupg_st_getval(aTHX_ st, i, 1), 0);
         FREETMPS;
     }
     return sv;
diff --git a/c/pgtypes.c b/c/pgtypes.c
index b307cc0..471b6d2 100644
--- a/c/pgtypes.c
+++ b/c/pgtypes.c
@@ -78,7 +78,7 @@ SENDFN(domain) { (void)out; SERR("domain type should not be handled by this func
 
 RECVFN(bool) {
     RLEN(1);
-    return *buf ? &PL_sv_yes : &PL_sv_no;
+    return *buf ? newSV_true() : newSV_false();
 }
 
 SENDFN(bool) {
@@ -89,7 +89,7 @@ SENDFN(bool) {
 RECVFN(void) {
     RLEN(0);
     (void)buf;
-    return &PL_sv_undef;
+    return newSV(0);
 }
 
 SENDFN(void) {
@@ -269,7 +269,7 @@ SENDFN(jsonpath) {
 #define ARRAY_MAXDIM 100
 
 static SV *fupg_recv_array_elem(pTHX_ const fupg_tio *elem, const char *header, U32 dim, U32 ndim, const char **buf, const char *end) {
-    SV *r = &PL_sv_undef;
+    SV *r;
     if (dim == ndim) {
         if (end - *buf < 4) fu_confess("Invalid array format");
         I32 len = fu_frombeI(32, *buf);
@@ -279,6 +279,8 @@ static SV *fupg_recv_array_elem(pTHX_ const fupg_tio *elem, const char *header,
         if (len >= 0) {
             r = elem->recv(aTHX_ elem, *buf, len);
             *buf += len;
+        } else {
+            r = newSV(0);
         }
 
     } else {
@@ -403,12 +405,14 @@ RECVFN(record) {
         if (oid != ctx->record.info->attrs[i].oid)
             RERR("expected field %d to be of type %u but got %u", i, ctx->record.info->attrs[i].oid, oid);
         I32 vlen = fu_frombeI(32, buf+4);
-        SV *r = &PL_sv_undef;
+        SV *r;
         buf += 8; len -= 8;
         if (vlen > len) RERR("input data too short");
         if (vlen >= 0) {
             r = ctx->record.tio[i].recv(aTHX_ ctx->record.tio+i, buf, vlen);
             buf += vlen; len -= vlen;
+        } else {
+            r = newSV(0);
         }
         hv_store(hv, ctx->record.info->attrs[i].name.n, -strlen(ctx->record.info->attrs[i].name.n), r, 0);
     }
diff --git a/t/json_parse.t b/t/json_parse.t
index 0c26dff..d01414f 100644
--- a/t/json_parse.t
+++ b/t/json_parse.t
@@ -2,7 +2,7 @@ use v5.36;
 use Test::More;
 use FU::Util 'json_parse';
 no warnings 'experimental::builtin';
-use builtin 'is_bool', 'created_as_number';
+use builtin 'is_bool', 'created_as_number', 'true', 'false';
 use Config;
 
 my @error = (
@@ -236,4 +236,10 @@ ok !eval { json_parse '{"":{"":{"":{"":1}}}}', max_depth => 4; 1 };
     ok !eval { json_parse '"string"', max_size => 7 };
 }
 
+# Mutable hashes/arrays
+my $d = json_parse('[true,false,null,{"a":true,"b":false,"c":null}]');
+is_deeply $d, [true,false,undef,{a => true, b => false, c => undef}];
+$_ = 1 for @{$d}[0,1,2], values $d->[3]->%*;
+is_deeply $d, [1,1,1,{a => 1, b => 1, c => 1}];
+
 done_testing;
diff --git a/t/pgconnect.t b/t/pgconnect.t
index 798734c..8536574 100644
--- a/t/pgconnect.t
+++ b/t/pgconnect.t
@@ -197,6 +197,7 @@ subtest '$st->kvv', sub {
     is_deeply $conn->q('SELECT 1 WHERE false')->kvv, {};
     is_deeply $conn->q('SELECT 1')->kvv, {1=>1};
     is_deeply $conn->q('SELECT 1, null UNION ALL SELECT 3, 2')->kvv, {1=>undef,3=>2};
+    $conn->q('SELECT 1')->kvv->{1} = 0;
 };
 
 subtest '$st->kva', sub {
diff --git a/t/pgtypes-dynamic.t b/t/pgtypes-dynamic.t
index b6954b5..2751a86 100644
--- a/t/pgtypes-dynamic.t
+++ b/t/pgtypes-dynamic.t
@@ -96,10 +96,15 @@ subtest 'custom types', sub {
         );
     _
 
-    is_deeply $txn->q(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val, [
+    $val = $txn->q(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val;
+    is_deeply $val, [
         { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
         { rec => { a => undef, aenum => undef, domain => undef }, dom => 'bb' },
     ];
+    $val->[0] = 0;
+    $val->[1]{rec}{a} = 0;
+    $val->[1]{rec} = 0;
+    $val->[1]{dom} = 0;
 
     is $txn->q('SELECT $1::fupg_test_table[]', [
         { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
diff --git a/t/pgtypes.t b/t/pgtypes.t
index 5a17a71..3a3252c 100644
--- a/t/pgtypes.t
+++ b/t/pgtypes.t
@@ -21,10 +21,12 @@ sub v($type, $p_in, @args) {
     my $test = "$type $s_in" =~ s/\n/\\n/rg;
     utf8::encode($test);
     {
-        my $res = $conn->q("SELECT \$1::$type", $s_in)->text_params->val;
+        my $array = $conn->q("SELECT \$1::$type", $s_in)->text_params->flat;
+        my $res = $array->[0];
         ok is_bool($res), "$test is bool" if $type eq 'bool';
         ok created_as_number($res), "$test is number" if $type =~ /^(int|float)\d/;
         is_deeply $res, $p_out, "$test text->bin";
+        $array->[0] = 0; # Must be writable
     }
     {
         my $res = $conn->q("SELECT \$1::$type", $p_in)->text_results->val;
@@ -41,7 +43,11 @@ sub f($type, $p_in) {
     ok !eval { $conn->q("SELECT \$1::$type", $p_in)->val; 1 }, "$test fail";
 }
 
-ok !defined $conn->q('SELECT pg_sleep(0)')->val; # void
+{ # void
+    my $array = $conn->q('SELECT pg_sleep(0)')->flat;
+    ok !defined $array->[0];
+    $array->[0] = 0;
+}
 
 v bool => true,  undef, 1, 't';
 v bool => false, undef, 0, 'f';
@@ -166,4 +172,10 @@ is $conn->q('SELECT ($1::int2[])[2]', [1,2,3,4])->val, 2;
 is $conn->q('SELECT ($1::int2vector)[1]', [1,2,3,4])->val, 2;
 is $conn->q('SELECT ($1::oidvector)[1]', [1,2,3,4])->val, 2;
 
+{
+    my $v = $conn->q("SELECT '{t,f,NULL}'::bool[]")->val;
+    is_deeply $v, [true, false, undef];
+    $_ = 0 for @$v;
+}
+
 done_testing;

From f8b0043e2248e981ea74bc08030fecb6fff93af2 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Tue, 29 Apr 2025 09:14:44 +0200
Subject: [PATCH 53/86] MultipartFormData: Bunch of parser fixes

---
 FU/MultipartFormData.pm | 11 ++++++-----
 t/multipart.t           | 15 ++++++++++++++-
 2 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index a92e4d7..7d9d77e 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -3,7 +3,7 @@ use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
 
-sub _arg($d) { $d =~ s/^\s+//r =~ s/\s+$//r =~ s/^"(.+)"$/$1/r }
+sub _arg($d) { $d =~ s{^"(.+)"$}{$1 =~ s/\\([\\"])/$1/rg}er }
 
 sub parse($pkg, $header, $data) {
     confess "Invalid multipart header '$header'"
@@ -26,13 +26,14 @@ sub parse($pkg, $header, $data) {
             start => pos $data,
         }, $pkg;
 
-        confess "Missing content-disposition header" if $hdrs !~ /content-disposition:\s*form-data;(.+)/i;
+        confess "Missing content-disposition header" if $hdrs !~ /content-disposition:\s*form-data(.+)/i;
         my $v = $1;
-        confess "Missing 'name' parameter" if $v !~ /[;\s]name=([^[;\s]+)/;
+        my $pvalue = qr/("(?:\\[\\"]|[^\\"\r\n]+)*"|[^\s;"]*)/;
+        confess "Missing 'name' parameter" if $v !~ /;\s*name\s*=\s*$pvalue/;
         $d->{name} = utf8_decode _arg $1;
-        $d->{filename} = utf8_decode _arg $1 if $v =~ /[;\s]filename=([^;\s]+)/;
+        $d->{filename} = utf8_decode _arg $1 if $v =~ /;\s*filename\s*=\s*$pvalue/;
 
-        if ($hdrs =~ /content-type:\s*([^;\s]+)(?:\s*;\s*charset=([^;\s]+))?/i) {
+        if ($hdrs =~ /content-type:\s*$pvalue(?:\s*;\s*charset\s*=\s*$pvalue)?/i) {
             $d->{mime} = utf8_decode _arg $1;
             $d->{charset} = utf8_decode _arg $2 if $2;
         }
diff --git a/t/multipart.t b/t/multipart.t
index 842b9cd..e045ff1 100644
--- a/t/multipart.t
+++ b/t/multipart.t
@@ -14,12 +14,17 @@ Content-Type: text
 Content-Disposition: form-data; filename="example.txt"; name=field2
 
 value2
+--delimiter12345
+Content-Type: something; charset = " a b \\ c "
+Content-Disposition: form-data; name = "field \"  name" ;filename= "月姫.jpg"
+
+
 --delimiter12345--
 _
 
 
 my $l = FU::MultipartFormData->parse('multipart/form-data;boundary="delimiter12345"', $t);
-is scalar @$l, 2;
+is scalar @$l, 3;
 
 my $v = $l->[0];
 is $v->name, 'field1';
@@ -44,4 +49,12 @@ is $v->charset, undef;
 is $v->length, 6;
 is $v->data, 'value2';
 
+$v = $l->[2];
+is $v->name, 'field "  name';
+is $v->filename, "\x{6708}\x{59eb}.jpg";
+is $v->mime, 'something';
+is $v->charset, ' a b \ c ';
+is $v->length, 0;
+is $v->data, '';
+
 done_testing;

From f52ad9a2e6ae0b55072f078e8d3668bf8ffe397a Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Tue, 29 Apr 2025 13:51:28 +0200
Subject: [PATCH 54/86] json_format(): Fix buffer overflow in float formatting

The ndigit argument to Gconvert() is the number of significant digits to
format, not the size of the output buffer. D'oh.
---
 c/jsonfmt.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/c/jsonfmt.c b/c/jsonfmt.c
index fff3e4f..a6d46a4 100644
--- a/c/jsonfmt.c
+++ b/c/jsonfmt.c
@@ -244,7 +244,7 @@ static void fujson_fmt(pTHX_ fujson_fmt_ctx *ctx, SV *val) {
         if (isinfnan(nv)) croak("unable to format floating point NaN or Inf as JSON");
         /* XXX: Cpanel::JSON::XS appears to always append a ".0" for round numbers, other modules do not. */
         /* XXX#2: This doesn't support quadmath. Makefile.PL checks for that */
-        fustr_reserve(ctx->out, NV_DIG+1);
+        fustr_reserve(ctx->out, NV_DIG+32);
         Gconvert(nv, NV_DIG, 0, ctx->out->cur);
         ctx->out->cur += strlen(ctx->out->cur);
     } else if (SvIOKp(val)) {

From af9340f908dcca0852d2c3f05809f6f2d44a9ffd Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 30 Apr 2025 15:00:21 +0200
Subject: [PATCH 55/86] DebugInfo: Styling + add request/response body and fu
 obj contents

Formatting is still shit.
---
 FU.pm           |  14 +++--
 FU/DebugImpl.pm | 139 ++++++++++++++++++++++++++++++++++--------------
 2 files changed, 110 insertions(+), 43 deletions(-)

diff --git a/FU.pm b/FU.pm
index 11c58ff..316f273 100644
--- a/FU.pm
+++ b/FU.pm
@@ -3,7 +3,7 @@ use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
 use POSIX ();
-use Time::HiRes 'clock_gettime', 'CLOCK_MONOTONIC';
+use Time::HiRes 'time', 'clock_gettime', 'CLOCK_MONOTONIC';
 use FU::Log 'log_write';
 use FU::Util;
 use FU::Validate;
@@ -318,7 +318,11 @@ sub _log_err($e) {
 }
 
 sub _do_req($c) {
-    local $REQ = { hdr => {}, trace_start => clock_gettime(CLOCK_MONOTONIC), trace_id => sprintf('%010x%08x%04x', int time, $$, int rand 1<<16) };
+    local $REQ = {
+        hdr => {},
+        trace_start => clock_gettime(CLOCK_MONOTONIC),
+        trace_id => sprintf('%012x%06x%04x', int(time*10000) % (1<<(12*4)), $$ % (1<<(6*4)), int rand 1<<16)
+    };
     local $fu = bless {}, 'FU::obj';
 
     $REQ->{ip} = $c->{client_sock} isa 'IO::Socket::INET' ? $c->{client_sock}->peerhost : '127.0.0.1';
@@ -648,8 +652,8 @@ sub log_verbose($,$msg) {
         length $r->{body} ? do {
             my $b = substr $r->{body}, 0, 4096;
             my $trunc = length $r->{body} > 4096 ? ', truncated' : '';
-            utf8::decode($b) ? ("Body (utf8$trunc)", _fmt_section($b =~ s/\r//rg =~ s/\n{4,}/\n[..]\n/rg))
-                             : ("Body (hex$trunc)", _fmt_section(unpack('H*', $b) =~ s/(.{128})/$1\n/rg))
+            utf8::decode($b) ? ("Body (utf8$trunc):", _fmt_section($b =~ s/\r//rg =~ s/\n{4,}/\n[..]\n/rg))
+                             : ("Body (hex$trunc):", _fmt_section(unpack('H*', $b) =~ s/(.{128})/$1\n/rg))
         } : (),
         'Message:', _fmt_section $msg
     );
@@ -898,10 +902,12 @@ sub _finalize {
         ) {
             push @vary, 'accept-encoding';
             if ($hasbrotli && ($r->{hdr}{'accept-encoding'}||'') =~ /\bbr\b/) {
+                $r->{resbody_orig} = $r->{resbody};
                 $r->{resbody} = FU::Util::brotli_compress(6, $r->{resbody});
                 $r->{reshdr}{'content-encoding'} = 'br';
 
             } elsif ($hasgzip && ($r->{hdr}{'accept-encoding'}||'') =~ /\bgzip\b/) {
+                $r->{resbody_orig} = $r->{resbody};
                 $r->{resbody} = FU::Util::gzip_compress(6, $r->{resbody});
                 $r->{reshdr}{'content-encoding'} = 'gzip';
             }
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index a0de7ea..f02ceed 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -16,27 +16,32 @@ sub loc_($loc) {
         my $l = $loc->[$_];
         my $f = $_ == $#$loc ? '(main)' : $loc->[$_+1][3];
         $f = "$l->[0]::$f" if $f !~ /^\Q$l->[0]/;
-        txt_ "$f @ $l->[1]:$l->[2]";
+        txt_ $f;
+        small_ " @ $l->[1]:$l->[2]";
     }
 }
 
-sub fmtpre_($code) {
-    lit_ xml_escape($code) =~ s/^\s+//r =~ s/\s+$//r =~ s/\n/<br>/rg;
-}
-
 sub clean_re($str) {
     # Regex formatting isn't stable, but this cleans up the crap I'm seeing a little bit.
     "$str" =~ s/^\(\?\^u:\^\(\?\^u://r =~ s/\)\$\)$//r;
 }
 
-my @tabs = (
+sub raw_data($str) {
+    my $d = substr $str, 0, 32*1024;
+    my $trunc = length $str > 32*1024 ? ', truncated' : '';
+    return utf8::decode($d) ? ("utf8$trunc", $d)
+         : ("hex$trunc", unpack('H*', $d) =~ s/(.{128})/$1\n/rg =~ s/(.{16})/$1 /rg);
+}
+
+my @sections = (
     req => sub {
+        my $r = $FU::REQ;
         table_ sub {
             tr_ sub { td_ 'Method'; td_ fu->method };
             tr_ sub { td_ 'Path'; td_ fu->path };
             tr_ sub { td_ 'Query'; td_ fu->query };
             tr_ sub { td_ 'Client IP'; td_ fu->ip };
-            tr_ sub { td_ 'Received'; td_ fmtts(time - (($FU::REQ->{trace_end}||clock_gettime(CLOCK_MONOTONIC)) - $FU::REQ->{trace_start})) };
+            tr_ sub { td_ 'Received'; td_ fmtts(time - (($r->{trace_end}||clock_gettime(CLOCK_MONOTONIC)) - $r->{trace_start})) };
         };
         h2_ 'Headers';
         table_ sub {
@@ -45,7 +50,38 @@ my @tabs = (
                 td_ fu->headers->{$_};
             } for sort keys fu->headers->%*;
         };
-        # TODO: Body? Certainly useful for JSON
+        if ((fu->header('content-length')||0) > 0) {
+            h2_ 'Body';
+            section_ class => 'tabs', sub {
+                my $json = eval { fu->json({type=>'any'}) };
+                details_ name => 'reqbody', open => !0, sub {
+                    summary_ 'JSON';
+                    pre_ FU::Util::json_format($json, pretty => 1, canonical => 1);
+                } if $json;
+                my $formdata = eval { fu->formdata({type=>'hash'}) };
+                details_ name => 'reqbody', open => !0, sub {
+                    summary_ 'Form data';
+                    table_ sub {
+                        for my $k (sort keys %$formdata) {
+                            tr_ sub {
+                                td_ $k;
+                                td_ $_;
+                            } for ref $formdata->{$k} ? $formdata->{$k}->@* : ($formdata->{$k});
+                        }
+                    };
+                } if $formdata;
+                my $multipart = eval { fu->multipart };
+                details_ name => 'reqbody', open => !0, sub {
+                    summary_ 'Multipart';
+                    pre_ join "\n", map $_->describe, @$multipart;
+                } if $multipart;
+                details_ name => 'reqbody', open => !0,sub {
+                    my($lbl, $data) = raw_data $r->{body};
+                    summary_ "Raw ($lbl)";
+                    pre_ $data;
+                };
+            }
+        }
         ('Request')
     },
 
@@ -84,11 +120,28 @@ my @tabs = (
                 } for !defined $v ? () : ref $v ? @$v : ($v);
             }
         };
+        my $body = $r->{resbody_orig} // $r->{resbody};
+        if (length $body) {
+            h2_ 'Body';
+            section_ class => 'tabs', sub {
+                my $json = ($r->{reshdr}{'content-type'}||'') =~ /^application\/json/ && eval { FU::Util::json_parse($body, utf8 => 1) };
+                details_ name => 'resbody', open => !0, sub {
+                    summary_ 'JSON';
+                    pre_ FU::Util::json_format($json, pretty => 1, canonical => 1);
+                } if $json;
+                details_ name => 'resbody', open => !0,sub {
+                    my($lbl, $data) = raw_data $body;
+                    summary_ "Raw ($lbl)";
+                    pre_ $data;
+                };
+            }
+        }
         ('Response')
     },
 
     sql => sub {
         return () if !$FU::REQ->{trace_sql};
+        # TODO: Summarize main table, expand to display full query, params table, interpolated query
         table_ sub {
             thead_ sub { tr_ sub {
                 td_ class => 'num', 'Exec';
@@ -100,8 +153,7 @@ my @tabs = (
                 td_ class => 'num', sprintf '%.1f ms', $_->{exec_time}*1000;
                 td_ class => 'num', !defined $_->{prepare_time} ? '-' : $_->{prepare_time} ? sprintf '%.1f ms', $_->{prepare_time}*1000 : 'cache';
                 td_ class => 'num', $_->{nrows};
-                td_ class => 'code', sub { fmtpre_ $_->{query} };
-                # TODO: Params, both separate and interpolated
+                td_ class => 'code', $_->{query};
             } for $FU::REQ->{trace_sql}->@*;
         };
         ('Queries', scalar $FU::REQ->{trace_sql}->@*)
@@ -109,7 +161,11 @@ my @tabs = (
 
     fu => sub {
         return () if !keys fu->%*;
-        # TODO: Contents of the 'fu' object
+        # TODO: This is kinda lazy, an expandable table might be nicer.
+        require Data::Dumper;
+        pre_ sub {
+            lit_ Data::Dumper->new([fu])->Sortkeys(1)->Terse(1)->Dump;
+        };
         ('fu obj')
     },
 
@@ -186,7 +242,7 @@ my @tabs = (
             } };
             tr_ sub {
                 td_ $_->[0];
-                td_ class => 'code', sub { fmtpre_ $_->[1] };
+                td_ class => 'code', $_->[1];
             } for @$lst;
         };
         ('Prepared statements', scalar @$lst)
@@ -196,9 +252,10 @@ my @tabs = (
 
 sub collect {
     my @t;
-    for my ($id, $sub) (@tabs) {
+    for my ($id, $sub) (@sections) {
         my($title, $num);
         my $html = fragment { ($title, $num) = $sub->() };
+        utf8::decode($html);
         push @t, { id => $id, title => $title, num => $num, html => $html } if $title;
     }
     \@t
@@ -215,42 +272,47 @@ sub framework_($data) {
               *, *:before, *:after { box-sizing: inherit }
               * { margin: 0; padding: 0; font: inherit; color: inherit }
 
-              body { display: grid; grid: 45px 400px / 220px auto; }
-              header { grid-column: 1 / 3; grid-row: 1 / 2 }
-              nav { grid-column: 1 / 2; grid-row: 2 / 3 }
-              main { grid-column: 2 / 3; grid-row: 2 / 3 }
+              /* Ugh, fixed positioning */
+              header { position: fixed; top: 0; left: 0; width: 100%; height: 40px; z-index: 2 }
+              nav { position: fixed; top: 38px; left: 0; width: 200px; z-index: 2 }
+              main { margin: 0 0 0 200px }
 
               header, nav { background: #eee }
-              main { border-top: 2px solid #009; border-left: 2px solid #009 }
-              nav { border-bottom: 2px solid #009 }
+              header { border-bottom: 2px solid #009 }
+              nav { border-bottom: 2px solid #009; border-right: 2px solid #009 }
 
-              header { display: flex; justify-content: space-between; padding: 10px }
-              header h1 { font-size: 20px; font-weight: bold }
+              header { display: flex; justify-content: space-between; align-items: baseline; padding: 5px 10px }
+              header h1 { font-size: 120%; font-weight: bold }
               header menu { list-style-type: none; display: flex; gap: 15px }
 
               body > input { display: none }
               nav { padding-top: 20px }
               nav menu { list-style-type: none }
-              nav label { display: block; width: 100%; padding: 2px 10px; cursor: pointer; white-space: nowrap }
-              nav label:hover { background-color: #fff }
-              nav label span { float: right; font-size: 80% }
+              nav a { display: block; width: 100%; text-decoration: none; padding: 2px 10px; cursor: pointer; white-space: nowrap }
+              nav a:hover { background-color: #fff }
+              nav a span { float: right; font-size: 80% }
 
-              main { padding: 10px 20px }
-              main h2 { margin: 30px 0 5px -10px; font-size: 20px; font-weight: bold }
-              main h2:first-child { margin-top: 0 }
+              main { padding: 0 10px 30px 10px }
+              main h1 { background: #eee; padding: 5px 10px 5px 205px; margin: 40px -10px 10px -210px; scroll-margin-top: 40px; font-size: 130%; font-weight: bold }
+              main h2 { margin: 20px 0 5px 0; font-size: 120%; font-weight: bold }
 
-              p, pre, table { margin: 5px 0 }
-              pre, .code { font-family: monospace; white-space: pre }
+              p, table, pre { margin: 5px 0 }
+              pre { font-family: monospace; white-space: pre; overflow-x: auto; padding-bottom: 15px; /* for the scrollbar, kinda browser-specific */ }
               table { border-collapse: collapse }
               td { padding: 1px 10px 1px 0; font-size: 12px; vertical-align: top }
+              td.code { font-family: monospace }
               tr:hover { background-color: #eee }
               thead { font-weight: bold }
               .num { text-align: right; white-space: nowrap }
+
+              section.tabs { position: relative; display: flex; flex-wrap: wrap; z-index: 1; }
+              section.tabs summary { cursor: pointer; order: 0; display: block; padding: 3px 5px; margin-right: 10px; background: #ddd }
+              section.tabs summary:hover, section.tabs details[open] summary { background: #eee }
+              section.tabs details { display: contents }
+              section.tabs details *:nth-child(2) { order: 1; width: 100% }
+
+              small { color: #555; font-size: 90% }
             _
-            style_ type => 'text/css', join "\n", map +(
-                "#tab_$_:checked ~ nav menu li label[for=tab_$_] { background-color: #fff }",
-                "#tab_$_:not(:checked) ~ main #tabc_$_ { display: none }",
-            ), map $_->{id}, @$data;
         };
         body_ sub {
             header_ sub {
@@ -261,22 +323,21 @@ sub framework_($data) {
                     li_ sub { a_ href => '?', 'Listing' };
                 };
             };
-            input_ type => 'radio', name => 'tab', id => "tab_$_->{id}", checked => $_ eq $data->[0] ? 'checked' : undef for @$data;
             nav_ sub {
                 menu_ sub {
                     li_ sub {
-                        label_ for => "tab_$_->{id}", sub {
+                        a_ href => "#$_->{id}", sub {
                             txt_ $_->{title};
                             span_ $_->{num} if defined $_->{num};
-                        }
+                        };
                     } for @$data;
                 };
             } if @$data;
             main_ sub {
-                div_ id => "tabc_$_->{id}", sub {
-                    h2_ $_->{title};
+                for (@$data) {
+                    h1_ id => $_->{id}, $_->{title};
                     lit_ $_->{html};
-                } for @$data;
+                }
             };
         };
     };

From beeefcf3373a11877d121f7eded08ab238b36189 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 30 Apr 2025 17:07:44 +0200
Subject: [PATCH 56/86] Pg: Add perl2bin() and bin2perl() conversion methods

---
 FU.xs       |  8 ++++++++
 FU/Pg.pm    | 26 +++++++++++++++++++++++++-
 c/pgconn.c  | 27 +++++++++++++++++++++++++++
 t/pgtypes.t | 10 +++++++++-
 4 files changed, 69 insertions(+), 2 deletions(-)

diff --git a/FU.xs b/FU.xs
index dc19870..68082a5 100644
--- a/FU.xs
+++ b/FU.xs
@@ -286,6 +286,14 @@ void _set_type(fupg_conn *c, SV *name, SV *sendsv, SV *recvsv)
     fupg_set_type(aTHX_ c, name, sendsv, recvsv);
     XSRETURN(1);
 
+void perl2bin(fupg_conn *c, int oid, SV *sv)
+  CODE:
+    ST(0) = fupg_perl2bin(aTHX_ c, oid, sv);
+
+void bin2perl(fupg_conn *c, int oid, SV *sv)
+  CODE:
+    ST(0) = fupg_bin2perl(aTHX_ c, oid, sv);
+
 
 MODULE = FU   PACKAGE = FU::Pg::txn
 
diff --git a/FU/Pg.pm b/FU/Pg.pm
index f43c7f8..d88a331 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -758,7 +758,31 @@ C<set_type()> to configure appropriate conversions for these types.
 
 =back
 
-I<TODO:> Methods to convert between the various formats.
+Utility functions:
+
+=over
+
+=item $conn->perl2bin($oid, $val)
+
+=item $conn->bin2perl($oid, $bin)
+
+Convert the value for a specific type between the Perl representation and the
+PostgreSQL binary format, using the current type configuration of the
+connection. This is the same conversion used internally by this module to send
+bind parameters and receive query results, and map to the C<send> and C<recv>
+functions of C<< $conn->set_type() >>.
+
+These methods throw an error if C<$oid> is not a known type or if the given
+data is not valid for the type. However, these methods should not be used for
+strict validation: the conversion routines are usually written under the
+assumption that the data has been received directly from Postgres or is about
+to be sent to (and further validated by) Postgres.  For some types,
+C<perl2bin()> may return invalid data on invalid input and C<bin2perl()> may
+accept invalid binary data.
+
+=back
+
+I<TODO:> Methods to convert between bin and text formats.
 
 I<TODO:> Methods to query type info.
 
diff --git a/c/pgconn.c b/c/pgconn.c
index 3a20b56..04423e6 100644
--- a/c/pgconn.c
+++ b/c/pgconn.c
@@ -626,3 +626,30 @@ static void fupg_tio_free(fupg_tio *tio) {
         safefree(tio->record.tio);
     }
 }
+
+
+
+
+static SV *fupg_perl2bin(pTHX_ fupg_conn *conn, Oid oid, SV *sv) {
+    int refresh_done = 0;
+    fupg_tio tio;
+    fustr buf;
+    memset(&tio, 0, sizeof(tio));
+    fupg_tio_setup(aTHX_ conn, &tio, FUPGT_SEND, oid, &refresh_done);
+    fustr_init(&buf, sv_newmortal(), SIZE_MAX);
+    tio.send(aTHX_ &tio, sv, &buf); /* XXX: Leaks 'tio' on error */
+    fupg_tio_free(&tio);
+    return fustr_done(&buf);
+}
+
+static SV *fupg_bin2perl(pTHX_ fupg_conn *conn, Oid oid, SV *sv) {
+    int refresh_done = 0;
+    fupg_tio tio;
+    STRLEN len;
+    const char *buf = SvPVbyte(sv, len);
+    memset(&tio, 0, sizeof(tio));
+    fupg_tio_setup(aTHX_ conn, &tio, FUPGT_RECV, oid, &refresh_done);
+    SV *r = tio.recv(aTHX_ &tio, buf, len); /* XXX: Leaks 'tio' on error */
+    fupg_tio_free(&tio);
+    return r;
+}
diff --git a/t/pgtypes.t b/t/pgtypes.t
index 3a3252c..67e566e 100644
--- a/t/pgtypes.t
+++ b/t/pgtypes.t
@@ -19,9 +19,12 @@ sub v($type, $p_in, @args) {
     my $s_out = @args > 2 && defined $args[2] ? $args[2] : $s_in;
 
     my $test = "$type $s_in" =~ s/\n/\\n/rg;
+    my $oid;
     utf8::encode($test);
     {
-        my $array = $conn->q("SELECT \$1::$type", $s_in)->text_params->flat;
+        my $st = $conn->q("SELECT \$1::$type", $s_in)->text_params;
+        $oid = $st->param_types->[0];
+        my $array = $st->flat;
         my $res = $array->[0];
         ok is_bool($res), "$test is bool" if $type eq 'bool';
         ok created_as_number($res), "$test is number" if $type =~ /^(int|float)\d/;
@@ -36,6 +39,11 @@ sub v($type, $p_in, @args) {
         my $res = $conn->q("SELECT \$1::$type", $p_in)->val;
         is_deeply $res, $p_out, "$test bin->bin";
     }
+    {
+        my $bin = $conn->perl2bin($oid, $p_in);
+        ok defined $bin;
+        is_deeply $conn->bin2perl($oid, $bin), $p_out if $type !~ /\(/;
+    }
 }
 sub f($type, $p_in) {
     my $test = "$type $p_in" =~ s/\n/\\n/rg;

From 76f55f277bd94155827ce6dcf23566f15128bab8 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 30 Apr 2025 20:02:12 +0200
Subject: [PATCH 57/86] Pg: Add text2bin() and bin2text() conversion methods

---
 FU.xs       |  8 ++++++++
 FU/Pg.pm    | 19 +++++++++++++++++--
 c/pgconn.c  | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 t/pgtypes.t | 19 ++++++++++++++++++-
 4 files changed, 93 insertions(+), 3 deletions(-)

diff --git a/FU.xs b/FU.xs
index 68082a5..1c342be 100644
--- a/FU.xs
+++ b/FU.xs
@@ -294,6 +294,14 @@ void bin2perl(fupg_conn *c, int oid, SV *sv)
   CODE:
     ST(0) = fupg_bin2perl(aTHX_ c, oid, sv);
 
+void bin2text(fupg_conn *c, ...)
+  CODE:
+    XSRETURN(fupg_bintext(aTHX_ c, 0, ax, items));
+
+void text2bin(fupg_conn *c, ...)
+  CODE:
+    XSRETURN(fupg_bintext(aTHX_ c, 1, ax, items));
+
 
 MODULE = FU   PACKAGE = FU::Pg::txn
 
diff --git a/FU/Pg.pm b/FU/Pg.pm
index d88a331..ff88d60 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -780,9 +780,24 @@ to be sent to (and further validated by) Postgres.  For some types,
 C<perl2bin()> may return invalid data on invalid input and C<bin2perl()> may
 accept invalid binary data.
 
-=back
+=item $conn->bin2text($oid, $bin, ...)
 
-I<TODO:> Methods to convert between bin and text formats.
+=item $conn->text2bin($oid, $text, ...)
+
+Convert between the binary format and the PostgreSQL text format. This
+conversion requires a round-trip to the server and throws an error if the
+connection state is not I<idle> or I<txn_idle>. Since it is Postgres doing the
+conversion, the input is properly validated and, in the case of C<bin2text()>,
+the result is guaranteed to be suitable for use as a textual bind parameter or
+for inclusion in an SQL query (but don't forget to use C<escape_literal()> in
+that case).
+
+Calling these methods many times can be pretty slow. If you have several values
+to convert, you can do that in a single call to speed things up:
+
+  my($text1, $text2, ..) = $conn->bin2text($oid1, $bin1, $oid2, $bin2, ..);
+
+=back
 
 I<TODO:> Methods to query type info.
 
diff --git a/c/pgconn.c b/c/pgconn.c
index 04423e6..607a9c6 100644
--- a/c/pgconn.c
+++ b/c/pgconn.c
@@ -653,3 +653,53 @@ static SV *fupg_bin2perl(pTHX_ fupg_conn *conn, Oid oid, SV *sv) {
     fupg_tio_free(&tio);
     return r;
 }
+
+
+static I32 fupg_bintext(pTHX_ fupg_conn *conn, int format, I32 ax, I32 argc) {
+    int vals = argc/2;
+
+    if (argc == 1 || argc % 2 == 0) croak("Usage: $conn->%s(oid, data, ...)", format ? "text2bin" : "bin2text");
+    if (vals > 1 && GIMME_V != G_LIST) {
+        ST(0) = sv_2mortal(newSViv(vals));
+        return 1;
+    }
+
+    Oid *paramtypes = safemalloc(vals * sizeof(*paramtypes));
+    const char **paramvalues = safemalloc(vals * sizeof(*paramvalues));
+    int *paramlengths = safemalloc(vals * sizeof(*paramlengths));
+    int *paramformats = safemalloc(vals * sizeof(*paramformats));
+
+    fustr sql;
+    fustr_init(&sql, NULL, SIZE_MAX);
+    fustr_write(&sql, "SELECT ", 7);
+
+    STRLEN len;
+    int i;
+    for (i=0; i<vals; i++) {
+        paramtypes[i] = SvIV(ST(i*2+1));
+        paramvalues[i] = format ? SvPVutf8(ST(i*2+2), len) : SvPVbyte(ST(i*2+2), len);
+        paramlengths[i] = len;
+        paramformats[i] = format ? 0 : 1;
+        if (i) fustr_write_ch(&sql, ',');
+        sql.cur -= 8 - sprintf(fustr_write_buf(&sql, 8), "$%d", i+1);
+    }
+    fustr_write_ch(&sql, 0);
+
+    PGresult *r = PQexecParams(conn->conn, fustr_start(&sql), vals,
+            paramtypes, paramvalues, paramlengths, paramformats, format);
+    safefree(paramtypes);
+    safefree(paramvalues);
+    safefree(paramlengths);
+    safefree(paramformats);
+    SvREFCNT_dec(sql.sv);
+
+    if (!r) fupg_conn_croak(conn, "exec");
+    if (PQresultStatus(r) != PGRES_TUPLES_OK) fupg_result_croak(r, "exec", sql.sv ? "SELECT $1, ..." : sql.sbuf);
+
+    /* The stack is guaranteed to be large enough, since we received 1+2*vals arguments */
+    for (i=0; i<vals; i++)
+        ST(i) = newSVpvn_flags(PQgetvalue(r, 0, i), PQgetlength(r, 0, i), SVs_TEMP | (format ? 0 : SVf_UTF8));
+
+    PQclear(r);
+    return vals;
+}
diff --git a/t/pgtypes.t b/t/pgtypes.t
index 67e566e..21c8eff 100644
--- a/t/pgtypes.t
+++ b/t/pgtypes.t
@@ -42,7 +42,11 @@ sub v($type, $p_in, @args) {
     {
         my $bin = $conn->perl2bin($oid, $p_in);
         ok defined $bin;
-        is_deeply $conn->bin2perl($oid, $bin), $p_out if $type !~ /\(/;
+        if ($type !~ /\(/) {
+            is_deeply $conn->bin2perl($oid, $bin), $p_out;
+            is $conn->bin2text($oid, $bin), $s_out;
+            is $conn->text2bin($oid, $s_out), $bin if $type ne 'jsonb'; # jsonb pretty-prints for some reason
+        }
     }
 }
 sub f($type, $p_in) {
@@ -180,6 +184,19 @@ is $conn->q('SELECT ($1::int2[])[2]', [1,2,3,4])->val, 2;
 is $conn->q('SELECT ($1::int2vector)[1]', [1,2,3,4])->val, 2;
 is $conn->q('SELECT ($1::oidvector)[1]', [1,2,3,4])->val, 2;
 
+is_deeply [$conn->bin2text(
+    16, $conn->perl2bin(16, 1),
+    25, 'Hello',
+    1007, $conn->perl2bin(1007, [-3,1,undef])
+)], ['t', 'Hello', '{-3,1,NULL}'];
+
+{
+    my($b,$s,$a) = $conn->text2bin(16, 't', 25, 'Hello', 1007, '{-3,1,NULL}');
+    is $conn->bin2perl(16, $b), 1;
+    is $conn->bin2perl(25, $s), 'Hello';
+    is_deeply $conn->bin2perl(1007, $a), [-3,1,undef];
+}
+
 {
     my $v = $conn->q("SELECT '{t,f,NULL}'::bool[]")->val;
     is_deeply $v, [true, false, undef];

From cbccf045b71f22696c5d235345271b00a1f1ce51 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 1 May 2025 11:48:08 +0200
Subject: [PATCH 58/86] DebugInfo: Expand queries table with params & details

Apart from the ugly implementation, this is pretty neat.
---
 FU.pm           |  23 ++++--
 FU/DebugImpl.pm | 189 ++++++++++++++++++++++++++++++++++--------------
 2 files changed, 153 insertions(+), 59 deletions(-)

diff --git a/FU.pm b/FU.pm
index 316f273..8b8eaa3 100644
--- a/FU.pm
+++ b/FU.pm
@@ -121,11 +121,24 @@ sub query_trace($st,@) {
     $REQ->{trace_nsqldirect}++ if !defined $st->prepare_time;
     $REQ->{trace_sqlexec} += $st->exec_time;
     $REQ->{trace_sqlprep} += $st->prepare_time if $st->prepare_time;
-    push $REQ->{trace_sql}->@*, {
-        query => $st->query, nrows => $st->nrows,
-        param_types => $st->param_types, param_values => $st->param_values,
-        exec_time => $st->exec_time, prepare_time => $st->prepare_time,
-    } if FU::debug;
+    if (FU::debug) {
+        my $t = $st->param_types;
+        my $v = $st->param_values;
+        my $txt = $st->get_text_params;
+        push $REQ->{trace_sql}->@*, {
+            query => $st->query, nrows => $st->nrows,
+            exec_time => $st->exec_time, prepare_time => $st->prepare_time,
+            # Store the binary value when we're in binary params mode, that way
+            # we don't have to keep a reference to the original perl value and
+            # we can defer & batch the conversion to text.
+            params => [ map +{
+                type => $t->[$_],
+                !defined $v->[$_] ? (text => undef) :
+                             $txt ? (text => "$v->[$_]")
+                                  : (bin => $DB->perl2bin($t->[$_], $v->[$_]))
+            }, 0..$#$v ],
+        };
+    }
 }
 sub _connect_db {
     $DB = ref $INIT_DB eq 'CODE' ? $INIT_DB->() : FU::Pg->connect($INIT_DB);
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index f02ceed..48c6cf1 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,6 +1,7 @@
 # Internal module used by FU.pm
 package FU::DebugImpl 0.5;
 use v5.36;
+use utf8;
 use experimental 'for_list';
 use FU;
 use FU::XMLWriter ':html5_', 'fragment', 'xml_escape';
@@ -140,23 +141,74 @@ my @sections = (
     },
 
     sql => sub {
-        return () if !$FU::REQ->{trace_sql};
-        # TODO: Summarize main table, expand to display full query, params table, interpolated query
-        table_ sub {
+        my $queries = $FU::REQ->{trace_sql};
+        return () if !$queries;
+
+        # Convert binary params to text.
+        # For queries with text_params, assume the params are already valid for the text format.
+        my @binparams = grep $_->{type} && !$_->{text}, map $_->{params}->@*, @$queries;
+        my @arg = map +($_->{type}, $_->{bin}), @binparams;
+        my @text;
+        my $ok = !@arg || eval { @text = $FU::DB->bin2text(@arg); 1 };
+        $binparams[$_]{text} = $text[$_] for 0..$#text;
+        pre_ "Error converting binary parameters:\n$@" if !$ok;
+
+        input_ type => 'checkbox', id => "row${_}_c" for 0..$#{$queries};
+        table_ class => 'sqlt', sub {
             thead_ sub { tr_ sub {
                 td_ class => 'num', 'Exec';
                 td_ class => 'num', 'Prep';
                 td_ class => 'num', 'Rows';
                 td_ 'Query';
             } };
+            my $rows = 0;
+            for my($i, $st) (builtin::indexed $queries->@*) {
+                $rows += $st->{nrows};
+                tr_ sub {
+                    td_ class => 'num', sprintf '%.1f ms', $st->{exec_time}*1000;
+                    td_ class => 'num', !defined $st->{prepare_time} ? '-' : $st->{prepare_time} ? sprintf '%.1f ms', $st->{prepare_time}*1000 : 'cache';
+                    td_ class => 'num', $st->{nrows};
+                    td_ class => 'sum', sub {
+                        label_ for => "row${i}_c", sub {
+                            span_ class => 'closed', '▶';
+                            span_ class => 'open', '▼';
+                            txt_ $st->{query} =~ s/[\r\n]/ /rg =~ s/\s\s+/ /rg =~ s/^\s+//r;
+                        };
+                    };
+                };
+                tr_ class => 'details', id => "row$i", sub {
+                    td_ '';
+                    td_ colspan => 3, sub {
+                        pre_ $st->{query};
+                        if ($st->{params}->@*) {
+                            strong_ 'Parameters:';
+                            table_ sub {
+                                tr_ sub {
+                                    td_ class => 'num', sprintf '$%d =', $_+1;
+                                    td_ class => 'code', sub {
+                                        my $p = $st->{params}[$_]{text};
+                                        !defined $p ? em_ 'null' : txt_ $p;
+                                    };
+                                } for (0..$#{$st->{params}});
+                            };
+                            # XXX: Buggy when the query contains string literals with $n variables.
+                            strong_ 'Interpolated:';
+                            pre_ $st->{query} =~ s{\$([1-9][0-9]*)}{
+                                my $v = $st->{params}[$1-1]{text};
+                                defined $v ? $FU::DB->escape_literal($v) : 'NULL'
+                            }egr;
+                        }
+                    };
+                };
+            }
             tr_ sub {
-                td_ class => 'num', sprintf '%.1f ms', $_->{exec_time}*1000;
-                td_ class => 'num', !defined $_->{prepare_time} ? '-' : $_->{prepare_time} ? sprintf '%.1f ms', $_->{prepare_time}*1000 : 'cache';
-                td_ class => 'num', $_->{nrows};
-                td_ class => 'code', $_->{query};
-            } for $FU::REQ->{trace_sql}->@*;
+                td_ class => 'num', sprintf '%.1f ms', $FU::REQ->{trace_sqlexec}*1000;
+                td_ class => 'num', !defined $FU::REQ->{trace_sqlprep} ? '-' : sprintf '%.1f ms', $FU::REQ->{trace_sqlprep}*1000;
+                td_ class => 'num', $rows;
+                td_ class => 'sum', 'total';
+            } if @$queries > 1;
         };
-        ('Queries', scalar $FU::REQ->{trace_sql}->@*)
+        ('Queries', scalar @$queries)
     },
 
     fu => sub {
@@ -245,7 +297,7 @@ my @sections = (
                 td_ class => 'code', $_->[1];
             } for @$lst;
         };
-        ('Prepared statements', scalar @$lst)
+        ('Prepared stmts', scalar @$lst)
     },
 );
 
@@ -267,51 +319,8 @@ sub framework_($data) {
         head_ sub {
             title_ 'FU Debugging Interface';
             meta_ name => 'viewport', content => 'width=device-width, initial-scale=1.0, user-scalable=yes';
+            link_ rel => 'stylesheet', type => 'text/css', media => 'all', href => '?css';
             style_ type => 'text/css', <<~_;
-              html { box-sizing: border-box; color: #000; background: #fff }
-              *, *:before, *:after { box-sizing: inherit }
-              * { margin: 0; padding: 0; font: inherit; color: inherit }
-
-              /* Ugh, fixed positioning */
-              header { position: fixed; top: 0; left: 0; width: 100%; height: 40px; z-index: 2 }
-              nav { position: fixed; top: 38px; left: 0; width: 200px; z-index: 2 }
-              main { margin: 0 0 0 200px }
-
-              header, nav { background: #eee }
-              header { border-bottom: 2px solid #009 }
-              nav { border-bottom: 2px solid #009; border-right: 2px solid #009 }
-
-              header { display: flex; justify-content: space-between; align-items: baseline; padding: 5px 10px }
-              header h1 { font-size: 120%; font-weight: bold }
-              header menu { list-style-type: none; display: flex; gap: 15px }
-
-              body > input { display: none }
-              nav { padding-top: 20px }
-              nav menu { list-style-type: none }
-              nav a { display: block; width: 100%; text-decoration: none; padding: 2px 10px; cursor: pointer; white-space: nowrap }
-              nav a:hover { background-color: #fff }
-              nav a span { float: right; font-size: 80% }
-
-              main { padding: 0 10px 30px 10px }
-              main h1 { background: #eee; padding: 5px 10px 5px 205px; margin: 40px -10px 10px -210px; scroll-margin-top: 40px; font-size: 130%; font-weight: bold }
-              main h2 { margin: 20px 0 5px 0; font-size: 120%; font-weight: bold }
-
-              p, table, pre { margin: 5px 0 }
-              pre { font-family: monospace; white-space: pre; overflow-x: auto; padding-bottom: 15px; /* for the scrollbar, kinda browser-specific */ }
-              table { border-collapse: collapse }
-              td { padding: 1px 10px 1px 0; font-size: 12px; vertical-align: top }
-              td.code { font-family: monospace }
-              tr:hover { background-color: #eee }
-              thead { font-weight: bold }
-              .num { text-align: right; white-space: nowrap }
-
-              section.tabs { position: relative; display: flex; flex-wrap: wrap; z-index: 1; }
-              section.tabs summary { cursor: pointer; order: 0; display: block; padding: 3px 5px; margin-right: 10px; background: #ddd }
-              section.tabs summary:hover, section.tabs details[open] summary { background: #eee }
-              section.tabs details { display: contents }
-              section.tabs details *:nth-child(2) { order: 1; width: 100% }
-
-              small { color: #555; font-size: 90% }
             _
         };
         body_ sub {
@@ -378,10 +387,23 @@ sub load($id) {
     fu->set_body(scalar <$fn>);
 }
 
+sub css {
+    # Awful CSS row hiding hack. I'm not sorry.
+    state $css = join '', <DATA>, map qq{
+        #row${_}_c:checked ~ * label[for=row${_}_c] .closed { display: none }
+        #row${_}_c:not(:checked) ~ * label[for=row${_}_c] .open { display: none }
+        #row${_}_c:not(:checked) ~ * #row${_} { display: none }
+    }, 0..1000;
+}
+
 sub render {
     my $q = fu->query;
     if (!$q) {
         fu->set_body(framework_ [{id => 'lst', title => 'Recent Requests', html => fragment \&listing_ }]);
+    } elsif ($q eq 'css') {
+        fu->set_header('content-type', 'text/css');
+        fu->set_header('cache-control', 'max-age=86400');
+        fu->set_body(css());
     } elsif ($q eq 'cur') {
         fu->set_body(framework_ collect);
     } elsif ($q eq 'last') {
@@ -415,3 +437,62 @@ sub save {
 }
 
 1;
+
+__DATA__
+html { box-sizing: border-box; color: #000; background: #fff }
+*, *:before, *:after { box-sizing: inherit }
+* { margin: 0; padding: 0; font: inherit; color: inherit }
+
+/* Ugh, fixed positioning */
+header { position: fixed; top: 0; left: 0; width: 100%; height: 40px; z-index: 2 }
+nav { position: fixed; top: 38px; left: 0; width: 200px; z-index: 2 }
+main { margin: 0 0 0 200px }
+
+header, nav { background: #eee }
+header { border-bottom: 2px solid #009 }
+nav { border-bottom: 2px solid #009; border-right: 2px solid #009 }
+
+header { display: flex; justify-content: space-between; align-items: baseline; padding: 5px 10px }
+header h1 { font-size: 120%; font-weight: bold }
+header menu { list-style-type: none; display: flex; gap: 15px }
+
+body > input { display: none }
+nav { padding-top: 20px }
+nav menu { list-style-type: none }
+nav a { display: block; width: 100%; text-decoration: none; padding: 2px 10px; cursor: pointer; white-space: nowrap }
+nav a:hover { background-color: #fff }
+nav a span { float: right; font-size: 80% }
+
+main { padding: 0 10px 30px 10px }
+main h1 { background: #eee; padding: 5px 10px 5px 205px; margin: 40px -10px 10px -210px; scroll-margin-top: 40px; font-size: 130%; font-weight: bold }
+main h2 { margin: 20px 0 5px 0; font-size: 120%; font-weight: bold }
+
+p, table, pre { margin: 5px 0 }
+pre { border-left: 2px dotted #999; padding-left: 5px; font-family: monospace; white-space: pre; overflow-x: auto; padding-bottom: 15px; /* for the scrollbar, kinda browser-specific */ }
+table { border-collapse: collapse }
+td { padding: 1px 10px 1px 0; font-size: 12px; vertical-align: top }
+td.code { font-family: monospace }
+tr:hover { background-color: #eee }
+thead { font-weight: bold }
+.num { text-align: right; white-space: nowrap }
+
+section.tabs { position: relative; display: flex; flex-wrap: wrap; z-index: 1; }
+section.tabs summary { cursor: pointer; order: 0; display: block; padding: 3px 5px; margin-right: 10px; background: #ddd }
+section.tabs summary:hover, section.tabs details[open] summary { background: #eee }
+section.tabs details { display: contents }
+section.tabs details *:nth-child(2) { order: 1; width: 100% }
+
+.sqlt { width: 100%; table-layout: fixed }
+.sqlt .num { width: 50px }
+.sqlt .num:first-child { width: 75px }
+.sqlt .num:nth-child(2) { width: 60px }
+.sqlt .sum { white-space: nowrap; font-family: monospace; overflow: hidden; text-overflow: ellipsis }
+.sqlt label { cursor: pointer }
+.sqlt label span { color: #555; display: inline-block; width: 15px }
+.sqlt tr.details { background: #fff }
+.sqlt tr.details > td { padding-bottom: 10px }
+input[id^=row] { display: none }
+
+small { color: #555; font-size: 90% }
+em { font-style: italic }
+strong { font-weight: bold }

From 6787f32fd9c4f22f20905e260f7aa48eb88e06a8 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 3 May 2025 12:32:50 +0200
Subject: [PATCH 59/86] DebugInfo: Fix handling of undef and falsy bind
 parameters

---
 FU/DebugImpl.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 48c6cf1..88e0bb5 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -146,7 +146,7 @@ my @sections = (
 
         # Convert binary params to text.
         # For queries with text_params, assume the params are already valid for the text format.
-        my @binparams = grep $_->{type} && !$_->{text}, map $_->{params}->@*, @$queries;
+        my @binparams = grep $_->{type} && !exists $_->{text}, map $_->{params}->@*, @$queries;
         my @arg = map +($_->{type}, $_->{bin}), @binparams;
         my @text;
         my $ok = !@arg || eval { @text = $FU::DB->bin2text(@arg); 1 };

From 52c36e0aeac242d960c357205f0fef30fde7dc90 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 4 May 2025 12:18:12 +0200
Subject: [PATCH 60/86] FU: Preserve existing headers on fu->redirect()

Allows setting custom headers (in particular, cookies) when redirecting.
This behavior is consistent with send_file().
---
 FU.pm | 1 -
 1 file changed, 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index 8b8eaa3..f81be88 100644
--- a/FU.pm
+++ b/FU.pm
@@ -862,7 +862,6 @@ sub send_file($, $root, $path) {
 
 sub redirect($, $code, $location) {
     state $alias = {qw/ perm 301  temp 302  tempget 303  tempsame 307  permsame 308 /};
-    fu->reset;
     fu->status($alias->{$code} // $code);
     fu->set_header(location => "$location");
     fu->set_header('content-type', 'text/plain');

From 6c54ee30911bb26dfd1fd0538cd5bd1d1280ac0f Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 9 May 2025 08:32:41 +0200
Subject: [PATCH 61/86] FU: Reject some invalid characters in path

---
 FU.pm      | 2 ++
 FU/Util.pm | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index f81be88..1a826e1 100644
--- a/FU.pm
+++ b/FU.pm
@@ -313,10 +313,12 @@ sub _read_req($c) {
     # Decode these into Unicode strings and check for special characters.
     eval { FU::Util::utf8_decode($_); 1} || fu->error(400, $@)
         for ($REQ->{path}, $REQ->{qs}, values $REQ->{hdr}->%*);
+    fu->error(400, 'Invalid character in path') if $REQ->{path} =~ /#/; # Some bots don't correctly split off the fragment
 
     ($REQ->{path}, my $qs) = split /\?/, $REQ->{path}//'', 2;
     $REQ->{qs} //= $qs;
     eval { $REQ->{path} = FU::Util::uri_unescape($REQ->{path}); 1; } || fu->error(400, $@);
+    fu->error(400, 'Invalid character in path') if $REQ->{path} =~ /[\r\n\t]/; # There are plenty other questionable characters, but newlines and tabs are definitely out
 }
 
 
diff --git a/FU/Util.pm b/FU/Util.pm
index 7d585d9..18db781 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -20,7 +20,7 @@ our @EXPORT_OK = qw/
 sub utf8_decode :prototype($) {
     return if !defined $_[0];
     confess 'Invalid UTF-8' if !utf8::decode($_[0]);
-    confess 'Invalid control character' if $_[0] =~ /[\x00-\x08\x0b\x0c\x0e-\x1f]/;
+    confess 'Invalid control character' if $_[0] =~ /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
     $_[0]
 }
 

From 8dbc17ab37858cca82410f222c71a56c427beb94 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 9 May 2025 09:53:43 +0200
Subject: [PATCH 62/86] FU: Fix error logging of formdata

---
 FU.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index 1a826e1..106322e 100644
--- a/FU.pm
+++ b/FU.pm
@@ -663,7 +663,7 @@ sub log_verbose($,$msg) {
         'Headers:', (map "  $_: $r->{hdr}{$_}", sort keys $r->{hdr}->%*),
         $r->{multipart} ? ('Body (multipart):', _fmt_section join "\n", map $_->describe, $r->{multipart}->@*) :
         $r->{json} ? ('Body (JSON):', _fmt_section FU::Util::json_format($r->{json}, pretty => 1, canonical => 1)) :
-        $r->{formdata} ? ('Body (formdata):', _fmt_section FU::Util::json_format($r->{json}, pretty => 1, canonical => 1)) :
+        $r->{formdata} ? ('Body (formdata):', _fmt_section FU::Util::json_format($r->{formdata}, pretty => 1, canonical => 1)) :
         length $r->{body} ? do {
             my $b = substr $r->{body}, 0, 4096;
             my $trunc = length $r->{body} > 4096 ? ', truncated' : '';

From 383ed8409cf823a2e7966f5692fa3aa6167039a6 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 11 May 2025 10:33:48 +0200
Subject: [PATCH 63/86] bench: version updates + add small Pg benchmark

---
 FU/Benchmarks.pod | 180 +++++++++++++++++++++++++++-------------------
 bench.PL          |  45 +++++++++++-
 2 files changed, 149 insertions(+), 76 deletions(-)

diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index 2479667..37415ff 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -26,9 +26,11 @@ The following module versions were used:
 
 =over
 
-=item L<Cpanel::JSON::XS> 4.38
+=item L<Cpanel::JSON::XS> 4.39
 
-=item L<FU> 0.1
+=item L<DBD::Pg> 3.18.0
+
+=item L<FU> 0.5
 
 =item L<HTML::Tiny> 1.08
 
@@ -40,6 +42,8 @@ The following module versions were used:
 
 =item L<JSON::XS> 4.03
 
+=item L<Pg::PQ> 0.15
+
 =item L<TUWF::XML> 1.5
 
 =item L<XML::Writer> 0.900
@@ -56,266 +60,294 @@ These benchmarks run on large-ish arrays with repeated values. JSON encoding is
 sufficiently fast that Perl function calling overhead tends to dominate for
 smaller inputs, but I don't find that overhead very interesting.
 
-Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
-SIMD parts are only used for parsing.
+Also worth noting that L<JSON::SIMD> formatting code is forked from
+L<JSON::XS>, the SIMD parts are only used for parsing.
 
 API object from L<JSON::XS> documentation.
 
                         Encode   Canonical      Decode
           JSON::PP      5312/s      5119/s      1290/s
         JSON::Tiny      7757/s           -      3426/s
-  Cpanel::JSON::XS    108187/s    101867/s    103575/s
+  Cpanel::JSON::XS    114802/s    104141/s    107274/s
         JSON::SIMD    130137/s    118948/s    115123/s
           JSON::XS    128421/s    120243/s    117940/s
-          FU::Util    133182/s    113275/s    118213/s
+          FU::Util    132890/s    111630/s    121124/s
 
 Object (small)
 
                         Encode   Canonical      Decode
           JSON::PP       907/s       829/s       202/s
         JSON::Tiny      1224/s           -       499/s
-  Cpanel::JSON::XS     43168/s     28114/s     19229/s
+  Cpanel::JSON::XS     45732/s     30862/s     20102/s
         JSON::SIMD     49019/s     30699/s     23267/s
           JSON::XS     49814/s     31326/s     25336/s
-          FU::Util     44110/s     26134/s     21144/s
+          FU::Util     43853/s     26568/s     20426/s
 
 Object (large)
 
                         Encode   Canonical      Decode
           JSON::PP       927/s       747/s       104/s
         JSON::Tiny      1108/s           -       392/s
-  Cpanel::JSON::XS     29672/s     12637/s     16609/s
+  Cpanel::JSON::XS     30587/s     11875/s     15515/s
         JSON::SIMD     24418/s     12388/s     22895/s
           JSON::XS     23192/s     13174/s     23553/s
-          FU::Util     39477/s     13567/s     17178/s
+          FU::Util     36455/s     11920/s     17370/s
 
 Object (large, mixed unicode)
 
                         Encode   Canonical      Decode
           JSON::PP       817/s       679/s        86/s
         JSON::Tiny      1036/s           -       402/s
-  Cpanel::JSON::XS     20437/s      1345/s      7408/s
+  Cpanel::JSON::XS     25333/s      1459/s      7480/s
         JSON::SIMD     25031/s      1331/s     15997/s
           JSON::XS     23580/s      1375/s      8526/s
-          FU::Util     34435/s     11916/s      9419/s
+          FU::Util     33085/s     12639/s      9375/s
 
 Small integers
 
                         Encode      Decode
           JSON::PP       113/s        29/s
         JSON::Tiny       160/s        86/s
-  Cpanel::JSON::XS      7137/s      6083/s
+  Cpanel::JSON::XS      7345/s      6151/s
         JSON::SIMD      7963/s      4361/s
           JSON::XS      7915/s      6058/s
-          FU::Util      8565/s      5639/s
+          FU::Util      7883/s      5671/s
 
 Large integers
 
                         Encode      Decode
           JSON::PP      2176/s       329/s
         JSON::Tiny      2999/s      1638/s
-  Cpanel::JSON::XS     31302/s     48892/s
+  Cpanel::JSON::XS     32545/s     50162/s
         JSON::SIMD     37201/s     51719/s
           JSON::XS     36722/s     50110/s
-          FU::Util    116188/s     62110/s
+          FU::Util    110210/s     61006/s
 
 ASCII strings
 
                         Encode      Decode
           JSON::PP      2934/s       336/s
         JSON::Tiny      4126/s      1439/s
-  Cpanel::JSON::XS    116744/s     43489/s
+  Cpanel::JSON::XS    116721/s     44560/s
         JSON::SIMD    134711/s     50429/s
           JSON::XS    135419/s     43976/s
-          FU::Util    182026/s     44312/s
+          FU::Util    164804/s     48163/s
 
 Unicode strings
 
                         Encode      Decode
           JSON::PP      5113/s       253/s
         JSON::Tiny      6603/s      2585/s
-  Cpanel::JSON::XS     91704/s     64489/s
+  Cpanel::JSON::XS     97039/s     67669/s
         JSON::SIMD    106928/s    102440/s
           JSON::XS    105473/s     60558/s
-          FU::Util    217135/s     58972/s
+          FU::Util    187489/s     61121/s
 
 String escaping (few)
 
                         Encode      Decode
           JSON::PP      4251/s       352/s
         JSON::Tiny      4704/s      1869/s
-  Cpanel::JSON::XS    131789/s    106306/s
+  Cpanel::JSON::XS    136755/s    118059/s
         JSON::SIMD    158171/s    153692/s
           JSON::XS    157261/s     97676/s
-          FU::Util    191699/s     91177/s
+          FU::Util    216443/s     96354/s
 
 String escaping (many)
 
                         Encode      Decode
           JSON::PP      2224/s       366/s
         JSON::Tiny      2884/s       984/s
-  Cpanel::JSON::XS    136583/s    100789/s
+  Cpanel::JSON::XS    140220/s    107040/s
         JSON::SIMD    152951/s    113242/s
           JSON::XS    153471/s    106269/s
-          FU::Util    142604/s     97984/s
+          FU::Util    153081/s    100279/s
 
 
 
 =head2 XML Writing
 
+L<FU::XMLWriter> is the only XS-based XML DSL that I'm aware of, so all direct
+competition is inherently slower by virtue of being pure perl. I'm sure some
+templating modules will perform better, though.
+
 HTML fragment
 
          TUWF::XML       795/s
        XML::Writer       833/s
         HTML::Tiny       423/s
-     FU::XMLWriter      5285/s
+     FU::XMLWriter      5396/s
+
+
+
+=head2 PostgreSQL client
+
+Fetching query results is highly unlikely to be a bottleneck in your code, this
+benchmark is mainly here to verify that L<FU::Pg> is not introducing a
+bottleneck where there shouldn't be one.
+
+Fetch and bitwise-or 20k integers
+
+                      Smallint      Bigint
+           DBD::Pg       194/s        22/s
+            Pg::PQ       226/s        19/s
+      FU::Pg (bin)       239/s        23/s
+     FU::Pg (text)       222/s        21/s
 
 
 
 =cut
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.
-json/api	Canonical	Cpanel::JSON::XS	101867
-json/api	Canonical	FU::Util	113275
+json/api	Canonical	Cpanel::JSON::XS	104141
+json/api	Canonical	FU::Util	111630
 json/api	Canonical	JSON::PP	5119
 json/api	Canonical	JSON::SIMD	118948
 json/api	Canonical	JSON::XS	120243
-json/api	Decode	Cpanel::JSON::XS	103575
-json/api	Decode	FU::Util	118213
+json/api	Decode	Cpanel::JSON::XS	107274
+json/api	Decode	FU::Util	121124
 json/api	Decode	JSON::PP	1290
 json/api	Decode	JSON::SIMD	115123
 json/api	Decode	JSON::Tiny	3426
 json/api	Decode	JSON::XS	117940
-json/api	Encode	Cpanel::JSON::XS	108187
-json/api	Encode	FU::Util	133182
+json/api	Encode	Cpanel::JSON::XS	114802
+json/api	Encode	FU::Util	132890
 json/api	Encode	JSON::PP	5312
 json/api	Encode	JSON::SIMD	130137
 json/api	Encode	JSON::Tiny	7757
 json/api	Encode	JSON::XS	128421
-json/intl	Decode	Cpanel::JSON::XS	48892
-json/intl	Decode	FU::Util	62110
+json/intl	Decode	Cpanel::JSON::XS	50162
+json/intl	Decode	FU::Util	61006
 json/intl	Decode	JSON::PP	329
 json/intl	Decode	JSON::SIMD	51719
 json/intl	Decode	JSON::Tiny	1638
 json/intl	Decode	JSON::XS	50110
-json/intl	Encode	Cpanel::JSON::XS	31302
-json/intl	Encode	FU::Util	116188
+json/intl	Encode	Cpanel::JSON::XS	32545
+json/intl	Encode	FU::Util	110210
 json/intl	Encode	JSON::PP	2176
 json/intl	Encode	JSON::SIMD	37201
 json/intl	Encode	JSON::Tiny	2999
 json/intl	Encode	JSON::XS	36722
-json/ints	Decode	Cpanel::JSON::XS	6083
-json/ints	Decode	FU::Util	5639
+json/ints	Decode	Cpanel::JSON::XS	6151
+json/ints	Decode	FU::Util	5671
 json/ints	Decode	JSON::PP	29
 json/ints	Decode	JSON::SIMD	4361
 json/ints	Decode	JSON::Tiny	86
 json/ints	Decode	JSON::XS	6058
-json/ints	Encode	Cpanel::JSON::XS	7137
-json/ints	Encode	FU::Util	8565
+json/ints	Encode	Cpanel::JSON::XS	7345
+json/ints	Encode	FU::Util	7883
 json/ints	Encode	JSON::PP	113
 json/ints	Encode	JSON::SIMD	7963
 json/ints	Encode	JSON::Tiny	160
 json/ints	Encode	JSON::XS	7915
-json/objl	Canonical	Cpanel::JSON::XS	12637
-json/objl	Canonical	FU::Util	13567
+json/objl	Canonical	Cpanel::JSON::XS	11875
+json/objl	Canonical	FU::Util	11920
 json/objl	Canonical	JSON::PP	747
 json/objl	Canonical	JSON::SIMD	12388
 json/objl	Canonical	JSON::XS	13174
-json/objl	Decode	Cpanel::JSON::XS	16609
-json/objl	Decode	FU::Util	17178
+json/objl	Decode	Cpanel::JSON::XS	15515
+json/objl	Decode	FU::Util	17370
 json/objl	Decode	JSON::PP	104
 json/objl	Decode	JSON::SIMD	22895
 json/objl	Decode	JSON::Tiny	392
 json/objl	Decode	JSON::XS	23553
-json/objl	Encode	Cpanel::JSON::XS	29672
-json/objl	Encode	FU::Util	39477
+json/objl	Encode	Cpanel::JSON::XS	30587
+json/objl	Encode	FU::Util	36455
 json/objl	Encode	JSON::PP	927
 json/objl	Encode	JSON::SIMD	24418
 json/objl	Encode	JSON::Tiny	1108
 json/objl	Encode	JSON::XS	23192
-json/objs	Canonical	Cpanel::JSON::XS	28114
-json/objs	Canonical	FU::Util	26134
+json/objs	Canonical	Cpanel::JSON::XS	30862
+json/objs	Canonical	FU::Util	26568
 json/objs	Canonical	JSON::PP	829
 json/objs	Canonical	JSON::SIMD	30699
 json/objs	Canonical	JSON::XS	31326
-json/objs	Decode	Cpanel::JSON::XS	19229
-json/objs	Decode	FU::Util	21144
+json/objs	Decode	Cpanel::JSON::XS	20102
+json/objs	Decode	FU::Util	20426
 json/objs	Decode	JSON::PP	202
 json/objs	Decode	JSON::SIMD	23267
 json/objs	Decode	JSON::Tiny	499
 json/objs	Decode	JSON::XS	25336
-json/objs	Encode	Cpanel::JSON::XS	43168
-json/objs	Encode	FU::Util	44110
+json/objs	Encode	Cpanel::JSON::XS	45732
+json/objs	Encode	FU::Util	43853
 json/objs	Encode	JSON::PP	907
 json/objs	Encode	JSON::SIMD	49019
 json/objs	Encode	JSON::Tiny	1224
 json/objs	Encode	JSON::XS	49814
-json/obju	Canonical	Cpanel::JSON::XS	1345
-json/obju	Canonical	FU::Util	11916
+json/obju	Canonical	Cpanel::JSON::XS	1459
+json/obju	Canonical	FU::Util	12639
 json/obju	Canonical	JSON::PP	679
 json/obju	Canonical	JSON::SIMD	1331
 json/obju	Canonical	JSON::XS	1375
-json/obju	Decode	Cpanel::JSON::XS	7408
-json/obju	Decode	FU::Util	9419
+json/obju	Decode	Cpanel::JSON::XS	7480
+json/obju	Decode	FU::Util	9375
 json/obju	Decode	JSON::PP	86
 json/obju	Decode	JSON::SIMD	15997
 json/obju	Decode	JSON::Tiny	402
 json/obju	Decode	JSON::XS	8526
-json/obju	Encode	Cpanel::JSON::XS	20437
-json/obju	Encode	FU::Util	34435
+json/obju	Encode	Cpanel::JSON::XS	25333
+json/obju	Encode	FU::Util	33085
 json/obju	Encode	JSON::PP	817
 json/obju	Encode	JSON::SIMD	25031
 json/obju	Encode	JSON::Tiny	1036
 json/obju	Encode	JSON::XS	23580
-json/strel	Decode	Cpanel::JSON::XS	100789
-json/strel	Decode	FU::Util	97984
+json/strel	Decode	Cpanel::JSON::XS	107040
+json/strel	Decode	FU::Util	100279
 json/strel	Decode	JSON::PP	366
 json/strel	Decode	JSON::SIMD	113242
 json/strel	Decode	JSON::Tiny	984
 json/strel	Decode	JSON::XS	106269
-json/strel	Encode	Cpanel::JSON::XS	136583
-json/strel	Encode	FU::Util	142604
+json/strel	Encode	Cpanel::JSON::XS	140220
+json/strel	Encode	FU::Util	153081
 json/strel	Encode	JSON::PP	2224
 json/strel	Encode	JSON::SIMD	152951
 json/strel	Encode	JSON::Tiny	2884
 json/strel	Encode	JSON::XS	153471
-json/stres	Decode	Cpanel::JSON::XS	106306
-json/stres	Decode	FU::Util	91177
+json/stres	Decode	Cpanel::JSON::XS	118059
+json/stres	Decode	FU::Util	96354
 json/stres	Decode	JSON::PP	352
 json/stres	Decode	JSON::SIMD	153692
 json/stres	Decode	JSON::Tiny	1869
 json/stres	Decode	JSON::XS	97676
-json/stres	Encode	Cpanel::JSON::XS	131789
-json/stres	Encode	FU::Util	191699
+json/stres	Encode	Cpanel::JSON::XS	136755
+json/stres	Encode	FU::Util	216443
 json/stres	Encode	JSON::PP	4251
 json/stres	Encode	JSON::SIMD	158171
 json/stres	Encode	JSON::Tiny	4704
 json/stres	Encode	JSON::XS	157261
-json/strs	Decode	Cpanel::JSON::XS	43489
-json/strs	Decode	FU::Util	44312
+json/strs	Decode	Cpanel::JSON::XS	44560
+json/strs	Decode	FU::Util	48163
 json/strs	Decode	JSON::PP	336
 json/strs	Decode	JSON::SIMD	50429
 json/strs	Decode	JSON::Tiny	1439
 json/strs	Decode	JSON::XS	43976
-json/strs	Encode	Cpanel::JSON::XS	116744
-json/strs	Encode	FU::Util	182026
+json/strs	Encode	Cpanel::JSON::XS	116721
+json/strs	Encode	FU::Util	164804
 json/strs	Encode	JSON::PP	2934
 json/strs	Encode	JSON::SIMD	134711
 json/strs	Encode	JSON::Tiny	4126
 json/strs	Encode	JSON::XS	135419
-json/stru	Decode	Cpanel::JSON::XS	64489
-json/stru	Decode	FU::Util	58972
+json/stru	Decode	Cpanel::JSON::XS	67669
+json/stru	Decode	FU::Util	61121
 json/stru	Decode	JSON::PP	253
 json/stru	Decode	JSON::SIMD	102440
 json/stru	Decode	JSON::Tiny	2585
 json/stru	Decode	JSON::XS	60558
-json/stru	Encode	Cpanel::JSON::XS	91704
-json/stru	Encode	FU::Util	217135
+json/stru	Encode	Cpanel::JSON::XS	97039
+json/stru	Encode	FU::Util	187489
 json/stru	Encode	JSON::PP	5113
 json/stru	Encode	JSON::SIMD	106928
 json/stru	Encode	JSON::Tiny	6603
 json/stru	Encode	JSON::XS	105473
-xml/a	Rate	FU::XMLWriter	5285
+pg/ints	Bigint	DBD::Pg	22
+pg/ints	Bigint	FU::Pg (bin)	23
+pg/ints	Bigint	FU::Pg (text)	21
+pg/ints	Bigint	Pg::PQ	19
+pg/ints	Smallint	DBD::Pg	194
+pg/ints	Smallint	FU::Pg (bin)	239
+pg/ints	Smallint	FU::Pg (text)	222
+pg/ints	Smallint	Pg::PQ	226
+xml/a	Rate	FU::XMLWriter	5396
 xml/a	Rate	HTML::Tiny	423
 xml/a	Rate	TUWF::XML	795
 xml/a	Rate	XML::Writer	833
diff --git a/bench.PL b/bench.PL
index fa53068..6ccd763 100755
--- a/bench.PL
+++ b/bench.PL
@@ -25,7 +25,10 @@ my %modules = map +($_, eval "require $_; \$${_}::VERSION"), qw/
     TUWF::XML
     HTML::Tiny
     XML::Writer
+    DBD::Pg
+    Pg::PQ
 /;
+use FU::Pg;
 
 my %data; # "id x y" => { id x y rate exists }
 my %oldmodules;
@@ -196,6 +199,32 @@ def 'xml/a', 'HTML fragment', [ 'Rate' ],
 
 
 
+{
+    die "FU_TEST_DB not set.\n" if !$ENV{FU_TEST_DB};
+    my $pq = Pg::PQ::Conn->new($ENV{FU_TEST_DB});
+    my $fu = FU::Pg->connect($ENV{FU_TEST_DB});
+    # XXX: Doesn't support all connection params this way
+    my $dbi = DBI->connect("dbi:Pg:dbname=".$pq->db, $pq->user, $pq->pass, {RaiseError => 1, PrintError => 0});
+
+    my $small = 'SELECT x, x+1, x+2, x+3, x+4, x+5, x+6, x+7, x+8, x+9 FROM generate_series(-10000::smallint, 9999, 10) x(x)';
+    my $big = 'SELECT x<<5, x<<10, x<<15, x<<20, x<<25, x<<30, x<<35, x<<40, x<<45, x<<50 FROM generate_series(1::bigint, 20000, 1) x(x)';
+
+    my sub dbi { my $sum = 0; my $st = $dbi->prepare_cached($_[0]); for my $row ($dbi->selectall_arrayref($st)->@*) { $sum ^= $_ for @$row; } }
+    my sub pq  { my $sum = 0; $pq->prepare('' => $_[0]); for my $row ($pq->execQueryPrepared('')->rows) { $sum ^= $_ for @$row; } }
+    my sub fub { my $sum = 0; for my $row ($fu->q($_[0])->alla->@*) { $sum ^= $_ for @$row; } }
+    my sub fut { my $sum = 0; for my $row ($fu->q($_[0])->text->alla->@*) { $sum ^= $_ for @$row; } }
+
+    def 'pg/ints', 'Fetch and bitwise-or 20k integers', [ 'Smallint', 'Bigint' ],
+        [ 'DBD::Pg', undef, sub { dbi($small) }, sub { dbi($big) } ],
+        [ 'Pg::PQ', undef, sub { pq($small) }, sub { pq($big) } ],
+        [ 'FU::Pg (bin)', 'FU', sub { fub($small) }, sub { fub($big) } ],
+        [ 'FU::Pg (text)', 'FU', sub { fut($small) }, sub { fut($big) } ];
+}
+
+
+
+
+
 delete @data{ grep !$data{$_}{exists}, keys %data };
 
 sub fmtbench($id, $text, $xs, $ys) {
@@ -276,15 +305,27 @@ These benchmarks run on large-ish arrays with repeated values. JSON encoding is
 sufficiently fast that Perl function calling overhead tends to dominate for
 smaller inputs, but I don't find that overhead very interesting.
 
-Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
-SIMD parts are only used for parsing.
+Also worth noting that L<JSON::SIMD> formatting code is forked from
+L<JSON::XS>, the SIMD parts are only used for parsing.
 
 :benches ^json
 
 %head2 XML Writing
 
+L<FU::XMLWriter> is the only XS-based XML DSL that I'm aware of, so all direct
+competition is inherently slower by virtue of being pure perl. I'm sure some
+templating modules will perform better, though.
+
 :benches ^xml
 
+%head2 PostgreSQL client
+
+Fetching query results is highly unlikely to be a bottleneck in your code, this
+benchmark is mainly here to verify that L<FU::Pg> is not introducing a
+bottleneck where there shouldn't be one.
+
+:benches ^pg
+
 %cut
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.

From 32c8fc1b898d5d505a55c1cb1dcdf1cc4822677c Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 11 May 2025 11:03:32 +0200
Subject: [PATCH 64/86] Version 1.0 + remove "experimental" notices

---
 ChangeLog               | 19 +++++++++++++++++++
 FU.pm                   | 15 ++++++---------
 FU/Benchmarks.pod       |  2 +-
 FU/DebugImpl.pm         |  2 +-
 FU/Log.pm               |  7 +------
 FU/MultipartFormData.pm |  2 +-
 FU/Pg.pm                |  7 +------
 FU/SQL.pm               |  7 +------
 FU/Util.pm              |  7 +------
 FU/Validate.pm          |  7 +------
 FU/XMLWriter.pm         |  7 +------
 FU/XS.pm                |  2 +-
 README.md               |  4 ----
 13 files changed, 35 insertions(+), 53 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 3f18b6b..29e387f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,22 @@
+1.0 - 2025-05-11
+	- FU::Util: Fix parsing of empty sections in query_decode()
+	- FU::Util: Fix buffer overflow in json_format() float formatting
+	- FU::Util: Reject `0x1f` in utf8_decode()
+	- FU::Pg: Add perl<->text and bin<->text type conversion methods
+	- FU::Validate: Improved error messages
+	- FU::MultipartFormData: Various parser fixes
+	- FU: Include request body in verbose error logs
+	- FU: Add fu->log_verbose()
+	- FU: Extend debug_info pages with request body, response body, 'fu'
+	  object dump, expandable query parameters and interpolated SQL queries
+	- FU: Improve styling of debug_info pages
+	- FU: Preserve headers on fu->redirect
+	- FU: Ignore HTTP_CONTENT_LENGTH and HTTP_CONTENT_TYPE FastCGI parameters
+	- FU: Suppress warning about missing files in FU::monitor_path
+	- FU: Reject hash character and newlines in request path
+	- Fix creating read-only undef/true/false in json_parse() and FU::Pg
+	- Benchmark updates
+
 0.5 - 2025-04-24
 	- FU::Util: Set O_CLOEXEC on fds received through `fdpass_recv()`
 	- FU::Util: Fix interpretation of false options in `json_format()` and
diff --git a/FU.pm b/FU.pm
index 106322e..440a14c 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 0.5;
+package FU 1.0;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
@@ -978,14 +978,6 @@ __END__
 
 FU - A Lean and Efficient Zero-Dependency Web Framework.
 
-=head1 EXPERIMENTAL
-
-This module is still in development: it's missing important functionality and
-there will likely be a few breaking API changes. This framework currently
-powers manned.org as a test. I'll do a stable 1.0 release once FU is used in
-production for vndb.org, which will take a few months in the best case
-scenario.
-
 =head1 SYNOPSIS
 
   use v5.36;
@@ -1011,6 +1003,11 @@ scenario.
 
 =head1 DESCRIPTION
 
+FU is the backend web framework developed for L<VNDB.org|https://vndb.org/> and
+L<Manned.org|https://manned.org/>, but is also perfectly suitable for other
+projects. Besides a web framework, this distrubion also includes a bunch of
+handy utility functions and modules.
+
 =head2 Distribution Overview
 
 This top-level C<FU> module is a web development framework. The C<FU>
diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index 37415ff..0762c7a 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -30,7 +30,7 @@ The following module versions were used:
 
 =item L<DBD::Pg> 3.18.0
 
-=item L<FU> 0.5
+=item L<FU> 1.0
 
 =item L<HTML::Tiny> 1.08
 
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 88e0bb5..ebf8c80 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 0.5;
+package FU::DebugImpl 1.0;
 use v5.36;
 use utf8;
 use experimental 'for_list';
diff --git a/FU/Log.pm b/FU/Log.pm
index e2da4a2..9606326 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 0.5;
+package FU::Log 1.0;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
@@ -65,11 +65,6 @@ __END__
 
 FU::Log - Extremely Basic Process-Wide Logging Infrastructure
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::Log 'log_write';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 7d9d77e..48ebb77 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 0.5;
+package FU::MultipartFormData 1.0;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index ff88d60..4732daf 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 0.5;
+package FU::Pg 1.0;
 use v5.36;
 use FU::XS;
 
@@ -35,11 +35,6 @@ __END__
 
 FU::Pg - The Ultimate (synchronous) Interface to PostgreSQL
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSYS
 
   use FU::Pg;
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 2f8566d..db8aff1 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 0.5;
+package FU::SQL 1.0;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
@@ -103,11 +103,6 @@ __END__
 
 FU::SQL - Small and Safe SQL Query Builder
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::SQL;
diff --git a/FU/Util.pm b/FU/Util.pm
index 18db781..922747d 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 0.5;
+package FU::Util 1.0;
 
 use v5.36;
 use FU::XS;
@@ -98,11 +98,6 @@ __END__
 
 FU::Util - Miscellaneous Utility Functions
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::Util qw/json_format/;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 2741ee6..a4544bf 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 0.5;
+package FU::Validate 1.0;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
@@ -447,11 +447,6 @@ __END__
 
 FU::Validate - Data and form validation and normalization
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 DESCRIPTION
 
 This module provides an easy and simple interface for data validation. It can
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 1e9bb90..fe755f1 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 0.5;
+package FU::XMLWriter 1.0;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
@@ -83,11 +83,6 @@ __END__
 
 FU::XMLWriter - Convenient and efficient XML and HTML generator.
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::XMLWriter ':html5_';
diff --git a/FU/XS.pm b/FU/XS.pm
index 52cc757..b583e00 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 0.5;
+package FU::XS 1.0;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');
diff --git a/README.md b/README.md
index d29e00c..8f25140 100644
--- a/README.md
+++ b/README.md
@@ -7,10 +7,6 @@ collection of handy utility modules.
 
 *Contributing:* Refer to my [contribution guidelines](https://dev.yorhel.nl/contributing).
 
-## Project Status
-
-**EXPERIMENTAL**; expect breaking changes.
-
 ## Build & Install
 
 ```sh

From 31994a4bf6a126aadae4bf736b40bfae60e9adf8 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 12 May 2025 12:38:23 +0200
Subject: [PATCH 65/86] Doc typos

---
 FU.pm           | 4 ++--
 FU/XMLWriter.pm | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/FU.pm b/FU.pm
index 440a14c..9c85edf 100644
--- a/FU.pm
+++ b/FU.pm
@@ -994,7 +994,7 @@ FU - A Lean and Efficient Zero-Dependency Web Framework.
   }
 
   FU::get qr{/hello/(.+)}, sub($who) {
-      my_html_ "Website title", sub {
+      myhtml_ "Website title", sub {
           h1_ "Hello, $who!";
       };
   };
@@ -1097,7 +1097,7 @@ returning strings deal with perl Unicode strings, not raw bytes.
 =item use FU -procname => $name
 
 When the C<-procname> import option is set, FU automatically updates the
-process name (as displayed in L<top(1)> and L<ps(1)>, see `$0`) with
+process name (as displayed in L<top(1)> and L<ps(1)>, see C<$0>) with
 information about the current process, prefixed with the given C<$name>.
 
 =item FU::init_db($info)
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index fe755f1..1b964ee 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -263,7 +263,7 @@ and C<"> are replaced with their XML entity.
 All of the functions mentioned in this document can be imported individually.
 There are also two import groups:
 
-  use FU::XMLWriter ':html_';
+  use FU::XMLWriter ':html5_';
 
 Exports C<tag_()>, C<html_()>, C<lit_()>, C<txt_()> and all of the C<<
 <html-tag>_ >> functions mentioned above.

From 81a3d3c608dd37214a94ad8381b4efe6232355e0 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 22 May 2025 09:22:05 +0200
Subject: [PATCH 66/86] SQL: Add IDENT() and quote_identifier options

Turns out VNDB has a few places where request data is directly used for
column names in VALUES/SET/WHERE clauses. These are already restricted
to known strings through the use of FU::Validate, but an extra layer of
protection seems warranted here.
---
 FU/SQL.pm | 46 +++++++++++++++++++++++++++++++++++++---------
 t/sql.t   | 10 ++++++++++
 2 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/FU/SQL.pm b/FU/SQL.pm
index db8aff1..63107b9 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -5,7 +5,7 @@ use Carp 'confess';
 use experimental 'builtin', 'for_list';
 
 our @EXPORT = qw/
-    P RAW SQL
+    P RAW IDENT SQL
     PARENS INTERSPERSE COMMA
     AND OR WHERE
     SET VALUES IN
@@ -16,6 +16,7 @@ sub _obj { bless [@_], 'FU::SQL::val' }
 
 sub P :prototype($) ($p) { bless \(my $x = $p), 'FU::SQL::p' }
 sub RAW :prototype($) ($s) { _obj "$s" }
+sub IDENT :prototype($) ($s) { bless \(my $x = "$s"), 'FU::SQL::i' }
 
 # These operate on $_ and must be called with &func syntax.
 # The readonly check can be finicky.
@@ -29,7 +30,7 @@ sub COMMA { INTERSPERSE ',', @_ }
 
 sub _conditions {
     @_ == 1 && ref $_[0] eq 'HASH'
-    ? map PARENS(RAW $_,
+    ? map PARENS(IDENT $_,
         !defined $_[0]{$_}              ? ('IS NULL') :
         ref($_[0]{$_}) eq 'FU::SQL::in' ? ($_[0]{$_})
                                         : ('=', $_[0]{$_})
@@ -41,11 +42,11 @@ sub AND { !@_ || (@_ == 1 && ref $_[0] eq 'HASH' && keys $_[0]->%* == 0) ? RAW '
 sub OR  { !@_ || (@_ == 1 && ref $_[0] eq 'HASH' && keys $_[0]->%* == 0) ? RAW '1=0' : INTERSPERSE 'OR',  _conditions @_ }
 sub WHERE { SQL 'WHERE', AND @_ }
 
-sub SET($h) { SQL 'SET', COMMA map SQL(RAW $_, '=', $h->{$_}), sort keys %$h }
+sub SET($h) { SQL 'SET', COMMA map SQL(IDENT $_, '=', $h->{$_}), sort keys %$h }
 
 sub VALUES {
     @_ == 1 && ref $_[0] eq 'HASH'
-    ? SQL '(', COMMA(map RAW $_, sort keys $_[0]->%*), ') VALUES (', COMMA(map $_[0]{$_}, sort keys $_[0]->%*), ')'
+    ? SQL '(', COMMA(map IDENT $_, sort keys $_[0]->%*), ') VALUES (', COMMA(map $_[0]{$_}, sort keys $_[0]->%*), ')'
     : @_ == 1 && ref $_[0] eq 'ARRAY'
     ? SQL 'VALUES (', COMMA($_[0]->@*), ')'
     : SQL 'VALUES (', COMMA(@_), ')';
@@ -71,6 +72,10 @@ sub FU::SQL::p::_compile($self, $opt, $sql, $params) {
     $$sql .= $opt->{placeholder_style} eq 'pg' ? '$'.@$params : '?';
 }
 
+sub FU::SQL::i::_compile($self, $opt, $sql, $params) {
+    $$sql .= $opt->{quote_identifier} ? $opt->{quote_identifier}->($$self) : $$self;
+}
+
 sub FU::SQL::in::_compile($self, $opt, $sql, $params) {
     if ($opt->{in_style} eq 'pg') {
         $$sql .= '= ANY(';
@@ -87,6 +92,7 @@ sub FU::SQL::in::_compile($self, $opt, $sql, $params) {
 }
 
 sub FU::SQL::val::compile($self, %opt) {
+    !/^(placeholder_style|in_style|quote_identifier)$/ && confess "Unknown flag: $_" for keys %opt;
     $opt{placeholder_style} ||= 'dbi';
     $opt{in_style} ||= 'dbi';
     my($sql, @params) = ('');
@@ -94,7 +100,7 @@ sub FU::SQL::val::compile($self, %opt) {
     ($sql, \@params)
 }
 
-*FU::SQL::p::compile = *FU::SQL::in::compile = \*FU::SQL::val::compile;
+*FU::SQL::p::compile = *FU::SQL::i::compile = *FU::SQL::in::compile = \*FU::SQL::val::compile;
 
 1;
 __END__
@@ -156,6 +162,16 @@ C<'pg'> when your SQL is going to L<FU::Pg> or L<Pg::PQ>.
 Set the style to use for C<IN> expressions, refer to the C<IN()> function below
 for details.
 
+=item quote_identifier => $func
+
+Set a function to perform quoting of SQL identifiers. When using DBI, you can
+do:
+
+  my($sql) = $obj->compile(quote_identifier => sub { $dbh->quote_identifier(@_) });
+
+If this option is not set, identifiers are included into the raw SQL string
+without any escaping.
+
 =back
 
 =back
@@ -176,7 +192,7 @@ types of supported arguments:
 
 =item 1.
 
-B<String literals> are interpreted as raw SQL fragments.
+I<String literals> are interpreted as raw SQL fragments.
 
 =item 2.
 
@@ -184,7 +200,7 @@ Objects returned by other functions listed below are included as SQL fragments.
 
 =item 3.
 
-B<Everything else> is considered a bind parameter.
+I<Everything else> is considered a bind parameter.
 
 =back
 
@@ -244,6 +260,18 @@ Force the given C<$sql> string to be included as SQL. For example:
 
 Never use this function with untrusted input.
 
+=item IDENT($string)
+
+Mark the given string as an SQL identifier. This function is only useful if you
+use potentially untrusted input to determine which column to select or which
+table to select from, for example:
+
+  SQL 'SELECT id,', IDENT $ENV{column}, 'FROM table';
+
+B<WARNING:> By default this function is equivalent to C<RAW()> and hence
+provides no safety whatsoever. Be sure to set the C<quote_identifier> option on
+C<compile()> to get more useful behavior.
+
 =item PARENS(@args)
 
 Like C<SQL()> but surrounds the expression by parens:
@@ -279,8 +307,8 @@ C<'1=1'> (i.e. true) if C<@conditions> is an empty list.
 =item AND($hashref)
 
 A special form of C<AND()> that tests the given columns for equality instead.
-The keys of the hashref are interpreted as raw SQL and the values as bind
-parameters.
+The keys of the hashref are interpreted as per C<IDENT()> and the values as
+bind parameters.
 
   AND { id => 1, number => RAW 'random()', x => undef }
   # '( id = ? ) AND ( number = random() ) AND ( x IS NULL )'
diff --git a/t/sql.t b/t/sql.t
index e6b7378..f9cee56 100644
--- a/t/sql.t
+++ b/t/sql.t
@@ -9,11 +9,15 @@ sub t($obj, $sql, $params, @opt) {
     is_deeply $gotparams, $params;
 }
 
+my @q_ident = (quote_identifier => sub($x) { $x =~ s/"/_/rg });
+
 my $x;
 t P '', '?', [''];
 t P '', '$1', [''], placeholder_style => 'pg';
 t P undef, '?', [undef];
 t RAW '', '', [];
+t IDENT '"hello"', '"hello"', [];
+t IDENT '"hello"', '_hello_', [], @q_ident;
 t SQL('select', '1'), 'select 1', [];
 t SQL('select', P '1'), 'select ?', [1];
 t SQL('select', $x = '1'), 'select ?', [1];
@@ -41,6 +45,7 @@ t WHERE($x, '1 = 2', SQL('x = ', $x)),
 t WHERE({ col1 => RAW 'NOW()', col2 => 'a'}),
   'WHERE ( col1 = NOW() ) AND ( col2 = ? )', ['a'];
 t WHERE(), 'WHERE 1=1', [];
+t WHERE({ '"x' => 1 }), 'WHERE ( _x = ? )', [1], @q_ident;
 
 t WHERE(AND('true', $x), OR($y, 'y'), AND, OR),
   'WHERE ( ( true ) AND ( ? ) ) AND ( ( ? ) OR ( y ) ) AND ( 1=1 ) AND ( 1=0 )', [$x, $y];
@@ -52,9 +57,11 @@ t SQL(SELECT => COMMA(qw/a b c/), FROM => 'table', WHERE { x => 1, a => undef })
 
 t SET({ a => 1, c => RAW 'NOW()', d => undef }),
   'SET a = ? , c = NOW() , d = ?', [1, undef];
+t SET({ '"x' => 1 }), 'SET _x = ?', [1], @q_ident;
 
 t VALUES({ a => 1, c => RAW 'NOW()', d => undef }),
   '( a , c , d ) VALUES ( ? , NOW() , ? )', [1, undef];
+t VALUES({ '"x' => 1 }), '( _x ) VALUES ( ? )', [1], @q_ident;
 
 t VALUES(1, $x, 'NOW()', RAW 'NOW()'), 'VALUES ( ? , ? , NOW() , NOW() )', [1, $x];
 t VALUES([1, $x, 'NOW()', RAW 'NOW()']), 'VALUES ( ? , ? , ? , NOW() )', [1, $x, 'NOW()'];
@@ -86,4 +93,7 @@ Hash::Util::lock_keys(%hash);
 Hash::Util::lock_value(%hash, 'v');
 t SQL($hash{v}), 'value', [];
 
+ok !eval { SQL('')->compile(oops => 1); 1 };
+like $@, qr/Unknown flag: oops/;
+
 done_testing;

From 2083ab2a6f3793ce560996dc2c8120c74c61a78b Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 22 May 2025 09:53:41 +0200
Subject: [PATCH 67/86] Pg: Set appropriate quote_identifier for $conn->Q()

---
 FU.xs               | 12 ++++++++++++
 FU/Pg.pm            |  9 +++++++--
 t/pgtypes-dynamic.t |  8 ++++++++
 3 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/FU.xs b/FU.xs
index 1c342be..7a387f9 100644
--- a/FU.xs
+++ b/FU.xs
@@ -217,6 +217,12 @@ void query_trace(fupg_conn *c, SV *cb)
     SvGETMAGIC(cb);
     c->trace = SvOK(cb) ? SvREFCNT_inc(cb) : NULL;
 
+void conn(fupg_conn *c)
+  CODE:
+    ST(0) = sv_newmortal();
+    sv_setrv_inc(ST(0), c->self);
+    sv_bless(ST(0), gv_stashpv("FU::Pg::conn", 0));
+
 void status(fupg_conn *c)
   CODE:
     ST(0) = sv_2mortal(newSVpv(fupg_conn_status(c), 0));
@@ -317,6 +323,12 @@ void cache(fupg_txn *x, ...)
   CODE:
     FUPG_STFLAGS;
 
+void conn(fupg_txn *t)
+  CODE:
+    ST(0) = sv_newmortal();
+    sv_setrv_inc(ST(0), t->conn->self);
+    sv_bless(ST(0), gv_stashpv("FU::Pg::conn", 0));
+
 void status(fupg_txn *t)
   CODE:
     ST(0) = sv_2mortal(newSVpv(fupg_txn_status(t), 0));
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 4732daf..2e7baf8 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -10,7 +10,11 @@ package FU::Pg::conn {
     sub Q {
         require FU::SQL;
         my $s = shift;
-        my($sql, $params) = FU::SQL::SQL(@_)->compile(placeholder_style => 'pg', in_style => 'pg');
+        my($sql, $params) = FU::SQL::SQL(@_)->compile(
+            placeholder_style => 'pg',
+            in_style          => 'pg',
+            quote_identifier  => sub { $s->conn->escape_identifier(@_) },
+        );
         $s->q($sql, @$params);
     }
 
@@ -208,7 +212,8 @@ used.
 =item $conn->Q(@args)
 
 Same as C<< $conn->q() >> but uses L<FU::SQL> to construct the query and bind
-parameters.
+parameters. Uses the 'pg' C<in_style> and C<< $conn->escape_identifier() >> for
+identifier quoting.
 
 =back
 
diff --git a/t/pgtypes-dynamic.t b/t/pgtypes-dynamic.t
index 2751a86..79abd92 100644
--- a/t/pgtypes-dynamic.t
+++ b/t/pgtypes-dynamic.t
@@ -127,6 +127,14 @@ subtest 'custom types', sub {
 };
 
 
+subtest 'identifier quoting', sub {
+    my $txn = $conn->txn;
+    $txn->exec('CREATE TEMPORARY TABLE fupg_test_tbl ("desc" int, ok int, "hello world" int)');
+    ok $txn->Q('INSERT INTO fupg_test_tbl', VALUES {desc => 5, ok => 10, 'hello world', 15})->exec;
+    is $txn->Q('SELECT', IDENT 'hello world', 'FROM fupg_test_tbl')->val, 15;
+};
+
+
 subtest 'vndbid', sub {
     plan skip_all => 'type not loaded in the database' if !$conn->q("SELECT 1 FROM pg_type WHERE typname = 'vndbtag'")->val;
 

From fd8332601b56e661c7c656cdfcd8d11fd65f3cc9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 22 May 2025 09:54:08 +0200
Subject: [PATCH 68/86] t/pgconnect: Fix ref leak in test

Apparently 'my sub' captured the $conn variable and held a ref on it
even beyond the parent sub scope. 'my $x = sub {}' doesn't do that.

Getting the ref counts right is important here for the last test to work.

(Found while I was inspecting the refcount effects of the new ->conn()
methods with Devel::Peek)
---
 t/pgconnect.t | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/t/pgconnect.t b/t/pgconnect.t
index 8536574..cec597d 100644
--- a/t/pgconnect.t
+++ b/t/pgconnect.t
@@ -370,18 +370,18 @@ subtest 'Prepared statement cache', sub {
     $conn->cache_size(2);
     my $txn = $conn->txn;
     $txn->cache;
-    my sub numexec($sql) {
+    my $numexec = sub($sql) {
         $txn->q('SELECT generic_plans + custom_plans FROM pg_prepared_statements WHERE statement = $1', $sql)->cache(0)->val
-    }
+    };
     is $txn->q('SELECT 1')->val, 1;
-    is numexec('SELECT 1'), 1;
+    is $numexec->('SELECT 1'), 1;
 
     my $sql = 'SELECT $1::int as a, $2::text as b';
-    ok !defined numexec($sql);
+    ok !defined $numexec->($sql);
 
     my $params = $txn->q($sql)->param_types;
     is_deeply $params, [23, 25];
-    is numexec($sql), 0;
+    is $numexec->($sql), 0;
     my $cparams = $txn->q($sql)->param_types;
     is_deeply $cparams, $params;
 
@@ -391,23 +391,23 @@ subtest 'Prepared statement cache', sub {
     is_deeply $ccols, $cols;
 
     $txn->q($sql, 0, '')->exec;
-    is numexec($sql), 1;
+    is $numexec->($sql), 1;
     $txn->q($sql, 0, '')->exec;
-    is numexec($sql), 2;
+    is $numexec->($sql), 2;
 
-    is numexec('SELECT 1'), 1;
+    is $numexec->('SELECT 1'), 1;
     $txn->q('SELECT 2')->exec;
-    ok !defined numexec('SELECT 1');
-    is numexec('SELECT 2'), 1;
+    ok !defined $numexec->('SELECT 1');
+    is $numexec->('SELECT 2'), 1;
 
     $conn->cache_size(1);
-    ok !defined numexec('SELECT 1');
-    ok !defined numexec($sql);
-    is numexec('SELECT 2'), 1;
+    ok !defined $numexec->('SELECT 1');
+    ok !defined $numexec->($sql);
+    is $numexec->('SELECT 2'), 1;
 
     $conn->cache_size(0);
-    ok !defined numexec($sql);
-    ok !defined numexec('SELECT 2');
+    ok !defined $numexec->($sql);
+    ok !defined $numexec->('SELECT 2');
 };
 
 

From f8cd8a6d8cbc687e452071b98f1457f546a55c08 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Tue, 27 May 2025 09:30:46 +0200
Subject: [PATCH 69/86] FU: Simplify --monitor file change detection

This changes the way that file changes are detected. The upside is that
it now correctly detects changes that happened after the code has loaded
but before the first request came in, the downside is that it now gets
stuck on reloading when a file has a future mtime.
---
 FU.pm | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/FU.pm b/FU.pm
index 9c85edf..6da28cb 100644
--- a/FU.pm
+++ b/FU.pm
@@ -217,17 +217,12 @@ sub monitor_path { push @monitor_paths, @_ }
 sub monitor_check :prototype(&) { $monitor_check = $_[0] }
 
 sub _monitor {
-    state %data;
     return 1 if $monitor_check && $monitor_check->();
 
     require File::Find;
     eval {
         File::Find::find({
-            wanted => sub {
-                my $m = (stat)[9];
-                $data{$_} //= $m;
-                die if $m > $data{$_};
-            },
+            wanted => sub { die if (-M) < 0 },
             no_chdir => 1
         }, grep -e, $scriptpath, values %INC, @monitor_paths);
         0

From a43dc70ff92b2baca45bb316b3bbce571054191d Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 2 Jun 2025 09:00:04 +0200
Subject: [PATCH 70/86] XMLWriter: Throw error when stringifying a bare
 reference

I can't think of a use case where Perl's default ref stringification is
something you actually want when writing XML/HTML - this pretty much
always points to a bug. One that I seem to be prone to making...
---
 c/xmlwr.c |  4 +++-
 t/xmlwr.t | 17 +++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/c/xmlwr.c b/c/xmlwr.c
index f81d94c..2d31fec 100644
--- a/c/xmlwr.c
+++ b/c/xmlwr.c
@@ -27,6 +27,8 @@ static void fuxmlwr_destroy(pTHX_ fuxmlwr *wr) {
 
 
 static void fuxmlwr_escape(pTHX_ fuxmlwr *wr, SV *sv) {
+    if (SvROK(sv) && !SvAMAGIC(sv)) fu_confess("Invalid attempt to output bare reference");
+
     STRLEN len;
     const unsigned char *str = (unsigned char *)SvPV_const(sv, len);
     const unsigned char *tmp, *end = str + len;
@@ -96,7 +98,7 @@ static void fuxmlwr_tag(pTHX_ fuxmlwr *wr, I32 ax, I32 offset, I32 argc, int sel
         val = ST(offset);
         offset++;
 
-        // Don't even try to stringify other arguments; non-string keys are always a bug.
+        // Don't even try to stringify attribute names; non-string keys are always a bug.
         if (!SvPOK(key)) fu_confess("Non-string attribute");
         keys = SvPVX(key);
 
diff --git a/t/xmlwr.t b/t/xmlwr.t
index e8b2d95..becb96c 100644
--- a/t/xmlwr.t
+++ b/t/xmlwr.t
@@ -65,4 +65,21 @@ sub t {
 
 is fragment { t 'arg' }, '<div attr1="arg"><span>ab&quot; &lt; c &amp;&lt; d</span><span><ok🥳ay></span>🥳</div>';
 
+ok !eval { fragment { tag_ 'hi', \1 } };
+like $@, qr/Invalid attempt to output bare reference/;
+
+ok !eval { fragment { tag_ 'hi', {} } };
+like $@, qr/Invalid attempt to output bare reference/;
+
+is fragment { tag_ 'hi', bless {}, 'XTEST1' }, '<hi>string</hi>';
+like fragment { tag_ 'hi', bless {}, 'XTEST2' }, qr{<hi>HASH\(.*\)</hi>}; # Yeah, whatever.
+like fragment { tag_ 'hi', ''.{} }, qr{<hi>HASH\(.*\)</hi>};
+
 done_testing;
+
+
+package XTEST1;
+use overload '""' => sub { 'string' };
+
+package XTEST2;
+use overload '""' => sub { {} };

From 55baa6c9a616e9a3a9223cc07826dc7c23ec6825 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Wed, 4 Jun 2025 18:48:06 +0200
Subject: [PATCH 71/86] json_parse(): Disallow control characters in strings by
 default

Deviating from the standard, but more consistent other FU functions.
---
 FU/Util.pm     | 16 +++++++++++-----
 c/jsonparse.c  | 10 ++++++++--
 t/json_parse.t | 12 +++++++++---
 3 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/FU/Util.pm b/FU/Util.pm
index 922747d..4b06f33 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -137,7 +137,7 @@ value for C<$val>, due to C<\0> and C<\1> being considered booleans.
 =head1 JSON Parsing & Formatting
 
 This module comes with a custom C-based JSON parser and formatter. These
-functions conform strictly to L<RFC-8259|https://tools.ietf.org/html/rfc8259>,
+functions conform to L<RFC-8259|https://tools.ietf.org/html/rfc8259>,
 non-standard extensions are not supported and never will be. It also happens to
 be pretty fast, refer to L<FU::Benchmarks> for some numbers.
 
@@ -171,6 +171,13 @@ Supported C<%options>:
 
 =over
 
+=item allow_control
+
+Boolean, set to true to allow (encoded) ASCII control characters in JSON
+strings, such as C<\u0000>, C<\b>, C<\u007f>, etc.  These characters are
+permitted per RFC-8259, but disallowed by this parser by default. See
+C<utf8_decode()> below.
+
 =item utf8
 
 Boolean, interpret the input C<$string> as a UTF-8 encoded byte string instead
@@ -251,10 +258,9 @@ value. There is no way to do that without violating JSON specs, so you should
 use entity escaping instead.
 
 Some JSON modules escape the forward slash (C</>) character instead, but that
-is, at best, B<only> sufficient for embedding inside a C<< <script> >> tag (I'm
-not sure how C<< <!-- >> and C<< <![CDATA[ >> are treated in that context). In
-any other context, you'll need the more thourough escaping provided by this
-C<html_safe> option.
+is I<only> sufficient for embedding inside a C<< <script> >> tag. In any other
+context, you'll need the more thourough escaping provided by this C<html_safe>
+option.
 
 =item max_size
 
diff --git a/c/jsonparse.c b/c/jsonparse.c
index 6dfee91..bf46557 100644
--- a/c/jsonparse.c
+++ b/c/jsonparse.c
@@ -2,6 +2,7 @@ typedef struct {
     const unsigned char *buf;
     const unsigned char *end;
     UV depth;
+    int allow_control;
 } fujson_parse_ctx;
 
 static SV *fujson_parse(pTHX_ fujson_parse_ctx *);
@@ -23,10 +24,10 @@ static inline int fujson_parse_string_escape(pTHX_ fujson_parse_ctx *ctx, fustr
         case '"': *(r->cur++) = '\"'; break;
         case '\\':*(r->cur++) = '\\'; break;
         case '/': *(r->cur++) = '/';  break; /* We don't escape this one */
-        case 'b': *(r->cur++) = 0x08; break;
+        case 'b': if (!ctx->allow_control) return 1; *(r->cur++) = 0x08; break;
         case 't': *(r->cur++) = 0x09; break;
         case 'n': *(r->cur++) = 0x0a; break;
-        case 'f': *(r->cur++) = 0x0c; break;
+        case 'f': if (!ctx->allow_control) return 1; *(r->cur++) = 0x0c; break;
         case 'r': *(r->cur++) = 0x0d; break;
         case 'u':
             /* (awful code adapted from ncdu) */
@@ -43,6 +44,9 @@ static inline int fujson_parse_string_escape(pTHX_ fujson_parse_ctx *ctx, fustr
                 n = 0x10000 + (((n & 0x03ff) << 10) | (s & 0x03ff));
                 ctx->buf += 6;
             }
+            if (!ctx->allow_control &&
+                    (n <= 8 || n == 0x0b || n == 0x0c || (n >= 0x0e && n <= 0x1f) || n == 0x7f))
+                return 1;
             r->cur = (char *)uvchr_to_utf8((U8 *)r->cur, n);
             if (n >= 0x80) r->setutf8 = 1;
             break;
@@ -265,6 +269,7 @@ static SV *fujson_parse_xs(pTHX_ I32 ax, I32 argc, SV *val) {
     fujson_parse_ctx ctx;
 
     ctx.depth = 0;
+    ctx.allow_control = 0;
     while (i < argc) {
         arg = SvPV_nolen(ST(i));
         i++;
@@ -275,6 +280,7 @@ static SV *fujson_parse_xs(pTHX_ I32 ax, I32 argc, SV *val) {
         if (strcmp(arg, "utf8") == 0) decutf8 = SvTRUEx(r);
         else if (strcmp(arg, "max_size") == 0) maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
+        else if (strcmp(arg, "allow_control") == 0) ctx.allow_control = SvTRUE(r);
         else if (strcmp(arg, "offset") == 0) offset = r;
         else croak("Unknown flag: '%s'", arg);
     }
diff --git a/t/json_parse.t b/t/json_parse.t
index d01414f..3ad3838 100644
--- a/t/json_parse.t
+++ b/t/json_parse.t
@@ -24,6 +24,11 @@ my @error = (
     '"\udc12\u1234"',
     "\"\x{110000}\"",
 
+    '"\u0000"',
+    '"\b"',
+    '"\f"',
+    '"\u007f"',
+
     '1.',
     '01',
     '1e',
@@ -82,9 +87,9 @@ sub str($in, $exp) {
 }
 str '""', '';
 str '"hello, world"', 'hello, world';
-str '"\u0000\u0099\u0234\u1234"', "\x{00}\x{99}\x{234}\x{1234}";
-str "\"\x{7f}\x{99}\x{234}\x{1234}\x{12345}\"", "\x{7f}\x{99}\x{234}\x{1234}\x{12345}";
-str '"\/\"\\\\\b\t\n\f\r"', "/\"\\\x{08}\x{09}\x{0a}\x{0c}\x{0d}";
+str '"\u0099\u0234\u1234"', "\x{99}\x{234}\x{1234}";
+str "\"\x{99}\x{234}\x{1234}\x{12345}\"", "\x{99}\x{234}\x{1234}\x{12345}";
+str '"\/\"\\\\\t\n\r"', "/\"\\\x{09}\x{0a}\x{0d}";
 str '"\uD83D\uDE03"', "\x{1F603}";
 
 sub num($in, $exp=$in) {
@@ -186,6 +191,7 @@ for (2000..2100, 4000..4200, 8100..8200, 12200..12300, 16300..16400) {
 ok !eval { json_parse '[[[[]]]]', max_depth => 4; 1 };
 ok !eval { json_parse '{"":{"":{"":{"":1}}}}', max_depth => 4; 1 };
 
+is json_parse('"\u0000\b\f\u007f"', allow_control => 1), "\x00\x08\x0c\x7f";
 
 # 500 depth
 {

From 5560d9af609c7666e6c4c08fd9cd08c763c2aa47 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 7 Jun 2025 09:49:38 +0200
Subject: [PATCH 72/86] Version 1.1

---
 ChangeLog               |   9 ++++
 FU.pm                   |   2 +-
 FU/Benchmarks.pod       | 116 ++++++++++++++++++++--------------------
 FU/DebugImpl.pm         |   2 +-
 FU/Log.pm               |   2 +-
 FU/MultipartFormData.pm |   2 +-
 FU/Pg.pm                |   2 +-
 FU/SQL.pm               |   2 +-
 FU/Util.pm              |   2 +-
 FU/Validate.pm          |   2 +-
 FU/XMLWriter.pm         |   2 +-
 FU/XS.pm                |   2 +-
 bench.PL                |   2 +-
 13 files changed, 78 insertions(+), 69 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index 29e387f..ed0d259 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,12 @@
+1.1 - 2025-06-07
+	- FU::SQL: Add IDENT function and `quote_identifier` option
+	- FU::Pg: Set appropriate `quote_identifier` for `$conn->Q()`
+	- FU: Improve `--monitor` file change detection
+	- FU::XMLWriter: Disallow stringification of bare Perl references
+	- FU::Util::json_parse(): Disallow control characters in strings, add
+	  `allow_control` option to revert to old behavior.
+	- Some doc fixes
+
 1.0 - 2025-05-11
 	- FU::Util: Fix parsing of empty sections in query_decode()
 	- FU::Util: Fix buffer overflow in json_format() float formatting
diff --git a/FU.pm b/FU.pm
index 6da28cb..4b0dfed 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 1.0;
+package FU 1.1;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index 0762c7a..3390fb1 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -30,7 +30,7 @@ The following module versions were used:
 
 =item L<DBD::Pg> 3.18.0
 
-=item L<FU> 1.0
+=item L<FU> 1.1
 
 =item L<HTML::Tiny> 1.08
 
@@ -71,7 +71,7 @@ API object from L<JSON::XS> documentation.
   Cpanel::JSON::XS    114802/s    104141/s    107274/s
         JSON::SIMD    130137/s    118948/s    115123/s
           JSON::XS    128421/s    120243/s    117940/s
-          FU::Util    132890/s    111630/s    121124/s
+          FU::Util    132067/s    111328/s    117781/s
 
 Object (small)
 
@@ -81,7 +81,7 @@ Object (small)
   Cpanel::JSON::XS     45732/s     30862/s     20102/s
         JSON::SIMD     49019/s     30699/s     23267/s
           JSON::XS     49814/s     31326/s     25336/s
-          FU::Util     43853/s     26568/s     20426/s
+          FU::Util     39684/s     24971/s     17998/s
 
 Object (large)
 
@@ -91,7 +91,7 @@ Object (large)
   Cpanel::JSON::XS     30587/s     11875/s     15515/s
         JSON::SIMD     24418/s     12388/s     22895/s
           JSON::XS     23192/s     13174/s     23553/s
-          FU::Util     36455/s     11920/s     17370/s
+          FU::Util     35489/s     13247/s     16571/s
 
 Object (large, mixed unicode)
 
@@ -101,7 +101,7 @@ Object (large, mixed unicode)
   Cpanel::JSON::XS     25333/s      1459/s      7480/s
         JSON::SIMD     25031/s      1331/s     15997/s
           JSON::XS     23580/s      1375/s      8526/s
-          FU::Util     33085/s     12639/s      9375/s
+          FU::Util     32519/s     12488/s      9290/s
 
 Small integers
 
@@ -111,7 +111,7 @@ Small integers
   Cpanel::JSON::XS      7345/s      6151/s
         JSON::SIMD      7963/s      4361/s
           JSON::XS      7915/s      6058/s
-          FU::Util      7883/s      5671/s
+          FU::Util      7851/s      5828/s
 
 Large integers
 
@@ -121,7 +121,7 @@ Large integers
   Cpanel::JSON::XS     32545/s     50162/s
         JSON::SIMD     37201/s     51719/s
           JSON::XS     36722/s     50110/s
-          FU::Util    110210/s     61006/s
+          FU::Util    109163/s     63176/s
 
 ASCII strings
 
@@ -131,7 +131,7 @@ ASCII strings
   Cpanel::JSON::XS    116721/s     44560/s
         JSON::SIMD    134711/s     50429/s
           JSON::XS    135419/s     43976/s
-          FU::Util    164804/s     48163/s
+          FU::Util    162246/s     44216/s
 
 Unicode strings
 
@@ -141,7 +141,7 @@ Unicode strings
   Cpanel::JSON::XS     97039/s     67669/s
         JSON::SIMD    106928/s    102440/s
           JSON::XS    105473/s     60558/s
-          FU::Util    187489/s     61121/s
+          FU::Util    201648/s     57397/s
 
 String escaping (few)
 
@@ -151,17 +151,17 @@ String escaping (few)
   Cpanel::JSON::XS    136755/s    118059/s
         JSON::SIMD    158171/s    153692/s
           JSON::XS    157261/s     97676/s
-          FU::Util    216443/s     96354/s
+          FU::Util    225259/s     92515/s
 
 String escaping (many)
 
                         Encode      Decode
-          JSON::PP      2224/s       366/s
-        JSON::Tiny      2884/s       984/s
-  Cpanel::JSON::XS    140220/s    107040/s
-        JSON::SIMD    152951/s    113242/s
-          JSON::XS    153471/s    106269/s
-          FU::Util    153081/s    100279/s
+          JSON::PP      3963/s       561/s
+        JSON::Tiny      4463/s      2175/s
+  Cpanel::JSON::XS    197154/s    133102/s
+        JSON::SIMD    199955/s    152557/s
+          JSON::XS    231905/s    125191/s
+          FU::Util    215367/s    118073/s
 
 
 
@@ -176,7 +176,7 @@ HTML fragment
          TUWF::XML       795/s
        XML::Writer       833/s
         HTML::Tiny       423/s
-     FU::XMLWriter      5396/s
+     FU::XMLWriter      5327/s
 
 
 
@@ -191,8 +191,8 @@ Fetch and bitwise-or 20k integers
                       Smallint      Bigint
            DBD::Pg       194/s        22/s
             Pg::PQ       226/s        19/s
-      FU::Pg (bin)       239/s        23/s
-     FU::Pg (text)       222/s        21/s
+      FU::Pg (bin)       245/s        22/s
+     FU::Pg (text)       217/s        20/s
 
 
 
@@ -200,154 +200,154 @@ Fetch and bitwise-or 20k integers
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.
 json/api	Canonical	Cpanel::JSON::XS	104141
-json/api	Canonical	FU::Util	111630
+json/api	Canonical	FU::Util	111328
 json/api	Canonical	JSON::PP	5119
 json/api	Canonical	JSON::SIMD	118948
 json/api	Canonical	JSON::XS	120243
 json/api	Decode	Cpanel::JSON::XS	107274
-json/api	Decode	FU::Util	121124
+json/api	Decode	FU::Util	117781
 json/api	Decode	JSON::PP	1290
 json/api	Decode	JSON::SIMD	115123
 json/api	Decode	JSON::Tiny	3426
 json/api	Decode	JSON::XS	117940
 json/api	Encode	Cpanel::JSON::XS	114802
-json/api	Encode	FU::Util	132890
+json/api	Encode	FU::Util	132067
 json/api	Encode	JSON::PP	5312
 json/api	Encode	JSON::SIMD	130137
 json/api	Encode	JSON::Tiny	7757
 json/api	Encode	JSON::XS	128421
 json/intl	Decode	Cpanel::JSON::XS	50162
-json/intl	Decode	FU::Util	61006
+json/intl	Decode	FU::Util	63176
 json/intl	Decode	JSON::PP	329
 json/intl	Decode	JSON::SIMD	51719
 json/intl	Decode	JSON::Tiny	1638
 json/intl	Decode	JSON::XS	50110
 json/intl	Encode	Cpanel::JSON::XS	32545
-json/intl	Encode	FU::Util	110210
+json/intl	Encode	FU::Util	109163
 json/intl	Encode	JSON::PP	2176
 json/intl	Encode	JSON::SIMD	37201
 json/intl	Encode	JSON::Tiny	2999
 json/intl	Encode	JSON::XS	36722
 json/ints	Decode	Cpanel::JSON::XS	6151
-json/ints	Decode	FU::Util	5671
+json/ints	Decode	FU::Util	5828
 json/ints	Decode	JSON::PP	29
 json/ints	Decode	JSON::SIMD	4361
 json/ints	Decode	JSON::Tiny	86
 json/ints	Decode	JSON::XS	6058
 json/ints	Encode	Cpanel::JSON::XS	7345
-json/ints	Encode	FU::Util	7883
+json/ints	Encode	FU::Util	7851
 json/ints	Encode	JSON::PP	113
 json/ints	Encode	JSON::SIMD	7963
 json/ints	Encode	JSON::Tiny	160
 json/ints	Encode	JSON::XS	7915
 json/objl	Canonical	Cpanel::JSON::XS	11875
-json/objl	Canonical	FU::Util	11920
+json/objl	Canonical	FU::Util	13247
 json/objl	Canonical	JSON::PP	747
 json/objl	Canonical	JSON::SIMD	12388
 json/objl	Canonical	JSON::XS	13174
 json/objl	Decode	Cpanel::JSON::XS	15515
-json/objl	Decode	FU::Util	17370
+json/objl	Decode	FU::Util	16571
 json/objl	Decode	JSON::PP	104
 json/objl	Decode	JSON::SIMD	22895
 json/objl	Decode	JSON::Tiny	392
 json/objl	Decode	JSON::XS	23553
 json/objl	Encode	Cpanel::JSON::XS	30587
-json/objl	Encode	FU::Util	36455
+json/objl	Encode	FU::Util	35489
 json/objl	Encode	JSON::PP	927
 json/objl	Encode	JSON::SIMD	24418
 json/objl	Encode	JSON::Tiny	1108
 json/objl	Encode	JSON::XS	23192
 json/objs	Canonical	Cpanel::JSON::XS	30862
-json/objs	Canonical	FU::Util	26568
+json/objs	Canonical	FU::Util	24971
 json/objs	Canonical	JSON::PP	829
 json/objs	Canonical	JSON::SIMD	30699
 json/objs	Canonical	JSON::XS	31326
 json/objs	Decode	Cpanel::JSON::XS	20102
-json/objs	Decode	FU::Util	20426
+json/objs	Decode	FU::Util	17998
 json/objs	Decode	JSON::PP	202
 json/objs	Decode	JSON::SIMD	23267
 json/objs	Decode	JSON::Tiny	499
 json/objs	Decode	JSON::XS	25336
 json/objs	Encode	Cpanel::JSON::XS	45732
-json/objs	Encode	FU::Util	43853
+json/objs	Encode	FU::Util	39684
 json/objs	Encode	JSON::PP	907
 json/objs	Encode	JSON::SIMD	49019
 json/objs	Encode	JSON::Tiny	1224
 json/objs	Encode	JSON::XS	49814
 json/obju	Canonical	Cpanel::JSON::XS	1459
-json/obju	Canonical	FU::Util	12639
+json/obju	Canonical	FU::Util	12488
 json/obju	Canonical	JSON::PP	679
 json/obju	Canonical	JSON::SIMD	1331
 json/obju	Canonical	JSON::XS	1375
 json/obju	Decode	Cpanel::JSON::XS	7480
-json/obju	Decode	FU::Util	9375
+json/obju	Decode	FU::Util	9290
 json/obju	Decode	JSON::PP	86
 json/obju	Decode	JSON::SIMD	15997
 json/obju	Decode	JSON::Tiny	402
 json/obju	Decode	JSON::XS	8526
 json/obju	Encode	Cpanel::JSON::XS	25333
-json/obju	Encode	FU::Util	33085
+json/obju	Encode	FU::Util	32519
 json/obju	Encode	JSON::PP	817
 json/obju	Encode	JSON::SIMD	25031
 json/obju	Encode	JSON::Tiny	1036
 json/obju	Encode	JSON::XS	23580
-json/strel	Decode	Cpanel::JSON::XS	107040
-json/strel	Decode	FU::Util	100279
-json/strel	Decode	JSON::PP	366
-json/strel	Decode	JSON::SIMD	113242
-json/strel	Decode	JSON::Tiny	984
-json/strel	Decode	JSON::XS	106269
-json/strel	Encode	Cpanel::JSON::XS	140220
-json/strel	Encode	FU::Util	153081
-json/strel	Encode	JSON::PP	2224
-json/strel	Encode	JSON::SIMD	152951
-json/strel	Encode	JSON::Tiny	2884
-json/strel	Encode	JSON::XS	153471
+json/strel	Decode	Cpanel::JSON::XS	133102
+json/strel	Decode	FU::Util	118073
+json/strel	Decode	JSON::PP	561
+json/strel	Decode	JSON::SIMD	152557
+json/strel	Decode	JSON::Tiny	2175
+json/strel	Decode	JSON::XS	125191
+json/strel	Encode	Cpanel::JSON::XS	197154
+json/strel	Encode	FU::Util	215367
+json/strel	Encode	JSON::PP	3963
+json/strel	Encode	JSON::SIMD	199955
+json/strel	Encode	JSON::Tiny	4463
+json/strel	Encode	JSON::XS	231905
 json/stres	Decode	Cpanel::JSON::XS	118059
-json/stres	Decode	FU::Util	96354
+json/stres	Decode	FU::Util	92515
 json/stres	Decode	JSON::PP	352
 json/stres	Decode	JSON::SIMD	153692
 json/stres	Decode	JSON::Tiny	1869
 json/stres	Decode	JSON::XS	97676
 json/stres	Encode	Cpanel::JSON::XS	136755
-json/stres	Encode	FU::Util	216443
+json/stres	Encode	FU::Util	225259
 json/stres	Encode	JSON::PP	4251
 json/stres	Encode	JSON::SIMD	158171
 json/stres	Encode	JSON::Tiny	4704
 json/stres	Encode	JSON::XS	157261
 json/strs	Decode	Cpanel::JSON::XS	44560
-json/strs	Decode	FU::Util	48163
+json/strs	Decode	FU::Util	44216
 json/strs	Decode	JSON::PP	336
 json/strs	Decode	JSON::SIMD	50429
 json/strs	Decode	JSON::Tiny	1439
 json/strs	Decode	JSON::XS	43976
 json/strs	Encode	Cpanel::JSON::XS	116721
-json/strs	Encode	FU::Util	164804
+json/strs	Encode	FU::Util	162246
 json/strs	Encode	JSON::PP	2934
 json/strs	Encode	JSON::SIMD	134711
 json/strs	Encode	JSON::Tiny	4126
 json/strs	Encode	JSON::XS	135419
 json/stru	Decode	Cpanel::JSON::XS	67669
-json/stru	Decode	FU::Util	61121
+json/stru	Decode	FU::Util	57397
 json/stru	Decode	JSON::PP	253
 json/stru	Decode	JSON::SIMD	102440
 json/stru	Decode	JSON::Tiny	2585
 json/stru	Decode	JSON::XS	60558
 json/stru	Encode	Cpanel::JSON::XS	97039
-json/stru	Encode	FU::Util	187489
+json/stru	Encode	FU::Util	201648
 json/stru	Encode	JSON::PP	5113
 json/stru	Encode	JSON::SIMD	106928
 json/stru	Encode	JSON::Tiny	6603
 json/stru	Encode	JSON::XS	105473
 pg/ints	Bigint	DBD::Pg	22
-pg/ints	Bigint	FU::Pg (bin)	23
-pg/ints	Bigint	FU::Pg (text)	21
+pg/ints	Bigint	FU::Pg (bin)	22
+pg/ints	Bigint	FU::Pg (text)	20
 pg/ints	Bigint	Pg::PQ	19
 pg/ints	Smallint	DBD::Pg	194
-pg/ints	Smallint	FU::Pg (bin)	239
-pg/ints	Smallint	FU::Pg (text)	222
+pg/ints	Smallint	FU::Pg (bin)	245
+pg/ints	Smallint	FU::Pg (text)	217
 pg/ints	Smallint	Pg::PQ	226
-xml/a	Rate	FU::XMLWriter	5396
+xml/a	Rate	FU::XMLWriter	5327
 xml/a	Rate	HTML::Tiny	423
 xml/a	Rate	TUWF::XML	795
 xml/a	Rate	XML::Writer	833
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index ebf8c80..083afc1 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 1.0;
+package FU::DebugImpl 1.1;
 use v5.36;
 use utf8;
 use experimental 'for_list';
diff --git a/FU/Log.pm b/FU/Log.pm
index 9606326..154be51 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 1.0;
+package FU::Log 1.1;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 48ebb77..7c99f5c 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 1.0;
+package FU::MultipartFormData 1.1;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 2e7baf8..4a75617 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 1.0;
+package FU::Pg 1.1;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 63107b9..bdf633a 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 1.0;
+package FU::SQL 1.1;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index 4b06f33..270c18b 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 1.0;
+package FU::Util 1.1;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index a4544bf..96dbd0d 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 1.0;
+package FU::Validate 1.1;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 1b964ee..6159198 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 1.0;
+package FU::XMLWriter 1.1;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index b583e00..0185e47 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 1.0;
+package FU::XS 1.1;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');
diff --git a/bench.PL b/bench.PL
index 6ccd763..f4309c9 100755
--- a/bench.PL
+++ b/bench.PL
@@ -115,7 +115,7 @@ defjson stru => 0, 'Unicode strings', do { use utf8;
     [ map +('グリザイアの果実 -LE FRUIT DE LA GRISAIA-', '💩', 'Я люблю нічого не робити'), 1..50 ];
 };
 defjson stres => 0, 'String escaping (few)', [ map 'This string needs to "be escaped" a little bit', 1..100 ];
-defjson strel => 0, 'String escaping (many)', [ map "This \" \\ needs \b\x01\x02\x03\x04 more", 1..100 ];
+defjson strel => 0, 'String escaping (many)', [ map "This \" \\ needs \n\x41\x42\x43\x44 more", 1..100 ];
 
 
 

From 02b1dcc328bb4bedbe35d29a2f5688b8e3769913 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 12 Jun 2025 09:12:19 +0200
Subject: [PATCH 73/86] FU: Set $0 when spawning a new worker

This is primarily useful in error state, to avoid having a worker
process waiting for an incoming connection to be marked as the
supervisor.
---
 FU.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/FU.pm b/FU.pm
index 4b0dfed..2b9244e 100644
--- a/FU.pm
+++ b/FU.pm
@@ -503,6 +503,7 @@ sub _supervisor($c) {
             die $! if !defined $pid;
             if (!$pid) { # child
                 $SIG{CHLD} = $SIG{HUP} = $SIG{INT} = $SIG{TERM} = undef;
+                $0 = sprintf '%s: starting', $procname if $procname;
                 # In error state, wait with loading the script until we've received a request.
                 # Otherwise we'll end up in an infinite spawning loop if the script doesn't start properly.
                 $client = $c->{listen_sock}->accept() or die $! if !$client && $err;

From a7868f74bf9f3fa9ce8356b8a5d5476269bc9aa9 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 12 Jun 2025 16:45:07 +0200
Subject: [PATCH 74/86] Pg: Be more strict with boolean bind parameters

Reason for this is that, with FU::SQL, it's possible to accidentally
introduce a bind parameter when a WHERE clause was intended (i.e.
"WHERE $1"). That's pretty bad, but can easily be caught by simply not
accepting *every* possible value as boolean.
---
 FU/Pg.pm    | 10 ++++++----
 c/pgtypes.c | 11 +++++++++--
 t/pgtypes.t | 13 +++++++++++--
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/FU/Pg.pm b/FU/Pg.pm
index 4a75617..e715d7b 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -631,10 +631,12 @@ Some built-in types deserve a few additional notes:
 
 =item bool
 
-Boolean values are converted to C<builtin::true> and C<builtin::false>. As bind
-parameters, Perl's idea of truthiness is used: C<0>, C<false> and C<""> are
-false, everything else is true. Objects that overload I<bool> are also
-supported. C<undef> always converts to SQL C<NULL>.
+Boolean values are converted to C<builtin::true> and C<builtin::false>.
+
+As bind parameters, values recognized by C<to_bool()> in L<FU::Util> are
+accepted, in addition to C<0>, C<"f"> and C<""> for false and C<1>, and C<"t">
+for true.  C<undef> always converts to SQL C<NULL>. Everything else throws an
+error.
 
 =item bytea
 
diff --git a/c/pgtypes.c b/c/pgtypes.c
index 471b6d2..6b9b14a 100644
--- a/c/pgtypes.c
+++ b/c/pgtypes.c
@@ -82,8 +82,15 @@ RECVFN(bool) {
 }
 
 SENDFN(bool) {
-    int r = fu_2bool(aTHX_ val); /* So that we also recognize \0 and \1 */
-    fustr_write_ch(out, r < 0 ? SvTRUE(val) : r);
+    int r = fu_2bool(aTHX_ val);
+    if (r < 0) {
+        STRLEN l;
+        const char *x = SvPV(val, l);
+        if (l == 0 || (l == 1 && (*x == '0' || *x == 'f'))) r = 0;
+        else if (l == 1 && (*x == '1' || *x == 't')) r = 1;
+        else SERR("invalid boolean value: %s", x);
+    }
+    fustr_write_ch(out, r);
 }
 
 RECVFN(void) {
diff --git a/t/pgtypes.t b/t/pgtypes.t
index 21c8eff..662de33 100644
--- a/t/pgtypes.t
+++ b/t/pgtypes.t
@@ -61,8 +61,17 @@ sub f($type, $p_in) {
     $array->[0] = 0;
 }
 
-v bool => true,  undef, 1, 't';
-v bool => false, undef, 0, 'f';
+v bool => true,  true, 1, 't';
+v bool => \1,    true, 1, 't';
+v bool => 1,     true, 1, 't';
+v bool => 't',   true, 1, 't';
+v bool => false, false, 0, 'f';
+v bool => \0,    false, 0, 'f';
+v bool => 0,     false, 0, 'f';
+v bool => '',    false, 0, 'f';
+v bool => 'f',   false, 0, 'f';
+f bool => 2;
+f bool => [];
 
 v int2 => $_ for (1, -1, -32768, 32767, '12345', -12345, 123.0);
 f int2 => $_ for (-32769, 32768, [], '', 'a', 1.5);

From 5a863c20c29f6f0bd82451c7836341c13fcbdb62 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 6 Jul 2025 07:25:37 +0200
Subject: [PATCH 75/86] Version 1.2

---
 ChangeLog               | 5 +++++
 FU.pm                   | 2 +-
 FU/Benchmarks.pod       | 2 +-
 FU/DebugImpl.pm         | 2 +-
 FU/Log.pm               | 2 +-
 FU/MultipartFormData.pm | 2 +-
 FU/Pg.pm                | 2 +-
 FU/SQL.pm               | 2 +-
 FU/Util.pm              | 2 +-
 FU/Validate.pm          | 2 +-
 FU/XMLWriter.pm         | 2 +-
 FU/XS.pm                | 2 +-
 12 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index ed0d259..a5cb275 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+1.2 - 2025-07-06
+	- FU::Pg: Throw error on non-boolean-looking Perl values for boolean bind
+	  parameters
+	- FU: Improve setting process status during startup
+
 1.1 - 2025-06-07
 	- FU::SQL: Add IDENT function and `quote_identifier` option
 	- FU::Pg: Set appropriate `quote_identifier` for `$conn->Q()`
diff --git a/FU.pm b/FU.pm
index 2b9244e..3a8c94c 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 1.1;
+package FU 1.2;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index 3390fb1..e23571f 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -30,7 +30,7 @@ The following module versions were used:
 
 =item L<DBD::Pg> 3.18.0
 
-=item L<FU> 1.1
+=item L<FU> 1.2
 
 =item L<HTML::Tiny> 1.08
 
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 083afc1..7e8ddac 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 1.1;
+package FU::DebugImpl 1.2;
 use v5.36;
 use utf8;
 use experimental 'for_list';
diff --git a/FU/Log.pm b/FU/Log.pm
index 154be51..64f95c8 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 1.1;
+package FU::Log 1.2;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 7c99f5c..e6b44e2 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 1.1;
+package FU::MultipartFormData 1.2;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index e715d7b..cd48ab2 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 1.1;
+package FU::Pg 1.2;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index bdf633a..7ff1242 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 1.1;
+package FU::SQL 1.2;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index 270c18b..3228694 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 1.1;
+package FU::Util 1.2;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index 96dbd0d..adca929 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 1.1;
+package FU::Validate 1.2;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 6159198..33cdea4 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 1.1;
+package FU::XMLWriter 1.2;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index 0185e47..a3d5337 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 1.1;
+package FU::XS 1.2;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');

From 2e9a40da69ee11b1e081e0136667dede19ca03f6 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 22 Aug 2025 09:21:06 +0200
Subject: [PATCH 76/86] More strict UTF-8 validation on input

---
 FU.pm      | 2 +-
 FU/Util.pm | 6 +++++-
 t/query.t  | 3 +++
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/FU.pm b/FU.pm
index 3a8c94c..f43dbaf 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1267,7 +1267,7 @@ handler being run. Any other exception is passed to the C<500> error handler.
 
 While the C<FU::> namespace is used for global configuration and utility
 functions, the C<fu> object is intended for methods that deal with request
-processing (although some are useful used outside of request handlers as well).
+processing (although some are useful outside of request handlers as well).
 
 The C<fu> object itself can be used to store request-local data. For example,
 the following is a valid approach to handle user authentication:
diff --git a/FU/Util.pm b/FU/Util.pm
index 3228694..2074ada 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -4,6 +4,7 @@ use v5.36;
 use FU::XS;
 use Carp 'confess';
 use Exporter 'import';
+use Encode ();
 use POSIX ();
 use experimental 'builtin';
 
@@ -19,7 +20,10 @@ our @EXPORT_OK = qw/
 
 sub utf8_decode :prototype($) {
     return if !defined $_[0];
-    confess 'Invalid UTF-8' if !utf8::decode($_[0]);
+    eval {
+        $_[0] = Encode::decode('UTF-8', $_[0], Encode::FB_CROAK);
+        1
+    } || confess($@ =~ s/ at .+\n$//r);
     confess 'Invalid control character' if $_[0] =~ /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
     $_[0]
 }
diff --git a/t/query.t b/t/query.t
index 80f2b00..9d1ca4a 100644
--- a/t/query.t
+++ b/t/query.t
@@ -10,6 +10,9 @@ is_deeply
 ok !eval { query_decode('%10'); 1 };
 like $@, qr/Invalid control character/;
 
+ok !eval { query_decode('a=%fe%83%bf%bf%bf%bf%bf%0a'); 1 };
+like $@, qr/does not map to Unicode/;
+
 is_deeply query_decode('&&&a=b'), { a => 'b' };
 
 is query_encode

From a8ac435f85724acd2caef107e907abcebc7c6e1b Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Fri, 22 Aug 2025 09:51:56 +0200
Subject: [PATCH 77/86] Move control character checking to FU::Validate,
 deprecate FU::Util::utf8_decode()

URI, JSON and formdata decoding no longer checks for control characters,
but FU::Validate now rejects control characters by default. This
decouples semantic validation from format parsing and gives better
control over when control characters are allowed.
---
 FU.pm                   |  2 +-
 FU/MultipartFormData.pm |  4 +---
 FU/Util.pm              | 35 ++++++++++-------------------------
 FU/Validate.pm          | 23 +++++++++++++++++++----
 c/jsonparse.c           | 11 +++--------
 t/json_parse.t          |  6 +-----
 t/query.t               |  3 ---
 t/validate.t            |  4 ++++
 8 files changed, 39 insertions(+), 49 deletions(-)

diff --git a/FU.pm b/FU.pm
index f43dbaf..2fabf2a 100644
--- a/FU.pm
+++ b/FU.pm
@@ -312,7 +312,7 @@ sub _read_req($c) {
 
     ($REQ->{path}, my $qs) = split /\?/, $REQ->{path}//'', 2;
     $REQ->{qs} //= $qs;
-    eval { $REQ->{path} = FU::Util::uri_unescape($REQ->{path}); 1; } || fu->error(400, $@);
+    eval { $REQ->{path} = FU::Util::uri_unescape($REQ->{path}); FU::Util::check_control($REQ->{path}); 1; } || fu->error(400, $@);
     fu->error(400, 'Invalid character in path') if $REQ->{path} =~ /[\r\n\t]/; # There are plenty other questionable characters, but newlines and tabs are definitely out
 }
 
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index e6b44e2..14ce323 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -175,9 +175,7 @@ this on large fields.
 
 =item value
 
-Returns a copy of the field value as a Unicode string. Uses C<utf8_decode()>
-from L<FU::Util>, so also throws an error if the value contains control
-characters.
+Returns a copy of the field value as a Unicode string.
 
 =item substr($off, $len)
 
diff --git a/FU/Util.pm b/FU/Util.pm
index 2074ada..77635a2 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -11,20 +11,26 @@ use experimental 'builtin';
 our @EXPORT_OK = qw/
     to_bool
     json_format json_parse
-    utf8_decode uri_escape uri_unescape
+    has_control check_control utf8_decode
+    uri_escape uri_unescape
     query_decode query_encode
     httpdate_format httpdate_parse
     gzip_lib gzip_compress brotli_compress
     fdpass_send fdpass_recv
 /;
 
+
+# Internal utility function
+sub has_control :prototype($) ($s) { defined $s && $s =~ /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/ }
+sub check_control :prototype($) ($s) { confess 'Invalid control character' if has_control $s; }
+
+# Deprecated, call Encode::decode() directly.
 sub utf8_decode :prototype($) {
     return if !defined $_[0];
     eval {
         $_[0] = Encode::decode('UTF-8', $_[0], Encode::FB_CROAK);
         1
     } || confess($@ =~ s/ at .+\n$//r);
-    confess 'Invalid control character' if $_[0] =~ /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/;
     $_[0]
 }
 
@@ -175,13 +181,6 @@ Supported C<%options>:
 
 =over
 
-=item allow_control
-
-Boolean, set to true to allow (encoded) ASCII control characters in JSON
-strings, such as C<\u0000>, C<\b>, C<\u007f>, etc.  These characters are
-permitted per RFC-8259, but disallowed by this parser by default. See
-C<utf8_decode()> below.
-
 =item utf8
 
 Boolean, interpret the input C<$string> as a UTF-8 encoded byte string instead
@@ -296,18 +295,6 @@ inputs, at the cost of flexibility.
 
 =over
 
-=item utf8_decode($bytes)
-
-Convert a (perl-UTF-8 encoded) byte string into a sanitized perl Unicode
-string. The conversion is performed in-place, so the C<$bytes> argument is
-turned into a Unicode string. Returns the same string for convenience.
-
-This function throws an error if the input is not valid UTF-8 or if it contains
-ASCII control characters - that is, any character between C<0x00> and C<0x1f>
-except for tab, newline and carriage return.
-
-(This is a tiny wrapper around C<utf8::decode()> with some extra checks)
-
 =item uri_escape($string)
 
 Takes an Unicode string and returns a percent-encoded ASCII string, suitable
@@ -316,8 +303,7 @@ for use in a query parameter.
 =item uri_unescape($string)
 
 Takes an Unicode string potentially containing percent-encoding and returns a
-decoded Unicode string. Also checks for ASCII control characters as per
-C<utf8_decode()>.
+decoded Unicode string.
 
 =item query_decode($string)
 
@@ -334,8 +320,7 @@ have a value are decoded as C<builtin::true>. Example:
     # }
 
 The input C<$string> is assumed to be a perl Unicode string. An error is thrown
-if the resulting data decodes into invalid UTF-8 or contains control
-characters, as per C<utf8_decode>.
+if the resulting data decodes into invalid UTF-8.
 
 =item query_encode($hashref)
 
diff --git a/FU/Validate.pm b/FU/Validate.pm
index adca929..fa3bdcc 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -4,7 +4,7 @@ use v5.36;
 use experimental 'builtin', 'for_list';
 use builtin qw/true false blessed trim/;
 use Carp 'confess';
-use FU::Util 'to_bool';
+use FU::Util 'to_bool', 'has_control';
 
 
 # Unavailable as custom validation names
@@ -12,7 +12,7 @@ my %builtin = map +($_,1), qw/
     type
     default
     onerror
-    trim
+    trim allow_control
     elems sort unique
     accept_scalar accept_array
     keys values unknown missing
@@ -296,8 +296,13 @@ sub _validate_input {
     $_[1] = $_[1]->@* == 0 ? undef : $c->{accept_array} eq 'first' ? $_[1][0] : $_[1][ $#{$_[1]} ]
         if $c->{accept_array} && ref $_[1] eq 'ARRAY';
 
-    # trim (needs to be done before the 'default' test)
-    $_[1] = trim $_[1] =~ s/\r//rg if defined $_[1] && !ref $_[1] && $type eq 'scalar' && (!exists $c->{trim} || $c->{trim});
+    # early scalar checks
+    if (defined $_[1] && !ref $_[1] && $type eq 'scalar') {
+        # trim needs to be done before the 'default' test
+        $_[1] = trim $_[1] =~ s/\r//rg if !exists $c->{trim} || $c->{trim};
+
+        return { validation => 'allow_control' } if !$c->{allow_control} && has_control $_[1];
+    }
 
     # default
     if (!defined $_[1] || (!ref $_[1] && $_[1] eq '')) {
@@ -403,6 +408,7 @@ sub _inval($t,$v) { sprintf 'invalid %s: %s', $t, _fmtval $v }
 # TODO: document.
 our %error_format = (
     required  => sub { 'required value missing' },
+    allow_control => sub { 'invalid control character' },
     type      => sub($e) { "invalid type, expected '$e->{expected}' but got '$e->{got}'" },
     unknown   => sub($e) { sprintf 'unknown key%s: %s', $e->{keys}->@* == 1 ? '' : 's', join ', ', map _fmtkey($_), $e->{keys}->@* },
     minlength => sub($e) { sprintf "input too short, expected minimum of %d but got %d", $e->{expected}, $e->{got} },
@@ -590,6 +596,9 @@ Upon failure, the error object will look something like:
     got        => 'scalar'
   }
 
+Beware: setting the type to I<any> causes the I<trim> and I<allow_control>
+validations to be skipped.
+
 =item default => $val
 
 If not set, or set to C<\'required'> (note: scalarref), then a value is required
@@ -623,6 +632,12 @@ By default, any whitespace around scalar-type input is removed before testing
 any other validations. Setting I<trim> to a false value will disable this
 behavior.
 
+=item allow_control => 0/1
+
+By default, ASCII control characters in the input are not permitted for scalar
+values and trigger a validation error. Set this to a positive value to disable
+the check.
+
 =item keys => $hashref
 
 Implies C<< type => 'hash' >>, this option specifies which keys are permitted,
diff --git a/c/jsonparse.c b/c/jsonparse.c
index bf46557..136034a 100644
--- a/c/jsonparse.c
+++ b/c/jsonparse.c
@@ -2,7 +2,6 @@ typedef struct {
     const unsigned char *buf;
     const unsigned char *end;
     UV depth;
-    int allow_control;
 } fujson_parse_ctx;
 
 static SV *fujson_parse(pTHX_ fujson_parse_ctx *);
@@ -24,10 +23,10 @@ static inline int fujson_parse_string_escape(pTHX_ fujson_parse_ctx *ctx, fustr
         case '"': *(r->cur++) = '\"'; break;
         case '\\':*(r->cur++) = '\\'; break;
         case '/': *(r->cur++) = '/';  break; /* We don't escape this one */
-        case 'b': if (!ctx->allow_control) return 1; *(r->cur++) = 0x08; break;
+        case 'b': *(r->cur++) = 0x08; break;
         case 't': *(r->cur++) = 0x09; break;
         case 'n': *(r->cur++) = 0x0a; break;
-        case 'f': if (!ctx->allow_control) return 1; *(r->cur++) = 0x0c; break;
+        case 'f': *(r->cur++) = 0x0c; break;
         case 'r': *(r->cur++) = 0x0d; break;
         case 'u':
             /* (awful code adapted from ncdu) */
@@ -44,9 +43,6 @@ static inline int fujson_parse_string_escape(pTHX_ fujson_parse_ctx *ctx, fustr
                 n = 0x10000 + (((n & 0x03ff) << 10) | (s & 0x03ff));
                 ctx->buf += 6;
             }
-            if (!ctx->allow_control &&
-                    (n <= 8 || n == 0x0b || n == 0x0c || (n >= 0x0e && n <= 0x1f) || n == 0x7f))
-                return 1;
             r->cur = (char *)uvchr_to_utf8((U8 *)r->cur, n);
             if (n >= 0x80) r->setutf8 = 1;
             break;
@@ -269,7 +265,6 @@ static SV *fujson_parse_xs(pTHX_ I32 ax, I32 argc, SV *val) {
     fujson_parse_ctx ctx;
 
     ctx.depth = 0;
-    ctx.allow_control = 0;
     while (i < argc) {
         arg = SvPV_nolen(ST(i));
         i++;
@@ -280,7 +275,7 @@ static SV *fujson_parse_xs(pTHX_ I32 ax, I32 argc, SV *val) {
         if (strcmp(arg, "utf8") == 0) decutf8 = SvTRUEx(r);
         else if (strcmp(arg, "max_size") == 0) maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
-        else if (strcmp(arg, "allow_control") == 0) ctx.allow_control = SvTRUE(r);
+        else if (strcmp(arg, "allow_control") == 0) {}
         else if (strcmp(arg, "offset") == 0) offset = r;
         else croak("Unknown flag: '%s'", arg);
     }
diff --git a/t/json_parse.t b/t/json_parse.t
index 3ad3838..686a1e6 100644
--- a/t/json_parse.t
+++ b/t/json_parse.t
@@ -24,11 +24,6 @@ my @error = (
     '"\udc12\u1234"',
     "\"\x{110000}\"",
 
-    '"\u0000"',
-    '"\b"',
-    '"\f"',
-    '"\u007f"',
-
     '1.',
     '01',
     '1e',
@@ -87,6 +82,7 @@ sub str($in, $exp) {
 }
 str '""', '';
 str '"hello, world"', 'hello, world';
+str '"\u0000\b"', "\x00\b";
 str '"\u0099\u0234\u1234"', "\x{99}\x{234}\x{1234}";
 str "\"\x{99}\x{234}\x{1234}\x{12345}\"", "\x{99}\x{234}\x{1234}\x{12345}";
 str '"\/\"\\\\\t\n\r"', "/\"\\\x{09}\x{0a}\x{0d}";
diff --git a/t/query.t b/t/query.t
index 9d1ca4a..96ae403 100644
--- a/t/query.t
+++ b/t/query.t
@@ -7,9 +7,6 @@ is_deeply
     query_decode('a&a&%c3%be=%26%3d%c3%be&a=3'),
     { a => [ builtin::true, builtin::true, 3 ], "\xfe" => "&=\xfe" };
 
-ok !eval { query_decode('%10'); 1 };
-like $@, qr/Invalid control character/;
-
 ok !eval { query_decode('a=%fe%83%bf%bf%bf%bf%bf%0a'); 1 };
 like $@, qr/does not map to Unicode/;
 
diff --git a/t/validate.t b/t/validate.t
index 26704cd..46dc173 100644
--- a/t/validate.t
+++ b/t/validate.t
@@ -79,6 +79,10 @@ t { trim => 0 }, " Va\rl id \n ", " Va\rl id \n ";
 f {}, '  ', { validation => 'required' }, 'required value missing';
 t { trim => 0 }, '  ', '  ';
 
+# allow_control
+f {}, "\b", { validation => 'allow_control' }, 'invalid control character';
+t { allow_control => 1 }, "\b", "\b";
+
 # accept_array
 t { default => undef, accept_array => 'first' }, [], undef;
 t { default => undef, accept_array => 'first' }, [' x '], 'x';

From 715f4a748b705456db479f4b44d01a4049fb95a3 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 4 Sep 2025 11:30:24 +0200
Subject: [PATCH 78/86] Version 1.3

---
 ChangeLog               | 8 ++++++++
 FU.pm                   | 2 +-
 FU/DebugImpl.pm         | 2 +-
 FU/Log.pm               | 2 +-
 FU/MultipartFormData.pm | 2 +-
 FU/Pg.pm                | 2 +-
 FU/SQL.pm               | 2 +-
 FU/Util.pm              | 2 +-
 FU/Validate.pm          | 2 +-
 FU/XMLWriter.pm         | 2 +-
 FU/XS.pm                | 2 +-
 11 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index a5cb275..f5ecc19 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+1.3 - 2025-09-04
+	- FU::Validate: Scalar validations now reject control characters by default
+	- FU::Validate: Add `allow_control` option to override above behavior
+	- FU::Util: JSON and URI parsing now always permit control characters
+	- FU::Util: More strict UTF-8 validation on path & URI decoding
+	- FU::Util: Deprecate `decode_utf8()`
+	- FU::Util: Deprecate `allow_control` option in `json_parse()`
+
 1.2 - 2025-07-06
 	- FU::Pg: Throw error on non-boolean-looking Perl values for boolean bind
 	  parameters
diff --git a/FU.pm b/FU.pm
index 2fabf2a..f1493a7 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 1.2;
+package FU 1.3;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 7e8ddac..13e4eb4 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 1.2;
+package FU::DebugImpl 1.3;
 use v5.36;
 use utf8;
 use experimental 'for_list';
diff --git a/FU/Log.pm b/FU/Log.pm
index 64f95c8..f3b1fcf 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 1.2;
+package FU::Log 1.3;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index 14ce323..ca21512 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 1.2;
+package FU::MultipartFormData 1.3;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index cd48ab2..75bc3af 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 1.2;
+package FU::Pg 1.3;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 7ff1242..3e0869c 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 1.2;
+package FU::SQL 1.3;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index 77635a2..da8b869 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 1.2;
+package FU::Util 1.3;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index fa3bdcc..f170bfc 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 1.2;
+package FU::Validate 1.3;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 33cdea4..5364c5e 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 1.2;
+package FU::XMLWriter 1.3;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index a3d5337..b22fbbb 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 1.2;
+package FU::XS 1.3;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');

From 144d88fc8b042df25a2b45112d461f837a98b5d5 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 4 Dec 2025 11:19:40 +0100
Subject: [PATCH 79/86] Misc doc fixes

---
 FU/Pg.pm  | 4 ++--
 FU/SQL.pm | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/FU/Pg.pm b/FU/Pg.pm
index 75bc3af..5c81b6b 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -71,7 +71,7 @@ C<$string> can either be in key=value format or a URI, refer to L<the
 PostgreSQL
 documentation|https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING>
 for the full list of supported formats and options. You may also pass an empty
-string and leave the configuration up L<environment
+string and leave the configuration up to L<environment
 variables|https://www.postgresql.org/docs/current/libpq-envars.html>.
 
 =item $conn->server_version
@@ -702,7 +702,7 @@ While C<null> is a valid JSON value, there's currently no way to distinguish
 that from SQL C<NULL>. When sending C<undef> as bind parameter, it is sent as
 SQL C<NULL>.
 
-If you prefer to work with JSON are raw text values instead, use:
+If you prefer to work with JSON as raw text values instead, use:
 
   $conn->set_type(json => 'text');
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 3e0869c..218a034 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -121,7 +121,7 @@ FU::SQL - Small and Safe SQL Query Builder
 
   my $sel = SQL 'SELECT id, name FROM table', WHERE { id => IN([1,2,3]) };
 
-  my($sql, @params) = $sel->compile;
+  my($sql, $params) = $sel->compile;
 
 =head1 DESCRIPTION
 

From 876613d03f09fdec7ebe3431d25dd63e63334bea Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 4 Dec 2025 11:20:09 +0100
Subject: [PATCH 80/86] FU: Fix useless warning on empty cookie sections

---
 FU.pm | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/FU.pm b/FU.pm
index f1493a7..002fec3 100644
--- a/FU.pm
+++ b/FU.pm
@@ -705,7 +705,8 @@ sub cookie {
         my %c;
         for my $c (split /; /, fu->header('cookie')||'') {
             my($n, $v) = split /=/, $c, 2;
-            if (!exists $c{$n}) { $c{$n} = $v }
+            if (!defined $v) {}
+            elsif (!exists $c{$n}) { $c{$n} = $v }
             elsif (ref $c{$n}) { push $c{$n}->@*, $v }
             else { $c{$n} = [ $c{$n}, $v ] }
         }

From 8140fefbca29702a5a39841007aad34de92ef734 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Thu, 4 Dec 2025 14:05:51 +0100
Subject: [PATCH 81/86] FU::Pg: Rename q() and Q() to sql() and SQL()

Because this easily confuses syntax highlighters and some humans with
the q// string syntax. Also for consistency with the 'fu->sql()'
aliases.

The old names are still available as alias.
---
 FU.pm               |   8 +-
 FU.xs               |   8 +-
 FU/DebugImpl.pm     |   2 +-
 FU/Pg.pm            |  68 ++++++++--------
 bench.PL            |   4 +-
 c/pgst.c            |   2 +-
 t/pgconnect.t       | 190 ++++++++++++++++++++++----------------------
 t/pgcopy.t          |   4 +-
 t/pgtypes-dynamic.t | 100 +++++++++++------------
 t/pgtypes.t         |  20 ++---
 10 files changed, 206 insertions(+), 200 deletions(-)

diff --git a/FU.pm b/FU.pm
index 002fec3..e4f4ac1 100644
--- a/FU.pm
+++ b/FU.pm
@@ -646,8 +646,8 @@ sub db {
     };
 }
 
-sub sql { shift->db->q(@_) }
-sub SQL { shift->db->Q(@_) }
+sub sql { shift->db->sql(@_) }
+sub SQL { shift->db->SQL(@_) }
 
 sub _fmt_section($s) { $s =~ s/^\s*/  /r =~ s/\s+$//r =~ s/\n/\n  /rg }
 
@@ -1306,11 +1306,11 @@ has successfully been processed, or rolled back if there was an error.
 
 =item fu->sql($query, @params)
 
-Convenient short-hand for C<< fu->db->q($query, @params) >>.
+Convenient short-hand for C<< fu->db->sql($query, @params) >>.
 
 =item fu->SQL(@args)
 
-Convenient short-hand for C<< fu->db->Q(@args) >>.
+Convenient short-hand for C<< fu->db->SQL(@args) >>.
 
 =item fu->log_verbose($message)
 
diff --git a/FU.xs b/FU.xs
index 7a387f9..60ef7a6 100644
--- a/FU.xs
+++ b/FU.xs
@@ -277,10 +277,10 @@ void exec(fupg_conn *c, SV *sv)
     FUPG_CONN_COOKIE;
     ST(0) = fupg_exec(aTHX_ c, SvPVutf8_nolen(sv));
 
-void q(fupg_conn *c, SV *sv, ...)
+void sql(fupg_conn *c, SV *sv, ...)
   CODE:
     FUPG_CONN_COOKIE;
-    ST(0) = fupg_q(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items);
+    ST(0) = fupg_sql(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items);
 
 void copy(fupg_conn *c, SV *sv)
   CODE:
@@ -353,10 +353,10 @@ void exec(fupg_txn *t, SV *sv)
     FUPG_TXN_COOKIE;
     ST(0) = fupg_exec(aTHX_ t->conn, SvPVutf8_nolen(sv));
 
-void q(fupg_txn *t, SV *sv, ...)
+void sql(fupg_txn *t, SV *sv, ...)
   CODE:
     FUPG_TXN_COOKIE;
-    ST(0) = fupg_q(aTHX_ t->conn, t->stflags, SvPVutf8_nolen(sv), ax, items);
+    ST(0) = fupg_sql(aTHX_ t->conn, t->stflags, SvPVutf8_nolen(sv), ax, items);
 
 # XXX: The copy object should probably keep a ref on the transaction
 void copy(fupg_txn *t, SV *sv)
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 13e4eb4..0ffe8cf 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -283,7 +283,7 @@ my @sections = (
 
     pgst => sub {
         return () if !$FU::DB;
-        my $lst = eval { $FU::DB->q(
+        my $lst = eval { $FU::DB->sql(
             'SELECT generic_plans + custom_plans, statement FROM pg_prepared_statements ORDER BY generic_plans + custom_plans DESC, statement'
         )->cache(0)->alla } || do { warn "Unable to collect prepared statement list: $@"; return () };
         return () if !@$lst;
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 5c81b6b..294481c 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -7,7 +7,7 @@ _load_libpq();
 package FU::Pg::conn {
     sub lib_version { FU::Pg::lib_version() }
 
-    sub Q {
+    sub SQL {
         require FU::SQL;
         my $s = shift;
         my($sql, $params) = FU::SQL::SQL(@_)->compile(
@@ -15,7 +15,7 @@ package FU::Pg::conn {
             in_style          => 'pg',
             quote_identifier  => sub { $s->conn->escape_identifier(@_) },
         );
-        $s->q($sql, @$params);
+        $s->sql($sql, @$params);
     }
 
     sub set_type($s, $n, @arg) {
@@ -26,7 +26,13 @@ package FU::Pg::conn {
     }
 };
 
-*FU::Pg::txn::Q = \*FU::Pg::conn::Q;
+*FU::Pg::txn::SQL = \*FU::Pg::conn::SQL;
+
+# Compat
+*FU::Pg::conn::q = \*FU::Pg::conn::sql;
+*FU::Pg::txn::q = \*FU::Pg::txn::sql;
+*FU::Pg::conn::Q = \*FU::Pg::conn::SQL;
+*FU::Pg::txn::Q = \*FU::Pg::txn::SQL;
 
 package FU::Pg::error {
     use overload '""' => sub($e, @) { $e->{full_message} };
@@ -47,10 +53,10 @@ FU::Pg - The Ultimate (synchronous) Interface to PostgreSQL
 
   $conn->exec('CREATE TABLE books (id SERIAL, title text, read bool)');
 
-  $conn->q('INSERT INTO books (title) VALUES ($1)', 'Revelation Space')->exec;
-  $conn->q('INSERT INTO books (title) VALUES ($1)', 'The Invincible')->exec;
+  $conn->sql('INSERT INTO books (title) VALUES ($1)', 'Revelation Space')->exec;
+  $conn->sql('INSERT INTO books (title) VALUES ($1)', 'The Invincible')->exec;
 
-  for my ($id, $title) ($conn->q('SELECT * FROM books')->flat->@*) {
+  for my ($id, $title) ($conn->sql('SELECT * FROM books')->flat->@*) {
       print "$id:  $title\n";
   }
 
@@ -141,7 +147,7 @@ a table, column, function, etc) in an SQL statement.
 
 =item $conn->text($enable)
 
-Set the default settings for new statements created with B<< $conn->q() >>.
+Set the default settings for new statements created with B<< $conn->sql() >>.
 
 =item $conn->cache_size($num)
 
@@ -169,7 +175,7 @@ Also worth noting that the subroutine is called from the context of the code
 executing the query, but I<before> the query results have been returned.
 
 The subroutine is (currently) only called for queries executed through C<<
-$conn->exec >>, C<< $conn->q >>, C<< $conn->Q >> and their C<$txn> variants;
+$conn->exec >>, C<< $conn->sql >>, C<< $conn->SQL >> and their C<$txn> variants;
 C<< $conn->copy >> statements and internal queries performed by this module
 (such as for transaction management, querying type information, etc) do not
 trigger the callback. Statements that result in an error being thrown during or
@@ -193,7 +199,7 @@ Execute one or more SQL commands, separated by a semicolon. Returns the number
 of rows affected by the last statement or I<undef> if that information is not
 available for the given command (like with C<CREATE TABLE>).
 
-=item $conn->q($sql, @params)
+=item $conn->sql($sql, @params)
 
 Create a new SQL statement with the given C<$sql> string and an optional list
 of bind parameters. C<$sql> can only hold a single statement.
@@ -209,15 +215,15 @@ Note that this method just creates a statement object, the query is not
 prepared or executed until the appropriate statement methods (see below) are
 used.
 
-=item $conn->Q(@args)
+=item $conn->SQL(@args)
 
-Same as C<< $conn->q() >> but uses L<FU::SQL> to construct the query and bind
+Same as C<< $conn->sql() >> but uses L<FU::SQL> to construct the query and bind
 parameters. Uses the 'pg' C<in_style> and C<< $conn->escape_identifier() >> for
 identifier quoting.
 
 =back
 
-Statement objects returned by C<< $conn->q() >> support the following
+Statement objects returned by C<< $conn->sql() >> support the following
 configuration parameters, which can be set before the statement is executed:
 
 =over
@@ -252,7 +258,7 @@ depending on how you'd like to obtain the results:
 Execute the query and return the number of rows affected. Similar to C<<
 $conn->exec >>.
 
-  my $v = $conn->q('UPDATE books SET read = true WHERE id = 1')->exec;
+  my $v = $conn->sql('UPDATE books SET read = true WHERE id = 1')->exec;
   # $v = 1
 
 =item $st->val
@@ -261,7 +267,7 @@ Return the first column of the first row. Throws an error if the query does not
 return exactly one column, or if multiple rows are returned. Returns I<undef>
 if no rows are returned or if its value is I<NULL>.
 
-  my $v = $conn->q('SELECT COUNT(*) FROM books')->val;
+  my $v = $conn->sql('SELECT COUNT(*) FROM books')->val;
   # $v = 2
 
 =item $st->rowl
@@ -269,7 +275,7 @@ if no rows are returned or if its value is I<NULL>.
 Return the first row as a list, or an empty list if no rows are returned.
 Throws an error if the query returned more than one row.
 
-  my($id, $title) = $conn->q('SELECT id, title FROM books LIMIT 1')->rowl;
+  my($id, $title) = $conn->sql('SELECT id, title FROM books LIMIT 1')->rowl;
   # ($id, $title) = (1, 'Revelation Space');
 
 =item $st->rowa
@@ -278,7 +284,7 @@ Return the first row as an arrayref, equivalent to C<< [$st->rowl] >> but might
 be slightly more efficient. Returns C<undef> if the query did not generate any
 rows.
 
-  my $row = $conn->q('SELECT id, title FROM books LIMIT 1')->rowa;
+  my $row = $conn->sql('SELECT id, title FROM books LIMIT 1')->rowa;
   # $row = [1, 'Revelation Space'];
 
 =item $st->rowh
@@ -287,14 +293,14 @@ Return the first row as a hashref. Returns C<undef> if the query did not
 generate any rows. Throws an error if the query returns multiple columns with
 the same name.
 
-  my $row = $conn->q('SELECT id, title FROM books LIMIT 1')->rowh;
+  my $row = $conn->sql('SELECT id, title FROM books LIMIT 1')->rowh;
   # $row = { id => 1, title => 'Revelation Space' };
 
 =item $st->alla
 
 Return all rows as an arrayref of arrayrefs.
 
-  my $data = $conn->q('SELECT id, title FROM books')->alla;
+  my $data = $conn->sql('SELECT id, title FROM books')->alla;
   # $data = [
   #   [ 1, 'Revelation Space' ],
   #   [ 2, 'The Invincible' ],
@@ -305,7 +311,7 @@ Return all rows as an arrayref of arrayrefs.
 Return all rows as an arrayref of hashrefs. Throws an error if the query
 returns multiple columns with the same name.
 
-  my $data = $conn->q('SELECT id, title FROM books')->allh;
+  my $data = $conn->sql('SELECT id, title FROM books')->allh;
   # $data = [
   #   { id => 1, title => 'Revelation Space' },
   #   { id => 2, title => 'The Invincible' },
@@ -315,7 +321,7 @@ returns multiple columns with the same name.
 
 Return an arrayref with all rows flattened.
 
-  my $data = $conn->q('SELECT id, title FROM books')->flat;
+  my $data = $conn->sql('SELECT id, title FROM books')->flat;
   # $data = [
   #   1, 'Revelation Space',
   #   2, 'The Invincible',
@@ -327,7 +333,7 @@ Return a hashref where the first result column is used as key and the second
 column as value. If the query only returns a single column, C<true> is used as
 value instead. An error is thrown if the query returns 3 or more columns.
 
-  my $data = $conn->q('SELECT id, title FROM books')->kvv;
+  my $data = $conn->sql('SELECT id, title FROM books')->kvv;
   # $data = {
   #   1 => 'Revelation Space',
   #   2 => 'The Invincible',
@@ -338,7 +344,7 @@ value instead. An error is thrown if the query returns 3 or more columns.
 Return a hashref where the first result column is used as key and the remaining
 columns are stored as arrayref.
 
-  my $data = $conn->q('SELECT id, title, read FROM books')->kva;
+  my $data = $conn->sql('SELECT id, title, read FROM books')->kva;
   # $data = {
   #   1 => [ 'Revelation Space', true ],
   #   2 => [ 'The Invincible', false ],
@@ -349,7 +355,7 @@ columns are stored as arrayref.
 Return a hashref where the first result column is used as key and the remaining
 columns are stored as hashref.
 
-  my $data = $conn->q('SELECT id, title, read FROM books')->kvh;
+  my $data = $conn->sql('SELECT id, title, read FROM books')->kvh;
   # $data = {
   #   1 => { title => 'Revelation Space', read => true },
   #   2 => { title => 'The Invincible', read => false },
@@ -361,7 +367,7 @@ The only time you actually need to assign a statement object to a variable is
 when you want to inspect the statement using one of the methods below, in all
 other cases you can chain the methods for more concise code. For example:
 
-  my $data = $conn->q('SELECT a, b FROM table')->cache(0)->text->alla;
+  my $data = $conn->sql('SELECT a, b FROM table')->cache(0)->text->alla;
 
 Statement objects can be inspected with the following methods (many of which
 only make sense after the query has been executed):
@@ -381,10 +387,10 @@ Returns the provided bind parameters as an arrayref.
 Returns an arrayref of integers indicating the type (as I<oid>) of each
 parameter in the given C<$sql> string. Example:
 
-  my $oids = $conn->q('SELECT id FROM books WHERE id = $1 AND title = $2')->param_types;
+  my $oids = $conn->sql('SELECT id FROM books WHERE id = $1 AND title = $2')->param_types;
   # $oids = [23,25]
 
-  my $oids = $conn->q('SELECT id FROM books')->params;
+  my $oids = $conn->sql('SELECT id FROM books')->params;
   # $oids = []
 
 This method can be called before the query has been executed, but will then
@@ -397,7 +403,7 @@ prepared statement caching is disabled and C<text_params> is enabled.
 Returns an arrayref of hashrefs describing each column that the statement
 returns.
 
-  my $cols = $conn->q('SELECT id, title FROM books')->columns;
+  my $cols = $conn->sql('SELECT id, title FROM books')->columns;
   # $cols = [
   #   { name => 'id', oid => 23 },
   #   { name => 'title', oid => 25 },
@@ -446,7 +452,7 @@ fail while a transaction object is alive.
     my $txn = $conn->txn;
 
     # run queries
-    $txn->q('DELETE FROM books WHERE id = $1', 1)->exec;
+    $txn->sql('DELETE FROM books WHERE id = $1', 1)->exec;
 
     # run commands in a subtransaction
     {
@@ -467,9 +473,9 @@ Transaction methods:
 
 =item $txn->exec(..)
 
-=item $txn->q(..)
+=item $txn->sql(..)
 
-=item $txn->Q(..)
+=item $txn->SQL(..)
 
 Run a query inside the transaction. These work the same as the respective
 methods on the parent C<$conn> object.
@@ -492,7 +498,7 @@ when the object goes out of scope.
 
 =item $txn->text($enable)
 
-Set the default settings for new statements created with B<< $txn->q() >>.
+Set the default settings for new statements created with B<< $txn->sql() >>.
 
 These settings are inherited from the main connection when the transaction is
 created. Subtransactions inherit these settings from their parent transaction.
diff --git a/bench.PL b/bench.PL
index f4309c9..4bee131 100755
--- a/bench.PL
+++ b/bench.PL
@@ -211,8 +211,8 @@ def 'xml/a', 'HTML fragment', [ 'Rate' ],
 
     my sub dbi { my $sum = 0; my $st = $dbi->prepare_cached($_[0]); for my $row ($dbi->selectall_arrayref($st)->@*) { $sum ^= $_ for @$row; } }
     my sub pq  { my $sum = 0; $pq->prepare('' => $_[0]); for my $row ($pq->execQueryPrepared('')->rows) { $sum ^= $_ for @$row; } }
-    my sub fub { my $sum = 0; for my $row ($fu->q($_[0])->alla->@*) { $sum ^= $_ for @$row; } }
-    my sub fut { my $sum = 0; for my $row ($fu->q($_[0])->text->alla->@*) { $sum ^= $_ for @$row; } }
+    my sub fub { my $sum = 0; for my $row ($fu->sql($_[0])->alla->@*) { $sum ^= $_ for @$row; } }
+    my sub fut { my $sum = 0; for my $row ($fu->sql($_[0])->text->alla->@*) { $sum ^= $_ for @$row; } }
 
     def 'pg/ints', 'Fetch and bitwise-or 20k integers', [ 'Smallint', 'Bigint' ],
         [ 'DBD::Pg', undef, sub { dbi($small) }, sub { dbi($big) } ],
diff --git a/c/pgst.c b/c/pgst.c
index e943450..21de36b 100644
--- a/c/pgst.c
+++ b/c/pgst.c
@@ -76,7 +76,7 @@ static SV *fupg_exec(pTHX_ fupg_conn *c, const char *sql) {
     return ret;
 }
 
-static SV *fupg_q(pTHX_ fupg_conn *c, int stflags, const char *query, I32 ax, I32 argc) {
+static SV *fupg_sql(pTHX_ fupg_conn *c, int stflags, const char *query, I32 ax, I32 argc) {
     fupg_st *st = safecalloc(1, sizeof(fupg_st));
     st->conn = c;
     st->cookie = c->cookie;
diff --git a/t/pgconnect.t b/t/pgconnect.t
index cec597d..a1717ab 100644
--- a/t/pgconnect.t
+++ b/t/pgconnect.t
@@ -37,7 +37,7 @@ subtest '$conn->exec', sub {
     ok !defined $conn->exec('');
     is $conn->exec('SELECT 1'), 1;
 
-    ok !eval { $conn->q('SELEXT')->param_types; };
+    ok !eval { $conn->sql('SELEXT')->param_types; };
     okerr ERROR => prepare => qr/syntax error/;
 
     is $conn->exec('SET client_encoding=utf8'), undef;
@@ -46,7 +46,7 @@ subtest '$conn->exec', sub {
 
 subtest '$st prepare & exec', sub {
     {
-        my $st = $conn->q('SELECT 1');
+        my $st = $conn->sql('SELECT 1');
         is_deeply $st->param_types, [];
         is_deeply $st->columns, [{ name => '?column?', oid => 23 }];
 
@@ -63,7 +63,7 @@ subtest '$st prepare & exec', sub {
     }
 
     {
-        my $st = $conn->q("SELECT \$1::int AS a, \$2::char(5) AS \"\x{1F603}\"", 1, 2);
+        my $st = $conn->sql("SELECT \$1::int AS a, \$2::char(5) AS \"\x{1F603}\"", 1, 2);
         is_deeply $st->param_types, [ 23, 1042 ];
         is_deeply $st->columns, [
             { oid => 23, name => 'a' },
@@ -74,28 +74,28 @@ subtest '$st prepare & exec', sub {
 
     is $conn->exec('SELECT 1 FROM pg_prepared_statements'), 0;
 
-    ok !eval { $conn->q('SELECT 1', 1)->exec; 1 };
+    ok !eval { $conn->sql('SELECT 1', 1)->exec; 1 };
     like $@, qr/bind message supplies 1 parameters, but prepared statement/;
 
-    ok !eval { $conn->q('SELECT $1')->exec; 1 };
+    ok !eval { $conn->sql('SELECT $1')->exec; 1 };
     like $@, qr/bind message supplies 0 parameters, but prepared statement/;
 
     # prepare + describe won't let us detect empty queries, hmm...
-    is_deeply $conn->q('')->param_types, [];
-    is_deeply $conn->q('')->columns, [];
+    is_deeply $conn->sql('')->param_types, [];
+    is_deeply $conn->sql('')->columns, [];
 
-    ok !eval { $conn->q('')->exec; 1 };
+    ok !eval { $conn->sql('')->exec; 1 };
     okerr FATAL => exec => qr/unexpected status code/;
 
-    is $conn->q('SET client_encoding=utf8')->exec, undef;
+    is $conn->sql('SET client_encoding=utf8')->exec, undef;
 
-    ok !eval { $conn->q('select 1; select 2')->exec; 1 };
+    ok !eval { $conn->sql('select 1; select 2')->exec; 1 };
     okerr ERROR => exec => qr/cannot insert multiple commands into a prepared statement/;
 
     # Interleaved
     {
-        my $x = $conn->q('SELECT 1 as a');
-        my $y = $conn->q('SELECT 2 as b');
+        my $x = $conn->sql('SELECT 1 as a');
+        my $y = $conn->sql('SELECT 2 as b');
         is_deeply $x->columns, [ { oid => 23, name => 'a' } ];
         is_deeply $y->columns, [ { oid => 23, name => 'b' } ];
         is $x->val, 1;
@@ -104,137 +104,137 @@ subtest '$st prepare & exec', sub {
 };
 
 subtest '$st->val', sub {
-    ok !eval { $conn->q('SELECT')->val; 1 };
+    ok !eval { $conn->sql('SELECT')->val; 1 };
     like $@, qr/on query returning no data/;
 
-    ok !eval { $conn->q('SELECT 1, 2')->val; 1 };
+    ok !eval { $conn->sql('SELECT 1, 2')->val; 1 };
     like $@, qr/on query returning more than one column/;
 
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->val; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->val; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !defined $conn->q('SELECT 1 WHERE false')->val;
-    ok !defined $conn->q('SELECT null')->val;
-    is $conn->q('SELECT $1::text', "\x{1F603}")->val, "\x{1F603}";
+    ok !defined $conn->sql('SELECT 1 WHERE false')->val;
+    ok !defined $conn->sql('SELECT null')->val;
+    is $conn->sql('SELECT $1::text', "\x{1F603}")->val, "\x{1F603}";
 };
 
 subtest '$st->rowl', sub {
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->rowl; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->rowl; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !eval { $conn->q('SELEXT')->rowl; 1; };
-    is scalar $conn->q('SELECT')->rowl, 0;
-    is scalar $conn->q('SELECT 1, 2')->rowl, 2;
-    is_deeply [$conn->q('SELECT')->rowl], [];
-    is_deeply [$conn->q('SELECT 1, null')->rowl], [1, undef];
-    is_deeply [$conn->q('SELECT 1, $1', undef)->rowl], [1, undef];
-    is_deeply [$conn->q('SELECT 1, $1::int', undef)->text_params(0)->rowl], [1, undef];
-    is_deeply [$conn->q('SELECT 1 WHERE false')->rowl], [];
+    ok !eval { $conn->sql('SELEXT')->rowl; 1; };
+    is scalar $conn->sql('SELECT')->rowl, 0;
+    is scalar $conn->sql('SELECT 1, 2')->rowl, 2;
+    is_deeply [$conn->sql('SELECT')->rowl], [];
+    is_deeply [$conn->sql('SELECT 1, null')->rowl], [1, undef];
+    is_deeply [$conn->sql('SELECT 1, $1', undef)->rowl], [1, undef];
+    is_deeply [$conn->sql('SELECT 1, $1::int', undef)->text_params(0)->rowl], [1, undef];
+    is_deeply [$conn->sql('SELECT 1 WHERE false')->rowl], [];
 };
 
 subtest '$st->rowa', sub {
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->rowa; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->rowa; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !eval { $conn->q('SELEXT')->rowa; 1; };
-    is $conn->q('SELECT 1 WHERE false')->rowa, undef;
-    is_deeply $conn->q('SELECT')->rowa, [];
-    is_deeply $conn->q('SELECT 1, 2')->rowa, [1, 2];
-    is_deeply $conn->q('SELECT 1, null')->rowa, [1, undef];
-    is_deeply $conn->q('SELECT 1, $1', undef)->rowa, [1, undef];
-    is_deeply $conn->q('SELECT 1, $1::int', undef)->text_params(0)->rowa, [1, undef];
+    ok !eval { $conn->sql('SELEXT')->rowa; 1; };
+    is $conn->sql('SELECT 1 WHERE false')->rowa, undef;
+    is_deeply $conn->sql('SELECT')->rowa, [];
+    is_deeply $conn->sql('SELECT 1, 2')->rowa, [1, 2];
+    is_deeply $conn->sql('SELECT 1, null')->rowa, [1, undef];
+    is_deeply $conn->sql('SELECT 1, $1', undef)->rowa, [1, undef];
+    is_deeply $conn->sql('SELECT 1, $1::int', undef)->text_params(0)->rowa, [1, undef];
 
 };
 
 subtest '$st->rowh', sub {
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->rowh; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->rowh; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !eval { $conn->q('SELECT 1 as a, 2 as a')->rowh; 1 };
+    ok !eval { $conn->sql('SELECT 1 as a, 2 as a')->rowh; 1 };
     like $@, qr/Query returns multiple columns with the same name/;
 
-    is $conn->q('SELECT 1 WHERE false')->rowh, undef;
-    is_deeply $conn->q('SELECT')->rowh, {};
-    is_deeply $conn->q('SELECT 1 as a, 2 as b')->rowh, {a => 1, b => 2};
-    is_deeply $conn->q('SELECT 1 as a, null as b')->rowh, {a => 1, b => undef};
-    is_deeply $conn->q('SELECT 1 as a, $1::int as b', undef)->rowh, {a => 1, b => undef};
+    is $conn->sql('SELECT 1 WHERE false')->rowh, undef;
+    is_deeply $conn->sql('SELECT')->rowh, {};
+    is_deeply $conn->sql('SELECT 1 as a, 2 as b')->rowh, {a => 1, b => 2};
+    is_deeply $conn->sql('SELECT 1 as a, null as b')->rowh, {a => 1, b => undef};
+    is_deeply $conn->sql('SELECT 1 as a, $1::int as b', undef)->rowh, {a => 1, b => undef};
 };
 
 subtest '$st->alla', sub {
-    is_deeply $conn->q('SELECT 1 WHERE false')->alla, [];
-    is_deeply $conn->q('SELECT')->alla, [[]];
-    is_deeply $conn->q('SELECT 1')->alla, [[1]];
-    is_deeply $conn->q('SELECT 1, null UNION ALL SELECT NULL, 2')->alla, [[1,undef],[undef,2]];
+    is_deeply $conn->sql('SELECT 1 WHERE false')->alla, [];
+    is_deeply $conn->sql('SELECT')->alla, [[]];
+    is_deeply $conn->sql('SELECT 1')->alla, [[1]];
+    is_deeply $conn->sql('SELECT 1, null UNION ALL SELECT NULL, 2')->alla, [[1,undef],[undef,2]];
 };
 
 subtest '$st->allh', sub {
-    ok !eval { $conn->q('SELECT 1 as a, 2 as a')->allh; 1 };
+    ok !eval { $conn->sql('SELECT 1 as a, 2 as a')->allh; 1 };
     like $@, qr/Query returns multiple columns with the same name/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->allh, [];
-    is_deeply $conn->q('SELECT')->allh, [{}];
-    is_deeply $conn->q('SELECT 1 a')->allh, [{a=>1}];
-    is_deeply $conn->q('SELECT 1 a, null b UNION ALL SELECT NULL, 2')->allh, [{a=>1,b=>undef},{a=>undef,b=>2}];
+    is_deeply $conn->sql('SELECT 1 WHERE false')->allh, [];
+    is_deeply $conn->sql('SELECT')->allh, [{}];
+    is_deeply $conn->sql('SELECT 1 a')->allh, [{a=>1}];
+    is_deeply $conn->sql('SELECT 1 a, null b UNION ALL SELECT NULL, 2')->allh, [{a=>1,b=>undef},{a=>undef,b=>2}];
 };
 
 subtest '$st->flat', sub {
-    is_deeply $conn->q('SELECT 1 WHERE false')->flat, [];
-    is_deeply $conn->q('SELECT')->flat, [];
-    is_deeply $conn->q('SELECT 1')->flat, [1];
-    is_deeply $conn->q('SELECT 1, null UNION ALL SELECT NULL, 2')->flat, [1,undef,undef,2];
+    is_deeply $conn->sql('SELECT 1 WHERE false')->flat, [];
+    is_deeply $conn->sql('SELECT')->flat, [];
+    is_deeply $conn->sql('SELECT 1')->flat, [1];
+    is_deeply $conn->sql('SELECT 1, null UNION ALL SELECT NULL, 2')->flat, [1,undef,undef,2];
 };
 
 subtest '$st->kvv', sub {
-    ok !eval { $conn->q('SELECT')->kvv; 1; };
+    ok !eval { $conn->sql('SELECT')->kvv; 1; };
     like $@, qr/returning no data/;
 
-    ok !eval { $conn->q('SELECT 1, 2, 3')->kvv; 1; };
+    ok !eval { $conn->sql('SELECT 1, 2, 3')->kvv; 1; };
     like $@, qr/returning more than two columns/;
 
-    ok !eval { $conn->q('SELECT 1 UNION ALL SELECT 1')->kvv; 1; };
+    ok !eval { $conn->sql('SELECT 1 UNION ALL SELECT 1')->kvv; 1; };
     like $@, qr/is duplicated/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->kvv, {};
-    is_deeply $conn->q('SELECT 1')->kvv, {1=>1};
-    is_deeply $conn->q('SELECT 1, null UNION ALL SELECT 3, 2')->kvv, {1=>undef,3=>2};
-    $conn->q('SELECT 1')->kvv->{1} = 0;
+    is_deeply $conn->sql('SELECT 1 WHERE false')->kvv, {};
+    is_deeply $conn->sql('SELECT 1')->kvv, {1=>1};
+    is_deeply $conn->sql('SELECT 1, null UNION ALL SELECT 3, 2')->kvv, {1=>undef,3=>2};
+    $conn->sql('SELECT 1')->kvv->{1} = 0;
 };
 
 subtest '$st->kva', sub {
-    ok !eval { $conn->q('SELECT')->kva; 1; };
+    ok !eval { $conn->sql('SELECT')->kva; 1; };
     like $@, qr/returning no data/;
 
-    ok !eval { $conn->q('SELECT 1 UNION ALL SELECT 1')->kva; 1; };
+    ok !eval { $conn->sql('SELECT 1 UNION ALL SELECT 1')->kva; 1; };
     like $@, qr/is duplicated/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->kva, {};
-    is_deeply $conn->q('SELECT 1')->kva, {1=>[]};
-    is_deeply $conn->q("SELECT 1, null, 'hi' UNION ALL SELECT 3, 2, 'ok'")->kva,
+    is_deeply $conn->sql('SELECT 1 WHERE false')->kva, {};
+    is_deeply $conn->sql('SELECT 1')->kva, {1=>[]};
+    is_deeply $conn->sql("SELECT 1, null, 'hi' UNION ALL SELECT 3, 2, 'ok'")->kva,
         {1=>[undef,'hi'], 3=>[2, 'ok']};
 };
 
 subtest '$st->kvh', sub {
-    ok !eval { $conn->q('SELECT')->kvh; 1; };
+    ok !eval { $conn->sql('SELECT')->kvh; 1; };
     like $@, qr/returning no data/;
 
-    ok !eval { $conn->q('SELECT 1 UNION ALL SELECT 1')->kvh; 1; };
+    ok !eval { $conn->sql('SELECT 1 UNION ALL SELECT 1')->kvh; 1; };
     like $@, qr/is duplicated/;
 
-    ok !eval { $conn->q('SELECT 1, 2, 3')->kvh; 1; };
+    ok !eval { $conn->sql('SELECT 1, 2, 3')->kvh; 1; };
     like $@, qr/Query returns multiple columns with the same name/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->kvh, {};
-    is_deeply $conn->q('SELECT 1')->kvh, {1=>{}};
-    is_deeply $conn->q("SELECT 1 as a , null as a, 'hi' as b UNION ALL SELECT 3, 2, 'ok'")->kvh,
+    is_deeply $conn->sql('SELECT 1 WHERE false')->kvh, {};
+    is_deeply $conn->sql('SELECT 1')->kvh, {1=>{}};
+    is_deeply $conn->sql("SELECT 1 as a , null as a, 'hi' as b UNION ALL SELECT 3, 2, 'ok'")->kvh,
         {1=>{a=>undef,b=>'hi'}, 3=>{a=>2,b=>'ok'}};
 };
 
 subtest 'txn', sub {
     $conn->exec('CREATE TEMPORARY TABLE fupg_tst (id int)');
     $conn->txn->exec('INSERT INTO fupg_tst VALUES (1)'); # rolled back
-    is $conn->q('SELECT COUNT(*) FROM fupg_tst')->val, 0;
+    is $conn->sql('SELECT COUNT(*) FROM fupg_tst')->val, 0;
 
-    my $st = $conn->q('SELECT COUNT(*) FROM fupg_tst');
+    my $st = $conn->sql('SELECT COUNT(*) FROM fupg_tst');
     my $sst;
     {
         my $txn = $conn->txn;
@@ -246,13 +246,13 @@ subtest 'txn', sub {
 
         ok !eval { $conn->exec('SELECT 1'); 1 };
         like $@, qr/Invalid operation on the top-level connection/;
-        ok !eval { $conn->q('SELECT 1'); 1 };
+        ok !eval { $conn->sql('SELECT 1'); 1 };
         like $@, qr/Invalid operation on the top-level connection/;
         ok !eval { $conn->txn; 1 };
         like $@, qr/Invalid operation on the top-level connection/;
 
         $txn->exec('INSERT INTO fupg_tst VALUES (1)');
-        $sst = $txn->q('SELECT 1');
+        $sst = $txn->sql('SELECT 1');
 
         is $conn->status, 'txn_idle';
         is $txn->status, 'idle';
@@ -268,7 +268,7 @@ subtest 'txn', sub {
         like $@, qr/Invalid operation on a transaction that has already been marked as done/;
         ok !eval { $txn->exec('select 1'); 1 };
         like $@, qr/Invalid operation on a transaction that has already been marked as done/;
-        ok !eval { $txn->q('select 1'); 1 };
+        ok !eval { $txn->sql('select 1'); 1 };
         like $@, qr/Invalid operation on a transaction that has already been marked as done/;
 
         ok !eval { $conn->exec('SELECT 1'); 1 };
@@ -295,7 +295,7 @@ subtest 'txn', sub {
 
     {
         my $txn = $conn->txn;
-        my $st = $txn->q('SELECT count(*) FROM fupg_tst WHERE id = 2');
+        my $st = $txn->sql('SELECT count(*) FROM fupg_tst WHERE id = 2');
         {
             my $sub = $txn->txn;
             is $conn->status, 'txn_idle';
@@ -316,7 +316,7 @@ subtest 'txn', sub {
         is $txn->status, 'idle';
         is $st->val, 0;
 
-        $st = $txn->q('SELECT count(*) FROM fupg_tst WHERE id = 2');
+        $st = $txn->sql('SELECT count(*) FROM fupg_tst WHERE id = 2');
         {
             my $sub = $txn->txn;
             $sub->exec('INSERT INTO fupg_tst VALUES (2)');
@@ -339,19 +339,19 @@ subtest 'txn', sub {
         $sub->commit;
     }
     # We didn't commit $txn, so $sub got aborted as well
-    is $conn->q('SELECT count(*) FROM fupg_tst WHERE id = 3')->val, 0;
+    is $conn->sql('SELECT count(*) FROM fupg_tst WHERE id = 3')->val, 0;
 };
 
 {
     local $_ = 'x';
-    my $st = $conn->q('SELECT $1', $_);
+    my $st = $conn->sql('SELECT $1', $_);
     $_ = 'y';
     is $st->val, 'x', 'shallow copy';
 }
 
 {
     my $x = [1,2];
-    my $st = $conn->q('SELECT $1::int[]', $x)->text(0);
+    my $st = $conn->sql('SELECT $1::int[]', $x)->text(0);
     $x->[1] = 3;
     is_deeply $st->val, [1,3], 'not deep copy';
 }
@@ -360,7 +360,7 @@ subtest 'txn', sub {
 {
     # Exact format returned by escape_literal() can differ between Postgres versions and configurations.
     my $x = q{"' \" \\};
-    is $conn->q('SELECT '.$conn->escape_literal($x))->val, $x;
+    is $conn->sql('SELECT '.$conn->escape_literal($x))->val, $x;
 
     # Format can also change, but unsure how to test this otherwise.
     is $conn->escape_identifier('hel\l"o'), '"hel\l""o"';
@@ -371,32 +371,32 @@ subtest 'Prepared statement cache', sub {
     my $txn = $conn->txn;
     $txn->cache;
     my $numexec = sub($sql) {
-        $txn->q('SELECT generic_plans + custom_plans FROM pg_prepared_statements WHERE statement = $1', $sql)->cache(0)->val
+        $txn->sql('SELECT generic_plans + custom_plans FROM pg_prepared_statements WHERE statement = $1', $sql)->cache(0)->val
     };
-    is $txn->q('SELECT 1')->val, 1;
+    is $txn->sql('SELECT 1')->val, 1;
     is $numexec->('SELECT 1'), 1;
 
     my $sql = 'SELECT $1::int as a, $2::text as b';
     ok !defined $numexec->($sql);
 
-    my $params = $txn->q($sql)->param_types;
+    my $params = $txn->sql($sql)->param_types;
     is_deeply $params, [23, 25];
     is $numexec->($sql), 0;
-    my $cparams = $txn->q($sql)->param_types;
+    my $cparams = $txn->sql($sql)->param_types;
     is_deeply $cparams, $params;
 
-    my $cols = $txn->q($sql)->columns;
+    my $cols = $txn->sql($sql)->columns;
     is_deeply $cols, [{ name => 'a', oid => 23 }, { name => 'b', oid => 25 }];
-    my $ccols = $txn->q($sql)->columns;
+    my $ccols = $txn->sql($sql)->columns;
     is_deeply $ccols, $cols;
 
-    $txn->q($sql, 0, '')->exec;
+    $txn->sql($sql, 0, '')->exec;
     is $numexec->($sql), 1;
-    $txn->q($sql, 0, '')->exec;
+    $txn->sql($sql, 0, '')->exec;
     is $numexec->($sql), 2;
 
     is $numexec->('SELECT 1'), 1;
-    $txn->q('SELECT 2')->exec;
+    $txn->sql('SELECT 2')->exec;
     ok !defined $numexec->('SELECT 1');
     is $numexec->('SELECT 2'), 1;
 
@@ -415,7 +415,7 @@ subtest 'Tracing', sub {
     my @log;
     $conn->query_trace(sub($st) { push @log, $st });
 
-    is_deeply $conn->q('SELECT 1 AS a, $1 AS b', 123)->text_params(0)->rowa, [ 1, 123 ];
+    is_deeply $conn->sql('SELECT 1 AS a, $1 AS b', 123)->text_params(0)->rowa, [ 1, 123 ];
     is scalar @log, 1;
     my $st = shift @log;
     is ref $st, 'FU::Pg::st';
@@ -451,7 +451,7 @@ subtest 'Tracing', sub {
 };
 
 {
-    my $st = $conn->q("SELECT 1");
+    my $st = $conn->sql("SELECT 1");
     undef $conn; # statement keeps the connection alive
     is $st->val, 1;
 }
diff --git a/t/pgcopy.t b/t/pgcopy.t
index 9c81349..5bd82cb 100644
--- a/t/pgcopy.t
+++ b/t/pgcopy.t
@@ -82,9 +82,9 @@ is $conn->status, 'idle';
     $c->write($bin);
     $c->close;
 
-    is $txn->q('SELECT sum(v) FROM fupg_copy_test')->val, 1+1+2+2+3+3;
+    is $txn->sql('SELECT sum(v) FROM fupg_copy_test')->val, 1+1+2+2+3+3;
     $txn->rollback;
 }
-is $conn->q('SELECT sum(v) FROM fupg_copy_test')->val, 1+2+3;
+is $conn->sql('SELECT sum(v) FROM fupg_copy_test')->val, 1+2+3;
 
 done_testing;
diff --git a/t/pgtypes-dynamic.t b/t/pgtypes-dynamic.t
index 79abd92..56d41eb 100644
--- a/t/pgtypes-dynamic.t
+++ b/t/pgtypes-dynamic.t
@@ -10,31 +10,31 @@ my $conn = FU::Pg->connect($ENV{FU_TEST_DB});
 $conn->_debug_trace(0);
 
 
-is_deeply $conn->Q('SELECT', 1, '::int')->param_types, [23];
-is_deeply $conn->Q('SELECT 1', IN([1,2,3]))->param_types, [1007];
-is $conn->Q('SELECT 1', IN([1,2,3]))->val, 1;
+is_deeply $conn->SQL('SELECT', 1, '::int')->param_types, [23];
+is_deeply $conn->SQL('SELECT 1', IN([1,2,3]))->param_types, [1007];
+is $conn->SQL('SELECT 1', IN([1,2,3]))->val, 1;
 
-ok !eval { $conn->q('SELECT $1::aclitem', '')->exec; 1 };
+ok !eval { $conn->sql('SELECT $1::aclitem', '')->exec; 1 };
 like $@, qr/Unable to send type/;
 
 
 subtest 'type overrides', sub {
     $conn->set_type(int4 => recv => 'bytea');
-    is $conn->q('SELECT 5::int4')->val, "\0\0\0\5";
-    is_deeply $conn->q('SELECT ARRAY[5::int4]')->val, ["\0\0\0\5"];
+    is $conn->sql('SELECT 5::int4')->val, "\0\0\0\5";
+    is_deeply $conn->sql('SELECT ARRAY[5::int4]')->val, ["\0\0\0\5"];
 
     $conn->set_type(int4 => send => 'bytea');
-    is $conn->q('SELECT $1::int4', "\0\0\0\5")->val, 5;
-    is_deeply $conn->q('SELECT $1::int4[]', ["\0\0\0\5"])->val, [5];
+    is $conn->sql('SELECT $1::int4', "\0\0\0\5")->val, 5;
+    is_deeply $conn->sql('SELECT $1::int4[]', ["\0\0\0\5"])->val, [5];
 
     $conn->set_type(int4 => 'int2');
-    ok !eval { $conn->q('SELECT 5::int4')->val };
+    ok !eval { $conn->sql('SELECT 5::int4')->val };
     like $@, qr/Error parsing value/;
-    ok !eval { $conn->q('SELECT $1::int4', 5)->val };
+    ok !eval { $conn->sql('SELECT $1::int4', 5)->val };
     like $@, qr/insufficient data left in message/;
 
     $conn->set_type(int4 => undef);
-    is $conn->q('SELECT 5::int4')->val, 5;
+    is $conn->sql('SELECT 5::int4')->val, 5;
 
     ok !eval { $conn->set_type(int4 => 1007); };
     like $@, qr/Cannot set a type to array/;
@@ -46,23 +46,23 @@ subtest 'type overrides', sub {
 
 subtest 'type override callback', sub {
     $conn->set_type(text => recv => sub { length $_[0] });
-    is $conn->q('SELECT $1', 'a')->val, 1;
-    is $conn->q('SELECT $1', 'ab')->val, 2;
-    is $conn->q('SELECT $1', 'abc')->val, 3;
-    is $conn->q('SELECT $1', 'abcd')->val, 4;
+    is $conn->sql('SELECT $1', 'a')->val, 1;
+    is $conn->sql('SELECT $1', 'ab')->val, 2;
+    is $conn->sql('SELECT $1', 'abc')->val, 3;
+    is $conn->sql('SELECT $1', 'abcd')->val, 4;
 
     $conn->set_type(text => send => sub { 'l'.length $_[0] });
-    is $conn->q('SELECT $1', 'a')->val, 'l1';
-    is $conn->q('SELECT $1', 'ab')->val, 'l2';
-    is $conn->q('SELECT $1', 'abc')->val, 'l3';
-    is $conn->q('SELECT $1', 'abcd')->val, 'l4';
+    is $conn->sql('SELECT $1', 'a')->val, 'l1';
+    is $conn->sql('SELECT $1', 'ab')->val, 'l2';
+    is $conn->sql('SELECT $1', 'abc')->val, 'l3';
+    is $conn->sql('SELECT $1', 'abcd')->val, 'l4';
 };
 
 
 subtest 'custom types', sub {
     my $txn = $conn->txn;
 
-    is $txn->Q('SELECT 1', IN([1,2,3]))->val, 1;
+    is $txn->SQL('SELECT 1', IN([1,2,3]))->val, 1;
 
     $txn->exec(<<~_);
         CREATE TYPE fupg_test_enum AS ENUM('aa', 'bb', 'ccccccccccccccccccc');
@@ -73,21 +73,21 @@ subtest 'custom types', sub {
             domain fupg_test_domain
         );
     _
-    is $txn->q("SELECT 'aa'::fupg_test_enum")->val, 'aa';
-    is $txn->q('SELECT $1::fupg_test_enum', 'ccccccccccccccccccc')->val, 'ccccccccccccccccccc';
+    is $txn->sql("SELECT 'aa'::fupg_test_enum")->val, 'aa';
+    is $txn->sql('SELECT $1::fupg_test_enum', 'ccccccccccccccccccc')->val, 'ccccccccccccccccccc';
 
-    is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_enum[]")->val, ['aa','bb',undef];
-    is $txn->q('SELECT $1::fupg_test_enum[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
+    is_deeply $txn->sql("SELECT '{aa,bb,null}'::fupg_test_enum[]")->val, ['aa','bb',undef];
+    is $txn->sql('SELECT $1::fupg_test_enum[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
 
-    is $txn->q("SELECT 'aa'::fupg_test_domain")->val, 'aa';
-    is $txn->q('SELECT $1::fupg_test_domain', 'bb')->val, 'bb';
+    is $txn->sql("SELECT 'aa'::fupg_test_domain")->val, 'aa';
+    is $txn->sql('SELECT $1::fupg_test_domain', 'bb')->val, 'bb';
 
-    is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_domain[]")->val, ['aa','bb',undef];
-    is $txn->q('SELECT $1::fupg_test_domain[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
+    is_deeply $txn->sql("SELECT '{aa,bb,null}'::fupg_test_domain[]")->val, ['aa','bb',undef];
+    is $txn->sql('SELECT $1::fupg_test_domain[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
 
     my $val = { a => undef, aenum => ['aa','bb'], domain => 'aa' };
-    is_deeply $txn->q("SELECT '(,\"{aa,bb}\",aa)'::fupg_test_record")->val, $val;
-    is $txn->q('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{aa,bb}",aa)';
+    is_deeply $txn->sql("SELECT '(,\"{aa,bb}\",aa)'::fupg_test_record")->val, $val;
+    is $txn->sql('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{aa,bb}",aa)';
 
     $txn->exec(<<~_);
         CREATE TEMPORARY TABLE fupg_test_table (
@@ -96,7 +96,7 @@ subtest 'custom types', sub {
         );
     _
 
-    $val = $txn->q(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val;
+    $val = $txn->sql(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val;
     is_deeply $val, [
         { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
         { rec => { a => undef, aenum => undef, domain => undef }, dom => 'bb' },
@@ -106,7 +106,7 @@ subtest 'custom types', sub {
     $val->[1]{rec} = 0;
     $val->[1]{dom} = 0;
 
-    is $txn->q('SELECT $1::fupg_test_table[]', [
+    is $txn->sql('SELECT $1::fupg_test_table[]', [
         { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
         { rec => {}, dom => 'bb', extra => 1 },
     ])->text_results->val, '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}';
@@ -114,46 +114,46 @@ subtest 'custom types', sub {
     # Wonky Postgres behavior: selecting a domain directly actually returns the
     # underlying type, but going through an array does work.
     $conn->set_type(fupg_test_domain => 21);
-    is_deeply $txn->q("SELECT ARRAY['aa'::fupg_test_domain]")->val, [0x6161];
+    is_deeply $txn->sql("SELECT ARRAY['aa'::fupg_test_domain]")->val, [0x6161];
 
     # Bind param type doesn't match column type, argh.
-    is $txn->q('SELECT $1::fupg_test_domain', 0x6161)->val, 'aa';
+    is $txn->sql('SELECT $1::fupg_test_domain', 0x6161)->val, 'aa';
 
     # Same for selecting from a table :(
     $txn->exec("INSERT INTO fupg_test_table VALUES (NULL, 'bb')");
-    is $txn->q("SELECT dom FROM fupg_test_table")->val, 'bb';
+    is $txn->sql("SELECT dom FROM fupg_test_table")->val, 'bb';
     $conn->set_type(fupg_test_enum => 21);
-    is $txn->q("SELECT dom FROM fupg_test_table")->val, 0x6262;
+    is $txn->sql("SELECT dom FROM fupg_test_table")->val, 0x6262;
 };
 
 
 subtest 'identifier quoting', sub {
     my $txn = $conn->txn;
     $txn->exec('CREATE TEMPORARY TABLE fupg_test_tbl ("desc" int, ok int, "hello world" int)');
-    ok $txn->Q('INSERT INTO fupg_test_tbl', VALUES {desc => 5, ok => 10, 'hello world', 15})->exec;
-    is $txn->Q('SELECT', IDENT 'hello world', 'FROM fupg_test_tbl')->val, 15;
+    ok $txn->SQL('INSERT INTO fupg_test_tbl', VALUES {desc => 5, ok => 10, 'hello world', 15})->exec;
+    is $txn->SQL('SELECT', IDENT 'hello world', 'FROM fupg_test_tbl')->val, 15;
 };
 
 
 subtest 'vndbid', sub {
-    plan skip_all => 'type not loaded in the database' if !$conn->q("SELECT 1 FROM pg_type WHERE typname = 'vndbtag'")->val;
+    plan skip_all => 'type not loaded in the database' if !$conn->sql("SELECT 1 FROM pg_type WHERE typname = 'vndbtag'")->val;
 
     for my $t (qw/a zz xxx/) {
-        is $conn->q('SELECT $1::vndbtag', $t)->val, $t;
-        is $conn->q('SELECT $1::vndbtag', $t)->text_params->val, $t;
-        is $conn->q('SELECT $1::vndbtag', $t)->text_results->val, $t;
+        is $conn->sql('SELECT $1::vndbtag', $t)->val, $t;
+        is $conn->sql('SELECT $1::vndbtag', $t)->text_params->val, $t;
+        is $conn->sql('SELECT $1::vndbtag', $t)->text_results->val, $t;
     }
-    ok !eval { $conn->q('SELECT $1::vndbtag', '')->val };
-    ok !eval { $conn->q('SELECT $1::vndbtag', 'abcd')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbtag', '')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbtag', 'abcd')->val };
 
     for my $t (qw/a123 zz992883231 xxx18388123/) {
-        is $conn->q('SELECT $1::vndbid', $t)->val, $t;
-        is $conn->q('SELECT $1::vndbid', $t)->text_params->val, $t;
-        is $conn->q('SELECT $1::vndbid', $t)->text_results->val, $t;
+        is $conn->sql('SELECT $1::vndbid', $t)->val, $t;
+        is $conn->sql('SELECT $1::vndbid', $t)->text_params->val, $t;
+        is $conn->sql('SELECT $1::vndbid', $t)->text_results->val, $t;
     }
-    ok !eval { $conn->q('SELECT $1::vndbid', '')->val };
-    ok !eval { $conn->q('SELECT $1::vndbid', 'ab')->val };
-    ok !eval { $conn->q('SELECT $1::vndbid', 'ab1219229999999999')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbid', '')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbid', 'ab')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbid', 'ab1219229999999999')->val };
 };
 
 done_testing;
diff --git a/t/pgtypes.t b/t/pgtypes.t
index 662de33..9d5f6d9 100644
--- a/t/pgtypes.t
+++ b/t/pgtypes.t
@@ -22,7 +22,7 @@ sub v($type, $p_in, @args) {
     my $oid;
     utf8::encode($test);
     {
-        my $st = $conn->q("SELECT \$1::$type", $s_in)->text_params;
+        my $st = $conn->sql("SELECT \$1::$type", $s_in)->text_params;
         $oid = $st->param_types->[0];
         my $array = $st->flat;
         my $res = $array->[0];
@@ -32,11 +32,11 @@ sub v($type, $p_in, @args) {
         $array->[0] = 0; # Must be writable
     }
     {
-        my $res = $conn->q("SELECT \$1::$type", $p_in)->text_results->val;
+        my $res = $conn->sql("SELECT \$1::$type", $p_in)->text_results->val;
         is $res, $s_out, "$test bin->text";
     }
     {
-        my $res = $conn->q("SELECT \$1::$type", $p_in)->val;
+        my $res = $conn->sql("SELECT \$1::$type", $p_in)->val;
         is_deeply $res, $p_out, "$test bin->bin";
     }
     {
@@ -52,11 +52,11 @@ sub v($type, $p_in, @args) {
 sub f($type, $p_in) {
     my $test = "$type $p_in" =~ s/\n/\\n/rg;
     utf8::encode($test);
-    ok !eval { $conn->q("SELECT \$1::$type", $p_in)->val; 1 }, "$test fail";
+    ok !eval { $conn->sql("SELECT \$1::$type", $p_in)->val; 1 }, "$test fail";
 }
 
 { # void
-    my $array = $conn->q('SELECT pg_sleep(0)')->flat;
+    my $array = $conn->sql('SELECT pg_sleep(0)')->flat;
     ok !defined $array->[0];
     $array->[0] = 0;
 }
@@ -187,11 +187,11 @@ f 'oidvector', [undef];
 
 # Example from https://www.postgresql.org/docs/17/arrays.html#ARRAYS-IO
 # Lower bounds are discarded.
-is_deeply $conn->q("SELECT '[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int[]")->val, [[[1,2,3],[4,5,6]]];
+is_deeply $conn->sql("SELECT '[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int[]")->val, [[[1,2,3],[4,5,6]]];
 
-is $conn->q('SELECT ($1::int2[])[2]', [1,2,3,4])->val, 2;
-is $conn->q('SELECT ($1::int2vector)[1]', [1,2,3,4])->val, 2;
-is $conn->q('SELECT ($1::oidvector)[1]', [1,2,3,4])->val, 2;
+is $conn->sql('SELECT ($1::int2[])[2]', [1,2,3,4])->val, 2;
+is $conn->sql('SELECT ($1::int2vector)[1]', [1,2,3,4])->val, 2;
+is $conn->sql('SELECT ($1::oidvector)[1]', [1,2,3,4])->val, 2;
 
 is_deeply [$conn->bin2text(
     16, $conn->perl2bin(16, 1),
@@ -207,7 +207,7 @@ is_deeply [$conn->bin2text(
 }
 
 {
-    my $v = $conn->q("SELECT '{t,f,NULL}'::bool[]")->val;
+    my $v = $conn->sql("SELECT '{t,f,NULL}'::bool[]")->val;
     is_deeply $v, [true, false, undef];
     $_ = 0 for @$v;
 }

From d300f4d791c80cdf091c1f9d2ddb50ed36a6c42a Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sun, 4 Jan 2026 10:46:46 +0100
Subject: [PATCH 82/86] FU: Log unclean worker shutdowns in supervisor

This is mainly to monitor kills from OOM or other situations that the
worker process itself is unable to log properly.
---
 FU.pm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/FU.pm b/FU.pm
index e4f4ac1..8e47c1d 100644
--- a/FU.pm
+++ b/FU.pm
@@ -491,6 +491,8 @@ sub _supervisor($c) {
             if (!$err && (!$childs{$pid} || $childs{$pid} != 2)) {
                 $err = 1;
                 log_write "Script exited before calling FU::run()\n";
+            } elsif ($?) {
+                log_write "Unclean shutdown of worker PID $pid status $?\n";
             }
             delete $childs{$pid};
         }

From 48fe393d5f05e9cec8ddb0c502d1a8e8c47d4c13 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 5 Jan 2026 08:57:48 +0100
Subject: [PATCH 83/86] FastCGI: Improve handling of EPIPE while writing
 response

That would previously result in the worker getting killed with SIGPIPE.
Which works, but we can also recover from that error without restarting
the process.
---
 FU.pm    | 11 +++++++++--
 FU.xs    |  6 +++---
 c/fcgi.c | 24 ++++++++++++++----------
 t/fcgi.t |  8 ++++++++
 4 files changed, 34 insertions(+), 15 deletions(-)

diff --git a/FU.pm b/FU.pm
index 8e47c1d..7d5df3b 100644
--- a/FU.pm
+++ b/FU.pm
@@ -292,7 +292,8 @@ sub _read_req($c) {
                : $r == -2 ? "I/O error while reading from FastCGI socket\n"
                : $r == -3 ? "FastCGI protocol error\n"
                : $r == -4 ? "Too long FastCGI parameter\n"
-               : $r == -5 ? "Too long request body\n" : undef if $r != -7;
+               : $r == -5 ? "Too long request body\n"
+               : $r == -8 ? "I/O error while writing to FastCGI socket\n" : undef if $r != -7;
             delete $c->{fcgi_obj};
             fu->error(-1);
         }
@@ -400,7 +401,13 @@ sub _do_req($c) {
     }
 
     $REQ->{trace_end} = clock_gettime(CLOCK_MONOTONIC);
-    fu->_flush($c->{fcgi_obj} || $c->{client_sock});
+    eval {
+        fu->_flush($c->{fcgi_obj} || $c->{client_sock});
+        1;
+    } || do {
+        log_write "Error writing response: $@\n";
+        $c->{client_sock} = $c->{fcgi_obj} = undef;
+    };
 
     if (debug && $REQ->{trace_id} && $debug_info->{history} && $debug_info->{storage}) {
         require FU::DebugImpl;
diff --git a/FU.xs b/FU.xs
index 60ef7a6..1477a0a 100644
--- a/FU.xs
+++ b/FU.xs
@@ -3,7 +3,7 @@
 #include <time.h> /* struct timespec & clock_gettime() */
 #include <string.h> /* strerror() */
 #include <arpa/inet.h> /* inet_ntop(), inet_ntoa() */
-#include <sys/socket.h> /* fd passing */
+#include <sys/socket.h> /* send(), fd passing */
 #include <sys/un.h> /* fd passing */
 #include <dlfcn.h> /* dlopen() etc */
 
@@ -170,11 +170,11 @@ void print(fufcgi *ctx, SV *sv)
   CODE:
     STRLEN len;
     const char *buf = SvPVbyte(sv, len);
-    fufcgi_print(ctx, buf, len);
+    fufcgi_print(aTHX_ ctx, buf, len);
 
 void flush(fufcgi *ctx)
   CODE:
-    fufcgi_done(ctx);
+    fufcgi_done(aTHX_ ctx);
 
 void DESTROY(fufcgi *ctx)
   CODE:
diff --git a/c/fcgi.c b/c/fcgi.c
index 4f886dd..efcce6e 100644
--- a/c/fcgi.c
+++ b/c/fcgi.c
@@ -18,6 +18,7 @@
 #define FUFE_CLEN  -5
 #define FUFE_ABORT -6  /* explicit abort or client-level EOF */
 #define FUFE_NOREQ -7  /* protocol-level EOF before we received anything */
+#define FUFE_SEND  -8  /* error in send() */
 
 #define FUFCGI_MAX_DATA 65535
 
@@ -177,8 +178,8 @@ static int fufcgi_write_record(fufcgi *ctx, fufcgi_rec *hdr, char *buf) {
     buf[7] = 0;
     int len = hdr->len + 8;
     while (len > 0) {
-        int r = write(ctx->fd, buf, len);
-        if (r <= 0) return r == 0 ? FUFE_EOF : FUFE_IO;
+        int r = send(ctx->fd, buf, len, MSG_NOSIGNAL);
+        if (r <= 0) return FUFE_SEND;
         buf += r;
         len -= r;
     }
@@ -409,18 +410,19 @@ static int fufcgi_read_req(pTHX_ fufcgi *ctx, SV *headers, SV *params) {
     }
 }
 
-static void fufcgi_flush(fufcgi *ctx) {
+static void fufcgi_flush(pTHX_ fufcgi *ctx) {
     fufcgi_rec hdr;
     if (ctx->len > 0) {
         hdr.len = ctx->len;
         hdr.type = FCGI_STDOUT;
         hdr.id = ctx->reqid;
-        fufcgi_write_record(ctx, &hdr, ctx->buf);
+        if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
+            croak("write error: %s", strerror(errno));
         ctx->len = 0;
     }
 }
 
-static void fufcgi_print(fufcgi *ctx, const char *buf, int len) {
+static void fufcgi_print(pTHX_ fufcgi *ctx, const char *buf, int len) {
     int r;
     while (len > 0) {
         r = len > FUFCGI_MAX_DATA - ctx->len ? FUFCGI_MAX_DATA - ctx->len : len;
@@ -428,23 +430,25 @@ static void fufcgi_print(fufcgi *ctx, const char *buf, int len) {
         ctx->len += r;
         len -= r;
         buf += r;
-        if (ctx->len >= FUFCGI_MAX_DATA) fufcgi_flush(ctx);
+        if (ctx->len >= FUFCGI_MAX_DATA) fufcgi_flush(aTHX_ ctx);
     }
 }
 
-static void fufcgi_done(fufcgi *ctx) {
+static void fufcgi_done(pTHX_ fufcgi *ctx) {
     fufcgi_rec hdr;
-    fufcgi_flush(ctx);
+    fufcgi_flush(aTHX_ ctx);
 
     hdr.len = 0;
     hdr.type = FCGI_STDOUT;
     hdr.id = ctx->reqid;
-    fufcgi_write_record(ctx, &hdr, ctx->buf);
+    if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
+        croak("write error: %s", strerror(errno));
 
     memcpy(ctx->buf+8, "\0\0\0\0\0\0\0\0", 8); /* FCGI_REQUEST_COMPLETE */
     hdr.type = FCGI_END_REQUEST;
     hdr.len = 8;
-    fufcgi_write_record(ctx, &hdr, ctx->buf);
+    if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
+        croak("write error: %s", strerror(errno));
 
     ctx->reqid = ctx->len = ctx->off = 0;
 }
diff --git a/t/fcgi.t b/t/fcgi.t
index d7860dc..85636e3 100644
--- a/t/fcgi.t
+++ b/t/fcgi.t
@@ -54,6 +54,11 @@ start;
 begin 1, 2;
 record 1, 4, "";
 
+start;
+begin 3, 2, 1;
+$remote->close;
+iserr -8;
+
 start;
 begin 3, 2, 1;
 begin 1, 1, 1;
@@ -173,6 +178,9 @@ record 1, 4, "\x13\x01HTTP_CONTENT_LENGTH3\x0e\x01CONTENT_LENGTH0\x13\x01HTTP_CO
 record 1, 4, "";
 record 1, 5, "";
 isrec {'content-length','0'}, {body => ''};
+$remote->close;
+ok !eval { $f->flush; 1 };
+like $@, qr/write error/;
 
 start;
 begin;

From a7e9fa1866d70edab049b33893975701dd473f1d Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Mon, 5 Jan 2026 13:20:59 +0100
Subject: [PATCH 84/86] FU: Less verbose and cryptic error message on write
 error

---
 c/fcgi.c | 6 +++---
 t/fcgi.t | 1 -
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/c/fcgi.c b/c/fcgi.c
index efcce6e..c7cf306 100644
--- a/c/fcgi.c
+++ b/c/fcgi.c
@@ -417,7 +417,7 @@ static void fufcgi_flush(pTHX_ fufcgi *ctx) {
         hdr.type = FCGI_STDOUT;
         hdr.id = ctx->reqid;
         if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
-            croak("write error: %s", strerror(errno));
+            croak("%s\n", strerror(errno));
         ctx->len = 0;
     }
 }
@@ -442,13 +442,13 @@ static void fufcgi_done(pTHX_ fufcgi *ctx) {
     hdr.type = FCGI_STDOUT;
     hdr.id = ctx->reqid;
     if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
-        croak("write error: %s", strerror(errno));
+        croak("%s\n", strerror(errno));
 
     memcpy(ctx->buf+8, "\0\0\0\0\0\0\0\0", 8); /* FCGI_REQUEST_COMPLETE */
     hdr.type = FCGI_END_REQUEST;
     hdr.len = 8;
     if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
-        croak("write error: %s", strerror(errno));
+        croak("%s\n", strerror(errno));
 
     ctx->reqid = ctx->len = ctx->off = 0;
 }
diff --git a/t/fcgi.t b/t/fcgi.t
index 85636e3..806182c 100644
--- a/t/fcgi.t
+++ b/t/fcgi.t
@@ -180,7 +180,6 @@ record 1, 5, "";
 isrec {'content-length','0'}, {body => ''};
 $remote->close;
 ok !eval { $f->flush; 1 };
-like $@, qr/write error/;
 
 start;
 begin;

From f50da04ba53244d9c5fad5ecdb027865dbca965b Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 10 Jan 2026 16:18:15 +0100
Subject: [PATCH 85/86] Benchmarks: Improve accuracy + re-run with latest
 versions

---
 FU/Benchmarks.pod | 448 +++++++++++++++++++++++-----------------------
 bench.PL          | 165 ++++++++++-------
 2 files changed, 327 insertions(+), 286 deletions(-)

diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index e23571f..e611832 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -26,25 +26,25 @@ The following module versions were used:
 
 =over
 
-=item L<Cpanel::JSON::XS> 4.39
+=item L<Cpanel::JSON::XS> 4.40
 
 =item L<DBD::Pg> 3.18.0
 
-=item L<FU> 1.2
+=item L<FU> 1.3
 
 =item L<HTML::Tiny> 1.08
 
 =item L<JSON::PP> 4.16
 
-=item L<JSON::SIMD> 1.06
+=item L<JSON::SIMD> 1.07
 
 =item L<JSON::Tiny> 0.58
 
-=item L<JSON::XS> 4.03
+=item L<JSON::XS> 4.04
 
 =item L<Pg::PQ> 0.15
 
-=item L<TUWF::XML> 1.5
+=item L<TUWF::XML> 1.6
 
 =item L<XML::Writer> 0.900
 
@@ -66,102 +66,102 @@ L<JSON::XS>, the SIMD parts are only used for parsing.
 API object from L<JSON::XS> documentation.
 
                         Encode   Canonical      Decode
-          JSON::PP      5312/s      5119/s      1290/s
-        JSON::Tiny      7757/s           -      3426/s
-  Cpanel::JSON::XS    114802/s    104141/s    107274/s
-        JSON::SIMD    130137/s    118948/s    115123/s
-          JSON::XS    128421/s    120243/s    117940/s
-          FU::Util    132067/s    111328/s    117781/s
+          JSON::PP      5136/s      4943/s      1240/s
+        JSON::Tiny      7617/s           -      3474/s
+  Cpanel::JSON::XS    108128/s     98734/s    105811/s
+        JSON::SIMD    125105/s    114822/s    118410/s
+          JSON::XS    128749/s    117518/s    120190/s
+          FU::Util    126909/s    109166/s    113983/s
 
 Object (small)
 
                         Encode   Canonical      Decode
-          JSON::PP       907/s       829/s       202/s
-        JSON::Tiny      1224/s           -       499/s
-  Cpanel::JSON::XS     45732/s     30862/s     20102/s
-        JSON::SIMD     49019/s     30699/s     23267/s
-          JSON::XS     49814/s     31326/s     25336/s
-          FU::Util     39684/s     24971/s     17998/s
+          JSON::PP       896/s       826/s       194/s
+        JSON::Tiny      1216/s           -       519/s
+  Cpanel::JSON::XS     44184/s     28190/s     19449/s
+        JSON::SIMD     52633/s     31157/s     23587/s
+          JSON::XS     50314/s     34276/s     25294/s
+          FU::Util     42121/s     25618/s     19203/s
 
 Object (large)
 
                         Encode   Canonical      Decode
-          JSON::PP       927/s       747/s       104/s
-        JSON::Tiny      1108/s           -       392/s
-  Cpanel::JSON::XS     30587/s     11875/s     15515/s
-        JSON::SIMD     24418/s     12388/s     22895/s
-          JSON::XS     23192/s     13174/s     23553/s
-          FU::Util     35489/s     13247/s     16571/s
+          JSON::PP       910/s       734/s        98/s
+        JSON::Tiny      1068/s           -       404/s
+  Cpanel::JSON::XS     27626/s     12484/s     15333/s
+        JSON::SIMD     34106/s     12808/s     23674/s
+          JSON::XS     35738/s     13099/s     22637/s
+          FU::Util     37663/s     13366/s     16292/s
 
 Object (large, mixed unicode)
 
                         Encode   Canonical      Decode
-          JSON::PP       817/s       679/s        86/s
-        JSON::Tiny      1036/s           -       402/s
-  Cpanel::JSON::XS     25333/s      1459/s      7480/s
-        JSON::SIMD     25031/s      1331/s     15997/s
-          JSON::XS     23580/s      1375/s      8526/s
-          FU::Util     32519/s     12488/s      9290/s
+          JSON::PP       835/s       664/s        82/s
+        JSON::Tiny      1028/s           -       427/s
+  Cpanel::JSON::XS     24123/s      1352/s      8694/s
+        JSON::SIMD     26008/s      1413/s     19707/s
+          JSON::XS     25444/s      1391/s     10442/s
+          FU::Util     33132/s     12006/s     11861/s
 
 Small integers
 
                         Encode      Decode
-          JSON::PP       113/s        29/s
-        JSON::Tiny       160/s        86/s
-  Cpanel::JSON::XS      7345/s      6151/s
-        JSON::SIMD      7963/s      4361/s
-          JSON::XS      7915/s      6058/s
-          FU::Util      7851/s      5828/s
+          JSON::PP       116/s        30/s
+        JSON::Tiny       158/s        86/s
+  Cpanel::JSON::XS      7426/s      5774/s
+        JSON::SIMD      8294/s      4375/s
+          JSON::XS      8526/s      6179/s
+          FU::Util      7996/s      5962/s
 
 Large integers
 
                         Encode      Decode
-          JSON::PP      2176/s       329/s
-        JSON::Tiny      2999/s      1638/s
-  Cpanel::JSON::XS     32545/s     50162/s
-        JSON::SIMD     37201/s     51719/s
-          JSON::XS     36722/s     50110/s
-          FU::Util    109163/s     63176/s
+          JSON::PP      2213/s       341/s
+        JSON::Tiny      2910/s      1661/s
+  Cpanel::JSON::XS     32616/s     53053/s
+        JSON::SIMD     37749/s     53032/s
+          JSON::XS     38644/s     55004/s
+          FU::Util    109930/s     63358/s
 
 ASCII strings
 
                         Encode      Decode
-          JSON::PP      2934/s       336/s
-        JSON::Tiny      4126/s      1439/s
-  Cpanel::JSON::XS    116721/s     44560/s
-        JSON::SIMD    134711/s     50429/s
-          JSON::XS    135419/s     43976/s
-          FU::Util    162246/s     44216/s
+          JSON::PP      2811/s       312/s
+        JSON::Tiny      3924/s      1506/s
+  Cpanel::JSON::XS    129468/s     51536/s
+        JSON::SIMD    140393/s     64499/s
+          JSON::XS    141149/s     56913/s
+          FU::Util    165938/s     55034/s
 
 Unicode strings
 
                         Encode      Decode
-          JSON::PP      5113/s       253/s
-        JSON::Tiny      6603/s      2585/s
-  Cpanel::JSON::XS     97039/s     67669/s
-        JSON::SIMD    106928/s    102440/s
-          JSON::XS    105473/s     60558/s
-          FU::Util    201648/s     57397/s
+          JSON::PP      5138/s       248/s
+        JSON::Tiny      6501/s      2677/s
+  Cpanel::JSON::XS     91004/s     64101/s
+        JSON::SIMD    101185/s     80941/s
+          JSON::XS    106312/s     61104/s
+          FU::Util    205716/s     52041/s
 
 String escaping (few)
 
                         Encode      Decode
-          JSON::PP      4251/s       352/s
-        JSON::Tiny      4704/s      1869/s
-  Cpanel::JSON::XS    136755/s    118059/s
-        JSON::SIMD    158171/s    153692/s
-          JSON::XS    157261/s     97676/s
-          FU::Util    225259/s     92515/s
+          JSON::PP      4269/s       329/s
+        JSON::Tiny      4878/s      2101/s
+  Cpanel::JSON::XS    152958/s    105597/s
+        JSON::SIMD    165340/s    130074/s
+          JSON::XS    165863/s     87872/s
+          FU::Util    228511/s     81599/s
 
 String escaping (many)
 
                         Encode      Decode
-          JSON::PP      3963/s       561/s
-        JSON::Tiny      4463/s      2175/s
-  Cpanel::JSON::XS    197154/s    133102/s
-        JSON::SIMD    199955/s    152557/s
-          JSON::XS    231905/s    125191/s
-          FU::Util    215367/s    118073/s
+          JSON::PP      4052/s       573/s
+        JSON::Tiny      4575/s      2274/s
+  Cpanel::JSON::XS    201958/s    102800/s
+        JSON::SIMD    242806/s    146341/s
+          JSON::XS    209689/s     98420/s
+          FU::Util    210713/s    100255/s
 
 
 
@@ -173,10 +173,10 @@ templating modules will perform better, though.
 
 HTML fragment
 
-         TUWF::XML       795/s
-       XML::Writer       833/s
-        HTML::Tiny       423/s
-     FU::XMLWriter      5327/s
+         TUWF::XML       787/s
+       XML::Writer       832/s
+        HTML::Tiny       403/s
+     FU::XMLWriter      5192/s
 
 
 
@@ -189,165 +189,165 @@ bottleneck where there shouldn't be one.
 Fetch and bitwise-or 20k integers
 
                       Smallint      Bigint
-           DBD::Pg       194/s        22/s
-            Pg::PQ       226/s        19/s
-      FU::Pg (bin)       245/s        22/s
-     FU::Pg (text)       217/s        20/s
+           DBD::Pg       346/s        33/s
+            Pg::PQ       270/s        24/s
+      FU::Pg (bin)       476/s        46/s
+     FU::Pg (text)       273/s        23/s
 
 
 
 =cut
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.
-json/api	Canonical	Cpanel::JSON::XS	104141
-json/api	Canonical	FU::Util	111328
-json/api	Canonical	JSON::PP	5119
-json/api	Canonical	JSON::SIMD	118948
-json/api	Canonical	JSON::XS	120243
-json/api	Decode	Cpanel::JSON::XS	107274
-json/api	Decode	FU::Util	117781
-json/api	Decode	JSON::PP	1290
-json/api	Decode	JSON::SIMD	115123
-json/api	Decode	JSON::Tiny	3426
-json/api	Decode	JSON::XS	117940
-json/api	Encode	Cpanel::JSON::XS	114802
-json/api	Encode	FU::Util	132067
-json/api	Encode	JSON::PP	5312
-json/api	Encode	JSON::SIMD	130137
-json/api	Encode	JSON::Tiny	7757
-json/api	Encode	JSON::XS	128421
-json/intl	Decode	Cpanel::JSON::XS	50162
-json/intl	Decode	FU::Util	63176
-json/intl	Decode	JSON::PP	329
-json/intl	Decode	JSON::SIMD	51719
-json/intl	Decode	JSON::Tiny	1638
-json/intl	Decode	JSON::XS	50110
-json/intl	Encode	Cpanel::JSON::XS	32545
-json/intl	Encode	FU::Util	109163
-json/intl	Encode	JSON::PP	2176
-json/intl	Encode	JSON::SIMD	37201
-json/intl	Encode	JSON::Tiny	2999
-json/intl	Encode	JSON::XS	36722
-json/ints	Decode	Cpanel::JSON::XS	6151
-json/ints	Decode	FU::Util	5828
-json/ints	Decode	JSON::PP	29
-json/ints	Decode	JSON::SIMD	4361
+json/api	Canonical	Cpanel::JSON::XS	98734
+json/api	Canonical	FU::Util	109166
+json/api	Canonical	JSON::PP	4943
+json/api	Canonical	JSON::SIMD	114822
+json/api	Canonical	JSON::XS	117518
+json/api	Decode	Cpanel::JSON::XS	105811
+json/api	Decode	FU::Util	113983
+json/api	Decode	JSON::PP	1240
+json/api	Decode	JSON::SIMD	118410
+json/api	Decode	JSON::Tiny	3474
+json/api	Decode	JSON::XS	120190
+json/api	Encode	Cpanel::JSON::XS	108128
+json/api	Encode	FU::Util	126909
+json/api	Encode	JSON::PP	5136
+json/api	Encode	JSON::SIMD	125105
+json/api	Encode	JSON::Tiny	7617
+json/api	Encode	JSON::XS	128749
+json/intl	Decode	Cpanel::JSON::XS	53053
+json/intl	Decode	FU::Util	63358
+json/intl	Decode	JSON::PP	341
+json/intl	Decode	JSON::SIMD	53032
+json/intl	Decode	JSON::Tiny	1661
+json/intl	Decode	JSON::XS	55004
+json/intl	Encode	Cpanel::JSON::XS	32616
+json/intl	Encode	FU::Util	109930
+json/intl	Encode	JSON::PP	2213
+json/intl	Encode	JSON::SIMD	37749
+json/intl	Encode	JSON::Tiny	2910
+json/intl	Encode	JSON::XS	38644
+json/ints	Decode	Cpanel::JSON::XS	5774
+json/ints	Decode	FU::Util	5962
+json/ints	Decode	JSON::PP	30
+json/ints	Decode	JSON::SIMD	4375
 json/ints	Decode	JSON::Tiny	86
-json/ints	Decode	JSON::XS	6058
-json/ints	Encode	Cpanel::JSON::XS	7345
-json/ints	Encode	FU::Util	7851
-json/ints	Encode	JSON::PP	113
-json/ints	Encode	JSON::SIMD	7963
-json/ints	Encode	JSON::Tiny	160
-json/ints	Encode	JSON::XS	7915
-json/objl	Canonical	Cpanel::JSON::XS	11875
-json/objl	Canonical	FU::Util	13247
-json/objl	Canonical	JSON::PP	747
-json/objl	Canonical	JSON::SIMD	12388
-json/objl	Canonical	JSON::XS	13174
-json/objl	Decode	Cpanel::JSON::XS	15515
-json/objl	Decode	FU::Util	16571
-json/objl	Decode	JSON::PP	104
-json/objl	Decode	JSON::SIMD	22895
-json/objl	Decode	JSON::Tiny	392
-json/objl	Decode	JSON::XS	23553
-json/objl	Encode	Cpanel::JSON::XS	30587
-json/objl	Encode	FU::Util	35489
-json/objl	Encode	JSON::PP	927
-json/objl	Encode	JSON::SIMD	24418
-json/objl	Encode	JSON::Tiny	1108
-json/objl	Encode	JSON::XS	23192
-json/objs	Canonical	Cpanel::JSON::XS	30862
-json/objs	Canonical	FU::Util	24971
-json/objs	Canonical	JSON::PP	829
-json/objs	Canonical	JSON::SIMD	30699
-json/objs	Canonical	JSON::XS	31326
-json/objs	Decode	Cpanel::JSON::XS	20102
-json/objs	Decode	FU::Util	17998
-json/objs	Decode	JSON::PP	202
-json/objs	Decode	JSON::SIMD	23267
-json/objs	Decode	JSON::Tiny	499
-json/objs	Decode	JSON::XS	25336
-json/objs	Encode	Cpanel::JSON::XS	45732
-json/objs	Encode	FU::Util	39684
-json/objs	Encode	JSON::PP	907
-json/objs	Encode	JSON::SIMD	49019
-json/objs	Encode	JSON::Tiny	1224
-json/objs	Encode	JSON::XS	49814
-json/obju	Canonical	Cpanel::JSON::XS	1459
-json/obju	Canonical	FU::Util	12488
-json/obju	Canonical	JSON::PP	679
-json/obju	Canonical	JSON::SIMD	1331
-json/obju	Canonical	JSON::XS	1375
-json/obju	Decode	Cpanel::JSON::XS	7480
-json/obju	Decode	FU::Util	9290
-json/obju	Decode	JSON::PP	86
-json/obju	Decode	JSON::SIMD	15997
-json/obju	Decode	JSON::Tiny	402
-json/obju	Decode	JSON::XS	8526
-json/obju	Encode	Cpanel::JSON::XS	25333
-json/obju	Encode	FU::Util	32519
-json/obju	Encode	JSON::PP	817
-json/obju	Encode	JSON::SIMD	25031
-json/obju	Encode	JSON::Tiny	1036
-json/obju	Encode	JSON::XS	23580
-json/strel	Decode	Cpanel::JSON::XS	133102
-json/strel	Decode	FU::Util	118073
-json/strel	Decode	JSON::PP	561
-json/strel	Decode	JSON::SIMD	152557
-json/strel	Decode	JSON::Tiny	2175
-json/strel	Decode	JSON::XS	125191
-json/strel	Encode	Cpanel::JSON::XS	197154
-json/strel	Encode	FU::Util	215367
-json/strel	Encode	JSON::PP	3963
-json/strel	Encode	JSON::SIMD	199955
-json/strel	Encode	JSON::Tiny	4463
-json/strel	Encode	JSON::XS	231905
-json/stres	Decode	Cpanel::JSON::XS	118059
-json/stres	Decode	FU::Util	92515
-json/stres	Decode	JSON::PP	352
-json/stres	Decode	JSON::SIMD	153692
-json/stres	Decode	JSON::Tiny	1869
-json/stres	Decode	JSON::XS	97676
-json/stres	Encode	Cpanel::JSON::XS	136755
-json/stres	Encode	FU::Util	225259
-json/stres	Encode	JSON::PP	4251
-json/stres	Encode	JSON::SIMD	158171
-json/stres	Encode	JSON::Tiny	4704
-json/stres	Encode	JSON::XS	157261
-json/strs	Decode	Cpanel::JSON::XS	44560
-json/strs	Decode	FU::Util	44216
-json/strs	Decode	JSON::PP	336
-json/strs	Decode	JSON::SIMD	50429
-json/strs	Decode	JSON::Tiny	1439
-json/strs	Decode	JSON::XS	43976
-json/strs	Encode	Cpanel::JSON::XS	116721
-json/strs	Encode	FU::Util	162246
-json/strs	Encode	JSON::PP	2934
-json/strs	Encode	JSON::SIMD	134711
-json/strs	Encode	JSON::Tiny	4126
-json/strs	Encode	JSON::XS	135419
-json/stru	Decode	Cpanel::JSON::XS	67669
-json/stru	Decode	FU::Util	57397
-json/stru	Decode	JSON::PP	253
-json/stru	Decode	JSON::SIMD	102440
-json/stru	Decode	JSON::Tiny	2585
-json/stru	Decode	JSON::XS	60558
-json/stru	Encode	Cpanel::JSON::XS	97039
-json/stru	Encode	FU::Util	201648
-json/stru	Encode	JSON::PP	5113
-json/stru	Encode	JSON::SIMD	106928
-json/stru	Encode	JSON::Tiny	6603
-json/stru	Encode	JSON::XS	105473
-pg/ints	Bigint	DBD::Pg	22
-pg/ints	Bigint	FU::Pg (bin)	22
-pg/ints	Bigint	FU::Pg (text)	20
-pg/ints	Bigint	Pg::PQ	19
-pg/ints	Smallint	DBD::Pg	194
-pg/ints	Smallint	FU::Pg (bin)	245
-pg/ints	Smallint	FU::Pg (text)	217
-pg/ints	Smallint	Pg::PQ	226
-xml/a	Rate	FU::XMLWriter	5327
-xml/a	Rate	HTML::Tiny	423
-xml/a	Rate	TUWF::XML	795
-xml/a	Rate	XML::Writer	833
+json/ints	Decode	JSON::XS	6179
+json/ints	Encode	Cpanel::JSON::XS	7426
+json/ints	Encode	FU::Util	7996
+json/ints	Encode	JSON::PP	116
+json/ints	Encode	JSON::SIMD	8294
+json/ints	Encode	JSON::Tiny	158
+json/ints	Encode	JSON::XS	8526
+json/objl	Canonical	Cpanel::JSON::XS	12484
+json/objl	Canonical	FU::Util	13366
+json/objl	Canonical	JSON::PP	734
+json/objl	Canonical	JSON::SIMD	12808
+json/objl	Canonical	JSON::XS	13099
+json/objl	Decode	Cpanel::JSON::XS	15333
+json/objl	Decode	FU::Util	16292
+json/objl	Decode	JSON::PP	98
+json/objl	Decode	JSON::SIMD	23674
+json/objl	Decode	JSON::Tiny	404
+json/objl	Decode	JSON::XS	22637
+json/objl	Encode	Cpanel::JSON::XS	27626
+json/objl	Encode	FU::Util	37663
+json/objl	Encode	JSON::PP	910
+json/objl	Encode	JSON::SIMD	34106
+json/objl	Encode	JSON::Tiny	1068
+json/objl	Encode	JSON::XS	35738
+json/objs	Canonical	Cpanel::JSON::XS	28190
+json/objs	Canonical	FU::Util	25618
+json/objs	Canonical	JSON::PP	826
+json/objs	Canonical	JSON::SIMD	31157
+json/objs	Canonical	JSON::XS	34276
+json/objs	Decode	Cpanel::JSON::XS	19449
+json/objs	Decode	FU::Util	19203
+json/objs	Decode	JSON::PP	194
+json/objs	Decode	JSON::SIMD	23587
+json/objs	Decode	JSON::Tiny	519
+json/objs	Decode	JSON::XS	25294
+json/objs	Encode	Cpanel::JSON::XS	44184
+json/objs	Encode	FU::Util	42121
+json/objs	Encode	JSON::PP	896
+json/objs	Encode	JSON::SIMD	52633
+json/objs	Encode	JSON::Tiny	1216
+json/objs	Encode	JSON::XS	50314
+json/obju	Canonical	Cpanel::JSON::XS	1352
+json/obju	Canonical	FU::Util	12006
+json/obju	Canonical	JSON::PP	664
+json/obju	Canonical	JSON::SIMD	1413
+json/obju	Canonical	JSON::XS	1391
+json/obju	Decode	Cpanel::JSON::XS	8694
+json/obju	Decode	FU::Util	11861
+json/obju	Decode	JSON::PP	82
+json/obju	Decode	JSON::SIMD	19707
+json/obju	Decode	JSON::Tiny	427
+json/obju	Decode	JSON::XS	10442
+json/obju	Encode	Cpanel::JSON::XS	24123
+json/obju	Encode	FU::Util	33132
+json/obju	Encode	JSON::PP	835
+json/obju	Encode	JSON::SIMD	26008
+json/obju	Encode	JSON::Tiny	1028
+json/obju	Encode	JSON::XS	25444
+json/strel	Decode	Cpanel::JSON::XS	102800
+json/strel	Decode	FU::Util	100255
+json/strel	Decode	JSON::PP	573
+json/strel	Decode	JSON::SIMD	146341
+json/strel	Decode	JSON::Tiny	2274
+json/strel	Decode	JSON::XS	98420
+json/strel	Encode	Cpanel::JSON::XS	201958
+json/strel	Encode	FU::Util	210713
+json/strel	Encode	JSON::PP	4052
+json/strel	Encode	JSON::SIMD	242806
+json/strel	Encode	JSON::Tiny	4575
+json/strel	Encode	JSON::XS	209689
+json/stres	Decode	Cpanel::JSON::XS	105597
+json/stres	Decode	FU::Util	81599
+json/stres	Decode	JSON::PP	329
+json/stres	Decode	JSON::SIMD	130074
+json/stres	Decode	JSON::Tiny	2101
+json/stres	Decode	JSON::XS	87872
+json/stres	Encode	Cpanel::JSON::XS	152958
+json/stres	Encode	FU::Util	228511
+json/stres	Encode	JSON::PP	4269
+json/stres	Encode	JSON::SIMD	165340
+json/stres	Encode	JSON::Tiny	4878
+json/stres	Encode	JSON::XS	165863
+json/strs	Decode	Cpanel::JSON::XS	51536
+json/strs	Decode	FU::Util	55034
+json/strs	Decode	JSON::PP	312
+json/strs	Decode	JSON::SIMD	64499
+json/strs	Decode	JSON::Tiny	1506
+json/strs	Decode	JSON::XS	56913
+json/strs	Encode	Cpanel::JSON::XS	129468
+json/strs	Encode	FU::Util	165938
+json/strs	Encode	JSON::PP	2811
+json/strs	Encode	JSON::SIMD	140393
+json/strs	Encode	JSON::Tiny	3924
+json/strs	Encode	JSON::XS	141149
+json/stru	Decode	Cpanel::JSON::XS	64101
+json/stru	Decode	FU::Util	52041
+json/stru	Decode	JSON::PP	248
+json/stru	Decode	JSON::SIMD	80941
+json/stru	Decode	JSON::Tiny	2677
+json/stru	Decode	JSON::XS	61104
+json/stru	Encode	Cpanel::JSON::XS	91004
+json/stru	Encode	FU::Util	205716
+json/stru	Encode	JSON::PP	5138
+json/stru	Encode	JSON::SIMD	101185
+json/stru	Encode	JSON::Tiny	6501
+json/stru	Encode	JSON::XS	106312
+pg/ints	Bigint	DBD::Pg	33
+pg/ints	Bigint	FU::Pg (bin)	46
+pg/ints	Bigint	FU::Pg (text)	23
+pg/ints	Bigint	Pg::PQ	24
+pg/ints	Smallint	DBD::Pg	346
+pg/ints	Smallint	FU::Pg (bin)	476
+pg/ints	Smallint	FU::Pg (text)	273
+pg/ints	Smallint	Pg::PQ	270
+xml/a	Rate	FU::XMLWriter	5192
+xml/a	Rate	HTML::Tiny	403
+xml/a	Rate	TUWF::XML	787
+xml/a	Rate	XML::Writer	832
diff --git a/bench.PL b/bench.PL
index 4bee131..fb95fb1 100755
--- a/bench.PL
+++ b/bench.PL
@@ -1,8 +1,9 @@
 #!/usr/bin/perl
 
 # Can be invoked as:
-#   ./bench.PL          # generates FU/Benchmarks.pod, running new benchmarks as necessary
-#   ./bench.PL id x y   # invalidate cache for the (regex-)matching benchmark IDs, x and y and re-run them
+#   ./bench.PL              # generates FU/Benchmarks.pod, running new benchmarks as necessary
+#   ./bench.PL id x y       # invalidate cache for the (regex-)matching benchmark IDs, x and y and re-run them
+#   ./bench.PL exec id x y  # Run just the given benchmark and exit
 #
 # This script obviously has more dependencies than the FU distribution itself.
 # It's supposed to be used by maintainers, not users.
@@ -30,30 +31,69 @@ my %modules = map +($_, eval "require $_; \$${_}::VERSION"), qw/
 /;
 use FU::Pg;
 
-my %data; # "id x y" => { id x y rate exists }
-my %oldmodules;
-{ if (open my $F, '<', 'FU/Benchmarks.pod') {
-    my $indata;
-    while (<$F>) {
-        chomp;
-        $oldmodules{$1} = $2 if /^=item L<([a-zA-Z0-9:]+)> ([0-9.]+)/;
-        $indata = 1 if /^# Cached data used by bench\.PL/;
-        next if !$indata || !$_ || /^#/;
-        my %d;
-        @d{qw/id x y rate/} = split /\t/;
-        $data{"$d{id} $d{x} $d{y}"} = \%d;
-    }
-} }
+my @exec = $ARGV[0] && $ARGV[0] eq 'exec' ? @ARGV[1..3] : ();
+my @run = !@exec && @ARGV && (qr/$ARGV[0]/i, $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/, $ARGV[2] ? qr/$ARGV[2]/i : qr/.*/);
 
-if (@ARGV) {
-    my $idre = qr/$ARGV[0]/i;
-    my $xre = $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/;
-    my $yre = $ARGV[2] ? qr/$ARGV[2]/i : qr/.*/;
-    delete $_->{rate} for grep $_->{id} =~ /$idre/ && $_->{x} =~ /$xre/ && $_->{y} =~ /$yre/, values %data;
+my %data; # "id x y" => { id x y rate exists }
+my @bench; # [ id, text, [ x_1, .. ], [ [ y_1, mod_1, sub_1, .. ], .. ] ]
+my %oldmodules;
+if (!@exec) {
+    if (open my $F, '<', 'FU/Benchmarks.pod') {
+        my $indata;
+        while (<$F>) {
+            chomp;
+            $oldmodules{$1} = $2 if /^=item L<([a-zA-Z0-9:]+)> ([0-9.]+)/;
+            $indata = 1 if /^# Cached data used by bench\.PL/;
+            next if !$indata || !$_ || /^#/;
+            my %d;
+            @d{qw/id x y rate/} = split /\t/;
+            $data{"$d{id} $d{x} $d{y}"} = \%d;
+        }
+    }
 }
 
 
-my @bench; # [ id, text, [ x_1, .. ], [ [ y_1, mod_1, sub_1, .. ], .. ] ]
+sub fmtbench($id, $text, $xs, $ys) {
+    my $r = "$text\n\n";
+    if (@$xs > 1) {
+        $r .= sprintf '%18s', '';
+        $r .= sprintf '%12s', $_ for @$xs;
+        $r .= "\n";
+    }
+    for my ($n, $yr) (builtin::indexed @$ys) {
+        my $x = $xs->[$n];
+        my ($y, $m, @ys) = @$yr;
+        $m ||= $y;
+        $r .= sprintf '%18s', $y;
+        for my $i (0..$#$xs) {
+            my $d = $data{"$id $xs->[$i] $y"};
+            $r .= $d && $d->{rate} ? sprintf '%10d/s', $d->{rate} : sprintf '%12s', '-';
+        }
+        $r .= "\n";
+    }
+    "$r\n"
+}
+
+$SIG{INT} = $SIG{HUP} = sub { exit };
+END {
+    exit if @exec;
+
+    open my $F, '>FU/Benchmarks.pod' or die $!;
+    select $F;
+    while (<DATA>) {
+        s/^%/=/;
+        s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
+        s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
+        print;
+    }
+    for (sort keys %data) {
+        my $b = $data{$_};
+        print join("\t", map $_//'', @{$b}{qw/ id x y rate /})."\n";
+    }
+}
+
+
+
 sub def($id, $text, $xs, @ys) {
     for my ($ya) (@ys) {
         my($y, $m, @sub) = @$ya;
@@ -64,12 +104,6 @@ sub def($id, $text, $xs, @ys) {
             $data{$d} ||= { id => $id, x => $x, y => $y };
             $d = $data{$d};
             $d->{exists} = 1;
-            delete $d->{rate} if !$oldmodules{$m} || $modules{$m} ne $oldmodules{$m};
-            if (!exists $d->{rate}) {
-                my $o = timethis -5, $sub[$i], 0, 'none';
-                $d->{rate} = sprintf '%.0f', $o->iters/$o->real;
-                printf "%-20s%-12s%-20s%10d/s\n", $id, $x, $y, $d->{rate};
-            }
         }
     }
     push @bench, [ $id, $text, $xs, \@ys ];
@@ -200,11 +234,11 @@ def 'xml/a', 'HTML fragment', [ 'Rate' ],
 
 
 {
-    die "FU_TEST_DB not set.\n" if !$ENV{FU_TEST_DB};
-    my $pq = Pg::PQ::Conn->new($ENV{FU_TEST_DB});
-    my $fu = FU::Pg->connect($ENV{FU_TEST_DB});
+    die "FU_TEST_DB not set.\n" if @exec && !$ENV{FU_TEST_DB};
+    my $pq = @exec && Pg::PQ::Conn->new($ENV{FU_TEST_DB});
+    my $fu = @exec && FU::Pg->connect($ENV{FU_TEST_DB});
     # XXX: Doesn't support all connection params this way
-    my $dbi = DBI->connect("dbi:Pg:dbname=".$pq->db, $pq->user, $pq->pass, {RaiseError => 1, PrintError => 0});
+    my $dbi = @exec && DBI->connect("dbi:Pg:dbname=".$pq->db, $pq->user, $pq->pass, {RaiseError => 1, PrintError => 0});
 
     my $small = 'SELECT x, x+1, x+2, x+3, x+4, x+5, x+6, x+7, x+8, x+9 FROM generate_series(-10000::smallint, 9999, 10) x(x)';
     my $big = 'SELECT x<<5, x<<10, x<<15, x<<20, x<<25, x<<30, x<<35, x<<40, x<<45, x<<50 FROM generate_series(1::bigint, 20000, 1) x(x)';
@@ -227,42 +261,49 @@ def 'xml/a', 'HTML fragment', [ 'Rate' ],
 
 delete @data{ grep !$data{$_}{exists}, keys %data };
 
-sub fmtbench($id, $text, $xs, $ys) {
-    my $r = "$text\n\n";
-    if (@$xs > 1) {
-        $r .= sprintf '%18s', '';
-        $r .= sprintf '%12s', $_ for @$xs;
-        $r .= "\n";
-    }
-    for my ($n, $yr) (builtin::indexed @$ys) {
-        my $x = $xs->[$n];
-        my ($y, $m, @ys) = @$yr;
-        $m ||= $y;
-        $r .= sprintf '%18s', $y;
-        for my $i (0..$#$xs) {
-            my $d = $data{"$id $xs->[$i] $y"};
-            $r .= $d ? sprintf '%10d/s', $d->{rate} : sprintf '%12s', '-';
-        }
-        $r .= "\n";
-    }
-    "$r\n"
+
+sub runbench($sub) {
+    my $o = timethis -1, $sub, 0, 'none';
+    printf "%.2f\n", $o->iters/$o->real;
+    exit;
 }
 
-{
-    open my $F, '>FU/Benchmarks.pod' or die $!;
-    select $F;
-    while (<DATA>) {
-        s/^%/=/;
-        s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
-        s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
-        print;
+sub execbench($d) {
+    my $sum = 0;
+    my $num = 1;
+    local $| = 1;
+    printf "%-20s%-12s%-20s", $d->{id}, $d->{x}, $d->{y};
+    for (1..$num) {
+        open my $P, '-|', $^X, (map "-I$_", @INC), $0, 'exec', $d->{id}, $d->{x}, $d->{y};
+        chomp(my $rate = <$P>);
+        printf "%10d", $rate;
+        $sum += $rate;
     }
-    for (sort keys %data) {
-        my $b = $data{$_};
-        print join("\t", @{$b}{qw/ id x y rate /})."\n";
+    printf " ->%10d\n", $sum/$num;
+    $d->{rate} = sprintf '%.0f', $sum/$num;
+}
+
+for my $b (@bench) {
+    my ($id, $text, $xs, $ys) = @$b;
+    for my ($ya) (@$ys) {
+        my($y, $m, @sub) = @$ya;
+        $m ||= $y;
+        for my($i, $x) (builtin::indexed @$xs) {
+            next if !$sub[$i];
+            if (@exec) {
+                runbench $sub[$i] if $exec[0] eq $id && $exec[1] eq $x && $exec[2] eq $y;
+            } else {
+                my $d = $data{"$id $x $y"};
+                execbench $d if !$oldmodules{$m} || $modules{$m} ne $oldmodules{$m}
+                    || (@run && $id =~ /$run[0]/ && $x =~ /$run[1]/ && $y =~ /$run[2]/);
+            }
+        }
     }
 }
 
+die if @exec;
+
+
 # s/^=/%/  to prevent tools from interpreting the below as POD
 __DATA__
 %head1 NAME

From 7980af731ecceb4ef58bf97af5e77ea1743bc6b0 Mon Sep 17 00:00:00 2001
From: Yorhel <git@yorhel.nl>
Date: Sat, 10 Jan 2026 17:30:41 +0100
Subject: [PATCH 86/86] Version 1.4

---
 ChangeLog               | 7 +++++++
 FU.pm                   | 2 +-
 FU/Benchmarks.pod       | 2 +-
 FU/DebugImpl.pm         | 2 +-
 FU/Log.pm               | 2 +-
 FU/MultipartFormData.pm | 2 +-
 FU/Pg.pm                | 2 +-
 FU/SQL.pm               | 2 +-
 FU/Util.pm              | 2 +-
 FU/Validate.pm          | 2 +-
 FU/XMLWriter.pm         | 2 +-
 FU/XS.pm                | 2 +-
 12 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/ChangeLog b/ChangeLog
index f5ecc19..0774d6b 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+1.4 - 2026-01-10
+	- FU::Pg: rename q() and Q() to sql() and SQL() (old names still work)
+	- FU: Improve handling of EPIPE when writing FastCGI response
+	- FU: Log unclean worker process shutdown
+	- FU: Fix warning when parsing empty cookie values
+	- Misc doc fixes
+
 1.3 - 2025-09-04
 	- FU::Validate: Scalar validations now reject control characters by default
 	- FU::Validate: Add `allow_control` option to override above behavior
diff --git a/FU.pm b/FU.pm
index 7d5df3b..cb73e10 100644
--- a/FU.pm
+++ b/FU.pm
@@ -1,4 +1,4 @@
-package FU 1.3;
+package FU 1.4;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
diff --git a/FU/Benchmarks.pod b/FU/Benchmarks.pod
index e611832..b4b9182 100644
--- a/FU/Benchmarks.pod
+++ b/FU/Benchmarks.pod
@@ -30,7 +30,7 @@ The following module versions were used:
 
 =item L<DBD::Pg> 3.18.0
 
-=item L<FU> 1.3
+=item L<FU> 1.4
 
 =item L<HTML::Tiny> 1.08
 
diff --git a/FU/DebugImpl.pm b/FU/DebugImpl.pm
index 0ffe8cf..4fd2a26 100644
--- a/FU/DebugImpl.pm
+++ b/FU/DebugImpl.pm
@@ -1,5 +1,5 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 1.3;
+package FU::DebugImpl 1.4;
 use v5.36;
 use utf8;
 use experimental 'for_list';
diff --git a/FU/Log.pm b/FU/Log.pm
index f3b1fcf..44f881c 100644
--- a/FU/Log.pm
+++ b/FU/Log.pm
@@ -1,4 +1,4 @@
-package FU::Log 1.3;
+package FU::Log 1.4;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
diff --git a/FU/MultipartFormData.pm b/FU/MultipartFormData.pm
index ca21512..46c6a6d 100644
--- a/FU/MultipartFormData.pm
+++ b/FU/MultipartFormData.pm
@@ -1,4 +1,4 @@
-package FU::MultipartFormData 1.3;
+package FU::MultipartFormData 1.4;
 use v5.36;
 use Carp 'confess';
 use FU::Util 'utf8_decode';
diff --git a/FU/Pg.pm b/FU/Pg.pm
index 294481c..465d076 100644
--- a/FU/Pg.pm
+++ b/FU/Pg.pm
@@ -1,4 +1,4 @@
-package FU::Pg 1.3;
+package FU::Pg 1.4;
 use v5.36;
 use FU::XS;
 
diff --git a/FU/SQL.pm b/FU/SQL.pm
index 218a034..c33d680 100644
--- a/FU/SQL.pm
+++ b/FU/SQL.pm
@@ -1,4 +1,4 @@
-package FU::SQL 1.3;
+package FU::SQL 1.4;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
diff --git a/FU/Util.pm b/FU/Util.pm
index da8b869..84f10d7 100644
--- a/FU/Util.pm
+++ b/FU/Util.pm
@@ -1,4 +1,4 @@
-package FU::Util 1.3;
+package FU::Util 1.4;
 
 use v5.36;
 use FU::XS;
diff --git a/FU/Validate.pm b/FU/Validate.pm
index f170bfc..74a50ce 100644
--- a/FU/Validate.pm
+++ b/FU/Validate.pm
@@ -1,4 +1,4 @@
-package FU::Validate 1.3;
+package FU::Validate 1.4;
 
 use v5.36;
 use experimental 'builtin', 'for_list';
diff --git a/FU/XMLWriter.pm b/FU/XMLWriter.pm
index 5364c5e..f80c20d 100644
--- a/FU/XMLWriter.pm
+++ b/FU/XMLWriter.pm
@@ -1,4 +1,4 @@
-package FU::XMLWriter 1.3;
+package FU::XMLWriter 1.4;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
diff --git a/FU/XS.pm b/FU/XS.pm
index b22fbbb..f6ed1f7 100644
--- a/FU/XS.pm
+++ b/FU/XS.pm
@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 1.3;
+package FU::XS 1.4;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');