Version 1.4

That would previously result in the worker getting killed with SIGPIPE.
Which works, but we can also recover from that error without restarting
the process.

ChangeLog

@@ -1,3 +1,96 @@
+1.4 - 2026-01-10
+	- FU::Pg: rename q() and Q() to sql() and SQL() (old names still work)
+	- FU: Improve handling of EPIPE when writing FastCGI response
+	- FU: Log unclean worker process shutdown
+	- FU: Fix warning when parsing empty cookie values
+	- Misc doc fixes
+
+1.3 - 2025-09-04
+	- FU::Validate: Scalar validations now reject control characters by default
+	- FU::Validate: Add `allow_control` option to override above behavior
+	- FU::Util: JSON and URI parsing now always permit control characters
+	- FU::Util: More strict UTF-8 validation on path & URI decoding
+	- FU::Util: Deprecate `decode_utf8()`
+	- FU::Util: Deprecate `allow_control` option in `json_parse()`
+
+1.2 - 2025-07-06
+	- FU::Pg: Throw error on non-boolean-looking Perl values for boolean bind
+	  parameters
+	- FU: Improve setting process status during startup
+
+1.1 - 2025-06-07
+	- FU::SQL: Add IDENT function and `quote_identifier` option
+	- FU::Pg: Set appropriate `quote_identifier` for `$conn->Q()`
+	- FU: Improve `--monitor` file change detection
+	- FU::XMLWriter: Disallow stringification of bare Perl references
+	- FU::Util::json_parse(): Disallow control characters in strings, add
+	  `allow_control` option to revert to old behavior.
+	- Some doc fixes
+
+1.0 - 2025-05-11
+	- FU::Util: Fix parsing of empty sections in query_decode()
+	- FU::Util: Fix buffer overflow in json_format() float formatting
+	- FU::Util: Reject `0x1f` in utf8_decode()
+	- FU::Pg: Add perl<->text and bin<->text type conversion methods
+	- FU::Validate: Improved error messages
+	- FU::MultipartFormData: Various parser fixes
+	- FU: Include request body in verbose error logs
+	- FU: Add fu->log_verbose()
+	- FU: Extend debug_info pages with request body, response body, 'fu'
+	  object dump, expandable query parameters and interpolated SQL queries
+	- FU: Improve styling of debug_info pages
+	- FU: Preserve headers on fu->redirect
+	- FU: Ignore HTTP_CONTENT_LENGTH and HTTP_CONTENT_TYPE FastCGI parameters
+	- FU: Suppress warning about missing files in FU::monitor_path
+	- FU: Reject hash character and newlines in request path
+	- Fix creating read-only undef/true/false in json_parse() and FU::Pg
+	- Benchmark updates
+
+0.5 - 2025-04-24
+	- FU::Util: Set O_CLOEXEC on fds received through `fdpass_recv()`
+	- FU::Util: Fix interpretation of false options in `json_format()` and
+	  `json_parse()`
+	- FU::Validate: Add `coerce()` and `empty()` utility methods
+	- FU::Validate: Limit values of int/uint input to 64 bits
+	- FU::Validate: Normalize num/int/uint inputs to Perl numeric types
+	- FU::Pg: Add `escape_literal()` and `escape_identifier()` methods
+	- FU::Pg: Use less memory for `kvv()`, `kva()` and `kvh()` methods
+	- FU::Pg: Disallow chaining of `cache()`, `text()`, `text_params()` and
+	  `text_results()` methods on connection and transaction objects
+	- FU: Throw and catch FU::Validate errors without wrapping in `fu->error()`
+	- FU: Add `-progname` option and add diagnostics to process names
+	- FU: Whole bunch of misc fixes
+	- Doc fixes
+	- Fix nul-termination of some XS-created strings
+
+0.4 - 2025-03-19
+	- FU::Validate: Support arrayref schemas
+	- FU::Validate: Rename 'values' option to 'elems'
+	- FU::Validate: Repurpose 'values' option to validate hash values
+	- FU::Validate: Merge nested 'elems', 'keys' and 'values' schemas during compile()
+	- FU::Validate: Rename 'scalar' to 'accept_scalar'
+	- FU::Validate: Add 'accept_array' option
+	- FU::Util: Add 'html_safe' option to json_format()
+	- FU::Util: Add gzip_compress() wrapper for libdeflate.so, zlib-ng.so or zlib.so
+	- FU::Util: Add brotli_compress() wrapper for libbrotlienc.so
+	- FU: Consistency fixes for fu->json() and fu->formdata()
+	- FU: Add fu->cookie() and fu->set_cookie()
+	- FU: Add support for brotli output compression
+	- FU: Use gzip_compress() for faster gzip output compression
+
+0.3 - 2025-03-10
+	- FU::Validate: Change API, ->validate() now returns data or throws error on failure
+	- FU::Validate: Rename 'rmwhitespace' to 'trim'
+	- FU::Validate: Support (more) human-readable error messages
+	- FU::Pg: Add support for COPY operations
+	- FU::Pg: Support types with dynamic OIDs
+	- FU: Add support for reading multipart/form-data
+	- FU: Add convenience methods for reading and writing JSON
+	- FU: Fix error in handling a 400
+	- FU::MultipartFormData: New helper module
+	- Fix some tests
+	- Some doc improvements
+
 0.2 - 2025-02-28
 	- FU: Add debug_info web interface
 	- FU: Add fu->denied and fu->notfound methods

FU.pm

@@ -1,21 +1,29 @@
-package FU 0.2;
+package FU 1.4;
 use v5.36;
 use Carp 'confess', 'croak';
 use IO::Socket;
 use POSIX ();
-use Time::HiRes 'time';
+use Time::HiRes 'time', 'clock_gettime', 'CLOCK_MONOTONIC';
 use FU::Log 'log_write';
 use FU::Util;
+use FU::Validate;
 
+my $procname;
+my $scriptpath = $0;
 
 sub import($pkg, @opt) {
     my $c = caller;
     no strict 'refs';
     *{$c.'::fu'} = \&fu;
+    my $spawn;
     for (@opt) {
-        if ($_ eq '-spawn') { _spawn() }
+        if (ref $procname eq 'FU::ARG') { $procname = $_ }
+        elsif ($_ eq '-procname') { $procname = bless {}, 'FU::ARG' }
+        elsif ($_ eq '-spawn') { $spawn = 1; }
         else { croak "Unknown import option: '$_'" }
     }
+    croak "Missing argument for -procname option" if ref $procname eq 'FU::ARG';
+    _spawn() if $spawn;
 }
 
 
@@ -113,15 +121,29 @@ sub query_trace($st,@) {
     $REQ->{trace_nsqldirect}++ if !defined $st->prepare_time;
     $REQ->{trace_sqlexec} += $st->exec_time;
     $REQ->{trace_sqlprep} += $st->prepare_time if $st->prepare_time;
+    if (FU::debug) {
+        my $t = $st->param_types;
+        my $v = $st->param_values;
+        my $txt = $st->get_text_params;
         push $REQ->{trace_sql}->@*, {
             query => $st->query, nrows => $st->nrows,
-        param_types => $st->param_types, param_values => $st->param_values,
             exec_time => $st->exec_time, prepare_time => $st->prepare_time,
-    } if FU::debug;
+            # Store the binary value when we're in binary params mode, that way
+            # we don't have to keep a reference to the original perl value and
+            # we can defer & batch the conversion to text.
+            params => [ map +{
+                type => $t->[$_],
+                !defined $v->[$_] ? (text => undef) :
+                             $txt ? (text => "$v->[$_]")
+                                  : (bin => $DB->perl2bin($t->[$_], $v->[$_]))
+            }, 0..$#$v ],
+        };
+    }
 }
 sub _connect_db {
     $DB = ref $INIT_DB eq 'CODE' ? $INIT_DB->() : FU::Pg->connect($INIT_DB);
     $DB->query_trace(\&query_trace);
+    $DB
 }
 sub init_db($info) {
     require FU::Pg;
@@ -195,25 +217,20 @@ sub monitor_path { push @monitor_paths, @_ }
 sub monitor_check :prototype(&) { $monitor_check = $_[0] }
 
 sub _monitor {
-    state %data;
     return 1 if $monitor_check && $monitor_check->();
 
     require File::Find;
     eval {
         File::Find::find({
-            wanted => sub {
+            wanted => sub { die if (-M) < 0 },
-                my $m = (stat)[9];
-                $data{$_} //= $m;
-                die if $m > $data{$_};
-            },
             no_chdir => 1
-        }, $0, values %INC, @monitor_paths);
+        }, grep -e, $scriptpath, values %INC, @monitor_paths);
         0
     } // 1;
 }
 
 
-our $debug_info = [];
+our $debug_info = {};
 sub debug_info($path, $storage=undef, $history=100) {
     $debug_info = { path => $path, storage => $storage, history => $history }
 }
@@ -253,8 +270,9 @@ sub _read_req_http($sock, $req) {
 
     $req->{body} = '';
     while ($len > 0) {
-        my $r = $sock->read($req->{body}, $len, -1);
+        my $r = $sock->read($req->{body}, $len, length $req->{body});
-        fu->error(400, 'Client disconnect before request was read') if !$r
+        fu->error(400, 'Client disconnect before request was read') if !$r;
+        $len -= $r;
     }
 }
 
@@ -274,7 +292,8 @@ sub _read_req($c) {
                : $r == -2 ? "I/O error while reading from FastCGI socket\n"
                : $r == -3 ? "FastCGI protocol error\n"
                : $r == -4 ? "Too long FastCGI parameter\n"
-               : $r == -5 ? "Too long request body\n" : undef if $r != -7;
+               : $r == -5 ? "Too long request body\n"
+               : $r == -8 ? "I/O error while writing to FastCGI socket\n" : undef if $r != -7;
             delete $c->{fcgi_obj};
             fu->error(-1);
         }
@@ -288,36 +307,33 @@ sub _read_req($c) {
 
     # The HTTP reader above and the FastCGI XS reader operate on bytes.
     # Decode these into Unicode strings and check for special characters.
-    eval { FU::Util::utf8_decode($_); 1} || fu->err(400, $@)
+    eval { FU::Util::utf8_decode($_); 1} || fu->error(400, $@)
         for ($REQ->{path}, $REQ->{qs}, values $REQ->{hdr}->%*);
+    fu->error(400, 'Invalid character in path') if $REQ->{path} =~ /#/; # Some bots don't correctly split off the fragment
 
     ($REQ->{path}, my $qs) = split /\?/, $REQ->{path}//'', 2;
     $REQ->{qs} //= $qs;
-    $REQ->{path} = FU::Util::uri_unescape($REQ->{path});
+    eval { $REQ->{path} = FU::Util::uri_unescape($REQ->{path}); FU::Util::check_control($REQ->{path}); 1; } || fu->error(400, $@);
+    fu->error(400, 'Invalid character in path') if $REQ->{path} =~ /[\r\n\t]/; # There are plenty other questionable characters, but newlines and tabs are definitely out
 }
 
 
-sub _is_done($e) { ref $@ eq 'FU::err' && $@->[0] == 200 }
+sub _is_done($e) { ref $e eq 'FU::err' && $e->[0] == 200 }
 
 sub _log_err($e) {
     return if !$e;
-    return if !debug && ref $@ eq 'FU::err' && $@->[0] != 500;
+    my $crit = $e isa 'FU::err' ? $e->[0] == 500 : !($e isa 'FU::Validate::err');
-    if (!$REQ->{full_err} && (ref $@ ne 'FU::err' || $@->[0] == 500)) {
+    return if !debug && !$crit;
-        $REQ->{full_err}++;
+    return fu->log_verbose($e) if $crit;
-        $e =~ s/^\s+//;
-        $e =~ s/\s+$//;
-        log_write join "\n",
-            'IP: '.($REQ->{ip}||'-'),
-            'Headers:', (map "  $_: $REQ->{hdr}{$_}", sort keys $REQ->{hdr}->%*),
-            'ERROR:'.($e =~ s/(^|\n)/\n  /rg);
-            # TODO: decoded body, if we have that.
-    } else {
     log_write $e;
 }
-}
 
 sub _do_req($c) {
-    local $REQ = { hdr => {}, trace_start => time, trace_id => sprintf('%010x%08x%04x', int time, $$, int rand 1<<16) };
+    local $REQ = {
+        hdr => {},
+        trace_start => clock_gettime(CLOCK_MONOTONIC),
+        trace_id => sprintf('%012x%06x%04x', int(time*10000) % (1<<(12*4)), $$ % (1<<(6*4)), int rand 1<<16)
+    };
     local $fu = bless {}, 'FU::obj';
 
     $REQ->{ip} = $c->{client_sock} isa 'IO::Socket::INET' ? $c->{client_sock}->peerhost : '127.0.0.1';
@@ -325,7 +341,7 @@ sub _do_req($c) {
 
     my $ok = eval {
         _read_req $c;
-        $REQ->{trace_start} = time;
+        $REQ->{trace_start} = clock_gettime(CLOCK_MONOTONIC);
 
         my $path = fu->path;
         my $method = fu->method eq 'HEAD' ? 'GET' : fu->method;
@@ -374,17 +390,24 @@ sub _do_req($c) {
     }
 
     if ($err) {
-        my($code, $msg) = ref $err eq 'FU::err' ? $err->@* : (500, $err);
+        my($code, $msg) = $err isa 'FU::err' ? @$err : $err isa 'FU::Validate::err' ? (400, $err) : (500, $err);
         fu->reset;
         fu->status($code);
-        eval {
+        my $ok = eval { ($onerr{$code} || $onerr{500})->($code, $msg) };
-            ($onerr{$code} || $onerr{500})->($code, $msg);
+        if (!$ok && !_is_done($@)) {
-            1;
+            _log_err $@;
-        } || _err_500();
+            _err_500();
+        }
     }
 
-    $REQ->{trace_end} = time;
+    $REQ->{trace_end} = clock_gettime(CLOCK_MONOTONIC);
+    eval {
         fu->_flush($c->{fcgi_obj} || $c->{client_sock});
+        1;
+    } || do {
+        log_write "Error writing response: $@\n";
+        $c->{client_sock} = $c->{fcgi_obj} = undef;
+    };
 
     if (debug && $REQ->{trace_id} && $debug_info->{history} && $debug_info->{storage}) {
         require FU::DebugImpl;
@@ -392,19 +415,20 @@ sub _do_req($c) {
     }
 
     my $proc_ms = ($REQ->{trace_end} - $REQ->{trace_start}) * 1000;
-    log_write(sprintf "%.0fms%s %s-%s %s-%d\n", $proc_ms,
+    log_write(sprintf "%.0fms%s %s-%s %d-%s\n", $proc_ms,
         $REQ->{trace_nsql} ?
             sprintf ' (sql %.0f+%.0fms, %d/%d/%d)',
             ($REQ->{trace_sqlexec}||0)*1000, ($REQ->{trace_sqlprep}||0)*1000,
             $REQ->{trace_nsqldirect}||0, $REQ->{trace_nsqlprep}||0, $REQ->{trace_nsql} : '',
         $REQ->{status}, ($REQ->{reshdr}{'content-type'}//'-') =~ s/;.+$//r,
-        $REQ->{reshdr}{'content-encoding'}//'bytes', length($REQ->{resbody}),
+        length($REQ->{resbody}), substr($REQ->{reshdr}{'content-encoding'}//'r', 0, 1)
     ) if FU::debug || $proc_ms > (FU::log_slow_reqs||1e10);
 }
 
 
 sub _run_loop($c) {
     my $stop = 0;
+    my $count = 0;
     local $SIG{HUP} = 'IGNORE';
     local $SIG{TERM} = $SIG{INT} = sub { $stop = 1 };
 
@@ -414,7 +438,13 @@ sub _run_loop($c) {
         exit;
     }
 
+    my sub setstate($state) {
+        $0 = sprintf "%s: %s [#%d%s]", $procname, $state, $count, $c->{max_reqs} ? "/$c->{max_reqs}" : '' if $procname;
+    }
+
     while (!$stop) {
+        setstate 'idle';
+
         $c->{client_sock} ||= $c->{listen_sock}->accept || next;
         $c->{fcgi_obj} ||= $c->{listen_proto} eq 'fcgi' && FU::fcgi::new(fileno $c->{client_sock}, $c->{proc});
 
@@ -423,11 +453,13 @@ sub _run_loop($c) {
             passclient;
         }
 
+        setstate 'working';
         _do_req $c;
 
         $c->{client_sock} = $c->{fcgi_obj} = undef if !($c->{fcgi_obj} && $c->{fcgi_obj}->keepalive);
 
-        passclient if $c->{max_reqs} && !--$c->{max_reqs};
+        $count++;
+        passclient if $c->{max_reqs} && $count >= $c->{max_reqs};
     }
 }
 
@@ -466,34 +498,36 @@ sub _supervisor($c) {
             if (!$err && (!$childs{$pid} || $childs{$pid} != 2)) {
                 $err = 1;
                 log_write "Script exited before calling FU::run()\n";
+            } elsif ($?) {
+                log_write "Unclean shutdown of worker PID $pid status $?\n";
             }
             delete $childs{$pid};
         }
 
         # Don't bother spawning more than 1 at a time while in error state
-        my $spawn = !$err ? $c->{proc} - keys %childs : (grep $_ == 1, values %childs) ? 0 : 1;
+        my $spawn = !$err ? $c->{proc} - keys %childs : !@client_fd && (grep $_ == 1, values %childs) ? 0 : 1;
         for (1..$spawn) {
-            my $client = shift @client_fd;
+            my $client = @client_fd ? IO::Socket->new_from_fd(shift(@client_fd), 'r') : undef;
             my $pid = fork;
             die $! if !defined $pid;
             if (!$pid) { # child
                 $SIG{CHLD} = $SIG{HUP} = $SIG{INT} = $SIG{TERM} = undef;
-                if ($client) {
+                $0 = sprintf '%s: starting', $procname if $procname;
-                    $ENV{FU_CLIENT_FD} = $client;
-                } elsif ($err) {
                 # In error state, wait with loading the script until we've received a request.
                 # Otherwise we'll end up in an infinite spawning loop if the script doesn't start properly.
-                    $client = $c->{listen_sock}->accept() or die $!;
+                $client = $c->{listen_sock}->accept() or die $! if !$client && $err;
+                if ($client) {
                     fcntl $client, Fcntl::F_SETFD, 0;
                     $ENV{FU_CLIENT_FD} = fileno $client;
                 }
-                exec $^X, (map "-I$_", @INC), $0;
+                exec $^X, (map "-I$_", @INC), $scriptpath;
                 exit 1;
             }
-            $client && IO::Socket->new_from_fd($client, 'r'); # close() the fd if we have one
             $childs{$pid} = 1;
         }
 
+        $0 = sprintf "%s: supervisor [%d/%d]", $procname, scalar keys %childs, $c->{proc} if $procname;
+
         my ($fd, $msgadd) = FU::Util::fdpass_recv(fileno($rsock), 500);
         push @client_fd, $fd if $fd;
         next if !defined $msgadd;
@@ -621,8 +655,29 @@ sub db {
     };
 }
 
-sub sql { shift->db->q(@_) }
+sub sql { shift->db->sql(@_) }
-sub SQL { shift->db->Q(@_) }
+sub SQL { shift->db->SQL(@_) }
+
+sub _fmt_section($s) { $s =~ s/^\s*/  /r =~ s/\s+$//r =~ s/\n/\n  /rg }
+
+sub log_verbose($,$msg) {
+    my $r = $FU::REQ;
+    return FU::Log::log_write($msg) if $r->{log_verbose}++;
+    FU::Log::log_write(join "\n",
+        'IP: '.($r->{ip}||'-'),
+        'Headers:', (map "  $_: $r->{hdr}{$_}", sort keys $r->{hdr}->%*),
+        $r->{multipart} ? ('Body (multipart):', _fmt_section join "\n", map $_->describe, $r->{multipart}->@*) :
+        $r->{json} ? ('Body (JSON):', _fmt_section FU::Util::json_format($r->{json}, pretty => 1, canonical => 1)) :
+        $r->{formdata} ? ('Body (formdata):', _fmt_section FU::Util::json_format($r->{formdata}, pretty => 1, canonical => 1)) :
+        length $r->{body} ? do {
+            my $b = substr $r->{body}, 0, 4096;
+            my $trunc = length $r->{body} > 4096 ? ', truncated' : '';
+            utf8::decode($b) ? ("Body (utf8$trunc):", _fmt_section($b =~ s/\r//rg =~ s/\n{4,}/\n[..]\n/rg))
+                             : ("Body (hex$trunc):", _fmt_section(unpack('H*', $b) =~ s/(.{128})/$1\n/rg))
+        } : (),
+        'Message:', _fmt_section $msg
+    );
+}
 
 
 
@@ -636,12 +691,13 @@ sub headers { $FU::REQ->{hdr} }
 sub ip { $FU::REQ->{ip} }
 
 sub _getfield($data, @a) {
-    return $data->{$a[0]} if @a == 1 && !ref $a[0];
+    if (@a == 1 && !ref $a[0]) {
-    require FU::Validate;
+        fu->error(400, "Expected top-level to be a hash") if ref $data ne 'HASH';
+        return $data->{$a[0]};
+    }
     my $schema = FU::Validate->compile(@a > 1 ? { keys => {@a} } : $a[0]);
     my $res = $schema->validate($data);
-    fu->error(400, "Input validation failed") if !$res; # TODO: More detailed error message
+    return @a == 2 ? $res->{$a[0]} : $res;
-    return @a == 2 ? $res->data->{$a[0]} : $res->data;
 }
 
 sub query {
@@ -651,18 +707,50 @@ sub query {
     _getfield $FU::REQ->{qs_parsed}, @_;
 }
 
+sub cookie {
+    shift;
+    return fu->header('cookie') if !@_;
+    $FU::REQ->{cookie} ||= do {
+        my %c;
+        for my $c (split /; /, fu->header('cookie')||'') {
+            my($n, $v) = split /=/, $c, 2;
+            if (!defined $v) {}
+            elsif (!exists $c{$n}) { $c{$n} = $v }
+            elsif (ref $c{$n}) { push $c{$n}->@*, $v }
+            else { $c{$n} = [ $c{$n}, $v ] }
+        }
+        \%c
+    };
+    _getfield $FU::REQ->{cookie}, @_;
+}
+
+sub json {
+    shift;
+    fu->error(400, "Invalid content type for json") if (fu->header('content-type')||'') !~ m{^application/json(?:;\s*charset=utf-?8)?$}i;
+    return FU::Util::utf8_decode(my $x = $FU::REQ->{body}) if !@_;
+    $FU::REQ->{json} ||= eval {
+        FU::Util::json_parse($FU::REQ->{body}, utf8 => 1)
+    } || fu->error(400, "JSON parse error: $@");
+    _getfield $FU::REQ->{json}, @_;
+}
+
 sub formdata {
     shift;
+    fu->error(400, "Invalid content type for form data") if (fu->header('content-type')||'') ne 'application/x-www-form-urlencoded';
+    return FU::Util::utf8_decode(my $x = $FU::REQ->{body}) if !@_;
     $FU::REQ->{formdata} ||= eval {
-        # TODO: Support multipart encoding
+        FU::Util::query_decode($FU::REQ->{body});
-        confess "Invalid content type for form data"
-            if (fu->header('content-type')||'') ne 'application/x-www-form-urlencoded';
-        FU::Util::query_decode($FU::REQ->{data});
     } || fu->error(400, $@);
-    # TODO: Accept schema validation thing.
     _getfield $FU::REQ->{formdata}, @_;
 }
 
+sub multipart {
+    require FU::MultipartFormData;
+    $FU::REQ->{multipart} ||= eval {
+        FU::MultipartFormData->parse(fu->header('content-type')||'', $FU::REQ->{body})
+    } || fu->error(400, $@);
+}
+
 
 
 
@@ -687,12 +775,13 @@ sub reset {
     $FU::REQ->{reshdr} = {
         'content-type', 'text/html',
     };
+    delete $FU::REQ->{rescookie};
 }
 
 
 sub _validate_header($hdr, $val) {
     confess "Invalid response header '$hdr'" if $hdr !~ /^$FU::hdrname_re$/;
-    confess "Invalid attempt to set response header containing a newline" if $val =~ /[\r\n]/;
+    confess "Invalid attempt to set response header containing a newline" if defined $val && $val =~ /[\r\n]/;
 }
 
 sub add_header($, $hdr, $val) {
@@ -709,6 +798,45 @@ sub set_header($, $hdr, $val=undef) {
     $FU::REQ->{reshdr}{ lc $hdr } = $val;
 }
 
+sub set_cookie($, $name, $val=undef, %opt) {
+    confess "Invalid cookie name '$name'" if $name !~ /^$FU::hdrname_re$/;
+    return delete $FU::REQ->{rescookie}{$name} if !defined $val;
+    confess "Invalid cookie value: $val" if $val =~ /[\0-\x1f\x7f-\x{10ffff}\s\r\n\t",;\\]/;
+    my $c = "$name=$val";
+    for my ($k,$v) (%opt) {
+        $k = lc $k; # attributes are case-insensitive
+        if ($k eq 'domain') {
+            confess "Invalid cookie domain: $v" if $v !~ $FU::Validate::re_domain;
+        } elsif ($k eq 'expires') {
+            confess "Cookie 'Expires' attribute should be a UNIX timestamp" if defined $v && $v !~ /^[0-9]+$/;
+            $v = FU::Util::httpdate_format($v || 0);
+        } elsif ($k eq 'httponly') {
+            $c .= "; $k" if $v;
+            next;
+        } elsif ($k eq 'max-age') {
+            confess "Invalid 'Max-Age' cookie attribute: $v" if $v !~ /^[0-9]+$/;
+        } elsif ($k eq 'partitioned') {
+            $c .= "; $k" if $v;
+            next;
+        } elsif ($k eq 'path') {
+            confess "Invalid 'Path' cookie attribute: $v" if $v =~ /[\0-\x1f\x7f-\x{10ffff}\s\r\n\t",;\\]/;
+        } elsif ($k eq 'secure') {
+            $c .= "; $k" if $v;
+            next;
+        } elsif ($k eq 'samesite') {
+            confess "Invalid 'SameSite' cookie attribute: $v" if $v !~ /^(?:Strict|Lax|None)$/;
+        }
+        $c .= "; $k=$v";
+    }
+    $FU::REQ->{rescookie}{$name} = $c;
+}
+
+sub send_json($, $data) {
+    fu->set_header('content-type', 'application/json');
+    fu->set_body(FU::Util::json_format($data, canonical => 1, utf8 => 1));
+    fu->done;
+}
+
 sub send_file($, $root, $path) {
     # This also catches files with '..' somewhere in the middle of the name.
     # Let's just disallow that to simplify this check, I'd err on the side of
@@ -742,7 +870,6 @@ sub send_file($, $root, $path) {
 
 sub redirect($, $code, $location) {
     state $alias = {qw/ perm 301  temp 302  tempget 303  tempsame 307  permsame 308 /};
-    fu->reset;
     fu->status($alias->{$code} // $code);
     fu->set_header(location => "$location");
     fu->set_header('content-type', 'text/plain');
@@ -775,9 +902,12 @@ sub _error_page($, $code, $title, $msg) {
 }
 
 sub _finalize {
-    state $haszlib = eval { require Compress::Raw::Zlib; 1 };
+    state $hasgzip = FU::Util::gzip_lib();
+    state $hasbrotli = eval { FU::Util::brotli_compress(6, ''); 1 };
     my $r = $FU::REQ;
 
+    fu->add_header('set-cookie', $_) for $r->{rescookie} ? sort values $r->{rescookie}->%* : ();
+
     if ($r->{status} == 204 || $r->{status} == 304) {
         delete $r->{reshdr}{'content-length'};
         delete $r->{reshdr}{'content-encoding'};
@@ -785,21 +915,24 @@ sub _finalize {
         $r->{resbody} = '';
 
     } else {
-        if ($haszlib && length($r->{resbody}) > 256
+        my @vary = ref $r->{reshdr}{vary} eq 'ARRAY' ? $r->{reshdr}{vary}->@* : defined $r->{reshdr}{vary} ? ($r->{reshdr}{vary}) : ();
-                && !defined $r->{reshdr}{'content-encoding'} && FU::compress_mimes->{$r->{reshdr}{'content-type'}}) {
+        if (($hasgzip || $hasbrotli) && length($r->{resbody}) > 256
+                && !defined $r->{reshdr}{'content-encoding'}
+                && FU::compress_mimes->{$r->{reshdr}{'content-type'}}
+        ) {
+            push @vary, 'accept-encoding';
+            if ($hasbrotli && ($r->{hdr}{'accept-encoding'}||'') =~ /\bbr\b/) {
+                $r->{resbody_orig} = $r->{resbody};
+                $r->{resbody} = FU::Util::brotli_compress(6, $r->{resbody});
+                $r->{reshdr}{'content-encoding'} = 'br';
 
-            $r->{reshdr}{'vary'} = ($r->{reshdr}{'vary'} ? $r->{reshdr}{'vary'}.', ' : '').'accept-encoding'
+            } elsif ($hasgzip && ($r->{hdr}{'accept-encoding'}||'') =~ /\bgzip\b/) {
-                if ($r->{reshdr}{'vary'}||'') !~ /accept-encoding/i;
+                $r->{resbody_orig} = $r->{resbody};
-
+                $r->{resbody} = FU::Util::gzip_compress(6, $r->{resbody});
-            if ($haszlib && ($r->{hdr}{'accept-encoding'}||'') =~ /gzip/) {
-                # Use lower-level API because the higher-level Compress::Zlib loads a whole bunch of other modules.
-                my $z = Compress::Raw::Zlib::Deflate->new(-WindowBits => Compress::Raw::Zlib::WANT_GZIP(), -Level => 3, -AppendOutput => 1);
-                $z->deflate($r->{resbody}, my $buf);
-                $z->flush($buf);
-                $r->{resbody} = $buf;
                 $r->{reshdr}{'content-encoding'} = 'gzip';
             }
         }
+        $r->{reshdr}{vary} = @vary ? join ', ', @vary : undef;
         $r->{reshdr}{'content-length'} = length $r->{resbody};
         $r->{resbody} = '' if (fu->method//'') eq 'HEAD';
     }
@@ -849,15 +982,7 @@ __END__
 
 =head1 NAME
 
-FU - Framework Ultimatum: A Lean and Efficient Zero-Dependency Web Framework.
+FU - A Lean and Efficient Zero-Dependency Web Framework.
-
-=head1 EXPERIMENTAL
-
-This module is still in development: it's missing important functionality and
-there will likely be a few breaking API changes. This framework currently
-powers manned.org as a test. I'll do a stable 1.0 release once FU is used in
-production for vndb.org, which will take a few months in the best case
-scenario.
 
 =head1 SYNOPSIS
 
@@ -875,7 +1000,7 @@ scenario.
   }
 
   FU::get qr{/hello/(.+)}, sub($who) {
-      my_html_ "Website title", sub {
+      myhtml_ "Website title", sub {
           h1_ "Hello, $who!";
       };
   };
@@ -884,6 +1009,11 @@ scenario.
 
 =head1 DESCRIPTION
 
+FU is the backend web framework developed for L<VNDB.org|https://vndb.org/> and
+L<Manned.org|https://manned.org/>, but is also perfectly suitable for other
+projects. Besides a web framework, this distrubion also includes a bunch of
+handy utility functions and modules.
+
 =head2 Distribution Overview
 
 This top-level C<FU> module is a web development framework. The C<FU>
@@ -916,6 +1046,12 @@ is). There are a few additional optional dependencies:
 =item * C<libpq.so> - required for L<FU::Pg>, dynamically loaded through
 C<dlopen()>.
 
+=item * C<libdeflate.so> or C<libz-ng.so> or C<libz.so> - required for
+C<gzip_compress()> in L<FU::Util> and used for HTTP output compression.
+
+=item * C<libbrotlienc.so> - required for C<brotli_compress()> in L<FU::Util>
+and used for HTTP output compression.
+
 =back
 
 
@@ -954,16 +1090,22 @@ certainly not great if you plan to transfer large files.
 =back
 
 The rest of this document is reference documentation; there's no easy
-introductionary cookbook-style docs yet, sorry about that.
+introductory cookbook-style docs yet, sorry about that.
 
 Unless specifically mentioned otherwise, all methods and functions taking or
 returning strings deal with perl Unicode strings, not raw bytes.
 
 
-=head2 Framework Configuration
+=head1 Framework Configuration
 
 =over
 
+=item use FU -procname => $name
+
+When the C<-procname> import option is set, FU automatically updates the
+process name (as displayed in L<top(1)> and L<ps(1)>, see C<$0>) with
+information about the current process, prefixed with the given C<$name>.
+
 =item FU::init_db($info)
 
 Set database configuration. C<$info> can either be a connection string for C<<
@@ -1057,7 +1199,7 @@ restart loop.
 =back
 
 
-=head2 Handlers & Routing
+=head1 Handlers & Routing
 
 =over
 
@@ -1124,12 +1266,18 @@ for a certain error code, C<500> is used as fallback.
 
 =back
 
+All of the above C<$sub> callbacks are allowed to throw an error. Special
+handling is given to exceptions generated by C<< fu->error() >>, which are
+relegated to the appropriate C<on_error> handler, and errors thrown by the
+C<validate()> method of L<FU::Validate>, which result in the C<400> error
+handler being run. Any other exception is passed to the C<500> error handler.
 
-=head2 The 'fu' Object
+
+=head1 The 'fu' Object
 
 While the C<FU::> namespace is used for global configuration and utility
 functions, the C<fu> object is intended for methods that deal with request
-processing (although some are useful used outside of request handlers as well).
+processing (although some are useful outside of request handlers as well).
 
 The C<fu> object itself can be used to store request-local data. For example,
 the following is a valid approach to handle user authentication:
@@ -1167,15 +1315,23 @@ has successfully been processed, or rolled back if there was an error.
 
 =item fu->sql($query, @params)
 
-Convenient short-hand for C<< fu->db->q($query, @params) >>.
+Convenient short-hand for C<< fu->db->sql($query, @params) >>.
 
 =item fu->SQL(@args)
 
-Convenient short-hand for C<< fu->db->Q(@args) >>.
+Convenient short-hand for C<< fu->db->SQL(@args) >>.
+
+=item fu->log_verbose($message)
+
+Write a verbose multi-line message to the log, including a full dump of
+information about the request: IP, headers and (potentially reformatted and/or
+truncated) body. This extra info is only written once per request, further
+calls to C<log_verbose()> just go directly to L<FU::Log>'s C<log_write()>
+instead.
 
 =back
 
-=head2 Request Information
+=head1 Request Information
 
 =over
 
@@ -1211,15 +1367,24 @@ C<https://example.com/some/path?query> this returns C<query>.
 =item fu->query($name)
 
 Parses the raw query string with C<query_decode> in L<FU::Util> and returns the
-value with the given $name. Beware: multiple values are returned as an array.
+value with the given $name. Beware: an array is returned if the given key is
-Prefer to use the C<$schema>-based validation methods below to reliably handle
+repeated in the query string.  Prefer to use the C<$schema>-based validation
-all sorts of query strings.
+methods below to reliably handle all sorts of query strings.
 
 =item fu->query($name => $schema)
 
 Parse, validate and return the query parameter identified by C<$name> with the
-given L<FU::Validate> schema. Calls C<< fu->error(400) >> with a useful error
+given L<FU::Validate> schema.
-message if validation fails.
+
+To fetch a query parameter that may have multiple values, use:
+
+  my $arrayref = fu->query(q => {accept_scalar => 1});
+
+  # OR:
+  my $first_value = fu->query(q => {accept_array => 'first'});
+
+  # OR:
+  my $last_value = fu->query(q => {accept_array => 'last'});
 
 =item fu->query($schema)
 
@@ -1236,22 +1401,40 @@ Parse, validate and return multiple query parameters.
   # Or, more concisely:
   my $data = fu->query(a => {anybool => 1}, b => {});
 
-=item fu->formdata($name)
+To fetch all query paramaters as decoded by C<query_decode()>, use:
 
-=item fu->formdata($schema)
+  my $data = fu->query({type=>'any'});
 
-Like C<< fu->query() >> but returns data from the POST request body.
+=item fu->cookie(...)
+
+Like C<< fu->query() >> but parses the C<Cookie> request header. Beware that,
+exactly like with query parameters, it's possible for a cookie to have multiple
+values and thus get represented as an array.
+
+=item fu->json(...)
+
+Like C<< fu->query() >> but parses the request body as JSON. Returns the raw
+(unvalidated!) JSON Unicode string if no arguments are given. To retrieve the
+decoded JSON data without performing further validation, use:
+
+  my $data = fu->json({type=>'any'});
+
+=item fu->formdata(...)
+
+Like C<< fu->query() >> but returns data from the POST request body. This
+method only supports form data encoded as C<application/x-www-form-urlencoded>,
+which is the default for HTML C<< <form> >>s. To handle multipart form data,
+use C<< fu->multipart >> instead.
+
+=item fu->multipart
+
+Parse the request body as C<multipart/form-data> and return an array of field
+objects.  Refer to L<FU::MultipartFormData> for more information.
 
 =back
 
-I<TODO:> Support C<multipart/form-data> and file uploads.
 
-I<TODO:> Support JSON bodies.
+=head1 Response Generation
-
-I<TODO:> Cookie parsing.
-
-
-=head2 Generating Responses
 
 =over
 
@@ -1295,6 +1478,31 @@ Add a response header, can be used to add multiple headers with the same name.
 Add a response header or overwrite the header with a new value if it already
 exists. Set C<$value> to undef to remove a previously set header.
 
+=item fu->set_cookie($name, $value, %attributes)
+
+Set or overwrite a cookie. Set C<$value> to undef to remove a previously set
+cookie. To fully remove a cookie from the user's browser, set the cookie with
+an empty value and zero C<Max-Age>:
+
+  fu->set_cookie(my_cookie => '', 'Max-Age' => 0);
+
+C<%attributes> can be any of the supported L<cookie
+attributes|https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Set-Cookie>.
+The C<Expires> attribute, when given, must be a UNIX timestamp. Boolean
+attributes are interpreted according to Perl's idea of truthiness. For example:
+
+  fu->set_cookie(auth => $auth_token,
+      Expires => time()+30*24*3600,
+      Domain => 'example.com',
+      Secure => 1,
+      SameSite => 'Lax'
+  );
+
+This method does not encode or escape the cookie value in any way. If you want
+to set a non-ASCII value or a value containing characters that are not
+permitted in the C<Set-Cookie> header, use C<uri_escape()> in L<FU::Util> or
+your favorite alternative cookie-safe encoding.
+
 =item fu->set_body($data)
 
 Set the (raw, binary) body of the response to C<$data>. This method is not very
@@ -1309,6 +1517,12 @@ templating system or L<FU::XMLWriter>:
       };
   });
 
+=item fu->send_json($data)
+
+Encode C<$data> as JSON (using C<json_format> in L<FU::Util>), set an
+appropriate C<Content-Type> header and send it to the client. Calls C<<
+fu->done >>.
+
 =item fu->send_file($root, $path)
 
 If a file identified by C<"$root/$path"> exists, set that as response and call
@@ -1360,12 +1574,9 @@ one of the following status codes or an alias:
 
 =back
 
-I<TODO:> Setting cookies.
-
-I<TODO:> JSON output.
 
 
-=head2 Running the Site
+=head1 Running the Site
 
 When your script is done setting L</"Framework Configuration"> and registering
 L</"Handlers & Routing">, it should call C<FU::run> to actually start serving

FU.xs

@@ -3,7 +3,7 @@
 #include <time.h> /* struct timespec & clock_gettime() */
 #include <string.h> /* strerror() */
 #include <arpa/inet.h> /* inet_ntop(), inet_ntoa() */
-#include <sys/socket.h> /* fd passing */
+#include <sys/socket.h> /* send(), fd passing */
 #include <sys/un.h> /* fd passing */
 #include <dlfcn.h> /* dlopen() etc */
 
@@ -18,7 +18,13 @@
 #define av_push_simple av_push
 #endif
 #ifndef BOOL_INTERNALS_sv_isbool_true
-#define BOOL_INTERNALS_sv_isbool_true(x) SvPVXtrue(x)
+#define BOOL_INTERNALS_sv_isbool_true(x) SvTRUEx(x)
+#endif
+#ifndef newSV_true
+#define newSV_true() newSVsv(&PL_sv_yes)
+#endif
+#ifndef newSV_false
+#define newSV_false() newSVsv(&PL_sv_no)
 #endif
 
 /* Disable key/value struct packing in khashl, so we can safely take a pointer
@@ -27,10 +33,12 @@
 
 #include "c/khashl.h"
 #include "c/common.c"
+
+#include "c/compress.c"
+#include "c/fcgi.c"
+#include "c/fdpass.c"
 #include "c/jsonfmt.c"
 #include "c/jsonparse.c"
-#include "c/fdpass.c"
-#include "c/fcgi.c"
 #include "c/xmlwr.c"
 
 #include "c/libpq.h"
@@ -53,7 +61,6 @@
         if (!ix) ix = FUPG_CACHE;\
         if (items == 1 || SvTRUE(ST(1))) x->stflags |= ix; \
         else x->stflags &= ~ix; \
-        XSRETURN(1); \
     } while(0)
 
 MODULE = FU
@@ -68,6 +75,7 @@ fuxmlwr *     FUXMLWR
 fupg_conn *   FUPG_CONN
 fupg_txn *    FUPG_TXN
 fupg_st *     FUPG_ST
+fupg_copy *   FUPG_COPY
 
 INPUT
 FUFCGI
@@ -89,6 +97,10 @@ FUPG_TXN
 FUPG_ST
     if (sv_derived_from($arg, \"FU::Pg::st\")) $var = (fupg_st *)SvIVX(SvRV($arg));
     else fu_confess(\"invalid statement object\");
+
+FUPG_COPY
+    if (sv_derived_from($arg, \"FU::Pg::copy\")) $var = (fupg_copy *)SvIVX(SvRV($arg));
+    else fu_confess(\"invalid COPY object\");
 #"
 EOT
 
@@ -110,6 +122,19 @@ void json_parse(SV *val, ...)
   CODE:
     ST(0) = fujson_parse_xs(aTHX_ ax, items, val);
 
+void gzip_lib()
+  PROTOTYPE:
+  CODE:
+    ST(0) = sv_2mortal(newSVpv(fugz_lib(), 0));
+
+void gzip_compress(IV level, SV *in)
+  CODE:
+    ST(0) = fugz_compress(aTHX_ level, in);
+
+void brotli_compress(IV level, SV *in)
+  CODE:
+    ST(0) = fubr_compress(aTHX_ level, in);
+
 void fdpass_send(int socket, int fd, SV *data)
   CODE:
     STRLEN buflen;
@@ -145,11 +170,11 @@ void print(fufcgi *ctx, SV *sv)
   CODE:
     STRLEN len;
     const char *buf = SvPVbyte(sv, len);
-    fufcgi_print(ctx, buf, len);
+    fufcgi_print(aTHX_ ctx, buf, len);
 
 void flush(fufcgi *ctx)
   CODE:
-    fufcgi_done(ctx);
+    fufcgi_done(aTHX_ ctx);
 
 void DESTROY(fufcgi *ctx)
   CODE:
@@ -192,10 +217,34 @@ void query_trace(fupg_conn *c, SV *cb)
     SvGETMAGIC(cb);
     c->trace = SvOK(cb) ? SvREFCNT_inc(cb) : NULL;
 
+void conn(fupg_conn *c)
+  CODE:
+    ST(0) = sv_newmortal();
+    sv_setrv_inc(ST(0), c->self);
+    sv_bless(ST(0), gv_stashpv("FU::Pg::conn", 0));
+
 void status(fupg_conn *c)
   CODE:
     ST(0) = sv_2mortal(newSVpv(fupg_conn_status(c), 0));
 
+void escape_literal(fupg_conn *c, SV *v)
+  CODE:
+    STRLEN len;
+    const char *str = SvPVutf8(v, len);
+    char *r = PQescapeLiteral(c->conn, str, len);
+    if (!r) fupg_conn_croak(c, "escapeLiteral");
+    ST(0) = newSVpvn_flags(r, strlen(r), SVf_UTF8|SVs_TEMP);
+    PQfreemem(r);
+
+void escape_identifier(fupg_conn *c, SV *v)
+  CODE:
+    STRLEN len;
+    const char *str = SvPVutf8(v, len);
+    char *r = PQescapeIdentifier(c->conn, str, len);
+    if (!r) fupg_conn_croak(c, "escapeIdentifier");
+    ST(0) = newSVpvn_flags(r, strlen(r), SVf_UTF8|SVs_TEMP);
+    PQfreemem(r);
+
 void cache(fupg_conn *x, ...)
   ALIAS:
     FU::Pg::conn::text_params  = FUPG_TEXT_PARAMS
@@ -228,16 +277,37 @@ void exec(fupg_conn *c, SV *sv)
     FUPG_CONN_COOKIE;
     ST(0) = fupg_exec(aTHX_ c, SvPVutf8_nolen(sv));
 
-void q(fupg_conn *c, SV *sv, ...)
+void sql(fupg_conn *c, SV *sv, ...)
   CODE:
     FUPG_CONN_COOKIE;
-    ST(0) = fupg_q(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items);
+    ST(0) = fupg_sql(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items);
+
+void copy(fupg_conn *c, SV *sv)
+  CODE:
+    FUPG_CONN_COOKIE;
+    ST(0) = fupg_copy_exec(aTHX_ c, SvPVutf8_nolen(sv));
 
 void _set_type(fupg_conn *c, SV *name, SV *sendsv, SV *recvsv)
   CODE:
     fupg_set_type(aTHX_ c, name, sendsv, recvsv);
     XSRETURN(1);
 
+void perl2bin(fupg_conn *c, int oid, SV *sv)
+  CODE:
+    ST(0) = fupg_perl2bin(aTHX_ c, oid, sv);
+
+void bin2perl(fupg_conn *c, int oid, SV *sv)
+  CODE:
+    ST(0) = fupg_bin2perl(aTHX_ c, oid, sv);
+
+void bin2text(fupg_conn *c, ...)
+  CODE:
+    XSRETURN(fupg_bintext(aTHX_ c, 0, ax, items));
+
+void text2bin(fupg_conn *c, ...)
+  CODE:
+    XSRETURN(fupg_bintext(aTHX_ c, 1, ax, items));
+
 
 MODULE = FU   PACKAGE = FU::Pg::txn
 
@@ -253,6 +323,12 @@ void cache(fupg_txn *x, ...)
   CODE:
     FUPG_STFLAGS;
 
+void conn(fupg_txn *t)
+  CODE:
+    ST(0) = sv_newmortal();
+    sv_setrv_inc(ST(0), t->conn->self);
+    sv_bless(ST(0), gv_stashpv("FU::Pg::conn", 0));
+
 void status(fupg_txn *t)
   CODE:
     ST(0) = sv_2mortal(newSVpv(fupg_txn_status(t), 0));
@@ -277,10 +353,16 @@ void exec(fupg_txn *t, SV *sv)
     FUPG_TXN_COOKIE;
     ST(0) = fupg_exec(aTHX_ t->conn, SvPVutf8_nolen(sv));
 
-void q(fupg_txn *t, SV *sv, ...)
+void sql(fupg_txn *t, SV *sv, ...)
   CODE:
     FUPG_TXN_COOKIE;
-    ST(0) = fupg_q(aTHX_ t->conn, t->stflags, SvPVutf8_nolen(sv), ax, items);
+    ST(0) = fupg_sql(aTHX_ t->conn, t->stflags, SvPVutf8_nolen(sv), ax, items);
+
+# XXX: The copy object should probably keep a ref on the transaction
+void copy(fupg_txn *t, SV *sv)
+  CODE:
+    FUPG_TXN_COOKIE;
+    ST(0) = fupg_copy_exec(aTHX_ t->conn, SvPVutf8_nolen(sv));
 
 
 
@@ -294,6 +376,7 @@ void cache(fupg_st *x, ...)
   CODE:
     if (ix == 0 && x->prepared) fu_confess("Invalid attempt to change statement configuration after it has already been prepared or executed");
     FUPG_STFLAGS;
+    XSRETURN(1);
 
 void exec(fupg_st *st)
   CODE:
@@ -393,6 +476,28 @@ void DESTROY(fupg_st *st)
     fupg_st_destroy(aTHX_ st);
 
 
+MODULE = FU   PACKAGE = FU::Pg::copy
+
+void write(fupg_copy *c, SV *sv)
+  CODE:
+    fupg_copy_write(aTHX_ c, sv);
+
+void read(fupg_copy *c)
+  CODE:
+    ST(0) = fupg_copy_read(aTHX_ c, 0);
+
+void is_binary(fupg_copy *c)
+  CODE:
+    ST(0) = c->bin ? &PL_sv_yes : &PL_sv_no;
+
+void close(fupg_copy *c)
+  CODE:
+    fupg_copy_close(aTHX_ c, 0);
+
+void DESTROY(fupg_copy *c)
+  CODE:
+    fupg_copy_destroy(aTHX_ c);
+
 
 MODULE = FU   PACKAGE = FU::XMLWriter
 

FU/Benchmarks.pod

@@ -26,21 +26,25 @@ The following module versions were used:
 
 =over
 
-=item L<Cpanel::JSON::XS> 4.38
+=item L<Cpanel::JSON::XS> 4.40
 
-=item L<FU> 0.1
+=item L<DBD::Pg> 3.18.0
+
+=item L<FU> 1.4
 
 =item L<HTML::Tiny> 1.08
 
 =item L<JSON::PP> 4.16
 
-=item L<JSON::SIMD> 1.06
+=item L<JSON::SIMD> 1.07
 
 =item L<JSON::Tiny> 0.58
 
-=item L<JSON::XS> 4.03
+=item L<JSON::XS> 4.04
 
-=item L<TUWF::XML> 1.5
+=item L<Pg::PQ> 0.15
+
+=item L<TUWF::XML> 1.6
 
 =item L<XML::Writer> 0.900
 
@@ -56,266 +60,294 @@ These benchmarks run on large-ish arrays with repeated values. JSON encoding is
 sufficiently fast that Perl function calling overhead tends to dominate for
 smaller inputs, but I don't find that overhead very interesting.
 
-Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
+Also worth noting that L<JSON::SIMD> formatting code is forked from
-SIMD parts are only used for parsing.
+L<JSON::XS>, the SIMD parts are only used for parsing.
 
 API object from L<JSON::XS> documentation.
 
                         Encode   Canonical      Decode
-          JSON::PP      5312/s      5119/s      1290/s
+          JSON::PP      5136/s      4943/s      1240/s
-        JSON::Tiny      7757/s           -      3426/s
+        JSON::Tiny      7617/s           -      3474/s
-  Cpanel::JSON::XS    108187/s    101867/s    103575/s
+  Cpanel::JSON::XS    108128/s     98734/s    105811/s
-        JSON::SIMD    130137/s    118948/s    115123/s
+        JSON::SIMD    125105/s    114822/s    118410/s
-          JSON::XS    128421/s    120243/s    117940/s
+          JSON::XS    128749/s    117518/s    120190/s
-          FU::Util    133182/s    113275/s    118213/s
+          FU::Util    126909/s    109166/s    113983/s
 
 Object (small)
 
                         Encode   Canonical      Decode
-          JSON::PP       907/s       829/s       202/s
+          JSON::PP       896/s       826/s       194/s
-        JSON::Tiny      1224/s           -       499/s
+        JSON::Tiny      1216/s           -       519/s
-  Cpanel::JSON::XS     43168/s     28114/s     19229/s
+  Cpanel::JSON::XS     44184/s     28190/s     19449/s
-        JSON::SIMD     49019/s     30699/s     23267/s
+        JSON::SIMD     52633/s     31157/s     23587/s
-          JSON::XS     49814/s     31326/s     25336/s
+          JSON::XS     50314/s     34276/s     25294/s
-          FU::Util     44110/s     26134/s     21144/s
+          FU::Util     42121/s     25618/s     19203/s
 
 Object (large)
 
                         Encode   Canonical      Decode
-          JSON::PP       927/s       747/s       104/s
+          JSON::PP       910/s       734/s        98/s
-        JSON::Tiny      1108/s           -       392/s
+        JSON::Tiny      1068/s           -       404/s
-  Cpanel::JSON::XS     29672/s     12637/s     16609/s
+  Cpanel::JSON::XS     27626/s     12484/s     15333/s
-        JSON::SIMD     24418/s     12388/s     22895/s
+        JSON::SIMD     34106/s     12808/s     23674/s
-          JSON::XS     23192/s     13174/s     23553/s
+          JSON::XS     35738/s     13099/s     22637/s
-          FU::Util     39477/s     13567/s     17178/s
+          FU::Util     37663/s     13366/s     16292/s
 
 Object (large, mixed unicode)
 
                         Encode   Canonical      Decode
-          JSON::PP       817/s       679/s        86/s
+          JSON::PP       835/s       664/s        82/s
-        JSON::Tiny      1036/s           -       402/s
+        JSON::Tiny      1028/s           -       427/s
-  Cpanel::JSON::XS     20437/s      1345/s      7408/s
+  Cpanel::JSON::XS     24123/s      1352/s      8694/s
-        JSON::SIMD     25031/s      1331/s     15997/s
+        JSON::SIMD     26008/s      1413/s     19707/s
-          JSON::XS     23580/s      1375/s      8526/s
+          JSON::XS     25444/s      1391/s     10442/s
-          FU::Util     34435/s     11916/s      9419/s
+          FU::Util     33132/s     12006/s     11861/s
 
 Small integers
 
                         Encode      Decode
-          JSON::PP       113/s        29/s
+          JSON::PP       116/s        30/s
-        JSON::Tiny       160/s        86/s
+        JSON::Tiny       158/s        86/s
-  Cpanel::JSON::XS      7137/s      6083/s
+  Cpanel::JSON::XS      7426/s      5774/s
-        JSON::SIMD      7963/s      4361/s
+        JSON::SIMD      8294/s      4375/s
-          JSON::XS      7915/s      6058/s
+          JSON::XS      8526/s      6179/s
-          FU::Util      8565/s      5639/s
+          FU::Util      7996/s      5962/s
 
 Large integers
 
                         Encode      Decode
-          JSON::PP      2176/s       329/s
+          JSON::PP      2213/s       341/s
-        JSON::Tiny      2999/s      1638/s
+        JSON::Tiny      2910/s      1661/s
-  Cpanel::JSON::XS     31302/s     48892/s
+  Cpanel::JSON::XS     32616/s     53053/s
-        JSON::SIMD     37201/s     51719/s
+        JSON::SIMD     37749/s     53032/s
-          JSON::XS     36722/s     50110/s
+          JSON::XS     38644/s     55004/s
-          FU::Util    116188/s     62110/s
+          FU::Util    109930/s     63358/s
 
 ASCII strings
 
                         Encode      Decode
-          JSON::PP      2934/s       336/s
+          JSON::PP      2811/s       312/s
-        JSON::Tiny      4126/s      1439/s
+        JSON::Tiny      3924/s      1506/s
-  Cpanel::JSON::XS    116744/s     43489/s
+  Cpanel::JSON::XS    129468/s     51536/s
-        JSON::SIMD    134711/s     50429/s
+        JSON::SIMD    140393/s     64499/s
-          JSON::XS    135419/s     43976/s
+          JSON::XS    141149/s     56913/s
-          FU::Util    182026/s     44312/s
+          FU::Util    165938/s     55034/s
 
 Unicode strings
 
                         Encode      Decode
-          JSON::PP      5113/s       253/s
+          JSON::PP      5138/s       248/s
-        JSON::Tiny      6603/s      2585/s
+        JSON::Tiny      6501/s      2677/s
-  Cpanel::JSON::XS     91704/s     64489/s
+  Cpanel::JSON::XS     91004/s     64101/s
-        JSON::SIMD    106928/s    102440/s
+        JSON::SIMD    101185/s     80941/s
-          JSON::XS    105473/s     60558/s
+          JSON::XS    106312/s     61104/s
-          FU::Util    217135/s     58972/s
+          FU::Util    205716/s     52041/s
 
 String escaping (few)
 
                         Encode      Decode
-          JSON::PP      4251/s       352/s
+          JSON::PP      4269/s       329/s
-        JSON::Tiny      4704/s      1869/s
+        JSON::Tiny      4878/s      2101/s
-  Cpanel::JSON::XS    131789/s    106306/s
+  Cpanel::JSON::XS    152958/s    105597/s
-        JSON::SIMD    158171/s    153692/s
+        JSON::SIMD    165340/s    130074/s
-          JSON::XS    157261/s     97676/s
+          JSON::XS    165863/s     87872/s
-          FU::Util    191699/s     91177/s
+          FU::Util    228511/s     81599/s
 
 String escaping (many)
 
                         Encode      Decode
-          JSON::PP      2224/s       366/s
+          JSON::PP      4052/s       573/s
-        JSON::Tiny      2884/s       984/s
+        JSON::Tiny      4575/s      2274/s
-  Cpanel::JSON::XS    136583/s    100789/s
+  Cpanel::JSON::XS    201958/s    102800/s
-        JSON::SIMD    152951/s    113242/s
+        JSON::SIMD    242806/s    146341/s
-          JSON::XS    153471/s    106269/s
+          JSON::XS    209689/s     98420/s
-          FU::Util    142604/s     97984/s
+          FU::Util    210713/s    100255/s
 
 
 
 =head2 XML Writing
 
+L<FU::XMLWriter> is the only XS-based XML DSL that I'm aware of, so all direct
+competition is inherently slower by virtue of being pure perl. I'm sure some
+templating modules will perform better, though.
+
 HTML fragment
 
-         TUWF::XML       795/s
+         TUWF::XML       787/s
-       XML::Writer       833/s
+       XML::Writer       832/s
-        HTML::Tiny       423/s
+        HTML::Tiny       403/s
-     FU::XMLWriter      5285/s
+     FU::XMLWriter      5192/s
+
+
+
+=head2 PostgreSQL client
+
+Fetching query results is highly unlikely to be a bottleneck in your code, this
+benchmark is mainly here to verify that L<FU::Pg> is not introducing a
+bottleneck where there shouldn't be one.
+
+Fetch and bitwise-or 20k integers
+
+                      Smallint      Bigint
+           DBD::Pg       346/s        33/s
+            Pg::PQ       270/s        24/s
+      FU::Pg (bin)       476/s        46/s
+     FU::Pg (text)       273/s        23/s
 
 
 
 =cut
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.
-json/api	Canonical	Cpanel::JSON::XS	101867
+json/api	Canonical	Cpanel::JSON::XS	98734
-json/api	Canonical	FU::Util	113275
+json/api	Canonical	FU::Util	109166
-json/api	Canonical	JSON::PP	5119
+json/api	Canonical	JSON::PP	4943
-json/api	Canonical	JSON::SIMD	118948
+json/api	Canonical	JSON::SIMD	114822
-json/api	Canonical	JSON::XS	120243
+json/api	Canonical	JSON::XS	117518
-json/api	Decode	Cpanel::JSON::XS	103575
+json/api	Decode	Cpanel::JSON::XS	105811
-json/api	Decode	FU::Util	118213
+json/api	Decode	FU::Util	113983
-json/api	Decode	JSON::PP	1290
+json/api	Decode	JSON::PP	1240
-json/api	Decode	JSON::SIMD	115123
+json/api	Decode	JSON::SIMD	118410
-json/api	Decode	JSON::Tiny	3426
+json/api	Decode	JSON::Tiny	3474
-json/api	Decode	JSON::XS	117940
+json/api	Decode	JSON::XS	120190
-json/api	Encode	Cpanel::JSON::XS	108187
+json/api	Encode	Cpanel::JSON::XS	108128
-json/api	Encode	FU::Util	133182
+json/api	Encode	FU::Util	126909
-json/api	Encode	JSON::PP	5312
+json/api	Encode	JSON::PP	5136
-json/api	Encode	JSON::SIMD	130137
+json/api	Encode	JSON::SIMD	125105
-json/api	Encode	JSON::Tiny	7757
+json/api	Encode	JSON::Tiny	7617
-json/api	Encode	JSON::XS	128421
+json/api	Encode	JSON::XS	128749
-json/intl	Decode	Cpanel::JSON::XS	48892
+json/intl	Decode	Cpanel::JSON::XS	53053
-json/intl	Decode	FU::Util	62110
+json/intl	Decode	FU::Util	63358
-json/intl	Decode	JSON::PP	329
+json/intl	Decode	JSON::PP	341
-json/intl	Decode	JSON::SIMD	51719
+json/intl	Decode	JSON::SIMD	53032
-json/intl	Decode	JSON::Tiny	1638
+json/intl	Decode	JSON::Tiny	1661
-json/intl	Decode	JSON::XS	50110
+json/intl	Decode	JSON::XS	55004
-json/intl	Encode	Cpanel::JSON::XS	31302
+json/intl	Encode	Cpanel::JSON::XS	32616
-json/intl	Encode	FU::Util	116188
+json/intl	Encode	FU::Util	109930
-json/intl	Encode	JSON::PP	2176
+json/intl	Encode	JSON::PP	2213
-json/intl	Encode	JSON::SIMD	37201
+json/intl	Encode	JSON::SIMD	37749
-json/intl	Encode	JSON::Tiny	2999
+json/intl	Encode	JSON::Tiny	2910
-json/intl	Encode	JSON::XS	36722
+json/intl	Encode	JSON::XS	38644
-json/ints	Decode	Cpanel::JSON::XS	6083
+json/ints	Decode	Cpanel::JSON::XS	5774
-json/ints	Decode	FU::Util	5639
+json/ints	Decode	FU::Util	5962
-json/ints	Decode	JSON::PP	29
+json/ints	Decode	JSON::PP	30
-json/ints	Decode	JSON::SIMD	4361
+json/ints	Decode	JSON::SIMD	4375
 json/ints	Decode	JSON::Tiny	86
-json/ints	Decode	JSON::XS	6058
+json/ints	Decode	JSON::XS	6179
-json/ints	Encode	Cpanel::JSON::XS	7137
+json/ints	Encode	Cpanel::JSON::XS	7426
-json/ints	Encode	FU::Util	8565
+json/ints	Encode	FU::Util	7996
-json/ints	Encode	JSON::PP	113
+json/ints	Encode	JSON::PP	116
-json/ints	Encode	JSON::SIMD	7963
+json/ints	Encode	JSON::SIMD	8294
-json/ints	Encode	JSON::Tiny	160
+json/ints	Encode	JSON::Tiny	158
-json/ints	Encode	JSON::XS	7915
+json/ints	Encode	JSON::XS	8526
-json/objl	Canonical	Cpanel::JSON::XS	12637
+json/objl	Canonical	Cpanel::JSON::XS	12484
-json/objl	Canonical	FU::Util	13567
+json/objl	Canonical	FU::Util	13366
-json/objl	Canonical	JSON::PP	747
+json/objl	Canonical	JSON::PP	734
-json/objl	Canonical	JSON::SIMD	12388
+json/objl	Canonical	JSON::SIMD	12808
-json/objl	Canonical	JSON::XS	13174
+json/objl	Canonical	JSON::XS	13099
-json/objl	Decode	Cpanel::JSON::XS	16609
+json/objl	Decode	Cpanel::JSON::XS	15333
-json/objl	Decode	FU::Util	17178
+json/objl	Decode	FU::Util	16292
-json/objl	Decode	JSON::PP	104
+json/objl	Decode	JSON::PP	98
-json/objl	Decode	JSON::SIMD	22895
+json/objl	Decode	JSON::SIMD	23674
-json/objl	Decode	JSON::Tiny	392
+json/objl	Decode	JSON::Tiny	404
-json/objl	Decode	JSON::XS	23553
+json/objl	Decode	JSON::XS	22637
-json/objl	Encode	Cpanel::JSON::XS	29672
+json/objl	Encode	Cpanel::JSON::XS	27626
-json/objl	Encode	FU::Util	39477
+json/objl	Encode	FU::Util	37663
-json/objl	Encode	JSON::PP	927
+json/objl	Encode	JSON::PP	910
-json/objl	Encode	JSON::SIMD	24418
+json/objl	Encode	JSON::SIMD	34106
-json/objl	Encode	JSON::Tiny	1108
+json/objl	Encode	JSON::Tiny	1068
-json/objl	Encode	JSON::XS	23192
+json/objl	Encode	JSON::XS	35738
-json/objs	Canonical	Cpanel::JSON::XS	28114
+json/objs	Canonical	Cpanel::JSON::XS	28190
-json/objs	Canonical	FU::Util	26134
+json/objs	Canonical	FU::Util	25618
-json/objs	Canonical	JSON::PP	829
+json/objs	Canonical	JSON::PP	826
-json/objs	Canonical	JSON::SIMD	30699
+json/objs	Canonical	JSON::SIMD	31157
-json/objs	Canonical	JSON::XS	31326
+json/objs	Canonical	JSON::XS	34276
-json/objs	Decode	Cpanel::JSON::XS	19229
+json/objs	Decode	Cpanel::JSON::XS	19449
-json/objs	Decode	FU::Util	21144
+json/objs	Decode	FU::Util	19203
-json/objs	Decode	JSON::PP	202
+json/objs	Decode	JSON::PP	194
-json/objs	Decode	JSON::SIMD	23267
+json/objs	Decode	JSON::SIMD	23587
-json/objs	Decode	JSON::Tiny	499
+json/objs	Decode	JSON::Tiny	519
-json/objs	Decode	JSON::XS	25336
+json/objs	Decode	JSON::XS	25294
-json/objs	Encode	Cpanel::JSON::XS	43168
+json/objs	Encode	Cpanel::JSON::XS	44184
-json/objs	Encode	FU::Util	44110
+json/objs	Encode	FU::Util	42121
-json/objs	Encode	JSON::PP	907
+json/objs	Encode	JSON::PP	896
-json/objs	Encode	JSON::SIMD	49019
+json/objs	Encode	JSON::SIMD	52633
-json/objs	Encode	JSON::Tiny	1224
+json/objs	Encode	JSON::Tiny	1216
-json/objs	Encode	JSON::XS	49814
+json/objs	Encode	JSON::XS	50314
-json/obju	Canonical	Cpanel::JSON::XS	1345
+json/obju	Canonical	Cpanel::JSON::XS	1352
-json/obju	Canonical	FU::Util	11916
+json/obju	Canonical	FU::Util	12006
-json/obju	Canonical	JSON::PP	679
+json/obju	Canonical	JSON::PP	664
-json/obju	Canonical	JSON::SIMD	1331
+json/obju	Canonical	JSON::SIMD	1413
-json/obju	Canonical	JSON::XS	1375
+json/obju	Canonical	JSON::XS	1391
-json/obju	Decode	Cpanel::JSON::XS	7408
+json/obju	Decode	Cpanel::JSON::XS	8694
-json/obju	Decode	FU::Util	9419
+json/obju	Decode	FU::Util	11861
-json/obju	Decode	JSON::PP	86
+json/obju	Decode	JSON::PP	82
-json/obju	Decode	JSON::SIMD	15997
+json/obju	Decode	JSON::SIMD	19707
-json/obju	Decode	JSON::Tiny	402
+json/obju	Decode	JSON::Tiny	427
-json/obju	Decode	JSON::XS	8526
+json/obju	Decode	JSON::XS	10442
-json/obju	Encode	Cpanel::JSON::XS	20437
+json/obju	Encode	Cpanel::JSON::XS	24123
-json/obju	Encode	FU::Util	34435
+json/obju	Encode	FU::Util	33132
-json/obju	Encode	JSON::PP	817
+json/obju	Encode	JSON::PP	835
-json/obju	Encode	JSON::SIMD	25031
+json/obju	Encode	JSON::SIMD	26008
-json/obju	Encode	JSON::Tiny	1036
+json/obju	Encode	JSON::Tiny	1028
-json/obju	Encode	JSON::XS	23580
+json/obju	Encode	JSON::XS	25444
-json/strel	Decode	Cpanel::JSON::XS	100789
+json/strel	Decode	Cpanel::JSON::XS	102800
-json/strel	Decode	FU::Util	97984
+json/strel	Decode	FU::Util	100255
-json/strel	Decode	JSON::PP	366
+json/strel	Decode	JSON::PP	573
-json/strel	Decode	JSON::SIMD	113242
+json/strel	Decode	JSON::SIMD	146341
-json/strel	Decode	JSON::Tiny	984
+json/strel	Decode	JSON::Tiny	2274
-json/strel	Decode	JSON::XS	106269
+json/strel	Decode	JSON::XS	98420
-json/strel	Encode	Cpanel::JSON::XS	136583
+json/strel	Encode	Cpanel::JSON::XS	201958
-json/strel	Encode	FU::Util	142604
+json/strel	Encode	FU::Util	210713
-json/strel	Encode	JSON::PP	2224
+json/strel	Encode	JSON::PP	4052
-json/strel	Encode	JSON::SIMD	152951
+json/strel	Encode	JSON::SIMD	242806
-json/strel	Encode	JSON::Tiny	2884
+json/strel	Encode	JSON::Tiny	4575
-json/strel	Encode	JSON::XS	153471
+json/strel	Encode	JSON::XS	209689
-json/stres	Decode	Cpanel::JSON::XS	106306
+json/stres	Decode	Cpanel::JSON::XS	105597
-json/stres	Decode	FU::Util	91177
+json/stres	Decode	FU::Util	81599
-json/stres	Decode	JSON::PP	352
+json/stres	Decode	JSON::PP	329
-json/stres	Decode	JSON::SIMD	153692
+json/stres	Decode	JSON::SIMD	130074
-json/stres	Decode	JSON::Tiny	1869
+json/stres	Decode	JSON::Tiny	2101
-json/stres	Decode	JSON::XS	97676
+json/stres	Decode	JSON::XS	87872
-json/stres	Encode	Cpanel::JSON::XS	131789
+json/stres	Encode	Cpanel::JSON::XS	152958
-json/stres	Encode	FU::Util	191699
+json/stres	Encode	FU::Util	228511
-json/stres	Encode	JSON::PP	4251
+json/stres	Encode	JSON::PP	4269
-json/stres	Encode	JSON::SIMD	158171
+json/stres	Encode	JSON::SIMD	165340
-json/stres	Encode	JSON::Tiny	4704
+json/stres	Encode	JSON::Tiny	4878
-json/stres	Encode	JSON::XS	157261
+json/stres	Encode	JSON::XS	165863
-json/strs	Decode	Cpanel::JSON::XS	43489
+json/strs	Decode	Cpanel::JSON::XS	51536
-json/strs	Decode	FU::Util	44312
+json/strs	Decode	FU::Util	55034
-json/strs	Decode	JSON::PP	336
+json/strs	Decode	JSON::PP	312
-json/strs	Decode	JSON::SIMD	50429
+json/strs	Decode	JSON::SIMD	64499
-json/strs	Decode	JSON::Tiny	1439
+json/strs	Decode	JSON::Tiny	1506
-json/strs	Decode	JSON::XS	43976
+json/strs	Decode	JSON::XS	56913
-json/strs	Encode	Cpanel::JSON::XS	116744
+json/strs	Encode	Cpanel::JSON::XS	129468
-json/strs	Encode	FU::Util	182026
+json/strs	Encode	FU::Util	165938
-json/strs	Encode	JSON::PP	2934
+json/strs	Encode	JSON::PP	2811
-json/strs	Encode	JSON::SIMD	134711
+json/strs	Encode	JSON::SIMD	140393
-json/strs	Encode	JSON::Tiny	4126
+json/strs	Encode	JSON::Tiny	3924
-json/strs	Encode	JSON::XS	135419
+json/strs	Encode	JSON::XS	141149
-json/stru	Decode	Cpanel::JSON::XS	64489
+json/stru	Decode	Cpanel::JSON::XS	64101
-json/stru	Decode	FU::Util	58972
+json/stru	Decode	FU::Util	52041
-json/stru	Decode	JSON::PP	253
+json/stru	Decode	JSON::PP	248
-json/stru	Decode	JSON::SIMD	102440
+json/stru	Decode	JSON::SIMD	80941
-json/stru	Decode	JSON::Tiny	2585
+json/stru	Decode	JSON::Tiny	2677
-json/stru	Decode	JSON::XS	60558
+json/stru	Decode	JSON::XS	61104
-json/stru	Encode	Cpanel::JSON::XS	91704
+json/stru	Encode	Cpanel::JSON::XS	91004
-json/stru	Encode	FU::Util	217135
+json/stru	Encode	FU::Util	205716
-json/stru	Encode	JSON::PP	5113
+json/stru	Encode	JSON::PP	5138
-json/stru	Encode	JSON::SIMD	106928
+json/stru	Encode	JSON::SIMD	101185
-json/stru	Encode	JSON::Tiny	6603
+json/stru	Encode	JSON::Tiny	6501
-json/stru	Encode	JSON::XS	105473
+json/stru	Encode	JSON::XS	106312
-xml/a	Rate	FU::XMLWriter	5285
+pg/ints	Bigint	DBD::Pg	33
-xml/a	Rate	HTML::Tiny	423
+pg/ints	Bigint	FU::Pg (bin)	46
-xml/a	Rate	TUWF::XML	795
+pg/ints	Bigint	FU::Pg (text)	23
-xml/a	Rate	XML::Writer	833
+pg/ints	Bigint	Pg::PQ	24
+pg/ints	Smallint	DBD::Pg	346
+pg/ints	Smallint	FU::Pg (bin)	476
+pg/ints	Smallint	FU::Pg (text)	273
+pg/ints	Smallint	Pg::PQ	270
+xml/a	Rate	FU::XMLWriter	5192
+xml/a	Rate	HTML::Tiny	403
+xml/a	Rate	TUWF::XML	787
+xml/a	Rate	XML::Writer	832

FU/DebugImpl.pm

@@ -1,10 +1,11 @@
 # Internal module used by FU.pm
-package FU::DebugImpl 0.2;
+package FU::DebugImpl 1.4;
 use v5.36;
+use utf8;
 use experimental 'for_list';
 use FU;
 use FU::XMLWriter ':html5_', 'fragment', 'xml_escape';
-use Time::HiRes 'time';
+use Time::HiRes 'time', 'clock_gettime', 'CLOCK_MONOTONIC';
 use POSIX 'strftime';
 
 sub fmtts { strftime '%Y-%m-%d %H:%M:%S UTC', gmtime shift }
@@ -15,27 +16,33 @@ sub loc_($loc) {
         br_ if $_;
         my $l = $loc->[$_];
         my $f = $_ == $#$loc ? '(main)' : $loc->[$_+1][3];
-        txt_ "$l->[1]:$l->[2] $f";
+        $f = "$l->[0]::$f" if $f !~ /^\Q$l->[0]/;
+        txt_ $f;
+        small_ " @ $l->[1]:$l->[2]";
     }
 }
 
-sub fmtpre_($code) {
-    lit_ xml_escape($code) =~ s/^\s+//r =~ s/\s+$//r =~ s/\n/<br>/rg;
-}
-
 sub clean_re($str) {
     # Regex formatting isn't stable, but this cleans up the crap I'm seeing a little bit.
     "$str" =~ s/^\(\?\^u:\^\(\?\^u://r =~ s/\)\$\)$//r;
 }
 
-my @tabs = (
+sub raw_data($str) {
+    my $d = substr $str, 0, 32*1024;
+    my $trunc = length $str > 32*1024 ? ', truncated' : '';
+    return utf8::decode($d) ? ("utf8$trunc", $d)
+         : ("hex$trunc", unpack('H*', $d) =~ s/(.{128})/$1\n/rg =~ s/(.{16})/$1 /rg);
+}
+
+my @sections = (
     req => sub {
+        my $r = $FU::REQ;
         table_ sub {
             tr_ sub { td_ 'Method'; td_ fu->method };
             tr_ sub { td_ 'Path'; td_ fu->path };
             tr_ sub { td_ 'Query'; td_ fu->query };
             tr_ sub { td_ 'Client IP'; td_ fu->ip };
-            tr_ sub { td_ 'Received'; td_ fmtts $FU::REQ->{trace_start} };
+            tr_ sub { td_ 'Received'; td_ fmtts(time - (($r->{trace_end}||clock_gettime(CLOCK_MONOTONIC)) - $r->{trace_start})) };
         };
         h2_ 'Headers';
         table_ sub {
@@ -44,7 +51,38 @@ my @tabs = (
                 td_ fu->headers->{$_};
             } for sort keys fu->headers->%*;
         };
-        # TODO: Body? Certainly useful for JSON
+        if ((fu->header('content-length')||0) > 0) {
+            h2_ 'Body';
+            section_ class => 'tabs', sub {
+                my $json = eval { fu->json({type=>'any'}) };
+                details_ name => 'reqbody', open => !0, sub {
+                    summary_ 'JSON';
+                    pre_ FU::Util::json_format($json, pretty => 1, canonical => 1);
+                } if $json;
+                my $formdata = eval { fu->formdata({type=>'hash'}) };
+                details_ name => 'reqbody', open => !0, sub {
+                    summary_ 'Form data';
+                    table_ sub {
+                        for my $k (sort keys %$formdata) {
+                            tr_ sub {
+                                td_ $k;
+                                td_ $_;
+                            } for ref $formdata->{$k} ? $formdata->{$k}->@* : ($formdata->{$k});
+                        }
+                    };
+                } if $formdata;
+                my $multipart = eval { fu->multipart };
+                details_ name => 'reqbody', open => !0, sub {
+                    summary_ 'Multipart';
+                    pre_ join "\n", map $_->describe, @$multipart;
+                } if $multipart;
+                details_ name => 'reqbody', open => !0,sub {
+                    my($lbl, $data) = raw_data $r->{body};
+                    summary_ "Raw ($lbl)";
+                    pre_ $data;
+                };
+            }
+        }
         ('Request')
     },
 
@@ -75,37 +113,111 @@ my @tabs = (
         };
         h2_ 'Headers';
         table_ sub {
+            for my $k (sort keys $r->{reshdr}->%*) {
+                my $v = $r->{reshdr}{$k};
                 tr_ sub {
+                    td_ $k;
                     td_ $_;
-                td_ $r->{reshdr}{$_};
+                } for !defined $v ? () : ref $v ? @$v : ($v);
-            } for sort keys $r->{reshdr}->%*;
+            }
         };
+        my $body = $r->{resbody_orig} // $r->{resbody};
+        if (length $body) {
+            h2_ 'Body';
+            section_ class => 'tabs', sub {
+                my $json = ($r->{reshdr}{'content-type'}||'') =~ /^application\/json/ && eval { FU::Util::json_parse($body, utf8 => 1) };
+                details_ name => 'resbody', open => !0, sub {
+                    summary_ 'JSON';
+                    pre_ FU::Util::json_format($json, pretty => 1, canonical => 1);
+                } if $json;
+                details_ name => 'resbody', open => !0,sub {
+                    my($lbl, $data) = raw_data $body;
+                    summary_ "Raw ($lbl)";
+                    pre_ $data;
+                };
+            }
+        }
         ('Response')
     },
 
     sql => sub {
-        return () if !$FU::REQ->{trace_sql};
+        my $queries = $FU::REQ->{trace_sql};
-        table_ sub {
+        return () if !$queries;
+
+        # Convert binary params to text.
+        # For queries with text_params, assume the params are already valid for the text format.
+        my @binparams = grep $_->{type} && !exists $_->{text}, map $_->{params}->@*, @$queries;
+        my @arg = map +($_->{type}, $_->{bin}), @binparams;
+        my @text;
+        my $ok = !@arg || eval { @text = $FU::DB->bin2text(@arg); 1 };
+        $binparams[$_]{text} = $text[$_] for 0..$#text;
+        pre_ "Error converting binary parameters:\n$@" if !$ok;
+
+        input_ type => 'checkbox', id => "row${_}_c" for 0..$#{$queries};
+        table_ class => 'sqlt', sub {
             thead_ sub { tr_ sub {
                 td_ class => 'num', 'Exec';
                 td_ class => 'num', 'Prep';
                 td_ class => 'num', 'Rows';
                 td_ 'Query';
             } };
+            my $rows = 0;
+            for my($i, $st) (builtin::indexed $queries->@*) {
+                $rows += $st->{nrows};
                 tr_ sub {
-                td_ class => 'num', sprintf '%.1f ms', $_->{exec_time}*1000;
+                    td_ class => 'num', sprintf '%.1f ms', $st->{exec_time}*1000;
-                td_ class => 'num', !defined $_->{prepare_time} ? '-' : $_->{prepare_time} ? sprintf '%.1f ms', $_->{prepare_time}*1000 : 'cache';
+                    td_ class => 'num', !defined $st->{prepare_time} ? '-' : $st->{prepare_time} ? sprintf '%.1f ms', $st->{prepare_time}*1000 : 'cache';
-                td_ class => 'num', $_->{nrows};
+                    td_ class => 'num', $st->{nrows};
-                td_ class => 'code', sub { fmtpre_ $_->{query} };
+                    td_ class => 'sum', sub {
-                # TODO: Params, both separate and interpolated
+                        label_ for => "row${i}_c", sub {
-            } for $FU::REQ->{trace_sql}->@*;
+                            span_ class => 'closed', '▶';
+                            span_ class => 'open', '▼';
+                            txt_ $st->{query} =~ s/[\r\n]/ /rg =~ s/\s\s+/ /rg =~ s/^\s+//r;
                         };
-        ('Queries', scalar $FU::REQ->{trace_sql}->@*)
+                    };
+                };
+                tr_ class => 'details', id => "row$i", sub {
+                    td_ '';
+                    td_ colspan => 3, sub {
+                        pre_ $st->{query};
+                        if ($st->{params}->@*) {
+                            strong_ 'Parameters:';
+                            table_ sub {
+                                tr_ sub {
+                                    td_ class => 'num', sprintf '$%d =', $_+1;
+                                    td_ class => 'code', sub {
+                                        my $p = $st->{params}[$_]{text};
+                                        !defined $p ? em_ 'null' : txt_ $p;
+                                    };
+                                } for (0..$#{$st->{params}});
+                            };
+                            # XXX: Buggy when the query contains string literals with $n variables.
+                            strong_ 'Interpolated:';
+                            pre_ $st->{query} =~ s{\$([1-9][0-9]*)}{
+                                my $v = $st->{params}[$1-1]{text};
+                                defined $v ? $FU::DB->escape_literal($v) : 'NULL'
+                            }egr;
+                        }
+                    };
+                };
+            }
+            tr_ sub {
+                td_ class => 'num', sprintf '%.1f ms', $FU::REQ->{trace_sqlexec}*1000;
+                td_ class => 'num', !defined $FU::REQ->{trace_sqlprep} ? '-' : sprintf '%.1f ms', $FU::REQ->{trace_sqlprep}*1000;
+                td_ class => 'num', $rows;
+                td_ class => 'sum', 'total';
+            } if @$queries > 1;
+        };
+        ('Queries', scalar @$queries)
     },
 
     fu => sub {
         return () if !keys fu->%*;
-        # TODO: Contents of the 'fu' object
+        # TODO: This is kinda lazy, an expandable table might be nicer.
+        require Data::Dumper;
+        pre_ sub {
+            lit_ Data::Dumper->new([fu])->Sortkeys(1)->Terse(1)->Dump;
+        };
         ('fu obj')
     },
 
@@ -171,7 +283,7 @@ my @tabs = (
 
     pgst => sub {
         return () if !$FU::DB;
-        my $lst = eval { $FU::DB->q(
+        my $lst = eval { $FU::DB->sql(
             'SELECT generic_plans + custom_plans, statement FROM pg_prepared_statements ORDER BY generic_plans + custom_plans DESC, statement'
         )->cache(0)->alla } || do { warn "Unable to collect prepared statement list: $@"; return () };
         return () if !@$lst;
@@ -182,19 +294,20 @@ my @tabs = (
             } };
             tr_ sub {
                 td_ $_->[0];
-                td_ class => 'code', sub { fmtpre_ $_->[1] };
+                td_ class => 'code', $_->[1];
             } for @$lst;
         };
-        ('Prepared statements', scalar @$lst)
+        ('Prepared stmts', scalar @$lst)
     },
 );
 
 
 sub collect {
     my @t;
-    for my ($id, $sub) (@tabs) {
+    for my ($id, $sub) (@sections) {
         my($title, $num);
         my $html = fragment { ($title, $num) = $sub->() };
+        utf8::decode($html);
         push @t, { id => $id, title => $title, num => $num, html => $html } if $title;
     }
     \@t
@@ -206,47 +319,9 @@ sub framework_($data) {
         head_ sub {
             title_ 'FU Debugging Interface';
             meta_ name => 'viewport', content => 'width=device-width, initial-scale=1.0, user-scalable=yes';
+            link_ rel => 'stylesheet', type => 'text/css', media => 'all', href => '?css';
             style_ type => 'text/css', <<~_;
-              html { box-sizing: border-box; color: #000; background: #fff }
-              *, *:before, *:after { box-sizing: inherit }
-              * { margin: 0; padding: 0; font: inherit; color: inherit }
-
-              body { display: grid; grid: 45px 400px / 220px auto; }
-              header { grid-column: 1 / 3; grid-row: 1 / 2 }
-              nav { grid-column: 1 / 2; grid-row: 2 / 3 }
-              main { grid-column: 2 / 3; grid-row: 2 / 3 }
-
-              header, nav { background: #eee }
-              main { border-top: 2px solid #009; border-left: 2px solid #009 }
-              nav { border-bottom: 2px solid #009 }
-
-              header { display: flex; justify-content: space-between; padding: 10px }
-              header h1 { font-size: 20px; font-weight: bold }
-              header menu { list-style-type: none; display: flex; gap: 15px }
-
-              body > input { display: none }
-              nav { padding-top: 20px }
-              nav menu { list-style-type: none }
-              nav label { display: block; width: 100%; padding: 2px 10px; cursor: pointer; white-space: nowrap }
-              nav label:hover { background-color: #fff }
-              nav label span { float: right; font-size: 80% }
-
-              main { padding: 10px 20px }
-              main h2 { margin: 30px 0 5px -10px; font-size: 20px; font-weight: bold }
-              main h2:first-child { margin-top: 0 }
-
-              p, pre, table { margin: 5px 0 }
-              pre, .code { font-family: monospace; white-space: pre }
-              table { border-collapse: collapse }
-              td { padding: 1px 10px 1px 0; font-size: 12px; vertical-align: top }
-              tr:hover { background-color: #eee }
-              thead { font-weight: bold }
-              .num { text-align: right; white-space: nowrap }
             _
-            style_ type => 'text/css', join "\n", map +(
-                "#tab_$_:checked ~ nav menu li label[for=tab_$_] { background-color: #fff }",
-                "#tab_$_:not(:checked) ~ main #tabc_$_ { display: none }",
-            ), map $_->{id}, @$data;
         };
         body_ sub {
             header_ sub {
@@ -257,22 +332,21 @@ sub framework_($data) {
                     li_ sub { a_ href => '?', 'Listing' };
                 };
             };
-            input_ type => 'radio', name => 'tab', id => "tab_$_->{id}", checked => $_ eq $data->[0] ? 'checked' : undef for @$data;
             nav_ sub {
                 menu_ sub {
                     li_ sub {
-                        label_ for => "tab_$_->{id}", sub {
+                        a_ href => "#$_->{id}", sub {
                             txt_ $_->{title};
                             span_ $_->{num} if defined $_->{num};
-                        }
+                        };
                     } for @$data;
                 };
             } if @$data;
             main_ sub {
-                div_ id => "tabc_$_->{id}", sub {
+                for (@$data) {
-                    h2_ $_->{title};
+                    h1_ id => $_->{id}, $_->{title};
                     lit_ $_->{html};
-                } for @$data;
+                }
             };
         };
     };
@@ -313,10 +387,23 @@ sub load($id) {
     fu->set_body(scalar <$fn>);
 }
 
+sub css {
+    # Awful CSS row hiding hack. I'm not sorry.
+    state $css = join '', <DATA>, map qq{
+        #row${_}_c:checked ~ * label[for=row${_}_c] .closed { display: none }
+        #row${_}_c:not(:checked) ~ * label[for=row${_}_c] .open { display: none }
+        #row${_}_c:not(:checked) ~ * #row${_} { display: none }
+    }, 0..1000;
+}
+
 sub render {
     my $q = fu->query;
     if (!$q) {
         fu->set_body(framework_ [{id => 'lst', title => 'Recent Requests', html => fragment \&listing_ }]);
+    } elsif ($q eq 'css') {
+        fu->set_header('content-type', 'text/css');
+        fu->set_header('cache-control', 'max-age=86400');
+        fu->set_body(css());
     } elsif ($q eq 'cur') {
         fu->set_body(framework_ collect);
     } elsif ($q eq 'last') {
@@ -342,7 +429,7 @@ sub save {
         return;
     };
     my $line = sprintf "%d %f %s %s %s\n",
-        time, time - $FU::REQ->{trace_start}, $FU::REQ->{status},
+        time, $FU::REQ->{trace_end} - $FU::REQ->{trace_start}, $FU::REQ->{status},
         fu->method, fu->path.(fu->query?'?'.fu->query:'');
     utf8::encode($line);
     print $fh $line;
@@ -350,3 +437,62 @@ sub save {
 }
 
 1;
+
+__DATA__
+html { box-sizing: border-box; color: #000; background: #fff }
+*, *:before, *:after { box-sizing: inherit }
+* { margin: 0; padding: 0; font: inherit; color: inherit }
+
+/* Ugh, fixed positioning */
+header { position: fixed; top: 0; left: 0; width: 100%; height: 40px; z-index: 2 }
+nav { position: fixed; top: 38px; left: 0; width: 200px; z-index: 2 }
+main { margin: 0 0 0 200px }
+
+header, nav { background: #eee }
+header { border-bottom: 2px solid #009 }
+nav { border-bottom: 2px solid #009; border-right: 2px solid #009 }
+
+header { display: flex; justify-content: space-between; align-items: baseline; padding: 5px 10px }
+header h1 { font-size: 120%; font-weight: bold }
+header menu { list-style-type: none; display: flex; gap: 15px }
+
+body > input { display: none }
+nav { padding-top: 20px }
+nav menu { list-style-type: none }
+nav a { display: block; width: 100%; text-decoration: none; padding: 2px 10px; cursor: pointer; white-space: nowrap }
+nav a:hover { background-color: #fff }
+nav a span { float: right; font-size: 80% }
+
+main { padding: 0 10px 30px 10px }
+main h1 { background: #eee; padding: 5px 10px 5px 205px; margin: 40px -10px 10px -210px; scroll-margin-top: 40px; font-size: 130%; font-weight: bold }
+main h2 { margin: 20px 0 5px 0; font-size: 120%; font-weight: bold }
+
+p, table, pre { margin: 5px 0 }
+pre { border-left: 2px dotted #999; padding-left: 5px; font-family: monospace; white-space: pre; overflow-x: auto; padding-bottom: 15px; /* for the scrollbar, kinda browser-specific */ }
+table { border-collapse: collapse }
+td { padding: 1px 10px 1px 0; font-size: 12px; vertical-align: top }
+td.code { font-family: monospace }
+tr:hover { background-color: #eee }
+thead { font-weight: bold }
+.num { text-align: right; white-space: nowrap }
+
+section.tabs { position: relative; display: flex; flex-wrap: wrap; z-index: 1; }
+section.tabs summary { cursor: pointer; order: 0; display: block; padding: 3px 5px; margin-right: 10px; background: #ddd }
+section.tabs summary:hover, section.tabs details[open] summary { background: #eee }
+section.tabs details { display: contents }
+section.tabs details *:nth-child(2) { order: 1; width: 100% }
+
+.sqlt { width: 100%; table-layout: fixed }
+.sqlt .num { width: 50px }
+.sqlt .num:first-child { width: 75px }
+.sqlt .num:nth-child(2) { width: 60px }
+.sqlt .sum { white-space: nowrap; font-family: monospace; overflow: hidden; text-overflow: ellipsis }
+.sqlt label { cursor: pointer }
+.sqlt label span { color: #555; display: inline-block; width: 15px }
+.sqlt tr.details { background: #fff }
+.sqlt tr.details > td { padding-bottom: 10px }
+input[id^=row] { display: none }
+
+small { color: #555; font-size: 90% }
+em { font-style: italic }
+strong { font-weight: bold }

FU/Log.pm

@@ -1,4 +1,4 @@
-package FU::Log 0.2;
+package FU::Log 1.4;
 use v5.36;
 use Exporter 'import';
 use POSIX 'strftime';
@@ -65,11 +65,6 @@ __END__
 
 FU::Log - Extremely Basic Process-Wide Logging Infrastructure
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::Log 'log_write';
@@ -89,7 +84,7 @@ interface either; the entire point of this module is that it only handles
 process-global logging. This module mainly exists for users of the L<FU>
 framework.
 
-=head2 Configuration
+=head1 Configuration
 
 =over
 
@@ -119,7 +114,7 @@ is then used instead. This is to avoid recursion.
 
 =back
 
-=head2 Exportable function
+=head1 Exportable function
 
 =over
 

FU/MultipartFormData.pm

@@ -0,0 +1,210 @@
+package FU::MultipartFormData 1.4;
+use v5.36;
+use Carp 'confess';
+use FU::Util 'utf8_decode';
+
+sub _arg($d) { $d =~ s{^"(.+)"$}{$1 =~ s/\\([\\"])/$1/rg}er }
+
+sub parse($pkg, $header, $data) {
+    confess "Invalid multipart header '$header'"
+        if $header !~ m{^multipart/form-data\s*;\s*boundary\s*=(.+)$};
+    my $boundary = _arg $1;
+    confess "Invalid multipart boundary '$boundary'" if $boundary !~ /^[\x21-\x7e]+$/;
+    utf8::encode($boundary);
+
+    my @a;
+    while ($data =~ m{--\Q$boundary\E(?:--\r\n|\r\n((?:.+\r\n)+)\r\n)}xg) {
+        my $hdrs = $1;
+        $a[$#a]{length} = $-[0] - 2 - $a[$#a]{start} if @a;
+        if (!$hdrs) {
+            confess "Trailing garbage" if pos $data != length $data;
+            last;
+        }
+
+        my $d = bless {
+            data => $data,
+            start => pos $data,
+        }, $pkg;
+
+        confess "Missing content-disposition header" if $hdrs !~ /content-disposition:\s*form-data(.+)/i;
+        my $v = $1;
+        my $pvalue = qr/("(?:\\[\\"]|[^\\"\r\n]+)*"|[^\s;"]*)/;
+        confess "Missing 'name' parameter" if $v !~ /;\s*name\s*=\s*$pvalue/;
+        $d->{name} = utf8_decode _arg $1;
+        $d->{filename} = utf8_decode _arg $1 if $v =~ /;\s*filename\s*=\s*$pvalue/;
+
+        if ($hdrs =~ /content-type:\s*$pvalue(?:\s*;\s*charset\s*=\s*$pvalue)?/i) {
+            $d->{mime} = utf8_decode _arg $1;
+            $d->{charset} = utf8_decode _arg $2 if $2;
+        }
+        push @a, $d;
+    }
+    confess "Missing end-of-data marker" if @a && !defined $a[$#a]{length};
+    \@a
+}
+
+sub name     { $_[0]{name} }
+sub filename { $_[0]{filename} }
+sub mime     { $_[0]{mime} }
+sub charset  { $_[0]{charset} }
+sub length   { $_[0]{length} }
+
+sub substr($o,$off,$len=undef) {
+    $off += $o->{length} if $off < 0;
+    $off = 0 if $off < 0;
+    $off = $o->{length} if $off > $o->{length};
+
+    $len //= $o->{length} - $off;
+    $len += $o->{length} - 1 if $len < 0;
+    $len = 0 if $len < 0;
+    $len = $o->{length} - $off if $len > $o->{length} - $off;
+
+    substr $o->{data}, $o->{start} + $off, $len;
+}
+
+sub data     { $_[0]->substr(0) }
+sub value    { utf8_decode $_[0]->data }
+
+sub syswrite($o, $fh) {
+    my $off = $o->{start};
+    my $end = $o->{start} + $o->{length};
+    while ($off < $end) {
+        my $r = syswrite $fh, $o->{data}, $end-$off, $off;
+        return if !defined $r;
+        $off += $r;
+    }
+    $o->{length};
+}
+
+sub save($o, $fn) {
+    open my $F, '>', $fn or confess "Error opening '$fn': $!";
+    defined $o->syswrite($F) or confess "Error writing to '$fn': $!";
+}
+
+sub describe($o) {
+    my $head = eval { utf8_decode $o->substr(0, 100) };
+    if (defined $head && $head =~ /\n/) {
+        ($head) = split /\n/, $head, 2;
+        $head .= '...';
+    } elsif (defined $head && $o->{length} > 100) {
+        $head .= '...';
+    }
+    $o->{name}.': '.join ' ',
+        $o->{filename} ? "filename=$o->{filename}" : (),
+        $o->{mime} ? "mime=$o->{mime}" : (),
+        $o->{charset} ? "charset=$o->{charset}" : (),
+        "length=$o->{length}",
+        defined $head ? "value=$head" : ();
+}
+
+1;
+__END__
+
+=head1 NAME
+
+FU::MultipartFormData - Parse multipart/form-data
+
+=head1 SYNOPSIS
+
+  my $fields = FU::MultipartFormData->parse($content_type_header, $request_body);
+
+  for my $f (@$fields) {
+      print "%s   %d\n", $f->name, $f->length;
+
+      $f->save('file.png') if $f->name eq 'image';
+  }
+
+=head1 DESCRIPTION
+
+This is a tiny module to parse an HTTP request body encoded as
+C<multipart/form-data>, which is typically used to handle file uploads.
+
+The entire request body is assumed to be in memory as a Perl string, but this
+module makes an attempt to avoid any further copies of data values.
+
+=head1 Parsing
+
+=over
+
+=item FU::MultipartFormData->parse($header, $body)
+
+Returns an array of field objects from the given C<$header>, which must be a
+valid value for the C<Content-Type> request header, and the given C<$body>,
+which must hold the request body as a byte string. An error is thrown if the
+header is not valid or parsing failed.
+
+This module is pretty lousy and does not fully comform to any HTTP standards,
+but it does happen to be able to parse POST data from any browser that I've
+tried.
+
+=back
+
+=head1 Field Object
+
+Each field is parsed into a field object that supports the following methods:
+
+=over
+
+=item name
+
+Returns the field name as a Perl Unicode string.
+
+=item filename
+
+Returns the filename as a Perl Unicode string, or C<undef> if no filename was
+provided.
+
+=item mime
+
+Returns the mime type extracted from the field's C<Content-Type> header, or
+C<undef> if none was present.
+
+=item charset
+
+Returns the charset extracted from the field's C<Content-Type> header, or
+C<undef> if none was present.
+
+=item length
+
+Returns the byte length of the field value.
+
+=item data
+
+Returns a copy of the field value as a byte string. You'll want to avoid using
+this on large fields.
+
+=item value
+
+Returns a copy of the field value as a Unicode string.
+
+=item substr($off, $len)
+
+Equivalent to calling C<substr()> on the string returned by C<data>, but avoids
+a copy of the entire field value.
+
+=item syswrite($fh)
+
+Write the field value to C<$fh> using Perl's C<syswrite()>, returns C<undef> on
+error or the number of bytes written on success.
+
+Can be used to write uploaded file data to a file or send it over a socket or
+pipe, without making a full in-memory copy of the data.
+
+=item save($fn)
+
+Save the field value to the file C<$fn>, throws an error on failure.
+
+=item describe
+
+Returns a human-readable string to describe this field. Mainly for debugging
+purposes, the exact format is subject to change.
+
+=back
+
+=head1 COPYRIGHT
+
+MIT.
+
+=head1 AUTHOR
+
+Yorhel <projects@yorhel.nl>

FU/Pg.pm

@@ -1,4 +1,4 @@
-package FU::Pg 0.2;
+package FU::Pg 1.4;
 use v5.36;
 use FU::XS;
 
@@ -7,11 +7,15 @@ _load_libpq();
 package FU::Pg::conn {
     sub lib_version { FU::Pg::lib_version() }
 
-    sub Q {
+    sub SQL {
         require FU::SQL;
         my $s = shift;
-        my($sql, $params) = FU::SQL::SQL(@_)->compile(placeholder_style => 'pg', in_style => 'pg');
+        my($sql, $params) = FU::SQL::SQL(@_)->compile(
-        $s->q($sql, @$params);
+            placeholder_style => 'pg',
+            in_style          => 'pg',
+            quote_identifier  => sub { $s->conn->escape_identifier(@_) },
+        );
+        $s->sql($sql, @$params);
     }
 
     sub set_type($s, $n, @arg) {
@@ -22,7 +26,13 @@ package FU::Pg::conn {
     }
 };
 
-*FU::Pg::txn::Q = \*FU::Pg::conn::Q;
+*FU::Pg::txn::SQL = \*FU::Pg::conn::SQL;
+
+# Compat
+*FU::Pg::conn::q = \*FU::Pg::conn::sql;
+*FU::Pg::txn::q = \*FU::Pg::txn::sql;
+*FU::Pg::conn::Q = \*FU::Pg::conn::SQL;
+*FU::Pg::txn::Q = \*FU::Pg::txn::SQL;
 
 package FU::Pg::error {
     use overload '""' => sub($e, @) { $e->{full_message} };
@@ -35,11 +45,6 @@ __END__
 
 FU::Pg - The Ultimate (synchronous) Interface to PostgreSQL
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSYS
 
   use FU::Pg;
@@ -48,10 +53,10 @@ changes, see the main L<FU> module for details.
 
   $conn->exec('CREATE TABLE books (id SERIAL, title text, read bool)');
 
-  $conn->q('INSERT INTO books (title) VALUES ($1)', 'Revelation Space')->exec;
+  $conn->sql('INSERT INTO books (title) VALUES ($1)', 'Revelation Space')->exec;
-  $conn->q('INSERT INTO books (title) VALUES ($1)', 'The Invincible')->exec;
+  $conn->sql('INSERT INTO books (title) VALUES ($1)', 'The Invincible')->exec;
 
-  for my ($id, $title) ($conn->q('SELECT * FROM books')->flat->@*) {
+  for my ($id, $title) ($conn->sql('SELECT * FROM books')->flat->@*) {
       print "$id:  $title\n";
   }
 
@@ -61,7 +66,7 @@ FU::Pg is a client module for PostgreSQL with a convenient high-level API and
 support for flexible and complex type conversions. This module interfaces
 directly with C<libpq>.
 
-=head2 Connection setup
+=head1 Connection setup
 
 =over
 
@@ -72,7 +77,7 @@ C<$string> can either be in key=value format or a URI, refer to L<the
 PostgreSQL
 documentation|https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING>
 for the full list of supported formats and options. You may also pass an empty
-string and leave the configuration up L<environment
+string and leave the configuration up to L<environment
 variables|https://www.postgresql.org/docs/current/libpq-envars.html>.
 
 =item $conn->server_version
@@ -112,12 +117,28 @@ Inside a transaction that is in an error state. The transaction must be rolled
 back in order to recover to a usable state. This happens automatically when the
 transaction object goes out of scope.
 
+=item active
+
+Currently executing a query. This state can only be observed during a L<COPY
+operation|/"COPY support">.
+
 =item bad
 
 Connection is dead or otherwise unusable.
 
 =back
 
+=item $conn->escape_literal($str)
+
+Return an escaped version of C<$str> suitable for use as a string literal in an
+SQL statement. You'll rarely need this, it's often better to pass data as bind
+parameters instead.
+
+=item $conn->escape_identifier($str)
+
+Return an escaped version of C<$str> suitable for use as an identifier (name of
+a table, column, function, etc) in an SQL statement.
+
 =item $conn->cache($enable)
 
 =item $conn->text_params($enable)
@@ -126,7 +147,7 @@ Connection is dead or otherwise unusable.
 
 =item $conn->text($enable)
 
-Set the default settings for new statements created with B<< $conn->q() >>.
+Set the default settings for new statements created with B<< $conn->sql() >>.
 
 =item $conn->cache_size($num)
 
@@ -154,11 +175,12 @@ Also worth noting that the subroutine is called from the context of the code
 executing the query, but I<before> the query results have been returned.
 
 The subroutine is (currently) only called for queries executed through C<<
-$conn->exec >>, C<< $conn->q >>, C<< $conn->Q >> and their C<$txn> variants;
+$conn->exec >>, C<< $conn->sql >>, C<< $conn->SQL >> and their C<$txn> variants;
-internal queries performed by this module (such as for transaction management,
+C<< $conn->copy >> statements and internal queries performed by this module
-querying type information, etc) do not trigger the callback. Statements that
+(such as for transaction management, querying type information, etc) do not
-result in an error being thrown during or before execution are also not
+trigger the callback. Statements that result in an error being thrown during or
-traceable this way. This behavior might change in the future.
+before execution are also not traceable this way. This behavior might change in
+the future.
 
 =item $conn->disconnect
 
@@ -167,7 +189,7 @@ attempts to use C<$conn> throw an error.
 
 =back
 
-=head2 Querying
+=head1 Querying
 
 =over
 
@@ -177,7 +199,7 @@ Execute one or more SQL commands, separated by a semicolon. Returns the number
 of rows affected by the last statement or I<undef> if that information is not
 available for the given command (like with C<CREATE TABLE>).
 
-=item $conn->q($sql, @params)
+=item $conn->sql($sql, @params)
 
 Create a new SQL statement with the given C<$sql> string and an optional list
 of bind parameters. C<$sql> can only hold a single statement.
@@ -193,14 +215,15 @@ Note that this method just creates a statement object, the query is not
 prepared or executed until the appropriate statement methods (see below) are
 used.
 
-=item $conn->Q(@args)
+=item $conn->SQL(@args)
 
-Same as C<< $conn->q() >> but uses L<FU::SQL> to construct the query and bind
+Same as C<< $conn->sql() >> but uses L<FU::SQL> to construct the query and bind
-parameters.
+parameters. Uses the 'pg' C<in_style> and C<< $conn->escape_identifier() >> for
+identifier quoting.
 
 =back
 
-Statement objects returned by C<< $conn->q() >> support the following
+Statement objects returned by C<< $conn->sql() >> support the following
 configuration parameters, which can be set before the statement is executed:
 
 =over
@@ -235,7 +258,7 @@ depending on how you'd like to obtain the results:
 Execute the query and return the number of rows affected. Similar to C<<
 $conn->exec >>.
 
-  my $v = $conn->q('UPDATE books SET read = true WHERE id = 1')->exec;
+  my $v = $conn->sql('UPDATE books SET read = true WHERE id = 1')->exec;
   # $v = 1
 
 =item $st->val
@@ -244,7 +267,7 @@ Return the first column of the first row. Throws an error if the query does not
 return exactly one column, or if multiple rows are returned. Returns I<undef>
 if no rows are returned or if its value is I<NULL>.
 
-  my $v = $conn->q('SELECT COUNT(*) FROM books')->val;
+  my $v = $conn->sql('SELECT COUNT(*) FROM books')->val;
   # $v = 2
 
 =item $st->rowl
@@ -252,7 +275,7 @@ if no rows are returned or if its value is I<NULL>.
 Return the first row as a list, or an empty list if no rows are returned.
 Throws an error if the query returned more than one row.
 
-  my($id, $title) = $conn->q('SELECT id, title FROM books LIMIT 1')->rowl;
+  my($id, $title) = $conn->sql('SELECT id, title FROM books LIMIT 1')->rowl;
   # ($id, $title) = (1, 'Revelation Space');
 
 =item $st->rowa
@@ -261,7 +284,7 @@ Return the first row as an arrayref, equivalent to C<< [$st->rowl] >> but might
 be slightly more efficient. Returns C<undef> if the query did not generate any
 rows.
 
-  my $row = $conn->q('SELECT id, title FROM books LIMIT 1')->rowa;
+  my $row = $conn->sql('SELECT id, title FROM books LIMIT 1')->rowa;
   # $row = [1, 'Revelation Space'];
 
 =item $st->rowh
@@ -270,14 +293,14 @@ Return the first row as a hashref. Returns C<undef> if the query did not
 generate any rows. Throws an error if the query returns multiple columns with
 the same name.
 
-  my $row = $conn->q('SELECT id, title FROM books LIMIT 1')->rowh;
+  my $row = $conn->sql('SELECT id, title FROM books LIMIT 1')->rowh;
   # $row = { id => 1, title => 'Revelation Space' };
 
 =item $st->alla
 
 Return all rows as an arrayref of arrayrefs.
 
-  my $data = $conn->q('SELECT id, title FROM books')->alla;
+  my $data = $conn->sql('SELECT id, title FROM books')->alla;
   # $data = [
   #   [ 1, 'Revelation Space' ],
   #   [ 2, 'The Invincible' ],
@@ -288,7 +311,7 @@ Return all rows as an arrayref of arrayrefs.
 Return all rows as an arrayref of hashrefs. Throws an error if the query
 returns multiple columns with the same name.
 
-  my $data = $conn->q('SELECT id, title FROM books')->allh;
+  my $data = $conn->sql('SELECT id, title FROM books')->allh;
   # $data = [
   #   { id => 1, title => 'Revelation Space' },
   #   { id => 2, title => 'The Invincible' },
@@ -298,7 +321,7 @@ returns multiple columns with the same name.
 
 Return an arrayref with all rows flattened.
 
-  my $data = $conn->q('SELECT id, title FROM books')->flat;
+  my $data = $conn->sql('SELECT id, title FROM books')->flat;
   # $data = [
   #   1, 'Revelation Space',
   #   2, 'The Invincible',
@@ -310,7 +333,7 @@ Return a hashref where the first result column is used as key and the second
 column as value. If the query only returns a single column, C<true> is used as
 value instead. An error is thrown if the query returns 3 or more columns.
 
-  my $data = $conn->q('SELECT id, title FROM books')->kvv;
+  my $data = $conn->sql('SELECT id, title FROM books')->kvv;
   # $data = {
   #   1 => 'Revelation Space',
   #   2 => 'The Invincible',
@@ -321,7 +344,7 @@ value instead. An error is thrown if the query returns 3 or more columns.
 Return a hashref where the first result column is used as key and the remaining
 columns are stored as arrayref.
 
-  my $data = $conn->q('SELECT id, title, read FROM books')->kva;
+  my $data = $conn->sql('SELECT id, title, read FROM books')->kva;
   # $data = {
   #   1 => [ 'Revelation Space', true ],
   #   2 => [ 'The Invincible', false ],
@@ -332,7 +355,7 @@ columns are stored as arrayref.
 Return a hashref where the first result column is used as key and the remaining
 columns are stored as hashref.
 
-  my $data = $conn->q('SELECT id, title, read FROM books')->kvh;
+  my $data = $conn->sql('SELECT id, title, read FROM books')->kvh;
   # $data = {
   #   1 => { title => 'Revelation Space', read => true },
   #   2 => { title => 'The Invincible', read => false },
@@ -344,7 +367,7 @@ The only time you actually need to assign a statement object to a variable is
 when you want to inspect the statement using one of the methods below, in all
 other cases you can chain the methods for more concise code. For example:
 
-  my $data = $conn->q('SELECT a, b FROM table')->cache(0)->text->alla;
+  my $data = $conn->sql('SELECT a, b FROM table')->cache(0)->text->alla;
 
 Statement objects can be inspected with the following methods (many of which
 only make sense after the query has been executed):
@@ -364,10 +387,10 @@ Returns the provided bind parameters as an arrayref.
 Returns an arrayref of integers indicating the type (as I<oid>) of each
 parameter in the given C<$sql> string. Example:
 
-  my $oids = $conn->q('SELECT id FROM books WHERE id = $1 AND title = $2')->param_types;
+  my $oids = $conn->sql('SELECT id FROM books WHERE id = $1 AND title = $2')->param_types;
   # $oids = [23,25]
 
-  my $oids = $conn->q('SELECT id FROM books')->params;
+  my $oids = $conn->sql('SELECT id FROM books')->params;
   # $oids = []
 
 This method can be called before the query has been executed, but will then
@@ -380,7 +403,7 @@ prepared statement caching is disabled and C<text_params> is enabled.
 Returns an arrayref of hashrefs describing each column that the statement
 returns.
 
-  my $cols = $conn->q('SELECT id, title FROM books')->columns;
+  my $cols = $conn->sql('SELECT id, title FROM books')->columns;
   # $cols = [
   #   { name => 'id', oid => 23 },
   #   { name => 'title', oid => 25 },
@@ -400,9 +423,7 @@ results into Perl values.
 
 Observed query preparation time, in seconds, including network round-trip.
 Returns 0 if a cached prepared statement was used or C<undef> if the query was
-executed without a separate preparation phase (currently only happens with C<<
+executed without a separate preparation phase.
-$conn->exec() >>, but support for direct query execution may be added for other
-queries in the future as well).
 
 =item $st->get_cache
 
@@ -416,7 +437,7 @@ Returns the respective configuration parameters.
 
 
 
-=head2 Transactions
+=head1 Transactions
 
 This module provides a convenient and safe API for I<scoped transactions> and
 I<subtransactions>. A new transaction can be started with C<< $conn->txn >>,
@@ -431,7 +452,7 @@ fail while a transaction object is alive.
     my $txn = $conn->txn;
 
     # run queries
-    $txn->q('DELETE FROM books WHERE id = $1', 1)->exec;
+    $txn->sql('DELETE FROM books WHERE id = $1', 1)->exec;
 
     # run commands in a subtransaction
     {
@@ -452,9 +473,9 @@ Transaction methods:
 
 =item $txn->exec(..)
 
-=item $txn->q(..)
+=item $txn->sql(..)
 
-=item $txn->Q(..)
+=item $txn->SQL(..)
 
 Run a query inside the transaction. These work the same as the respective
 methods on the parent C<$conn> object.
@@ -477,7 +498,7 @@ when the object goes out of scope.
 
 =item $txn->text($enable)
 
-Set the default settings for new statements created with B<< $txn->q() >>.
+Set the default settings for new statements created with B<< $txn->sql() >>.
 
 These settings are inherited from the main connection when the transaction is
 created. Subtransactions inherit these settings from their parent transaction.
@@ -521,6 +542,11 @@ current implementation does not track subtransactions that closely)
 
 A subtransaction is in error state and awaiting to be rolled back.
 
+=item active
+
+Currently executing a query. This state can only be observed during a L<COPY
+operation|/"COPY support">.
+
 =item bad
 
 Connection is dead or otherwise unusable.
@@ -540,7 +566,7 @@ Just don't try to use transaction objects and manual transaction commands at
 the same time, that won't end well.
 
 
-=head2 Formats and Types
+=head1 Formats and Types
 
 The PostgreSQL wire protocol supports sending bind parameters and receiving
 query results in two different formats: text and binary. While the exact wire
@@ -611,10 +637,12 @@ Some built-in types deserve a few additional notes:
 
 =item bool
 
-Boolean values are converted to C<builtin::true> and C<builtin::false>. As bind
+Boolean values are converted to C<builtin::true> and C<builtin::false>.
-parameters, Perl's idea of truthiness is used: C<0>, C<false> and C<""> are
+
-false, everything else is true. Objects that overload I<bool> are also
+As bind parameters, values recognized by C<to_bool()> in L<FU::Util> are
-supported. C<undef> always converts to SQL C<NULL>.
+accepted, in addition to C<0>, C<"f"> and C<""> for false and C<1>, and C<"t">
+for true.  C<undef> always converts to SQL C<NULL>. Everything else throws an
+error.
 
 =item bytea
 
@@ -666,8 +694,8 @@ module does not.
 
 Converted between floating point seconds since C<00:00:00>, supporting
 microsecond precision. This format allows for easy comparison against Unix
-timestamps (time of day = C<$timestamp % 86400>) and can be added to an integer
+timestamps (time of day in UTC = C<$timestamp % 86400>) and can be added to an
-date value to form a complete timestamp.
+integer date value to form a complete timestamp.
 
 (There's no support for the string format yet)
 
@@ -680,7 +708,7 @@ While C<null> is a valid JSON value, there's currently no way to distinguish
 that from SQL C<NULL>. When sending C<undef> as bind parameter, it is sent as
 SQL C<NULL>.
 
-If you prefer to work with JSON are raw text values instead, use:
+If you prefer to work with JSON as raw text values instead, use:
 
   $conn->set_type(json => 'text');
 
@@ -738,11 +766,115 @@ C<set_type()> to configure appropriate conversions for these types.
 
 =back
 
-I<TODO:> Methods to convert between the various formats.
+Utility functions:
+
+=over
+
+=item $conn->perl2bin($oid, $val)
+
+=item $conn->bin2perl($oid, $bin)
+
+Convert the value for a specific type between the Perl representation and the
+PostgreSQL binary format, using the current type configuration of the
+connection. This is the same conversion used internally by this module to send
+bind parameters and receive query results, and map to the C<send> and C<recv>
+functions of C<< $conn->set_type() >>.
+
+These methods throw an error if C<$oid> is not a known type or if the given
+data is not valid for the type. However, these methods should not be used for
+strict validation: the conversion routines are usually written under the
+assumption that the data has been received directly from Postgres or is about
+to be sent to (and further validated by) Postgres.  For some types,
+C<perl2bin()> may return invalid data on invalid input and C<bin2perl()> may
+accept invalid binary data.
+
+=item $conn->bin2text($oid, $bin, ...)
+
+=item $conn->text2bin($oid, $text, ...)
+
+Convert between the binary format and the PostgreSQL text format. This
+conversion requires a round-trip to the server and throws an error if the
+connection state is not I<idle> or I<txn_idle>. Since it is Postgres doing the
+conversion, the input is properly validated and, in the case of C<bin2text()>,
+the result is guaranteed to be suitable for use as a textual bind parameter or
+for inclusion in an SQL query (but don't forget to use C<escape_literal()> in
+that case).
+
+Calling these methods many times can be pretty slow. If you have several values
+to convert, you can do that in a single call to speed things up:
+
+  my($text1, $text2, ..) = $conn->bin2text($oid1, $bin1, $oid2, $bin2, ..);
+
+=back
 
 I<TODO:> Methods to query type info.
 
-=head2 Errors
+
+=head1 COPY support
+
+You can use L<COPY
+statements|https://www.postgresql.org/docs/current/sql-copy.html> for efficient
+bulk data transfers between your application and the PostgreSQL server:
+
+=over
+
+=item $copy = $conn->copy($statement)
+
+=item $copy = $txn->copy($statement)
+
+Execute C<$statement> and return a C<FU::Pg::copy> object that lets you
+transfer data to or from Postgres.
+
+It is not possible to execute any other queries on the same connection while a
+copy operation is in progress. When used on a transaction object, C<$txn> must
+be kept alive long enough to finish the copy operation.
+
+=back
+
+A C<$copy> object supports the following methods:
+
+=over
+
+=item $copy->is_binary
+
+Returns true if the transfer is performed in the binary format, false for text.
+
+=item $copy->write($data)
+
+Send C<$data> to the server. An error is thrown if this is not a C<COPY FROM
+STDIN> operation. An error may be thrown if C<$data> is not a valid format
+understood by Postgres, but such errors can also be deferred to C<close()>.
+
+C<$data> is interpreted as a Perl Unicode string for textual transfers and as a
+binary string for binary transfers.
+
+=item $copy->read
+
+Return the next row read from the Postgres server, or C<undef> if no more data
+is coming. In the text format, a single line - including trailing newline - is
+returned as a Perl Unicode string. In the binary format, a single row is
+returned as a byte string. An error is thrown if this is not a C<COPY TO
+STDOUT> operation.
+
+=item $copy->close
+
+Marks the end of the copy operation. Does not return anything but throws an
+error if something went wrong.
+
+It is possible to close a read-copy operation before all data has been
+consumed, but that causes all data to still be read and discarded during
+C<close()>. If you really want to interrupt a large read operation, a more
+efficient approach is to call C<< $conn->disconnect >> and discard the entire
+connection.
+
+It is not I<necessary> to call this method, simply letting the C<$copy> object
+run out of scope will do the trick as well, but in that case errors are
+silently discarded. An explicit C<close()> is recommended to catch errors.
+
+=back
+
+
+=head1 Errors
 
 All methods can throw an exception on error. When possible, the error message
 is constructed using L<Carp>'s C<confess()>, including a full stack trace.
@@ -825,32 +957,17 @@ to it after C<connect()> is always safe:
 
 =item * Only works with blocking (synchronous) calls, not very suitable for use
 in asynchronous frameworks unless you know your queries are fast and you have a
-low-latency connection with the Postgres server.
+low-latency connection with the Postgres server. This is unlikely to improve in
+future versions, Perl's async story is somewhat awkward in general, and fully
+supporting async operation might require a fundamental redesign of how this
+module works.
 
-=back
+=item * LISTEN support is still missing. May be added in a future version, as
+this seems doable without supporting full async.
 
-Missing features:
+=item * Pipelining support is also missing. I have some ideas for an API, but
-
+doubt I'll ever implement it. Suffers from the same awkwardness and complexity
-=over
+as asynchronous calls.
-
-=item COPY support
-
-I hope to implement this someday.
-
-=item LISTEN support
-
-Would be nice to have, most likely doable without going full async.
-
-=item Asynchronous calls
-
-Probably won't happen. Perl's async story is slightly awkward in general, and
-fully supporting async operation might require a fundamental redesign of how
-this module works. It certainly won't I<simplify> the implementation.
-
-=item Pipelining
-
-I have some ideas for an API, but doubt I'll ever implement it. Suffers from
-the same awkwardness and complexity as asynchronous calls.
 
 =back
 

FU/SQL.pm

@@ -1,11 +1,11 @@
-package FU::SQL 0.2;
+package FU::SQL 1.4;
 use v5.36;
 use Exporter 'import';
 use Carp 'confess';
 use experimental 'builtin', 'for_list';
 
 our @EXPORT = qw/
-    P RAW SQL
+    P RAW IDENT SQL
     PARENS INTERSPERSE COMMA
     AND OR WHERE
     SET VALUES IN
@@ -16,6 +16,7 @@ sub _obj { bless [@_], 'FU::SQL::val' }
 
 sub P :prototype($) ($p) { bless \(my $x = $p), 'FU::SQL::p' }
 sub RAW :prototype($) ($s) { _obj "$s" }
+sub IDENT :prototype($) ($s) { bless \(my $x = "$s"), 'FU::SQL::i' }
 
 # These operate on $_ and must be called with &func syntax.
 # The readonly check can be finicky.
@@ -29,7 +30,7 @@ sub COMMA { INTERSPERSE ',', @_ }
 
 sub _conditions {
     @_ == 1 && ref $_[0] eq 'HASH'
-    ? map PARENS(RAW $_,
+    ? map PARENS(IDENT $_,
         !defined $_[0]{$_}              ? ('IS NULL') :
         ref($_[0]{$_}) eq 'FU::SQL::in' ? ($_[0]{$_})
                                         : ('=', $_[0]{$_})
@@ -41,11 +42,11 @@ sub AND { !@_ || (@_ == 1 && ref $_[0] eq 'HASH' && keys $_[0]->%* == 0) ? RAW '
 sub OR  { !@_ || (@_ == 1 && ref $_[0] eq 'HASH' && keys $_[0]->%* == 0) ? RAW '1=0' : INTERSPERSE 'OR',  _conditions @_ }
 sub WHERE { SQL 'WHERE', AND @_ }
 
-sub SET($h) { SQL 'SET', COMMA map SQL(RAW $_, '=', $h->{$_}), sort keys %$h }
+sub SET($h) { SQL 'SET', COMMA map SQL(IDENT $_, '=', $h->{$_}), sort keys %$h }
 
 sub VALUES {
     @_ == 1 && ref $_[0] eq 'HASH'
-    ? SQL '(', COMMA(map RAW $_, sort keys $_[0]->%*), ') VALUES (', COMMA(map $_[0]{$_}, sort keys $_[0]->%*), ')'
+    ? SQL '(', COMMA(map IDENT $_, sort keys $_[0]->%*), ') VALUES (', COMMA(map $_[0]{$_}, sort keys $_[0]->%*), ')'
     : @_ == 1 && ref $_[0] eq 'ARRAY'
     ? SQL 'VALUES (', COMMA($_[0]->@*), ')'
     : SQL 'VALUES (', COMMA(@_), ')';
@@ -71,6 +72,10 @@ sub FU::SQL::p::_compile($self, $opt, $sql, $params) {
     $$sql .= $opt->{placeholder_style} eq 'pg' ? '$'.@$params : '?';
 }
 
+sub FU::SQL::i::_compile($self, $opt, $sql, $params) {
+    $$sql .= $opt->{quote_identifier} ? $opt->{quote_identifier}->($$self) : $$self;
+}
+
 sub FU::SQL::in::_compile($self, $opt, $sql, $params) {
     if ($opt->{in_style} eq 'pg') {
         $$sql .= '= ANY(';
@@ -87,6 +92,7 @@ sub FU::SQL::in::_compile($self, $opt, $sql, $params) {
 }
 
 sub FU::SQL::val::compile($self, %opt) {
+    !/^(placeholder_style|in_style|quote_identifier)$/ && confess "Unknown flag: $_" for keys %opt;
     $opt{placeholder_style} ||= 'dbi';
     $opt{in_style} ||= 'dbi';
     my($sql, @params) = ('');
@@ -94,7 +100,7 @@ sub FU::SQL::val::compile($self, %opt) {
     ($sql, \@params)
 }
 
-*FU::SQL::p::compile = *FU::SQL::in::compile = \*FU::SQL::val::compile;
+*FU::SQL::p::compile = *FU::SQL::i::compile = *FU::SQL::in::compile = \*FU::SQL::val::compile;
 
 1;
 __END__
@@ -103,11 +109,6 @@ __END__
 
 FU::SQL - Small and Safe SQL Query Builder
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::SQL;
@@ -120,11 +121,11 @@ changes, see the main L<FU> module for details.
 
   my $sel = SQL 'SELECT id, name FROM table', WHERE { id => IN([1,2,3]) };
 
-  my($sql, @params) = $sel->compile;
+  my($sql, $params) = $sel->compile;
 
 =head1 DESCRIPTION
 
-=head2 Compiling SQL
+=head1 Compiling SQL
 
 All functions listed under L</"Constructing SQL"> return an object that can be
 passed to other construction functions or compiled into SQL and bind
@@ -161,11 +162,21 @@ C<'pg'> when your SQL is going to L<FU::Pg> or L<Pg::PQ>.
 Set the style to use for C<IN> expressions, refer to the C<IN()> function below
 for details.
 
+=item quote_identifier => $func
+
+Set a function to perform quoting of SQL identifiers. When using DBI, you can
+do:
+
+  my($sql) = $obj->compile(quote_identifier => sub { $dbh->quote_identifier(@_) });
+
+If this option is not set, identifiers are included into the raw SQL string
+without any escaping.
+
 =back
 
 =back
 
-=head2 Constructing SQL
+=head1 Constructing SQL
 
 All of the functions below return an object with a C<compile()> method. All
 functions are exported by default.
@@ -181,7 +192,7 @@ types of supported arguments:
 
 =item 1.
 
-B<String literals> are interpreted as raw SQL fragments.
+I<String literals> are interpreted as raw SQL fragments.
 
 =item 2.
 
@@ -189,7 +200,7 @@ Objects returned by other functions listed below are included as SQL fragments.
 
 =item 3.
 
-B<Everything else> is considered a bind parameter.
+I<Everything else> is considered a bind parameter.
 
 =back
 
@@ -249,6 +260,18 @@ Force the given C<$sql> string to be included as SQL. For example:
 
 Never use this function with untrusted input.
 
+=item IDENT($string)
+
+Mark the given string as an SQL identifier. This function is only useful if you
+use potentially untrusted input to determine which column to select or which
+table to select from, for example:
+
+  SQL 'SELECT id,', IDENT $ENV{column}, 'FROM table';
+
+B<WARNING:> By default this function is equivalent to C<RAW()> and hence
+provides no safety whatsoever. Be sure to set the C<quote_identifier> option on
+C<compile()> to get more useful behavior.
+
 =item PARENS(@args)
 
 Like C<SQL()> but surrounds the expression by parens:
@@ -284,8 +307,8 @@ C<'1=1'> (i.e. true) if C<@conditions> is an empty list.
 =item AND($hashref)
 
 A special form of C<AND()> that tests the given columns for equality instead.
-The keys of the hashref are interpreted as raw SQL and the values as bind
+The keys of the hashref are interpreted as per C<IDENT()> and the values as
-parameters.
+bind parameters.
 
   AND { id => 1, number => RAW 'random()', x => undef }
   # '( id = ? ) AND ( number = random() ) AND ( x IS NULL )'
@@ -356,12 +379,12 @@ values. This function results in different SQL depending on the C<in_style>
 option given to C<compile()>. The default C<'dbi'> style passes each value as a
 bind parameter:
 
-  SQL 'WHERE id', IN([1, 2, 3, 4]);
+  SQL 'WHERE id', IN [1, 2, 3, 4];
   # 'WHERE id IN(?, ?, ?, ?)', parameters: 1, 2, 3, 4
 
 The C<'pg'> style passes the entire array as a single bind parameter instead:
 
-  SQL 'WHERE id', IN([1, 2, 3, 4]);
+  SQL 'WHERE id', IN [1, 2, 3, 4];
   # 'WHERE id = ANY(?)', parameter: [1, 2, 3, 4]
 
 The C<'pg'> style allows for more efficient re-use of cached prepared
@@ -372,7 +395,7 @@ with L<DBD::Pg> or L<Pg::PQ>.
 
 Can be used in the C<$hashref> versions of C<AND>, C<OR> and C<WHERE> as well:
 
-  WHERE { id => IN([1, 2]) }
+  WHERE { id => IN [1, 2] }
   # 'WHERE id IN(?, ?)'
 
 =back

FU/Util.pm

@@ -1,25 +1,36 @@
-package FU::Util 0.2;
+package FU::Util 1.4;
 
 use v5.36;
 use FU::XS;
 use Carp 'confess';
 use Exporter 'import';
+use Encode ();
 use POSIX ();
 use experimental 'builtin';
 
 our @EXPORT_OK = qw/
     to_bool
     json_format json_parse
-    utf8_decode uri_escape uri_unescape
+    has_control check_control utf8_decode
+    uri_escape uri_unescape
     query_decode query_encode
     httpdate_format httpdate_parse
+    gzip_lib gzip_compress brotli_compress
     fdpass_send fdpass_recv
 /;
 
+
+# Internal utility function
+sub has_control :prototype($) ($s) { defined $s && $s =~ /[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]/ }
+sub check_control :prototype($) ($s) { confess 'Invalid control character' if has_control $s; }
+
+# Deprecated, call Encode::decode() directly.
 sub utf8_decode :prototype($) {
     return if !defined $_[0];
-    confess 'Invalid UTF-8' if !utf8::decode($_[0]);
+    eval {
-    confess 'Invalid control character' if $_[0] =~ /[\x00-\x08\x0b\x0c\x0e-\x1f]/;
+        $_[0] = Encode::decode('UTF-8', $_[0], Encode::FB_CROAK);
+        1
+    } || confess($@ =~ s/ at .+\n$//r);
     $_[0]
 }
 
@@ -32,6 +43,7 @@ sub uri_escape :prototype($) ($s) {
 sub uri_unescape :prototype($) ($s) {
     return if !defined $s;
     utf8::encode($s);
+    $s =~ tr/+/ /;
     $s =~ s/%([0-9A-Fa-f]{2})/chr(hex($1))/eg;
     utf8_decode $s;
 }
@@ -39,6 +51,7 @@ sub uri_unescape :prototype($) ($s) {
 sub query_decode :prototype($) ($s) {
     my %o;
     for (split /&/, $s//'') {
+        next if !length;
         my($k,$v) = map uri_unescape($_), split /=/, $_, 2;
         $v //= builtin::true;
         if (ref $o{$k}) { push $o{$k}->@*, $v }
@@ -93,15 +106,7 @@ __END__
 
 =head1 NAME
 
-FU::Util - Miscellaneous utility functions that really should have been part of
+FU::Util - Miscellaneous Utility Functions
-a core Perl installation but aren't for some reason because the Perl community
-doesn't believe in the concept of a "batteries included" standard library.
-</rant>
-
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
 
 =head1 SYNOPSIS
 
@@ -111,7 +116,11 @@ changes, see the main L<FU> module for details.
 
 =head1 DESCRIPTION
 
-=head2 Boolean Stuff
+A bunch of functions that are too small (or I'm too lazy) to split out into
+separate modules. Some of these functions really ought to be part of Perl core.
+
+
+=head1 Boolean Stuff
 
 Perl has had a builtin boolean type since version 5.36 and FU uses that where
 appropriate, but there's still a lot of older code out there using different
@@ -135,10 +144,10 @@ value for C<$val>, due to C<\0> and C<\1> being considered booleans.
 
 =back
 
-=head2 JSON parsing & formatting
+=head1 JSON Parsing & Formatting
 
 This module comes with a custom C-based JSON parser and formatter. These
-functions conform strictly to L<RFC-8259|https://tools.ietf.org/html/rfc8259>,
+functions conform to L<RFC-8259|https://tools.ietf.org/html/rfc8259>,
 non-standard extensions are not supported and never will be. It also happens to
 be pretty fast, refer to L<FU::Benchmarks> for some numbers.
 
@@ -212,13 +221,6 @@ roughly similar to:
 
   JSON::PP->new->allow_nonref->core_bools->convert_blessed->encode($scalar);
 
-Some modules escape the slash character in encoded strings to prevent a
-potential XSS vulnerability when embedding JSON inside C<< <script> ..
-</script> >> tags.  This function does I<not> do that because it might not even
-be sufficient. The following is probably an improvement:
-
-  json_format($data) =~ s{</}{<\\/}rg =~ s/<!--/<\\u0021--/rg;
-
 This function generates invalid JSON if you pass it a string with invalid
 Unicode characters; I don't see how you'd ever accidentally end up with such a
 string, anyway.
@@ -244,6 +246,25 @@ versions.
 
 Boolean, returns a UTF-8 encoded byte string instead of a Perl Unicode string.
 
+=item html_safe
+
+Boolean. When set, the encoded JSON is safe for (unescaped) inclusion into HTML
+or XML content. This encodes C<< < >>, C<< > >> and C<< & >> as Unicode escapes.
+Commonly used to embed data inside a HTML page:
+
+  $html = '<script id="site_data" type="application/json">'
+        . json_format($data, html_safe => 1)
+        . '</script>';
+
+This option does NOT make it safe to include the encoded JSON as an attribute
+value. There is no way to do that without violating JSON specs, so you should
+use entity escaping instead.
+
+Some JSON modules escape the forward slash (C</>) character instead, but that
+is I<only> sufficient for embedding inside a C<< <script> >> tag. In any other
+context, you'll need the more thourough escaping provided by this C<html_safe>
+option.
+
 =item max_size
 
 Maximum permitted size, in bytes, of the generated JSON string. Defaults to 1 GiB.
@@ -266,7 +287,7 @@ functions, L<JSON::PP> and L<Cpanel::JSON::XS> are perfectly fine alternatives.
 L<JSON::SIMD> and L<JSON::Tiny> also look like good and maintained candidates.)
 
 
-=head2 URI-Related Functions
+=head1 URI-Related Functions
 
 While URIs are capable of encoding arbitrary binary data, the functions below
 assume you're only dealing with text. This makes them more robust against weird
@@ -274,18 +295,6 @@ inputs, at the cost of flexibility.
 
 =over
 
-=item utf8_decode($bytes)
-
-Convert a (perl-UTF-8 encoded) byte string into a sanitized perl Unicode
-string. The conversion is performed in-place, so the C<$bytes> argument is
-turned into a Unicode string. Returns the same string for convenience.
-
-This function throws an error if the input is not valid UTF-8 or if it contains
-ASCII control characters - that is, any character between C<0x00> and C<0x1f>
-except for tab, newline and carriage return.
-
-(This is a tiny wrapper around C<utf8::decode()> with some extra checks)
-
 =item uri_escape($string)
 
 Takes an Unicode string and returns a percent-encoded ASCII string, suitable
@@ -294,8 +303,7 @@ for use in a query parameter.
 =item uri_unescape($string)
 
 Takes an Unicode string potentially containing percent-encoding and returns a
-decoded Unicode string. Also checks for ASCII control characters as per
+decoded Unicode string.
-C<utf8_decode()>.
 
 =item query_decode($string)
 
@@ -312,8 +320,7 @@ have a value are decoded as C<builtin::true>. Example:
     # }
 
 The input C<$string> is assumed to be a perl Unicode string. An error is thrown
-if the resulting data decodes into invalid UTF-8 or contains control
+if the resulting data decodes into invalid UTF-8.
-characters, as per C<utf8_decode>.
 
 =item query_encode($hashref)
 
@@ -328,7 +335,7 @@ then encoded.
 =back
 
 
-=head2 HTTP Date Formatting
+=head1 HTTP Date Formatting
 
 The HTTP date format is utter garbage, but with the right tools it doesn't
 require I<too> much code to work with.
@@ -353,7 +360,67 @@ This will not happen if your local timezone is UTC.
 =back
 
 
-=head2 File Descriptor Passing
+=head1 Gzip Compression
+
+Gzip compression can be done with a few different libraries. The canonical one
+is I<zlib>, which is old and not well optimized for modern systems. There's
+also I<zlib-ng>, a (much) more performant reimplementation that remains
+API-compatible with I<zlib>. And there's I<libdeflate>, which offers a
+different API that does not support streaming compression but is, in exchange,
+even faster than I<zlib-ng>.
+
+There are more implementations, of course, but this module only supports those
+three and (attempts to) pick the best one that's available on your system.
+
+=over
+
+=item gzip_lib()
+
+Returns an empty string if no supported gzip library was found on your system
+(unlikely but possible), otherwise returns the selected implementation: either
+C<"libdeflate">, C<"zlib-ng"> or C<"zlib">.
+
+This function does not try very hard to differentiate between I<zlib> and
+I<zlib-ng>, so it may report that I<zlib> is being used on systems where
+C<libz.so> is, in fact, I<zlib-ng>.
+
+=item gzip_compress($level, $data)
+
+Returns a byte string with the gzip-compressed version of C<$data> at the given
+gzip C<$level>, which is a number between 0 (no compression) and 12 (strongest
+compression). Only I<libdeflate> supports levels higher than 9, for
+I<zlib(-ng)> the level is capped at 9. 6 is typically used as a default.
+
+Throws an error if no suitable library was found.
+
+This function is B<NOT> safe to use from multiple threads!
+
+=back
+
+This module does not currently implement decompression. If you need that, or
+streaming, or other functionality not provided here, there's
+L<Compress::Raw::Zlib> and L<Compress::Zlib> in the core Perl distribution and
+L<Gzip::Faster>, L<Gzip::Zopfli> and L<Gzip::Libdeflate> on CPAN.
+
+
+=head1 Brotli Compression
+
+Just a small wrapper around C<libbrotlienc.so>'s one-shot compression
+interface.
+
+=over
+
+=item brotli_compress($level, $data)
+
+Returns a byte string with the brotli-compressed version of C<$data> at the
+given quality C<$level> (between 0 and 11).
+
+Throws an error if C<libbrotlienc.so> could not be found or loaded.
+
+=back
+
+
+=head1 File Descriptor Passing
 
 UNIX sockets (see L<IO::Socket::UNIX>) have the fancy property of letting you
 send file descriptors over them, allowing you to pass, for example, a socket
@@ -382,10 +449,9 @@ Like regular socket I/O, a single C<fdpass_send()> message may be split across
 multiple C<fdpass_recv()> calls; in that case the C<$fd> is only received on
 the first call.
 
-Don't use this function if the sender may include multiple file descriptors in
+The C<O_CLOEXEC> flag is set on received file descriptors. Don't use this
-a single message, weird things can happen. File descriptors received this way
+function if the sender may include multiple file descriptors in a single
-do not have the C<CLOEXEC> flag and will thus survive a call to C<exec()>.
+message, weird things can happen. Refer to L<this wonderful
-Refer to L<this wonderful
 discussion|https://gist.github.com/kentonv/bc7592af98c68ba2738f4436920868dc>
 for more weirdness and edge cases.
 

FU/XMLWriter.pm

@@ -1,4 +1,4 @@
-package FU::XMLWriter 0.2;
+package FU::XMLWriter 1.4;
 use v5.36;
 use Carp 'confess';
 use Exporter 'import';
@@ -83,11 +83,6 @@ __END__
 
 FU::XMLWriter - Convenient and efficient XML and HTML generator.
 
-=head1 EXPERIMENTAL
-
-This module is still in development and there will likely be a few breaking API
-changes, see the main L<FU> module for details.
-
 =head1 SYNOPSIS
 
   use FU::XMLWriter ':html5_';
@@ -138,7 +133,7 @@ other XML writing modules on CPAN that I tried, but whether this approach is
 faster than typical templating solutions... I've no idea. Check out
 L<FU::Benchmarks> for some benchmarks.
 
-=head2 Top-level functions
+=head1 Top-level functions
 
 These functions all return a byte string with (UTF-8) encoded XML.
 
@@ -146,7 +141,7 @@ These functions all return a byte string with (UTF-8) encoded XML.
 
 =item fragment($block)
 
-Executes C<$block> and captures the output of all I</"Output functions">
+Executes C<$block> and captures the output of all L</"Output functions">
 called within the same scope into a string. This function can be safely nested:
 
   my $string = fragment {
@@ -170,7 +165,7 @@ passed to the C<tag_()> call for the top-level C<< <html> >> element.
 
 =back
 
-=head2 Output functions
+=head1 Output functions
 
 =over
 
@@ -252,7 +247,7 @@ provided it defaults to C<undef>.
 
 =back
 
-=head2 Utility function
+=head1 Utility function
 
 =over
 
@@ -263,12 +258,12 @@ and C<"> are replaced with their XML entity.
 
 =back
 
-=head2 Import options
+=head1 Import options
 
 All of the functions mentioned in this document can be imported individually.
 There are also two import groups:
 
-  use FU::XMLWriter ':html_';
+  use FU::XMLWriter ':html5_';
 
 Exports C<tag_()>, C<html_()>, C<lit_()>, C<txt_()> and all of the C<<
 <html-tag>_ >> functions mentioned above.

FU/XS.pm

@@ -1,5 +1,5 @@
 # This module is for internal use by other FU modules.
-package FU::XS 0.2;
+package FU::XS 1.4;
 use Carp; # may be called by XS.
 use XSLoader;
 XSLoader::load('FU');

Makefile.PL

@@ -2,7 +2,7 @@ use ExtUtils::MakeMaker;
 use Config;
 
 os_unsupported if $^O eq 'MSWin32'; # I don't know on which OS'es the code will work exactly, but this one I can easily rule out.
-os_unsupported if $Config{ivsize} < 8;
+os_unsupported if $Config{ptrsize} < 8;
 os_unsupported if $Config{usequadmath};
 
 WriteMakefile(
@@ -15,12 +15,17 @@ WriteMakefile(
     MIN_PERL_VERSION => 'v5.36',
     META_MERGE => {
         dynamic_config => 0,
+        'meta-spec' => { version => 2 },
         resources => {
-            repository => 'https://code.blicky.net/yorhel/fu',
+            homepage => 'https://dev.yorhel.nl/fu',
-            bugtracker => 'https://code.blicky.net/yorhel/fu/issues',
+            repository => {
+                web => 'https://code.blicky.net/yorhel/fu',
+                type => 'git',
+            },
+            bugtracker => {
+                web => 'https://code.blicky.net/yorhel/fu/issues',
+                mailto => 'projects@yorhel.nl',
             },
-        no_index => {
-            file => 'bench.PL',
         },
     },
     depend => { '$(OBJECT)', 'c/*.c' },

README.md

@@ -1,14 +1,12 @@
-# FU - Framework Ultimatum: A Lean and Efficient Zero-Dependency Web Framework
+# FU - A Lean and Efficient Zero-Dependency Web Framework
 
-FU is a web development framework for Perl and a collection of handy utility
+FU (Framework Ultimatum) is a web development framework for Perl and a
-modules.
+collection of handy utility modules.
+
+*Website:* More information @ [dev.yorhel.nl/fu](https://dev.yorhel.nl/fu).
 
 *Contributing:* Refer to my [contribution guidelines](https://dev.yorhel.nl/contributing).
 
-## Project Status
-
-**EXPERIMENTAL**; expect breaking changes.
-
 ## Build & Install
 
 ```sh
@@ -23,7 +21,6 @@ Things that may or may not happen:
 
 - FU::JSON - JSON::{XS,PP,etc}-compatible wrapper around FU::Util's JSON functions? I prolly won't need this myself, but could be handy.
 - FU::DBI - DBI wrapper with a FU::Pg-like API, for easier integration into FU.
-- FU::Mailer - Simple sendmail wrapper
 
 # License
 

bench.PL

@@ -3,6 +3,7 @@
 # Can be invoked as:
 #   ./bench.PL              # generates FU/Benchmarks.pod, running new benchmarks as necessary
 #   ./bench.PL id x y       # invalidate cache for the (regex-)matching benchmark IDs, x and y and re-run them
+#   ./bench.PL exec id x y  # Run just the given benchmark and exit
 #
 # This script obviously has more dependencies than the FU distribution itself.
 # It's supposed to be used by maintainers, not users.
@@ -25,11 +26,19 @@ my %modules = map +($_, eval "require $_; \$${_}::VERSION"), qw/
     TUWF::XML
     HTML::Tiny
     XML::Writer
+    DBD::Pg
+    Pg::PQ
 /;
+use FU::Pg;
+
+my @exec = $ARGV[0] && $ARGV[0] eq 'exec' ? @ARGV[1..3] : ();
+my @run = !@exec && @ARGV && (qr/$ARGV[0]/i, $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/, $ARGV[2] ? qr/$ARGV[2]/i : qr/.*/);
 
 my %data; # "id x y" => { id x y rate exists }
+my @bench; # [ id, text, [ x_1, .. ], [ [ y_1, mod_1, sub_1, .. ], .. ] ]
 my %oldmodules;
-{ if (open my $F, '<', 'FU/Benchmarks.pod') {
+if (!@exec) {
+    if (open my $F, '<', 'FU/Benchmarks.pod') {
         my $indata;
         while (<$F>) {
             chomp;
@@ -40,17 +49,51 @@ my %oldmodules;
             @d{qw/id x y rate/} = split /\t/;
             $data{"$d{id} $d{x} $d{y}"} = \%d;
         }
-} }
+    }
-
-if (@ARGV) {
-    my $idre = qr/$ARGV[0]/i;
-    my $xre = $ARGV[1] ? qr/$ARGV[1]/i : qr/.*/;
-    my $yre = $ARGV[2] ? qr/$ARGV[2]/i : qr/.*/;
-    delete $_->{rate} for grep $_->{id} =~ /$idre/ && $_->{x} =~ /$xre/ && $_->{y} =~ /$yre/, values %data;
 }
 
 
-my @bench; # [ id, text, [ x_1, .. ], [ [ y_1, mod_1, sub_1, .. ], .. ] ]
+sub fmtbench($id, $text, $xs, $ys) {
+    my $r = "$text\n\n";
+    if (@$xs > 1) {
+        $r .= sprintf '%18s', '';
+        $r .= sprintf '%12s', $_ for @$xs;
+        $r .= "\n";
+    }
+    for my ($n, $yr) (builtin::indexed @$ys) {
+        my $x = $xs->[$n];
+        my ($y, $m, @ys) = @$yr;
+        $m ||= $y;
+        $r .= sprintf '%18s', $y;
+        for my $i (0..$#$xs) {
+            my $d = $data{"$id $xs->[$i] $y"};
+            $r .= $d && $d->{rate} ? sprintf '%10d/s', $d->{rate} : sprintf '%12s', '-';
+        }
+        $r .= "\n";
+    }
+    "$r\n"
+}
+
+$SIG{INT} = $SIG{HUP} = sub { exit };
+END {
+    exit if @exec;
+
+    open my $F, '>FU/Benchmarks.pod' or die $!;
+    select $F;
+    while (<DATA>) {
+        s/^%/=/;
+        s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
+        s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
+        print;
+    }
+    for (sort keys %data) {
+        my $b = $data{$_};
+        print join("\t", map $_//'', @{$b}{qw/ id x y rate /})."\n";
+    }
+}
+
+
+
 sub def($id, $text, $xs, @ys) {
     for my ($ya) (@ys) {
         my($y, $m, @sub) = @$ya;
@@ -61,12 +104,6 @@ sub def($id, $text, $xs, @ys) {
             $data{$d} ||= { id => $id, x => $x, y => $y };
             $d = $data{$d};
             $d->{exists} = 1;
-            delete $d->{rate} if !$oldmodules{$m} || $modules{$m} ne $oldmodules{$m};
-            if (!exists $d->{rate}) {
-                my $o = timethis -5, $sub[$i], 0, 'none';
-                $d->{rate} = sprintf '%.0f', $o->iters/$o->real;
-                printf "%-20s%-12s%-20s%10d/s\n", $id, $x, $y, $d->{rate};
-            }
         }
     }
     push @bench, [ $id, $text, $xs, \@ys ];
@@ -112,7 +149,7 @@ defjson stru => 0, 'Unicode strings', do { use utf8;
     [ map +('グリザイアの果実 -LE FRUIT DE LA GRISAIA-', '💩', 'Я люблю нічого не робити'), 1..50 ];
 };
 defjson stres => 0, 'String escaping (few)', [ map 'This string needs to "be escaped" a little bit', 1..100 ];
-defjson strel => 0, 'String escaping (many)', [ map "This \" \\ needs \b\x01\x02\x03\x04 more", 1..100 ];
+defjson strel => 0, 'String escaping (many)', [ map "This \" \\ needs \n\x41\x42\x43\x44 more", 1..100 ];
 
 
 
@@ -196,49 +233,84 @@ def 'xml/a', 'HTML fragment', [ 'Rate' ],
 
 
 
+{
+    die "FU_TEST_DB not set.\n" if @exec && !$ENV{FU_TEST_DB};
+    my $pq = @exec && Pg::PQ::Conn->new($ENV{FU_TEST_DB});
+    my $fu = @exec && FU::Pg->connect($ENV{FU_TEST_DB});
+    # XXX: Doesn't support all connection params this way
+    my $dbi = @exec && DBI->connect("dbi:Pg:dbname=".$pq->db, $pq->user, $pq->pass, {RaiseError => 1, PrintError => 0});
+
+    my $small = 'SELECT x, x+1, x+2, x+3, x+4, x+5, x+6, x+7, x+8, x+9 FROM generate_series(-10000::smallint, 9999, 10) x(x)';
+    my $big = 'SELECT x<<5, x<<10, x<<15, x<<20, x<<25, x<<30, x<<35, x<<40, x<<45, x<<50 FROM generate_series(1::bigint, 20000, 1) x(x)';
+
+    my sub dbi { my $sum = 0; my $st = $dbi->prepare_cached($_[0]); for my $row ($dbi->selectall_arrayref($st)->@*) { $sum ^= $_ for @$row; } }
+    my sub pq  { my $sum = 0; $pq->prepare('' => $_[0]); for my $row ($pq->execQueryPrepared('')->rows) { $sum ^= $_ for @$row; } }
+    my sub fub { my $sum = 0; for my $row ($fu->sql($_[0])->alla->@*) { $sum ^= $_ for @$row; } }
+    my sub fut { my $sum = 0; for my $row ($fu->sql($_[0])->text->alla->@*) { $sum ^= $_ for @$row; } }
+
+    def 'pg/ints', 'Fetch and bitwise-or 20k integers', [ 'Smallint', 'Bigint' ],
+        [ 'DBD::Pg', undef, sub { dbi($small) }, sub { dbi($big) } ],
+        [ 'Pg::PQ', undef, sub { pq($small) }, sub { pq($big) } ],
+        [ 'FU::Pg (bin)', 'FU', sub { fub($small) }, sub { fub($big) } ],
+        [ 'FU::Pg (text)', 'FU', sub { fut($small) }, sub { fut($big) } ];
+}
+
+
+
+
+
 delete @data{ grep !$data{$_}{exists}, keys %data };
 
-sub fmtbench($id, $text, $xs, $ys) {
+
-    my $r = "$text\n\n";
+sub runbench($sub) {
-    if (@$xs > 1) {
+    my $o = timethis -1, $sub, 0, 'none';
-        $r .= sprintf '%18s', '';
+    printf "%.2f\n", $o->iters/$o->real;
-        $r .= sprintf '%12s', $_ for @$xs;
+    exit;
-        $r .= "\n";
 }
-    for my ($n, $yr) (builtin::indexed @$ys) {
+
-        my $x = $xs->[$n];
+sub execbench($d) {
-        my ($y, $m, @ys) = @$yr;
+    my $sum = 0;
+    my $num = 1;
+    local $| = 1;
+    printf "%-20s%-12s%-20s", $d->{id}, $d->{x}, $d->{y};
+    for (1..$num) {
+        open my $P, '-|', $^X, (map "-I$_", @INC), $0, 'exec', $d->{id}, $d->{x}, $d->{y};
+        chomp(my $rate = <$P>);
+        printf "%10d", $rate;
+        $sum += $rate;
+    }
+    printf " ->%10d\n", $sum/$num;
+    $d->{rate} = sprintf '%.0f', $sum/$num;
+}
+
+for my $b (@bench) {
+    my ($id, $text, $xs, $ys) = @$b;
+    for my ($ya) (@$ys) {
+        my($y, $m, @sub) = @$ya;
         $m ||= $y;
-        $r .= sprintf '%18s', $y;
+        for my($i, $x) (builtin::indexed @$xs) {
-        for my $i (0..$#$xs) {
+            next if !$sub[$i];
-            my $d = $data{"$id $xs->[$i] $y"};
+            if (@exec) {
-            $r .= $d ? sprintf '%10d/s', $d->{rate} : sprintf '%12s', '-';
+                runbench $sub[$i] if $exec[0] eq $id && $exec[1] eq $x && $exec[2] eq $y;
+            } else {
+                my $d = $data{"$id $x $y"};
+                execbench $d if !$oldmodules{$m} || $modules{$m} ne $oldmodules{$m}
+                    || (@run && $id =~ /$run[0]/ && $x =~ /$run[1]/ && $y =~ /$run[2]/);
             }
-        $r .= "\n";
         }
-    "$r\n"
-}
-
-{
-    open my $F, '>FU/Benchmarks.pod' or die $!;
-    select $F;
-    while (<DATA>) {
-        s#^:modules#join '', map sprintf("=item L<%s> %s\n\n", $_, $modules{$_}), sort keys %modules#e;
-        s#^:benches (.+)#join '', map fmtbench(@$_), grep $_->[0] =~ /$1/, @bench#e;
-        print;
-    }
-    for (sort keys %data) {
-        my $b = $data{$_};
-        print join("\t", @{$b}{qw/ id x y rate /})."\n";
     }
 }
 
+die if @exec;
+
+
+# s/^=/%/  to prevent tools from interpreting the below as POD
 __DATA__
-=head1 NAME
+%head1 NAME
 
 FU::Benchmarks - A bunch of automated benchmark results.
 
-=head1 DESCRIPTION
+%head1 DESCRIPTION
 
 This file is automatically generated from 'bench.PL' in the L<FU> distribution.
 These benchmarks compare performance of some FU functionality against similar
@@ -256,33 +328,45 @@ real-world use.
 B<DISCLAIMER#3:> Many of these benchmarks exists solely to test edge case
 performance, these numbers are not representative for real-world use.
 
-=head1 MODULE VERSIONS
+%head1 MODULE VERSIONS
 
 The following module versions were used:
 
-=over
+%over
 
 :modules
 
-=back
+%back
 
-=head1 BENCHMARKS
+%head1 BENCHMARKS
 
-=head2 JSON Parsing & Formatting
+%head2 JSON Parsing & Formatting
 
 These benchmarks run on large-ish arrays with repeated values. JSON encoding is
 sufficiently fast that Perl function calling overhead tends to dominate for
 smaller inputs, but I don't find that overhead very interesting.
 
-Also worth noting that JSON::SIMD formatting code is forked from JSON::XS, the
+Also worth noting that L<JSON::SIMD> formatting code is forked from
-SIMD parts are only used for parsing.
+L<JSON::XS>, the SIMD parts are only used for parsing.
 
 :benches ^json
 
-=head2 XML Writing
+%head2 XML Writing
+
+L<FU::XMLWriter> is the only XS-based XML DSL that I'm aware of, so all direct
+competition is inherently slower by virtue of being pure perl. I'm sure some
+templating modules will perform better, though.
 
 :benches ^xml
 
-=cut
+%head2 PostgreSQL client
+
+Fetching query results is highly unlikely to be a bottleneck in your code, this
+benchmark is mainly here to verify that L<FU::Pg> is not introducing a
+bottleneck where there shouldn't be one.
+
+:benches ^pg
+
+%cut
 
 # Cached data used by bench.PL. Same as the formatted tables above but easier to parse.

c/compress.c

@@ -0,0 +1,164 @@
+static const char *fugz_imps[] = {"", "libdeflate", "zlib-ng", "zlib"};
+static int fugz_imp = -1;
+
+
+/* zlib & zlib-ng */
+
+typedef struct {
+    const char *next_in;
+    unsigned int avail_in;
+    unsigned long total_in;
+    char *next_out;
+    unsigned int avail_out;
+    unsigned long total_out;
+    const char *msg;
+    struct internal_state *state;
+    void *zalloc;
+    void *zfree;
+    void *opaque;
+    int data_type;
+    unsigned long adler;
+    unsigned long reserved;
+} z_stream;
+
+static int (*deflate)(z_stream *, int);
+static int (*deflateEnd)(z_stream *);
+static int (*deflateInit2)(z_stream *, int, int, int, int, int);
+static int (*deflateInit2_)(z_stream *, int, int, int, int, int, const char *, int);
+static unsigned long (*compressBound)(unsigned long);
+
+
+/* libdeflate */
+
+static struct libdeflate_compressor *fugz_ld_ctx;
+static int fugz_ld_comp = -1;
+
+static struct libdeflate_compressor *(*libdeflate_alloc_compressor)(int);
+static void (*libdeflate_free_compressor)(struct libdeflate_compressor *);
+static size_t (*libdeflate_gzip_compress_bound)(struct libdeflate_compressor *, size_t);
+static size_t (*libdeflate_gzip_compress)(struct libdeflate_compressor *, const void *, size_t, void *, size_t);
+
+
+
+static const char *fugz_lib() {
+    if (fugz_imp >= 0) goto done;
+
+    void *handle;
+    if ((handle = dlopen("libdeflate.so", RTLD_LAZY))) {
+        if ((libdeflate_alloc_compressor = dlsym(handle, "libdeflate_alloc_compressor"))
+           && (libdeflate_free_compressor = dlsym(handle, "libdeflate_free_compressor"))
+           && (libdeflate_gzip_compress_bound = dlsym(handle, "libdeflate_gzip_compress_bound"))
+           && (libdeflate_gzip_compress = dlsym(handle, "libdeflate_gzip_compress"))) {
+            fugz_imp = 1;
+            goto done;
+        }
+    }
+
+    int i;
+    for (i=2; i<=3; i++) {
+        if ((handle = dlopen(i == 2 ? "libz-ng.so" : "libz.so", RTLD_LAZY))) {
+            if (((deflate = dlsym(handle, "zng_deflate")) || (deflate = dlsym(handle, "deflate")))
+               && ((deflateEnd = dlsym(handle, "zng_deflateEnd")) || (deflateEnd = dlsym(handle, "deflateEnd")))
+               && ((deflateInit2 = dlsym(handle, "zng_deflateInit2")) || (deflateInit2_ = dlsym(handle, "deflateInit2_")))
+               && ((compressBound = dlsym(handle, "zng_compressBound")) || (compressBound = dlsym(handle, "compressBound")))) {
+                fugz_imp = i;
+                goto done;
+            }
+        }
+    }
+    fugz_imp = 0;
+
+done:
+    return fugz_imps[fugz_imp];
+}
+
+
+static SV *fugz_compress_ld(pTHX_ int level, const char *bytes, size_t inlen) {
+    if (fugz_ld_comp != level) {
+        if (fugz_ld_ctx) libdeflate_free_compressor(fugz_ld_ctx);
+        fugz_ld_ctx = NULL;
+        fugz_ld_comp = level;
+    }
+    if (!fugz_ld_ctx) fugz_ld_ctx = libdeflate_alloc_compressor(level);
+
+    size_t outlen = libdeflate_gzip_compress_bound(fugz_ld_ctx, inlen);
+    SV *out = sv_2mortal(newSV(outlen));
+    SvPOK_only(out);
+    size_t len = libdeflate_gzip_compress(fugz_ld_ctx, bytes, inlen, SvPVX(out), outlen);
+    if (!len) fu_confess("Libdeflate compression failed"); /* Shouldn't happen */
+    SvCUR_set(out, len);
+    SvPVX(out)[len] = 0;
+    return out;
+}
+
+
+static SV *fugz_compress_zlib(pTHX_ int level, const char *bytes, size_t inlen) {
+    z_stream stream;
+    memset(&stream, 0, sizeof(stream));
+
+    int r = deflateInit2
+        ? deflateInit2(&stream, level > 9 ? 9 : level, 8, 16+15, 9, 0)
+        : deflateInit2_(&stream, level > 9 ? 9 : level, 8, 16+15, 9, 0, "1.3.1", (int)sizeof(stream));
+    if (r) fu_confess("Zlib compression failed (%d)", r);
+
+    stream.avail_out = compressBound(inlen) + 64; /* compressBound() does not include the gzip header */
+    SV *out = sv_2mortal(newSV(stream.avail_out));
+    SvPOK_only(out);
+    stream.next_out = SvPVX(out);
+    stream.next_in = bytes;
+    stream.avail_in = inlen;
+
+    if ((r = deflate(&stream, 4)) != 1) fu_confess("Zlib compression failed (%d)", r);
+
+    SvCUR_set(out, stream.total_out);
+    SvPVX(out)[stream.total_out] = 0;
+    deflateEnd(&stream);
+    return out;
+}
+
+
+static SV *fugz_compress(pTHX_ IV level, SV *in) {
+    if (level < 0 || level > 12) fu_confess("Invalid compression level: %"IVdf, level);
+    if (!*fugz_lib()) fu_confess("Unable to load a suitable compression library");
+
+    STRLEN inlen;
+    const char *bytes = SvPVbyte(in, inlen);
+
+    if (fugz_imp == 1) return fugz_compress_ld(aTHX_ level, bytes, inlen);
+    else return fugz_compress_zlib(aTHX_ level, bytes, inlen);
+}
+
+
+
+/* Brotli */
+
+typedef enum { BROTLI_MODE_GENERIC = 0, BROTLI_MODE_TEXT = 1, BROTLI_MODE_FONT = 2 } BrotliEncoderMode;
+
+static size_t (*BrotliEncoderMaxCompressedSize)(size_t);
+static int (*BrotliEncoderCompress)(int, int, BrotliEncoderMode, size_t, const char *, size_t *, char *);
+
+static SV *fubr_compress(pTHX_ IV level, SV *in) {
+    if (!BrotliEncoderCompress) {
+        void *handle;
+        if (!(handle = dlopen("libbrotlienc.so", RTLD_LAZY))
+         || !(BrotliEncoderMaxCompressedSize = dlsym(handle, "BrotliEncoderMaxCompressedSize"))
+         || !(BrotliEncoderCompress = dlsym(handle, "BrotliEncoderCompress")))
+            fu_confess("Unable to load libbrotlienc.so: %s", dlerror());
+    }
+    if (level < 0 || level > 11) fu_confess("Invalid compression level: %"IVdf, level);
+
+    STRLEN inlen;
+    const char *bytes = SvPVbyte(in, inlen);
+
+    size_t outlen = BrotliEncoderMaxCompressedSize(inlen);
+    /* "Result is only valid if quality is at least 2", so let's use a (more conservative?) fallback */
+    if (level < 2 && outlen < inlen + 256) outlen = inlen + 256;
+
+    SV *out = sv_2mortal(newSV(outlen));
+    SvPOK_only(out);
+    if (!BrotliEncoderCompress(level, 22, BROTLI_MODE_GENERIC, inlen, bytes, &outlen, SvPVX(out)))
+        fu_confess("Brotli compression failed");
+    SvCUR_set(out, outlen);
+    SvPVX(out)[outlen] = 0;
+    return out;
+}

c/fcgi.c

@@ -18,6 +18,7 @@
 #define FUFE_CLEN  -5
 #define FUFE_ABORT -6  /* explicit abort or client-level EOF */
 #define FUFE_NOREQ -7  /* protocol-level EOF before we received anything */
+#define FUFE_SEND  -8  /* error in send() */
 
 #define FUFCGI_MAX_DATA 65535
 
@@ -177,8 +178,8 @@ static int fufcgi_write_record(fufcgi *ctx, fufcgi_rec *hdr, char *buf) {
     buf[7] = 0;
     int len = hdr->len + 8;
     while (len > 0) {
-        int r = write(ctx->fd, buf, len);
+        int r = send(ctx->fd, buf, len, MSG_NOSIGNAL);
-        if (r <= 0) return r == 0 ? FUFE_EOF : FUFE_IO;
+        if (r <= 0) return FUFE_SEND;
         buf += r;
         len -= r;
     }
@@ -319,8 +320,11 @@ static int fufcgi_read_params(pTHX_ fufcgi *ctx, fufcgi_rec *rec) {
                 p.name += 5;
                 for (r=0; r<p.namelen; r++)
                     p.name[r] = p.name[r] == '_' ? '-' : p.name[r] >= 'A' && p.name[r] <= 'Z' ? p.name[r] | 0x20 : p.name[r];
+                if (!(p.namelen == 14 && memcmp(p.name, "content-length", 14) == 0)
+                        && !(p.namelen == 12 && memcmp(p.name, "content-type", 12) == 0)) {
                     valsv = newSV(p.vallen+1);
                     hv_store(ctx->headers, p.name, p.namelen, valsv, 0);
+                }
 
             } else if (p.namelen == 14 && memcmp(p.name, "CONTENT_LENGTH", 14) == 0) {
                 valsv = newSV(p.vallen+1);
@@ -406,18 +410,19 @@ static int fufcgi_read_req(pTHX_ fufcgi *ctx, SV *headers, SV *params) {
     }
 }
 
-static void fufcgi_flush(fufcgi *ctx) {
+static void fufcgi_flush(pTHX_ fufcgi *ctx) {
     fufcgi_rec hdr;
     if (ctx->len > 0) {
         hdr.len = ctx->len;
         hdr.type = FCGI_STDOUT;
         hdr.id = ctx->reqid;
-        fufcgi_write_record(ctx, &hdr, ctx->buf);
+        if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
+            croak("%s\n", strerror(errno));
         ctx->len = 0;
     }
 }
 
-static void fufcgi_print(fufcgi *ctx, const char *buf, int len) {
+static void fufcgi_print(pTHX_ fufcgi *ctx, const char *buf, int len) {
     int r;
     while (len > 0) {
         r = len > FUFCGI_MAX_DATA - ctx->len ? FUFCGI_MAX_DATA - ctx->len : len;
@@ -425,23 +430,25 @@ static void fufcgi_print(fufcgi *ctx, const char *buf, int len) {
         ctx->len += r;
         len -= r;
         buf += r;
-        if (ctx->len >= FUFCGI_MAX_DATA) fufcgi_flush(ctx);
+        if (ctx->len >= FUFCGI_MAX_DATA) fufcgi_flush(aTHX_ ctx);
     }
 }
 
-static void fufcgi_done(fufcgi *ctx) {
+static void fufcgi_done(pTHX_ fufcgi *ctx) {
     fufcgi_rec hdr;
-    fufcgi_flush(ctx);
+    fufcgi_flush(aTHX_ ctx);
 
     hdr.len = 0;
     hdr.type = FCGI_STDOUT;
     hdr.id = ctx->reqid;
-    fufcgi_write_record(ctx, &hdr, ctx->buf);
+    if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
+        croak("%s\n", strerror(errno));
 
     memcpy(ctx->buf+8, "\0\0\0\0\0\0\0\0", 8); /* FCGI_REQUEST_COMPLETE */
     hdr.type = FCGI_END_REQUEST;
     hdr.len = 8;
-    fufcgi_write_record(ctx, &hdr, ctx->buf);
+    if (fufcgi_write_record(ctx, &hdr, ctx->buf) != FUFE_OK)
+        croak("%s\n", strerror(errno));
 
     ctx->reqid = ctx->len = ctx->off = 0;
 }

c/fdpass.c

@@ -53,7 +53,7 @@ static int fufdpass_recv(pTHX_ I32 ax, int socket, size_t len) {
     msg.msg_controllen = sizeof(cmsgbuf.buf);
     msg.msg_flags = 0;
 
-    ssize_t r = recvmsg(socket, &msg, 0);
+    ssize_t r = recvmsg(socket, &msg, MSG_CMSG_CLOEXEC);
     if (r < 0) {
         ST(0) = &PL_sv_undef;
         ST(1) = &PL_sv_undef;
@@ -71,6 +71,7 @@ static int fufdpass_recv(pTHX_ I32 ax, int socket, size_t len) {
     }
 
     SvCUR_set(buf, r);
+    SvPVX(buf)[r] = 0;
     ST(1) = buf;
     return 2;
 }

c/jsonfmt.c

@@ -3,6 +3,7 @@ typedef struct {
     UV depth;
     int canon;
     int pretty; /* <0 when disabled, current nesting level otherwise */
+    int htmlsafe;
 } fujson_fmt_ctx;
 
 static void fujson_fmt(pTHX_ fujson_fmt_ctx *, SV *);
@@ -27,21 +28,22 @@ static void fujson_fmt_str(pTHX_ fujson_fmt_ctx *ctx, const char *stri, size_t l
     while (off < len) {
         /* Fast path: no escaping needed */
         loff = off;
+
+#define SKIPUNTIL(cond) \
+                while (off < len) { \
+                    x = str[off]; \
+                    if (x <= 0x1f || x == '"' || x == '\\' || cond) break; \
+                    off++;\
+                }
         if (utf8) {
-            /* assume >=0x80 is valid utf8 */
+            if (!ctx->htmlsafe) { SKIPUNTIL(x == 0x7f) }
-            while (off < len) {
+            else                { SKIPUNTIL(x == 0x7f || x == '<' || x == '>' || x == '&') }
-                x = str[off];
-                if (x <= 0x1f || x == '"' || x == '\\' || x == 0x7f) break;
-                off++;
-            }
         } else {
-            /* binary strings need special handling for >=0x80 */
+            if (!ctx->htmlsafe) { SKIPUNTIL(x >= 0x7f) }
-            while (off < len) {
+            else                { SKIPUNTIL(x >= 0x7f || x == '<' || x == '>' || x == '&') }
-                x = str[off];
-                if (x <= 0x1f || x == '"' || x == '\\' || x >= 0x7f) break;
-                off++;
-            }
         }
+#undef SKIPUNTIL
+
         fustr_write(ctx->out, (char *)str+loff, off-loff);
 
         if (off < len) { /* early break, which means current byte needs special processing */
@@ -242,7 +244,7 @@ static void fujson_fmt(pTHX_ fujson_fmt_ctx *ctx, SV *val) {
         if (isinfnan(nv)) croak("unable to format floating point NaN or Inf as JSON");
         /* XXX: Cpanel::JSON::XS appears to always append a ".0" for round numbers, other modules do not. */
         /* XXX#2: This doesn't support quadmath. Makefile.PL checks for that */
-        fustr_reserve(ctx->out, NV_DIG+1);
+        fustr_reserve(ctx->out, NV_DIG+32);
         Gconvert(nv, NV_DIG, 0, ctx->out->cur);
         ctx->out->cur += strlen(ctx->out->cur);
     } else if (SvIOKp(val)) {
@@ -279,7 +281,7 @@ static SV *fujson_fmt_xs(pTHX_ I32 ax, I32 argc, SV *val) {
     ctx.out = &out;
     ctx.depth = 0;
     ctx.pretty = INT_MIN;
-    ctx.canon = 0;
+    ctx.canon = ctx.htmlsafe = 0;
     while (i < argc) {
         arg = SvPV_nolen(ST(i));
         i++;
@@ -287,9 +289,10 @@ static SV *fujson_fmt_xs(pTHX_ I32 ax, I32 argc, SV *val) {
         r = ST(i);
         i++;
 
-        if (strcmp(arg, "canonical") == 0) ctx.canon = SvPVXtrue(r);
+        if (strcmp(arg, "canonical") == 0) ctx.canon = SvTRUEx(r);
-        else if (strcmp(arg, "pretty") == 0) ctx.pretty = SvPVXtrue(r) ? 0 : INT_MIN;
+        else if (strcmp(arg, "pretty") == 0) ctx.pretty = SvTRUEx(r) ? 0 : INT_MIN;
-        else if (strcmp(arg, "utf8") == 0) encutf8 = SvPVXtrue(r);
+        else if (strcmp(arg, "html_safe") == 0) ctx.htmlsafe = !!SvTRUEx(r);
+        else if (strcmp(arg, "utf8") == 0) encutf8 = SvTRUEx(r);
         else if (strcmp(arg, "max_size") == 0) out.maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
         else croak("Unknown flag: '%s'", arg);

c/jsonparse.c

@@ -236,12 +236,12 @@ static SV *fujson_parse(pTHX_ fujson_parse_ctx *ctx) {
             if (ctx->end - ctx->buf < 4) return NULL;
             if (memcmp(ctx->buf, "true", 4) != 0) return NULL;
             ctx->buf += 4;
-            return &PL_sv_yes;
+            return newSV_true();
         case 'f':
             if (ctx->end - ctx->buf < 5) return NULL;
             if (memcmp(ctx->buf, "false", 5) != 0) return NULL;
             ctx->buf += 5;
-            return &PL_sv_no;
+            return newSV_false();
         case 'n':
             if (ctx->end - ctx->buf < 4) return NULL;
             if (memcmp(ctx->buf, "null", 4) != 0) return NULL;
@@ -272,9 +272,10 @@ static SV *fujson_parse_xs(pTHX_ I32 ax, I32 argc, SV *val) {
         r = ST(i);
         i++;
 
-        if (strcmp(arg, "utf8") == 0) decutf8 = SvPVXtrue(r);
+        if (strcmp(arg, "utf8") == 0) decutf8 = SvTRUEx(r);
         else if (strcmp(arg, "max_size") == 0) maxlen = SvUV(r);
         else if (strcmp(arg, "max_depth") == 0) ctx.depth = SvUV(r);
+        else if (strcmp(arg, "allow_control") == 0) {}
         else if (strcmp(arg, "offset") == 0) offset = r;
         else croak("Unknown flag: '%s'", arg);
     }

c/libpq.h

@@ -36,12 +36,15 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
 #define PG_DIAG_SOURCE_FUNCTION 'R'
 
 #define PG_FUNCS \
+    X(PQbinaryTuples, int, const PGresult *) \
     X(PQclear, void, PGresult *) \
     X(PQclosePrepared, PGresult *, PGconn *, const char *) \
     X(PQcmdTuples, char *, PGresult *) \
     X(PQconnectdb, PGconn *, const char *) \
     X(PQenterPipelineMode, int, PGconn *) \
     X(PQerrorMessage, char *, const PGconn *) \
+    X(PQescapeIdentifier, char *, PGconn *, const char *, size_t) \
+    X(PQescapeLiteral, char *, PGconn *, const char *, size_t) \
     X(PQexec, PGresult *, PGconn *, const char *) \
     X(PQexecParams, PGresult *, PGconn *, const char *, int, const Oid *, const char * const *, const int *, const int *, int) \
     X(PQexecPrepared, PGresult *, PGconn *, const char *, int, const char * const *, const int *, const int *, int) \
@@ -51,9 +54,10 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
     X(PQfname, char *, const PGresult *, int) \
     X(PQfreemem, void, void *) \
     X(PQftype, Oid, const PGresult *, int) \
+    X(PQgetCopyData, int, PGconn *, char **, int) \
+    X(PQgetResult, PGresult *, PGconn *) \
     X(PQgetisnull, int, const PGresult *, int, int) \
     X(PQgetlength, int, const PGresult *, int, int) \
-    X(PQgetResult, PGresult *, PGconn *) \
     X(PQgetvalue, char *, const PGresult *, int, int) \
     X(PQlibVersion, int, void) \
     X(PQnfields, int, const PGresult *) \
@@ -61,6 +65,8 @@ typedef enum { PQTRANS_IDLE, PQTRANS_ACTIVE, PQTRANS_INTRANS, PQTRANS_INERROR, P
     X(PQntuples, int, const PGresult *) \
     X(PQparamtype, Oid, const PGresult *, int) \
     X(PQpipelineSync, int, PGconn *) \
+    X(PQputCopyData, int, PGconn *, const char *, int) \
+    X(PQputCopyEnd, int, PGconn *, const char *) \
     X(PQresStatus, char *, ExecStatusType) \
     X(PQresultErrorField, char *, const PGresult *, int) \
     X(PQresultErrorMessage, char *, const PGresult *) \

c/pgconn.c

@@ -471,6 +471,7 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) {
         snprintf(t->name.n, sizeof(t->name.n), "%s", PQgetvalue(r, i, 1));
         char typ = *PQgetvalue(r, i, 2);
         t->elemoid = fu_frombeU(32, PQgetvalue(r, i, 3));
+        const fupg_type *builtin;
 
         if (t->elemoid) {
             if (typ == 'd') { /* domain */
@@ -487,13 +488,14 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) {
             /* enum, can use text send/recv */
             t->send = fupg_send_text;
             t->recv = fupg_recv_text;
-        } else {
+        } else if ((builtin = fupg_builtin_byoid(t->oid))) {
-            /* TODO: (multi)ranges, custom overrides, by-name lookup for dynamic-oid types */
-            const fupg_type *builtin = fupg_builtin_byoid(t->oid);
-            if (builtin) {
             t->send = builtin->send;
             t->recv = builtin->recv;
-            }
+        } else if ((builtin = fupg_dynoid_byname(t->name.n))) {
+            t->send = builtin->send;
+            t->recv = builtin->recv;
+        } else {
+            /* TODO: (multi)ranges */
         }
     }
     PQclear(r);
@@ -571,11 +573,8 @@ static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid
         return;
     }
 
-    /* Minor wart? When the type is overridden by oid, the name & oid in error
+    /* Minor wart? When the type is overridden by oid, its name in error
-     * messages will be that of the builtin type.  When overridden by name, the
+     * messages will be that of the builtin type instead of the actual type. */
-     * name will be correct but the oid is still of the builtin type.
-     * Some send/recv functions have slightly different behavior based on oid,
-     * in those cases this behavior is useful. */
 
     SV *cb = NULL;
     const fupg_type *e, *t;
@@ -627,3 +626,80 @@ static void fupg_tio_free(fupg_tio *tio) {
         safefree(tio->record.tio);
     }
 }
+
+
+
+
+static SV *fupg_perl2bin(pTHX_ fupg_conn *conn, Oid oid, SV *sv) {
+    int refresh_done = 0;
+    fupg_tio tio;
+    fustr buf;
+    memset(&tio, 0, sizeof(tio));
+    fupg_tio_setup(aTHX_ conn, &tio, FUPGT_SEND, oid, &refresh_done);
+    fustr_init(&buf, sv_newmortal(), SIZE_MAX);
+    tio.send(aTHX_ &tio, sv, &buf); /* XXX: Leaks 'tio' on error */
+    fupg_tio_free(&tio);
+    return fustr_done(&buf);
+}
+
+static SV *fupg_bin2perl(pTHX_ fupg_conn *conn, Oid oid, SV *sv) {
+    int refresh_done = 0;
+    fupg_tio tio;
+    STRLEN len;
+    const char *buf = SvPVbyte(sv, len);
+    memset(&tio, 0, sizeof(tio));
+    fupg_tio_setup(aTHX_ conn, &tio, FUPGT_RECV, oid, &refresh_done);
+    SV *r = tio.recv(aTHX_ &tio, buf, len); /* XXX: Leaks 'tio' on error */
+    fupg_tio_free(&tio);
+    return r;
+}
+
+
+static I32 fupg_bintext(pTHX_ fupg_conn *conn, int format, I32 ax, I32 argc) {
+    int vals = argc/2;
+
+    if (argc == 1 || argc % 2 == 0) croak("Usage: $conn->%s(oid, data, ...)", format ? "text2bin" : "bin2text");
+    if (vals > 1 && GIMME_V != G_LIST) {
+        ST(0) = sv_2mortal(newSViv(vals));
+        return 1;
+    }
+
+    Oid *paramtypes = safemalloc(vals * sizeof(*paramtypes));
+    const char **paramvalues = safemalloc(vals * sizeof(*paramvalues));
+    int *paramlengths = safemalloc(vals * sizeof(*paramlengths));
+    int *paramformats = safemalloc(vals * sizeof(*paramformats));
+
+    fustr sql;
+    fustr_init(&sql, NULL, SIZE_MAX);
+    fustr_write(&sql, "SELECT ", 7);
+
+    STRLEN len;
+    int i;
+    for (i=0; i<vals; i++) {
+        paramtypes[i] = SvIV(ST(i*2+1));
+        paramvalues[i] = format ? SvPVutf8(ST(i*2+2), len) : SvPVbyte(ST(i*2+2), len);
+        paramlengths[i] = len;
+        paramformats[i] = format ? 0 : 1;
+        if (i) fustr_write_ch(&sql, ',');
+        sql.cur -= 8 - sprintf(fustr_write_buf(&sql, 8), "$%d", i+1);
+    }
+    fustr_write_ch(&sql, 0);
+
+    PGresult *r = PQexecParams(conn->conn, fustr_start(&sql), vals,
+            paramtypes, paramvalues, paramlengths, paramformats, format);
+    safefree(paramtypes);
+    safefree(paramvalues);
+    safefree(paramlengths);
+    safefree(paramformats);
+    SvREFCNT_dec(sql.sv);
+
+    if (!r) fupg_conn_croak(conn, "exec");
+    if (PQresultStatus(r) != PGRES_TUPLES_OK) fupg_result_croak(r, "exec", sql.sv ? "SELECT $1, ..." : sql.sbuf);
+
+    /* The stack is guaranteed to be large enough, since we received 1+2*vals arguments */
+    for (i=0; i<vals; i++)
+        ST(i) = newSVpvn_flags(PQgetvalue(r, 0, i), PQgetlength(r, 0, i), SVs_TEMP | (format ? 0 : SVf_UTF8));
+
+    PQclear(r);
+    return vals;
+}

c/pgst.c

@@ -76,7 +76,7 @@ static SV *fupg_exec(pTHX_ fupg_conn *c, const char *sql) {
     return ret;
 }
 
-static SV *fupg_q(pTHX_ fupg_conn *c, int stflags, const char *query, I32 ax, I32 argc) {
+static SV *fupg_sql(pTHX_ fupg_conn *c, int stflags, const char *query, I32 ax, I32 argc) {
     fupg_st *st = safecalloc(1, sizeof(fupg_st));
     st->conn = c;
     st->cookie = c->cookie;
@@ -460,9 +460,11 @@ static SV *fupg_st_kvv(pTHX_ fupg_st *st) {
     HV *hv = newHV();
     SV *sv = sv_2mortal(newRV_noinc((SV *)hv));
     for (i=0; i<nrows; i++) {
+        SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kvv() query results", SvPV_nolen(key));
-        hv_store_ent(hv, key, st->nfields == 1 ? &PL_sv_yes : fupg_st_getval(aTHX_ st, i, 1), 0);
+        hv_store_ent(hv, key, st->nfields == 1 ? newSV_true() : fupg_st_getval(aTHX_ st, i, 1), 0);
+        FREETMPS;
     }
     return sv;
 }
@@ -474,10 +476,12 @@ static SV *fupg_st_kva(pTHX_ fupg_st *st) {
     HV *hv = newHV();
     SV *sv = sv_2mortal(newRV_noinc((SV *)hv));
     for (i=0; i<nrows; i++) {
+        SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kva() query results", SvPV_nolen(key));
         AV *row = st->nfields == 1 ? newAV() : newAV_alloc_x(st->nfields-1);
         hv_store_ent(hv, key, newRV_noinc((SV *)row), 0);
+        FREETMPS;
         for (j=1; j<st->nfields; j++)
             av_push_simple(row, fupg_st_getval(aTHX_ st, i, j));
     }
@@ -492,10 +496,12 @@ static SV *fupg_st_kvh(pTHX_ fupg_st *st) {
     HV *hv = newHV();
     SV *sv = sv_2mortal(newRV_noinc((SV *)hv));
     for (i=0; i<nrows; i++) {
+        SAVETMPS;
         SV *key = sv_2mortal(fupg_st_getval(aTHX_ st, i, 0));
         if (hv_exists_ent(hv, key, 0)) fu_confess("Key '%s' is duplicated in $st->kvh() query results", SvPV_nolen(key));
         HV *row = newHV();
         hv_store_ent(hv, key, newRV_noinc((SV *)row), 0);
+        FREETMPS;
         for (j=1; j<st->nfields; j++) {
             const char *key = PQfname(st->result, j);
             hv_store(row, key, -strlen(key), fupg_st_getval(aTHX_ st, i, j), 0);
@@ -503,3 +509,82 @@ static SV *fupg_st_kvh(pTHX_ fupg_st *st) {
     }
     return sv;
 }
+
+
+
+
+/* COPY support */
+
+typedef struct {
+    SV *self;
+    fupg_conn *conn;
+    char in;
+    char bin;
+    char rddone;
+    char closed;
+} fupg_copy;
+
+static SV *fupg_copy_exec(pTHX_ fupg_conn *c, const char *sql) {
+    PGresult *r = PQexec(c->conn, sql);
+
+    if (!r) fupg_conn_croak(c, "exec");
+    int s = PQresultStatus(r);
+    switch (s) {
+        case PGRES_COPY_OUT:
+        case PGRES_COPY_IN:
+            break;
+        default: fupg_result_croak(r, "exec", sql);
+    }
+
+    fupg_copy *copy = safecalloc(1, sizeof(fupg_copy));
+    copy->conn = c;
+    SvREFCNT_inc(c->self);
+    copy->bin = !!PQbinaryTuples(r);
+    copy->in = s == PGRES_COPY_IN;
+    PQclear(r);
+    return fu_selfobj(copy, "FU::Pg::copy");
+}
+
+static void fupg_copy_write(pTHX_ fupg_copy *c, SV *data) {
+    STRLEN len;
+    const char *buf = c->bin ? SvPVbyte(data, len) : SvPVutf8(data, len);
+    if (PQputCopyData(c->conn->conn, buf, len) < 0) fupg_conn_croak(c->conn, "copy");
+}
+
+static SV *fupg_copy_read(pTHX_ fupg_copy *c, int discard) {
+    char *buf = NULL;
+    int len = PQgetCopyData(c->conn->conn, &buf, 0);
+    if (len == -1) {
+        c->rddone = 1;
+        return &PL_sv_undef;
+    } else if (len < 0) {
+        if (discard) c->rddone = 1;
+        else fupg_conn_croak(c->conn, "copy");
+    }
+    SV *r = discard ? &PL_sv_undef : newSVpvn_flags(buf, len, SVs_TEMP | (c->bin ? 0 : SVf_UTF8));
+    PQfreemem(buf);
+    return r;
+}
+
+static void fupg_copy_close(pTHX_ fupg_copy *c, int ignerror) {
+    if (c->closed) return;
+    c->closed = 1; /* Mark as closed even on error, a second attempt won't help anyway */
+
+    if (c->in && PQputCopyEnd(c->conn->conn, NULL) < 0 && !ignerror)
+        fupg_conn_croak(c->conn, "copyEnd");
+
+    while (!c->in && !c->rddone) fupg_copy_read(aTHX_ c, 1);
+
+    PGresult *r = PQgetResult(c->conn->conn);
+    if (!ignerror && !r) fupg_conn_croak(c->conn, "copyEnd");
+    if (!ignerror && PQresultStatus(r) != PGRES_COMMAND_OK) fupg_result_croak(r, "copy", "");
+    PQclear(r);
+
+    while ((r = PQgetResult(c->conn->conn))) PQclear(r);
+}
+
+static void fupg_copy_destroy(pTHX_ fupg_copy *c) {
+    fupg_copy_close(aTHX_ c, 1);
+    SvREFCNT_dec(c->conn->self);
+    safefree(c);
+}

c/pgtypes.c

@@ -78,18 +78,25 @@ SENDFN(domain) { (void)out; SERR("domain type should not be handled by this func
 
 RECVFN(bool) {
     RLEN(1);
-    return *buf ? &PL_sv_yes : &PL_sv_no;
+    return *buf ? newSV_true() : newSV_false();
 }
 
 SENDFN(bool) {
-    int r = fu_2bool(aTHX_ val); /* So that we also recognize \0 and \1 */
+    int r = fu_2bool(aTHX_ val);
-    fustr_write_ch(out, r < 0 ? SvTRUE(val) : r);
+    if (r < 0) {
+        STRLEN l;
+        const char *x = SvPV(val, l);
+        if (l == 0 || (l == 1 && (*x == '0' || *x == 'f'))) r = 0;
+        else if (l == 1 && (*x == '1' || *x == 't')) r = 1;
+        else SERR("invalid boolean value: %s", x);
+    }
+    fustr_write_ch(out, r);
 }
 
 RECVFN(void) {
     RLEN(0);
     (void)buf;
-    return &PL_sv_undef;
+    return newSV(0);
 }
 
 SENDFN(void) {
@@ -166,6 +173,7 @@ RECVFN(hex) {
         *out++ = PL_hexdigit[(in[i] >> 4) & 0x0f];
         *out++ = PL_hexdigit[in[i] & 0x0f];
     }
+    *out = 0;
     SvCUR_set(r, len * 2);
     return r;
 }
@@ -268,7 +276,7 @@ SENDFN(jsonpath) {
 #define ARRAY_MAXDIM 100
 
 static SV *fupg_recv_array_elem(pTHX_ const fupg_tio *elem, const char *header, U32 dim, U32 ndim, const char **buf, const char *end) {
-    SV *r = &PL_sv_undef;
+    SV *r;
     if (dim == ndim) {
         if (end - *buf < 4) fu_confess("Invalid array format");
         I32 len = fu_frombeI(32, *buf);
@@ -278,6 +286,8 @@ static SV *fupg_recv_array_elem(pTHX_ const fupg_tio *elem, const char *header,
         if (len >= 0) {
             r = elem->recv(aTHX_ elem, *buf, len);
             *buf += len;
+        } else {
+            r = newSV(0);
         }
 
     } else {
@@ -402,12 +412,14 @@ RECVFN(record) {
         if (oid != ctx->record.info->attrs[i].oid)
             RERR("expected field %d to be of type %u but got %u", i, ctx->record.info->attrs[i].oid, oid);
         I32 vlen = fu_frombeI(32, buf+4);
-        SV *r = &PL_sv_undef;
+        SV *r;
         buf += 8; len -= 8;
         if (vlen > len) RERR("input data too short");
         if (vlen >= 0) {
             r = ctx->record.tio[i].recv(aTHX_ ctx->record.tio+i, buf, vlen);
             buf += vlen; len -= vlen;
+        } else {
+            r = newSV(0);
         }
         hv_store(hv, ctx->record.info->attrs[i].name.n, -strlen(ctx->record.info->attrs[i].name.n), r, 0);
     }
@@ -456,8 +468,8 @@ RECVFN(perlcb) {
     call_sv(ctx->cb, G_SCALAR);
     SPAGAIN;
 
-    SV *ret = newSV(0);
+    SV *ret = POPs;
-    sv_setsv(ret, POPs);
+    SvREFCNT_inc(ret);
     PUTBACK;
 
     FREETMPS;
@@ -638,6 +650,72 @@ SENDFN(time) {
     fustr_writebeI(64, out, SvNV(val) * 1000000);
 }
 
+
+
+/* VNDB types */
+
+const char vndbtag_alpha[] = "\0""abcdefghijklmnopqrstuvwxyz?????";
+
+static I16 vndbtag_parse(char **str) {
+    I16 tag = 0;
+    if (**str >= 'a' && **str <= 'z') {
+        tag = (**str - 'a' + 1) << 10;
+        (*str)++;
+        if (**str >= 'a' && **str <= 'z') {
+            tag |= (**str - 'a' + 1) << 5;
+            (*str)++;
+            if (**str >= 'a' && **str <= 'z') {
+                tag |= **str - 'a' + 1;
+                (*str)++;
+            }
+        }
+    }
+    return tag;
+}
+
+void vndbtag_fmt(I16 tag, char *out) {
+    out[0] = vndbtag_alpha[(tag >> 10) & 31];
+    out[1] = vndbtag_alpha[(tag >>  5) & 31];
+    out[2] = vndbtag_alpha[(tag >>  0) & 31];
+    out[3] = 0;
+}
+
+RECVFN(vndbtag) {
+    RLEN(2);
+    SV *r = newSV(4);
+    SvPOK_only(r);
+    vndbtag_fmt(fu_frombeI(16, buf), SvPVX(r));
+    SvCUR_set(r, strlen(SvPVX(r)));
+    return r;
+}
+
+SENDFN(vndbtag) {
+    char *t = SvPV_nolen(val);
+    I16 v = vndbtag_parse(&t);
+    if (*t) SERR("Invalid vndbtag: '%s'", SvPV_nolen(val));
+    fustr_writebeI(16, out, v);
+}
+
+
+#define VNDBID2_MAXNUM (((I64)1<<48)-1)
+
+RECVFN(vndbid) {
+    RLEN(8);
+    I64 v = fu_frombeI(64, buf);
+    char tbuf[4];
+    vndbtag_fmt(v >> 48, tbuf);
+    return newSVpvf("%s%"UVuf, tbuf, (UV)(v & VNDBID2_MAXNUM));
+}
+
+SENDFN(vndbid) {
+    char *ostr = SvPV_nolen(val), *str = ostr;
+    UV num;
+    I16 tag = vndbtag_parse(&str);
+    if (!grok_atoUV(str, &num, NULL) || num > VNDBID2_MAXNUM) SERR("invalid vndbid '%s'", ostr);
+    fustr_writebeI(64, out, ((I64)tag)<<48 | num);
+}
+
+
 #undef SIV
 #undef RLEN
 #undef RECVFN
@@ -818,7 +896,24 @@ static const fupg_type fupg_builtin[] = {
 #define FUPG_BUILTIN (sizeof(fupg_builtin) / sizeof(fupg_type))
 
 
+/* List of types identified by name */
+
+#define DYNOID\
+    T("vndbtag",   vndbtag)\
+    T("vndbid",    vndbid)
+
+static const fupg_type fupg_dynoid[] = {
+#define T(name, fun) { 0, 0, {name"\0"}, fupg_send_##fun, fupg_recv_##fun },
+    DYNOID
+#undef T
+};
+
+#undef DYNOID
+#define FUPG_DYNOID (sizeof(fupg_dynoid) / sizeof(fupg_type))
+
+
 /* List of special types for use with set_type() */
+
 #define SPECIALS\
     T("$date_str",   date_str)\
     T("$hex",        hex     )
@@ -851,8 +946,19 @@ static const fupg_type *fupg_builtin_byoid(Oid oid) {
     return fupg_type_byoid(fupg_builtin, FUPG_BUILTIN, oid);
 }
 
+static const fupg_type *fupg_dynoid_byname(const char *name) {
+    size_t i;
+    for (i=0; i<FUPG_DYNOID; i++)
+        if (strcmp(fupg_dynoid[i].name.n, name) == 0)
+            return fupg_dynoid+i;
+    return NULL;
+}
+
 static const fupg_type *fupg_builtin_byname(const char *name) {
     size_t i;
+    const fupg_type *r = fupg_dynoid_byname(name);
+    if (r) return r;
+
     /* XXX: Can use binary search here if the list of specials grows.
      * That list does not have to be ordered by oid. */
     for (i=0; i<FUPG_SPECIALS; i++)

c/xmlwr.c

@@ -27,6 +27,8 @@ static void fuxmlwr_destroy(pTHX_ fuxmlwr *wr) {
 
 
 static void fuxmlwr_escape(pTHX_ fuxmlwr *wr, SV *sv) {
+    if (SvROK(sv) && !SvAMAGIC(sv)) fu_confess("Invalid attempt to output bare reference");
+
     STRLEN len;
     const unsigned char *str = (unsigned char *)SvPV_const(sv, len);
     const unsigned char *tmp, *end = str + len;
@@ -96,7 +98,7 @@ static void fuxmlwr_tag(pTHX_ fuxmlwr *wr, I32 ax, I32 offset, I32 argc, int sel
         val = ST(offset);
         offset++;
 
-        // Don't even try to stringify other arguments; non-string keys are always a bug.
+        // Don't even try to stringify attribute names; non-string keys are always a bug.
         if (!SvPOK(key)) fu_confess("Non-string attribute");
         keys = SvPVX(key);
 

t/compress.t

@@ -0,0 +1,66 @@
+use v5.36;
+use Test::More;
+use FU::Util qw/gzip_lib gzip_compress brotli_compress/;
+
+like gzip_lib, qr/^(|libdeflate|zlib-ng|zlib)$/, gzip_lib;
+
+my $incompressible;
+
+subtest 'gzip_compress', sub {
+    plan skip_all => 'No suitable gzip library found' if !gzip_lib;
+    plan skip_all => 'Compress::Zlib not found' if !eval { require Compress::Zlib };
+
+    $incompressible = Compress::Zlib::memGzip(join '', map chr(rand 256), 0..93123);
+
+    for my $str ('', 'Hello world!', 'x'x4096, $incompressible) {
+        is Compress::Zlib::memGunzip(gzip_compress(0, $str)), $str;
+        is Compress::Zlib::memGunzip(gzip_compress(12, $str)), $str;
+    }
+};
+
+
+subtest 'brotli_compress', sub {
+    plan skip_all => 'libbrotlienc not available'
+        if !eval { brotli_compress 6, '' } && $@ =~ /Unable to load/;
+
+    ok length(brotli_compress 0, '') > 0;
+    ok length(brotli_compress 11, '') > 0;
+    # '0' does not disable compression...
+    ok length(brotli_compress 0, 'Hello world!'x100) < 200;
+    ok length(brotli_compress 11, 'Hello world!'x100) < 100;
+};
+
+
+done_testing;
+
+
+__END__
+
+# Test for leaks:
+
+use Test::LeakTrace;
+diag count_sv;
+for (0..1000) {
+    for my $str ('', 'Hello world!', 'x'x4096, $incompressible) {
+        local $_ = gzip_lib;
+        $_ = gzip_compress(0, $str);
+        $_ = gzip_compress(12, $str);
+        $_ = brotli_compress(0, $str);
+        $_ = brotli_compress(11, $str);
+    }
+}
+diag count_sv;
+
+
+# Compare performance:
+
+use Benchmark 'cmpthese';
+open my $F, '<', 'FU.pm';
+local $/ = undef;
+my $data = <$F>;
+
+cmpthese -3, {
+    memGzip => 'Compress::Zlib::memGzip($data)',
+    gzip_compress => 'gzip_compress(6, $data)',
+    brotli_compress => 'brotli_compress(6, $data)',
+};

t/fcgi.t

@@ -54,6 +54,11 @@ start;
 begin 1, 2;
 record 1, 4, "";
 
+start;
+begin 3, 2, 1;
+$remote->close;
+iserr -8;
+
 start;
 begin 3, 2, 1;
 begin 1, 1, 1;
@@ -167,6 +172,15 @@ record 1, 4, "\x0c\x05CONTENT_TYPEsomet";
 record 1, 2, "";
 isrec {'content-type','somet'}, {body => ''}, -6;
 
+start;
+begin;
+record 1, 4, "\x13\x01HTTP_CONTENT_LENGTH3\x0e\x01CONTENT_LENGTH0\x13\x01HTTP_CONTENT_LENGTH5";
+record 1, 4, "";
+record 1, 5, "";
+isrec {'content-length','0'}, {body => ''};
+$remote->close;
+ok !eval { $f->flush; 1 };
+
 start;
 begin;
 record 1, 4, "\x0e\x05CONTENT_LENGTH65536";

t/json_format.t

@@ -55,7 +55,7 @@ my @tests = (
     ''.$$, '"'.$$.'"',
     do { my $x = 12; utf8::decode($x); $x }, '"12"',
     do { no warnings 'numeric'; my $x = '19a'; $x += 0; $x }, '19',
-    1844674407370955161 / 10, $Config{uselongdouble} ? 184467440737095516 : '1.84467440737096e+17',
+    $Config{uselongdouble} ? () : ( 1844674407370955161 / 10, '1.84467440737096e+17' ),
 );
 
 my @errors = (
@@ -110,6 +110,7 @@ is json_format(
 }
 _
 
+is json_format('<hello & world>', html_safe => 1), '"\u003chello \u0026 world\u003e"';
 
 eval { json_format [[]], max_depth => 2 };
 like $@, qr/max_depth exceeded while formatting JSON/;

t/json_parse.t

@@ -2,7 +2,7 @@ use v5.36;
 use Test::More;
 use FU::Util 'json_parse';
 no warnings 'experimental::builtin';
-use builtin 'is_bool', 'created_as_number';
+use builtin 'is_bool', 'created_as_number', 'true', 'false';
 use Config;
 
 my @error = (
@@ -82,9 +82,10 @@ sub str($in, $exp) {
 }
 str '""', '';
 str '"hello, world"', 'hello, world';
-str '"\u0000\u0099\u0234\u1234"', "\x{00}\x{99}\x{234}\x{1234}";
+str '"\u0000\b"', "\x00\b";
-str "\"\x{7f}\x{99}\x{234}\x{1234}\x{12345}\"", "\x{7f}\x{99}\x{234}\x{1234}\x{12345}";
+str '"\u0099\u0234\u1234"', "\x{99}\x{234}\x{1234}";
-str '"\/\"\\\\\b\t\n\f\r"', "/\"\\\x{08}\x{09}\x{0a}\x{0c}\x{0d}";
+str "\"\x{99}\x{234}\x{1234}\x{12345}\"", "\x{99}\x{234}\x{1234}\x{12345}";
+str '"\/\"\\\\\t\n\r"', "/\"\\\x{09}\x{0a}\x{0d}";
 str '"\uD83D\uDE03"', "\x{1F603}";
 
 sub num($in, $exp=$in) {
@@ -97,8 +98,10 @@ num ' -0 ', 0;
 num '-9223372036854775808';
 num '9223372036854775807';
 num '18446744073709551615';
-num '-9223372036854775809', $Config{uselongdouble} ? -9.22337203685477581e+18 : -9.22337203685478e+18;
+if (!$Config{uselongdouble}) { # Behavior of longdouble is architecture-dependent
-num '18446744073709551616', $Config{uselongdouble} ? 1.84467440737095516e+19 : 1.84467440737096e+19;
+    num '-9223372036854775809', -9.22337203685478e+18;
+    num '18446744073709551616', 1.84467440737096e+19;
+}
 num '1.234';
 num '1e5', 100000;
 num '1e+5', 100000;
@@ -184,6 +187,7 @@ for (2000..2100, 4000..4200, 8100..8200, 12200..12300, 16300..16400) {
 ok !eval { json_parse '[[[[]]]]', max_depth => 4; 1 };
 ok !eval { json_parse '{"":{"":{"":{"":1}}}}', max_depth => 4; 1 };
 
+is json_parse('"\u0000\b\f\u007f"', allow_control => 1), "\x00\x08\x0c\x7f";
 
 # 500 depth
 {
@@ -234,4 +238,10 @@ ok !eval { json_parse '{"":{"":{"":{"":1}}}}', max_depth => 4; 1 };
     ok !eval { json_parse '"string"', max_size => 7 };
 }
 
+# Mutable hashes/arrays
+my $d = json_parse('[true,false,null,{"a":true,"b":false,"c":null}]');
+is_deeply $d, [true,false,undef,{a => true, b => false, c => undef}];
+$_ = 1 for @{$d}[0,1,2], values $d->[3]->%*;
+is_deeply $d, [1,1,1,{a => 1, b => 1, c => 1}];
+
 done_testing;

t/multipart.t

@@ -0,0 +1,60 @@
+use v5.36;
+use Test::More;
+use FU::MultipartFormData;
+
+# Example based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/POST
+my $t = <<'_' =~ s/\n/\r\n/rg;
+--delimiter12345
+Content-Disposition: form-data; name="field1"
+content-type: hello; charset=x
+
+value1
+--delimiter12345
+Content-Type: text
+Content-Disposition: form-data; filename="example.txt"; name=field2
+
+value2
+--delimiter12345
+Content-Type: something; charset = " a b \\ c "
+Content-Disposition: form-data; name = "field \"  name" ;filename= "月姫.jpg"
+
+
+--delimiter12345--
+_
+
+
+my $l = FU::MultipartFormData->parse('multipart/form-data;boundary="delimiter12345"', $t);
+is scalar @$l, 3;
+
+my $v = $l->[0];
+is $v->name, 'field1';
+is $v->filename, undef;
+is $v->mime, 'hello';
+is $v->charset, 'x';
+is $v->length, 6;
+is $v->data, 'value1';
+
+is $v->substr(4), 'e1';
+is $v->substr(1, 2), 'al';
+is $v->substr(-2, 1), 'e';
+is $v->substr(-2, 5), 'e1';
+is $v->substr(-100, 2), 'va';
+is $v->substr(1, -3), 'al';
+
+$v = $l->[1];
+is $v->name, 'field2';
+is $v->filename, 'example.txt';
+is $v->mime, 'text';
+is $v->charset, undef;
+is $v->length, 6;
+is $v->data, 'value2';
+
+$v = $l->[2];
+is $v->name, 'field "  name';
+is $v->filename, "\x{6708}\x{59eb}.jpg";
+is $v->mime, 'something';
+is $v->charset, ' a b \ c ';
+is $v->length, 0;
+is $v->data, '';
+
+done_testing;

t/pgconnect.t

@@ -17,7 +17,9 @@ okerr FATAL => connect => qr/missing "=" after "invalid"/;
 
 ok FU::Pg::lib_version() > 100000;
 
-my $conn = FU::Pg->connect($ENV{FU_TEST_DB})->text->cache(0);
+my $conn = FU::Pg->connect($ENV{FU_TEST_DB});
+$conn->text;
+$conn->cache(0);
 $conn->_debug_trace(0);
 
 is ref $conn, 'FU::Pg::conn';
@@ -35,7 +37,7 @@ subtest '$conn->exec', sub {
     ok !defined $conn->exec('');
     is $conn->exec('SELECT 1'), 1;
 
-    ok !eval { $conn->q('SELEXT')->param_types; };
+    ok !eval { $conn->sql('SELEXT')->param_types; };
     okerr ERROR => prepare => qr/syntax error/;
 
     is $conn->exec('SET client_encoding=utf8'), undef;
@@ -44,7 +46,7 @@ subtest '$conn->exec', sub {
 
 subtest '$st prepare & exec', sub {
     {
-        my $st = $conn->q('SELECT 1');
+        my $st = $conn->sql('SELECT 1');
         is_deeply $st->param_types, [];
         is_deeply $st->columns, [{ name => '?column?', oid => 23 }];
 
@@ -61,7 +63,7 @@ subtest '$st prepare & exec', sub {
     }
 
     {
-        my $st = $conn->q("SELECT \$1::int AS a, \$2::char(5) AS \"\x{1F603}\"", 1, 2);
+        my $st = $conn->sql("SELECT \$1::int AS a, \$2::char(5) AS \"\x{1F603}\"", 1, 2);
         is_deeply $st->param_types, [ 23, 1042 ];
         is_deeply $st->columns, [
             { oid => 23, name => 'a' },
@@ -72,28 +74,28 @@ subtest '$st prepare & exec', sub {
 
     is $conn->exec('SELECT 1 FROM pg_prepared_statements'), 0;
 
-    ok !eval { $conn->q('SELECT 1', 1)->exec; 1 };
+    ok !eval { $conn->sql('SELECT 1', 1)->exec; 1 };
     like $@, qr/bind message supplies 1 parameters, but prepared statement/;
 
-    ok !eval { $conn->q('SELECT $1')->exec; 1 };
+    ok !eval { $conn->sql('SELECT $1')->exec; 1 };
     like $@, qr/bind message supplies 0 parameters, but prepared statement/;
 
     # prepare + describe won't let us detect empty queries, hmm...
-    is_deeply $conn->q('')->param_types, [];
+    is_deeply $conn->sql('')->param_types, [];
-    is_deeply $conn->q('')->columns, [];
+    is_deeply $conn->sql('')->columns, [];
 
-    ok !eval { $conn->q('')->exec; 1 };
+    ok !eval { $conn->sql('')->exec; 1 };
     okerr FATAL => exec => qr/unexpected status code/;
 
-    is $conn->q('SET client_encoding=utf8')->exec, undef;
+    is $conn->sql('SET client_encoding=utf8')->exec, undef;
 
-    ok !eval { $conn->q('select 1; select 2')->exec; 1 };
+    ok !eval { $conn->sql('select 1; select 2')->exec; 1 };
     okerr ERROR => exec => qr/cannot insert multiple commands into a prepared statement/;
 
     # Interleaved
     {
-        my $x = $conn->q('SELECT 1 as a');
+        my $x = $conn->sql('SELECT 1 as a');
-        my $y = $conn->q('SELECT 2 as b');
+        my $y = $conn->sql('SELECT 2 as b');
         is_deeply $x->columns, [ { oid => 23, name => 'a' } ];
         is_deeply $y->columns, [ { oid => 23, name => 'b' } ];
         is $x->val, 1;
@@ -102,136 +104,137 @@ subtest '$st prepare & exec', sub {
 };
 
 subtest '$st->val', sub {
-    ok !eval { $conn->q('SELECT')->val; 1 };
+    ok !eval { $conn->sql('SELECT')->val; 1 };
     like $@, qr/on query returning no data/;
 
-    ok !eval { $conn->q('SELECT 1, 2')->val; 1 };
+    ok !eval { $conn->sql('SELECT 1, 2')->val; 1 };
     like $@, qr/on query returning more than one column/;
 
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->val; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->val; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !defined $conn->q('SELECT 1 WHERE false')->val;
+    ok !defined $conn->sql('SELECT 1 WHERE false')->val;
-    ok !defined $conn->q('SELECT null')->val;
+    ok !defined $conn->sql('SELECT null')->val;
-    is $conn->q('SELECT $1::text', "\x{1F603}")->val, "\x{1F603}";
+    is $conn->sql('SELECT $1::text', "\x{1F603}")->val, "\x{1F603}";
 };
 
 subtest '$st->rowl', sub {
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->rowl; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->rowl; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !eval { $conn->q('SELEXT')->rowl; 1; };
+    ok !eval { $conn->sql('SELEXT')->rowl; 1; };
-    is scalar $conn->q('SELECT')->rowl, 0;
+    is scalar $conn->sql('SELECT')->rowl, 0;
-    is scalar $conn->q('SELECT 1, 2')->rowl, 2;
+    is scalar $conn->sql('SELECT 1, 2')->rowl, 2;
-    is_deeply [$conn->q('SELECT')->rowl], [];
+    is_deeply [$conn->sql('SELECT')->rowl], [];
-    is_deeply [$conn->q('SELECT 1, null')->rowl], [1, undef];
+    is_deeply [$conn->sql('SELECT 1, null')->rowl], [1, undef];
-    is_deeply [$conn->q('SELECT 1, $1', undef)->rowl], [1, undef];
+    is_deeply [$conn->sql('SELECT 1, $1', undef)->rowl], [1, undef];
-    is_deeply [$conn->q('SELECT 1, $1::int', undef)->text_params(0)->rowl], [1, undef];
+    is_deeply [$conn->sql('SELECT 1, $1::int', undef)->text_params(0)->rowl], [1, undef];
-    is_deeply [$conn->q('SELECT 1 WHERE false')->rowl], [];
+    is_deeply [$conn->sql('SELECT 1 WHERE false')->rowl], [];
 };
 
 subtest '$st->rowa', sub {
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->rowa; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->rowa; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !eval { $conn->q('SELEXT')->rowa; 1; };
+    ok !eval { $conn->sql('SELEXT')->rowa; 1; };
-    is $conn->q('SELECT 1 WHERE false')->rowa, undef;
+    is $conn->sql('SELECT 1 WHERE false')->rowa, undef;
-    is_deeply $conn->q('SELECT')->rowa, [];
+    is_deeply $conn->sql('SELECT')->rowa, [];
-    is_deeply $conn->q('SELECT 1, 2')->rowa, [1, 2];
+    is_deeply $conn->sql('SELECT 1, 2')->rowa, [1, 2];
-    is_deeply $conn->q('SELECT 1, null')->rowa, [1, undef];
+    is_deeply $conn->sql('SELECT 1, null')->rowa, [1, undef];
-    is_deeply $conn->q('SELECT 1, $1', undef)->rowa, [1, undef];
+    is_deeply $conn->sql('SELECT 1, $1', undef)->rowa, [1, undef];
-    is_deeply $conn->q('SELECT 1, $1::int', undef)->text_params(0)->rowa, [1, undef];
+    is_deeply $conn->sql('SELECT 1, $1::int', undef)->text_params(0)->rowa, [1, undef];
 
 };
 
 subtest '$st->rowh', sub {
-    ok !eval { $conn->q('SELECT 1 UNION SELECT 2')->rowh; 1 };
+    ok !eval { $conn->sql('SELECT 1 UNION SELECT 2')->rowh; 1 };
     like $@, qr/on query returning more than one row/;
 
-    ok !eval { $conn->q('SELECT 1 as a, 2 as a')->rowh; 1 };
+    ok !eval { $conn->sql('SELECT 1 as a, 2 as a')->rowh; 1 };
     like $@, qr/Query returns multiple columns with the same name/;
 
-    is $conn->q('SELECT 1 WHERE false')->rowh, undef;
+    is $conn->sql('SELECT 1 WHERE false')->rowh, undef;
-    is_deeply $conn->q('SELECT')->rowh, {};
+    is_deeply $conn->sql('SELECT')->rowh, {};
-    is_deeply $conn->q('SELECT 1 as a, 2 as b')->rowh, {a => 1, b => 2};
+    is_deeply $conn->sql('SELECT 1 as a, 2 as b')->rowh, {a => 1, b => 2};
-    is_deeply $conn->q('SELECT 1 as a, null as b')->rowh, {a => 1, b => undef};
+    is_deeply $conn->sql('SELECT 1 as a, null as b')->rowh, {a => 1, b => undef};
-    is_deeply $conn->q('SELECT 1 as a, $1::int as b', undef)->rowh, {a => 1, b => undef};
+    is_deeply $conn->sql('SELECT 1 as a, $1::int as b', undef)->rowh, {a => 1, b => undef};
 };
 
 subtest '$st->alla', sub {
-    is_deeply $conn->q('SELECT 1 WHERE false')->alla, [];
+    is_deeply $conn->sql('SELECT 1 WHERE false')->alla, [];
-    is_deeply $conn->q('SELECT')->alla, [[]];
+    is_deeply $conn->sql('SELECT')->alla, [[]];
-    is_deeply $conn->q('SELECT 1')->alla, [[1]];
+    is_deeply $conn->sql('SELECT 1')->alla, [[1]];
-    is_deeply $conn->q('SELECT 1, null UNION ALL SELECT NULL, 2')->alla, [[1,undef],[undef,2]];
+    is_deeply $conn->sql('SELECT 1, null UNION ALL SELECT NULL, 2')->alla, [[1,undef],[undef,2]];
 };
 
 subtest '$st->allh', sub {
-    ok !eval { $conn->q('SELECT 1 as a, 2 as a')->allh; 1 };
+    ok !eval { $conn->sql('SELECT 1 as a, 2 as a')->allh; 1 };
     like $@, qr/Query returns multiple columns with the same name/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->allh, [];
+    is_deeply $conn->sql('SELECT 1 WHERE false')->allh, [];
-    is_deeply $conn->q('SELECT')->allh, [{}];
+    is_deeply $conn->sql('SELECT')->allh, [{}];
-    is_deeply $conn->q('SELECT 1 a')->allh, [{a=>1}];
+    is_deeply $conn->sql('SELECT 1 a')->allh, [{a=>1}];
-    is_deeply $conn->q('SELECT 1 a, null b UNION ALL SELECT NULL, 2')->allh, [{a=>1,b=>undef},{a=>undef,b=>2}];
+    is_deeply $conn->sql('SELECT 1 a, null b UNION ALL SELECT NULL, 2')->allh, [{a=>1,b=>undef},{a=>undef,b=>2}];
 };
 
 subtest '$st->flat', sub {
-    is_deeply $conn->q('SELECT 1 WHERE false')->flat, [];
+    is_deeply $conn->sql('SELECT 1 WHERE false')->flat, [];
-    is_deeply $conn->q('SELECT')->flat, [];
+    is_deeply $conn->sql('SELECT')->flat, [];
-    is_deeply $conn->q('SELECT 1')->flat, [1];
+    is_deeply $conn->sql('SELECT 1')->flat, [1];
-    is_deeply $conn->q('SELECT 1, null UNION ALL SELECT NULL, 2')->flat, [1,undef,undef,2];
+    is_deeply $conn->sql('SELECT 1, null UNION ALL SELECT NULL, 2')->flat, [1,undef,undef,2];
 };
 
 subtest '$st->kvv', sub {
-    ok !eval { $conn->q('SELECT')->kvv; 1; };
+    ok !eval { $conn->sql('SELECT')->kvv; 1; };
     like $@, qr/returning no data/;
 
-    ok !eval { $conn->q('SELECT 1, 2, 3')->kvv; 1; };
+    ok !eval { $conn->sql('SELECT 1, 2, 3')->kvv; 1; };
     like $@, qr/returning more than two columns/;
 
-    ok !eval { $conn->q('SELECT 1 UNION ALL SELECT 1')->kvv; 1; };
+    ok !eval { $conn->sql('SELECT 1 UNION ALL SELECT 1')->kvv; 1; };
     like $@, qr/is duplicated/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->kvv, {};
+    is_deeply $conn->sql('SELECT 1 WHERE false')->kvv, {};
-    is_deeply $conn->q('SELECT 1')->kvv, {1=>1};
+    is_deeply $conn->sql('SELECT 1')->kvv, {1=>1};
-    is_deeply $conn->q('SELECT 1, null UNION ALL SELECT 3, 2')->kvv, {1=>undef,3=>2};
+    is_deeply $conn->sql('SELECT 1, null UNION ALL SELECT 3, 2')->kvv, {1=>undef,3=>2};
+    $conn->sql('SELECT 1')->kvv->{1} = 0;
 };
 
 subtest '$st->kva', sub {
-    ok !eval { $conn->q('SELECT')->kva; 1; };
+    ok !eval { $conn->sql('SELECT')->kva; 1; };
     like $@, qr/returning no data/;
 
-    ok !eval { $conn->q('SELECT 1 UNION ALL SELECT 1')->kva; 1; };
+    ok !eval { $conn->sql('SELECT 1 UNION ALL SELECT 1')->kva; 1; };
     like $@, qr/is duplicated/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->kva, {};
+    is_deeply $conn->sql('SELECT 1 WHERE false')->kva, {};
-    is_deeply $conn->q('SELECT 1')->kva, {1=>[]};
+    is_deeply $conn->sql('SELECT 1')->kva, {1=>[]};
-    is_deeply $conn->q("SELECT 1, null, 'hi' UNION ALL SELECT 3, 2, 'ok'")->kva,
+    is_deeply $conn->sql("SELECT 1, null, 'hi' UNION ALL SELECT 3, 2, 'ok'")->kva,
         {1=>[undef,'hi'], 3=>[2, 'ok']};
 };
 
 subtest '$st->kvh', sub {
-    ok !eval { $conn->q('SELECT')->kvh; 1; };
+    ok !eval { $conn->sql('SELECT')->kvh; 1; };
     like $@, qr/returning no data/;
 
-    ok !eval { $conn->q('SELECT 1 UNION ALL SELECT 1')->kvh; 1; };
+    ok !eval { $conn->sql('SELECT 1 UNION ALL SELECT 1')->kvh; 1; };
     like $@, qr/is duplicated/;
 
-    ok !eval { $conn->q('SELECT 1, 2, 3')->kvh; 1; };
+    ok !eval { $conn->sql('SELECT 1, 2, 3')->kvh; 1; };
     like $@, qr/Query returns multiple columns with the same name/;
 
-    is_deeply $conn->q('SELECT 1 WHERE false')->kvh, {};
+    is_deeply $conn->sql('SELECT 1 WHERE false')->kvh, {};
-    is_deeply $conn->q('SELECT 1')->kvh, {1=>{}};
+    is_deeply $conn->sql('SELECT 1')->kvh, {1=>{}};
-    is_deeply $conn->q("SELECT 1 as a , null as a, 'hi' as b UNION ALL SELECT 3, 2, 'ok'")->kvh,
+    is_deeply $conn->sql("SELECT 1 as a , null as a, 'hi' as b UNION ALL SELECT 3, 2, 'ok'")->kvh,
         {1=>{a=>undef,b=>'hi'}, 3=>{a=>2,b=>'ok'}};
 };
 
 subtest 'txn', sub {
     $conn->exec('CREATE TEMPORARY TABLE fupg_tst (id int)');
     $conn->txn->exec('INSERT INTO fupg_tst VALUES (1)'); # rolled back
-    is $conn->q('SELECT COUNT(*) FROM fupg_tst')->val, 0;
+    is $conn->sql('SELECT COUNT(*) FROM fupg_tst')->val, 0;
 
-    my $st = $conn->q('SELECT COUNT(*) FROM fupg_tst');
+    my $st = $conn->sql('SELECT COUNT(*) FROM fupg_tst');
     my $sst;
     {
         my $txn = $conn->txn;
@@ -243,13 +246,13 @@ subtest 'txn', sub {
 
         ok !eval { $conn->exec('SELECT 1'); 1 };
         like $@, qr/Invalid operation on the top-level connection/;
-        ok !eval { $conn->q('SELECT 1'); 1 };
+        ok !eval { $conn->sql('SELECT 1'); 1 };
         like $@, qr/Invalid operation on the top-level connection/;
         ok !eval { $conn->txn; 1 };
         like $@, qr/Invalid operation on the top-level connection/;
 
         $txn->exec('INSERT INTO fupg_tst VALUES (1)');
-        $sst = $txn->q('SELECT 1');
+        $sst = $txn->sql('SELECT 1');
 
         is $conn->status, 'txn_idle';
         is $txn->status, 'idle';
@@ -265,7 +268,7 @@ subtest 'txn', sub {
         like $@, qr/Invalid operation on a transaction that has already been marked as done/;
         ok !eval { $txn->exec('select 1'); 1 };
         like $@, qr/Invalid operation on a transaction that has already been marked as done/;
-        ok !eval { $txn->q('select 1'); 1 };
+        ok !eval { $txn->sql('select 1'); 1 };
         like $@, qr/Invalid operation on a transaction that has already been marked as done/;
 
         ok !eval { $conn->exec('SELECT 1'); 1 };
@@ -292,7 +295,7 @@ subtest 'txn', sub {
 
     {
         my $txn = $conn->txn;
-        my $st = $txn->q('SELECT count(*) FROM fupg_tst WHERE id = 2');
+        my $st = $txn->sql('SELECT count(*) FROM fupg_tst WHERE id = 2');
         {
             my $sub = $txn->txn;
             is $conn->status, 'txn_idle';
@@ -313,7 +316,7 @@ subtest 'txn', sub {
         is $txn->status, 'idle';
         is $st->val, 0;
 
-        $st = $txn->q('SELECT count(*) FROM fupg_tst WHERE id = 2');
+        $st = $txn->sql('SELECT count(*) FROM fupg_tst WHERE id = 2');
         {
             my $sub = $txn->txn;
             $sub->exec('INSERT INTO fupg_tst VALUES (2)');
@@ -336,63 +339,75 @@ subtest 'txn', sub {
         $sub->commit;
     }
     # We didn't commit $txn, so $sub got aborted as well
-    is $conn->q('SELECT count(*) FROM fupg_tst WHERE id = 3')->val, 0;
+    is $conn->sql('SELECT count(*) FROM fupg_tst WHERE id = 3')->val, 0;
 };
 
 {
     local $_ = 'x';
-    my $st = $conn->q('SELECT $1', $_);
+    my $st = $conn->sql('SELECT $1', $_);
     $_ = 'y';
     is $st->val, 'x', 'shallow copy';
 }
 
 {
     my $x = [1,2];
-    my $st = $conn->q('SELECT $1::int[]', $x)->text(0);
+    my $st = $conn->sql('SELECT $1::int[]', $x)->text(0);
     $x->[1] = 3;
     is_deeply $st->val, [1,3], 'not deep copy';
 }
 
-subtest 'Prepared statement cache', sub {
+
-    my $txn = $conn->cache_size(2)->txn->cache;
+{
-    my sub numexec($sql) {
+    # Exact format returned by escape_literal() can differ between Postgres versions and configurations.
-        $txn->q('SELECT generic_plans + custom_plans FROM pg_prepared_statements WHERE statement = $1', $sql)->cache(0)->val
+    my $x = q{"' \" \\};
+    is $conn->sql('SELECT '.$conn->escape_literal($x))->val, $x;
+
+    # Format can also change, but unsure how to test this otherwise.
+    is $conn->escape_identifier('hel\l"o'), '"hel\l""o"';
 }
-    is $txn->q('SELECT 1')->val, 1;
+
-    is numexec('SELECT 1'), 1;
+subtest 'Prepared statement cache', sub {
+    $conn->cache_size(2);
+    my $txn = $conn->txn;
+    $txn->cache;
+    my $numexec = sub($sql) {
+        $txn->sql('SELECT generic_plans + custom_plans FROM pg_prepared_statements WHERE statement = $1', $sql)->cache(0)->val
+    };
+    is $txn->sql('SELECT 1')->val, 1;
+    is $numexec->('SELECT 1'), 1;
 
     my $sql = 'SELECT $1::int as a, $2::text as b';
-    ok !defined numexec($sql);
+    ok !defined $numexec->($sql);
 
-    my $params = $txn->q($sql)->param_types;
+    my $params = $txn->sql($sql)->param_types;
     is_deeply $params, [23, 25];
-    is numexec($sql), 0;
+    is $numexec->($sql), 0;
-    my $cparams = $txn->q($sql)->param_types;
+    my $cparams = $txn->sql($sql)->param_types;
     is_deeply $cparams, $params;
 
-    my $cols = $txn->q($sql)->columns;
+    my $cols = $txn->sql($sql)->columns;
     is_deeply $cols, [{ name => 'a', oid => 23 }, { name => 'b', oid => 25 }];
-    my $ccols = $txn->q($sql)->columns;
+    my $ccols = $txn->sql($sql)->columns;
     is_deeply $ccols, $cols;
 
-    $txn->q($sql, 0, '')->exec;
+    $txn->sql($sql, 0, '')->exec;
-    is numexec($sql), 1;
+    is $numexec->($sql), 1;
-    $txn->q($sql, 0, '')->exec;
+    $txn->sql($sql, 0, '')->exec;
-    is numexec($sql), 2;
+    is $numexec->($sql), 2;
 
-    is numexec('SELECT 1'), 1;
+    is $numexec->('SELECT 1'), 1;
-    $txn->q('SELECT 2')->exec;
+    $txn->sql('SELECT 2')->exec;
-    ok !defined numexec('SELECT 1');
+    ok !defined $numexec->('SELECT 1');
-    is numexec('SELECT 2'), 1;
+    is $numexec->('SELECT 2'), 1;
 
     $conn->cache_size(1);
-    ok !defined numexec('SELECT 1');
+    ok !defined $numexec->('SELECT 1');
-    ok !defined numexec($sql);
+    ok !defined $numexec->($sql);
-    is numexec('SELECT 2'), 1;
+    is $numexec->('SELECT 2'), 1;
 
     $conn->cache_size(0);
-    ok !defined numexec($sql);
+    ok !defined $numexec->($sql);
-    ok !defined numexec('SELECT 2');
+    ok !defined $numexec->('SELECT 2');
 };
 
 
@@ -400,7 +415,7 @@ subtest 'Tracing', sub {
     my @log;
     $conn->query_trace(sub($st) { push @log, $st });
 
-    is_deeply $conn->q('SELECT 1 AS a, $1 AS b', 123)->text_params(0)->rowa, [ 1, 123 ];
+    is_deeply $conn->sql('SELECT 1 AS a, $1 AS b', 123)->text_params(0)->rowa, [ 1, 123 ];
     is scalar @log, 1;
     my $st = shift @log;
     is ref $st, 'FU::Pg::st';
@@ -436,7 +451,7 @@ subtest 'Tracing', sub {
 };
 
 {
-    my $st = $conn->q("SELECT 1");
+    my $st = $conn->sql("SELECT 1");
     undef $conn; # statement keeps the connection alive
     is $st->val, 1;
 }

t/pgcopy.t

@@ -0,0 +1,90 @@
+use v5.36;
+use Test::More;
+
+plan skip_all => $@ if !eval { require FU::Pg; } && $@ =~ /Unable to load libpq/;
+die $@ if $@;
+plan skip_all => 'Please set FU_TEST_DB to a PostgreSQL connection string to run these tests' if !$ENV{FU_TEST_DB};
+
+my $conn = FU::Pg->connect($ENV{FU_TEST_DB});
+$conn->_debug_trace(0);
+
+ok !eval { $conn->copy('SELECT 1') };
+like $@, qr/unexpected status code/;
+
+ok !eval { $conn->copy('COPX') };
+like $@, qr/syntax error/;
+
+$conn->exec('CREATE TEMPORARY TABLE fupg_copy_test (v int)');
+
+is $conn->status, 'idle';
+{
+    my $c = $conn->copy('COPY (SELECT 1) TO STDOUT');
+    is $conn->status, 'active';
+    $c->close;
+}
+is $conn->status, 'idle';
+$conn->copy('COPY (SELECT 1) TO STDOUT');
+is $conn->status, 'idle';
+
+{
+    my $c = $conn->copy('COPY fupg_copy_test FROM STDIN');
+    is $conn->status, 'active';
+    $c->close;
+}
+is $conn->status, 'idle';
+$conn->copy('COPY fupg_copy_test FROM STDIN');
+is $conn->status, 'idle';
+
+{
+    my $c = $conn->copy('COPY fupg_copy_test FROM STDIN');
+    ok !$c->is_binary;
+    ok !eval { $c->{read} };
+    $c->write("1");
+    $c->write("\n2\n3\n");
+    $c->close;
+    ok !eval { $c->read };
+    ok !eval { $c->write('') };
+    $c->close;
+}
+is $conn->status, 'idle';
+
+{
+    my $c = $conn->copy('COPY (SELECT * FROM fupg_copy_test ORDER BY v) TO STDOUT');
+    ok !$c->is_binary;
+    ok !eval { $c->write('') };
+    is $c->read, "1\n";
+    is $c->read, "2\n";
+    is $c->read, "3\n";
+    is $c->read, undef;
+    $c->close;
+    ok !eval { $c->read };
+    ok !eval { $c->write('') };
+    $c->close;
+}
+is $conn->status, 'idle';
+
+my $bin = '';
+{
+    my $c = $conn->copy('COPY fupg_copy_test TO STDOUT (FORMAT binary)');
+    ok $c->is_binary;
+    while (my $d = $c->read) {
+        $bin .= $d;
+    }
+    $c->close;
+}
+is $conn->status, 'idle';
+
+{
+    my $txn = $conn->txn;
+    my $c = $txn->copy('COPY fupg_copy_test FROM STDIN (FORMAT binary)');
+    is $txn->status, 'active';
+    ok $c->is_binary;
+    $c->write($bin);
+    $c->close;
+
+    is $txn->sql('SELECT sum(v) FROM fupg_copy_test')->val, 1+1+2+2+3+3;
+    $txn->rollback;
+}
+is $conn->sql('SELECT sum(v) FROM fupg_copy_test')->val, 1+2+3;
+
+done_testing;

t/pgtypes-dynamic.t

@@ -10,31 +10,31 @@ my $conn = FU::Pg->connect($ENV{FU_TEST_DB});
 $conn->_debug_trace(0);
 
 
-is_deeply $conn->Q('SELECT', 1, '::int')->param_types, [23];
+is_deeply $conn->SQL('SELECT', 1, '::int')->param_types, [23];
-is_deeply $conn->Q('SELECT 1', IN([1,2,3]))->param_types, [1007];
+is_deeply $conn->SQL('SELECT 1', IN([1,2,3]))->param_types, [1007];
-is $conn->Q('SELECT 1', IN([1,2,3]))->val, 1;
+is $conn->SQL('SELECT 1', IN([1,2,3]))->val, 1;
 
-ok !eval { $conn->q('SELECT $1::aclitem', '')->exec; 1 };
+ok !eval { $conn->sql('SELECT $1::aclitem', '')->exec; 1 };
 like $@, qr/Unable to send type/;
 
 
 subtest 'type overrides', sub {
     $conn->set_type(int4 => recv => 'bytea');
-    is $conn->q('SELECT 5::int4')->val, "\0\0\0\5";
+    is $conn->sql('SELECT 5::int4')->val, "\0\0\0\5";
-    is_deeply $conn->q('SELECT ARRAY[5::int4]')->val, ["\0\0\0\5"];
+    is_deeply $conn->sql('SELECT ARRAY[5::int4]')->val, ["\0\0\0\5"];
 
     $conn->set_type(int4 => send => 'bytea');
-    is $conn->q('SELECT $1::int4', "\0\0\0\5")->val, 5;
+    is $conn->sql('SELECT $1::int4', "\0\0\0\5")->val, 5;
-    is_deeply $conn->q('SELECT $1::int4[]', ["\0\0\0\5"])->val, [5];
+    is_deeply $conn->sql('SELECT $1::int4[]', ["\0\0\0\5"])->val, [5];
 
     $conn->set_type(int4 => 'int2');
-    ok !eval { $conn->q('SELECT 5::int4')->val };
+    ok !eval { $conn->sql('SELECT 5::int4')->val };
     like $@, qr/Error parsing value/;
-    ok !eval { $conn->q('SELECT $1::int4', 5)->val };
+    ok !eval { $conn->sql('SELECT $1::int4', 5)->val };
     like $@, qr/insufficient data left in message/;
 
     $conn->set_type(int4 => undef);
-    is $conn->q('SELECT 5::int4')->val, 5;
+    is $conn->sql('SELECT 5::int4')->val, 5;
 
     ok !eval { $conn->set_type(int4 => 1007); };
     like $@, qr/Cannot set a type to array/;
@@ -46,23 +46,23 @@ subtest 'type overrides', sub {
 
 subtest 'type override callback', sub {
     $conn->set_type(text => recv => sub { length $_[0] });
-    is $conn->q('SELECT $1', 'a')->val, 1;
+    is $conn->sql('SELECT $1', 'a')->val, 1;
-    is $conn->q('SELECT $1', 'ab')->val, 2;
+    is $conn->sql('SELECT $1', 'ab')->val, 2;
-    is $conn->q('SELECT $1', 'abc')->val, 3;
+    is $conn->sql('SELECT $1', 'abc')->val, 3;
-    is $conn->q('SELECT $1', 'abcd')->val, 4;
+    is $conn->sql('SELECT $1', 'abcd')->val, 4;
 
     $conn->set_type(text => send => sub { 'l'.length $_[0] });
-    is $conn->q('SELECT $1', 'a')->val, 'l1';
+    is $conn->sql('SELECT $1', 'a')->val, 'l1';
-    is $conn->q('SELECT $1', 'ab')->val, 'l2';
+    is $conn->sql('SELECT $1', 'ab')->val, 'l2';
-    is $conn->q('SELECT $1', 'abc')->val, 'l3';
+    is $conn->sql('SELECT $1', 'abc')->val, 'l3';
-    is $conn->q('SELECT $1', 'abcd')->val, 'l4';
+    is $conn->sql('SELECT $1', 'abcd')->val, 'l4';
 };
 
 
 subtest 'custom types', sub {
     my $txn = $conn->txn;
 
-    is $txn->Q('SELECT 1', IN([1,2,3]))->val, 1;
+    is $txn->SQL('SELECT 1', IN([1,2,3]))->val, 1;
 
     $txn->exec(<<~_);
         CREATE TYPE fupg_test_enum AS ENUM('aa', 'bb', 'ccccccccccccccccccc');
@@ -73,21 +73,21 @@ subtest 'custom types', sub {
             domain fupg_test_domain
         );
     _
-    is $txn->q("SELECT 'aa'::fupg_test_enum")->val, 'aa';
+    is $txn->sql("SELECT 'aa'::fupg_test_enum")->val, 'aa';
-    is $txn->q('SELECT $1::fupg_test_enum', 'ccccccccccccccccccc')->val, 'ccccccccccccccccccc';
+    is $txn->sql('SELECT $1::fupg_test_enum', 'ccccccccccccccccccc')->val, 'ccccccccccccccccccc';
 
-    is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_enum[]")->val, ['aa','bb',undef];
+    is_deeply $txn->sql("SELECT '{aa,bb,null}'::fupg_test_enum[]")->val, ['aa','bb',undef];
-    is $txn->q('SELECT $1::fupg_test_enum[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
+    is $txn->sql('SELECT $1::fupg_test_enum[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
 
-    is $txn->q("SELECT 'aa'::fupg_test_domain")->val, 'aa';
+    is $txn->sql("SELECT 'aa'::fupg_test_domain")->val, 'aa';
-    is $txn->q('SELECT $1::fupg_test_domain', 'bb')->val, 'bb';
+    is $txn->sql('SELECT $1::fupg_test_domain', 'bb')->val, 'bb';
 
-    is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_domain[]")->val, ['aa','bb',undef];
+    is_deeply $txn->sql("SELECT '{aa,bb,null}'::fupg_test_domain[]")->val, ['aa','bb',undef];
-    is $txn->q('SELECT $1::fupg_test_domain[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
+    is $txn->sql('SELECT $1::fupg_test_domain[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
 
     my $val = { a => undef, aenum => ['aa','bb'], domain => 'aa' };
-    is_deeply $txn->q("SELECT '(,\"{aa,bb}\",aa)'::fupg_test_record")->val, $val;
+    is_deeply $txn->sql("SELECT '(,\"{aa,bb}\",aa)'::fupg_test_record")->val, $val;
-    is $txn->q('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{aa,bb}",aa)';
+    is $txn->sql('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{aa,bb}",aa)';
 
     $txn->exec(<<~_);
         CREATE TEMPORARY TABLE fupg_test_table (
@@ -96,12 +96,17 @@ subtest 'custom types', sub {
         );
     _
 
-    is_deeply $txn->q(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val, [
+    $val = $txn->sql(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val;
+    is_deeply $val, [
         { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
         { rec => { a => undef, aenum => undef, domain => undef }, dom => 'bb' },
     ];
+    $val->[0] = 0;
+    $val->[1]{rec}{a} = 0;
+    $val->[1]{rec} = 0;
+    $val->[1]{dom} = 0;
 
-    is $txn->q('SELECT $1::fupg_test_table[]', [
+    is $txn->sql('SELECT $1::fupg_test_table[]', [
         { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
         { rec => {}, dom => 'bb', extra => 1 },
     ])->text_results->val, '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}';
@@ -109,16 +114,46 @@ subtest 'custom types', sub {
     # Wonky Postgres behavior: selecting a domain directly actually returns the
     # underlying type, but going through an array does work.
     $conn->set_type(fupg_test_domain => 21);
-    is_deeply $txn->q("SELECT ARRAY['aa'::fupg_test_domain]")->val, [0x6161];
+    is_deeply $txn->sql("SELECT ARRAY['aa'::fupg_test_domain]")->val, [0x6161];
 
     # Bind param type doesn't match column type, argh.
-    is $txn->q('SELECT $1::fupg_test_domain', 0x6161)->val, 'aa';
+    is $txn->sql('SELECT $1::fupg_test_domain', 0x6161)->val, 'aa';
 
     # Same for selecting from a table :(
     $txn->exec("INSERT INTO fupg_test_table VALUES (NULL, 'bb')");
-    is $txn->q("SELECT dom FROM fupg_test_table")->val, 'bb';
+    is $txn->sql("SELECT dom FROM fupg_test_table")->val, 'bb';
     $conn->set_type(fupg_test_enum => 21);
-    is $txn->q("SELECT dom FROM fupg_test_table")->val, 0x6262;
+    is $txn->sql("SELECT dom FROM fupg_test_table")->val, 0x6262;
+};
+
+
+subtest 'identifier quoting', sub {
+    my $txn = $conn->txn;
+    $txn->exec('CREATE TEMPORARY TABLE fupg_test_tbl ("desc" int, ok int, "hello world" int)');
+    ok $txn->SQL('INSERT INTO fupg_test_tbl', VALUES {desc => 5, ok => 10, 'hello world', 15})->exec;
+    is $txn->SQL('SELECT', IDENT 'hello world', 'FROM fupg_test_tbl')->val, 15;
+};
+
+
+subtest 'vndbid', sub {
+    plan skip_all => 'type not loaded in the database' if !$conn->sql("SELECT 1 FROM pg_type WHERE typname = 'vndbtag'")->val;
+
+    for my $t (qw/a zz xxx/) {
+        is $conn->sql('SELECT $1::vndbtag', $t)->val, $t;
+        is $conn->sql('SELECT $1::vndbtag', $t)->text_params->val, $t;
+        is $conn->sql('SELECT $1::vndbtag', $t)->text_results->val, $t;
+    }
+    ok !eval { $conn->sql('SELECT $1::vndbtag', '')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbtag', 'abcd')->val };
+
+    for my $t (qw/a123 zz992883231 xxx18388123/) {
+        is $conn->sql('SELECT $1::vndbid', $t)->val, $t;
+        is $conn->sql('SELECT $1::vndbid', $t)->text_params->val, $t;
+        is $conn->sql('SELECT $1::vndbid', $t)->text_results->val, $t;
+    }
+    ok !eval { $conn->sql('SELECT $1::vndbid', '')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbid', 'ab')->val };
+    ok !eval { $conn->sql('SELECT $1::vndbid', 'ab1219229999999999')->val };
 };
 
 done_testing;

t/pgtypes.t

@@ -19,32 +19,59 @@ sub v($type, $p_in, @args) {
     my $s_out = @args > 2 && defined $args[2] ? $args[2] : $s_in;
 
     my $test = "$type $s_in" =~ s/\n/\\n/rg;
+    my $oid;
     utf8::encode($test);
     {
-        my $res = $conn->q("SELECT \$1::$type", $s_in)->text_params->val;
+        my $st = $conn->sql("SELECT \$1::$type", $s_in)->text_params;
+        $oid = $st->param_types->[0];
+        my $array = $st->flat;
+        my $res = $array->[0];
         ok is_bool($res), "$test is bool" if $type eq 'bool';
         ok created_as_number($res), "$test is number" if $type =~ /^(int|float)\d/;
         is_deeply $res, $p_out, "$test text->bin";
+        $array->[0] = 0; # Must be writable
     }
     {
-        my $res = $conn->q("SELECT \$1::$type", $p_in)->text_results->val;
+        my $res = $conn->sql("SELECT \$1::$type", $p_in)->text_results->val;
         is $res, $s_out, "$test bin->text";
     }
     {
-        my $res = $conn->q("SELECT \$1::$type", $p_in)->val;
+        my $res = $conn->sql("SELECT \$1::$type", $p_in)->val;
         is_deeply $res, $p_out, "$test bin->bin";
     }
+    {
+        my $bin = $conn->perl2bin($oid, $p_in);
+        ok defined $bin;
+        if ($type !~ /\(/) {
+            is_deeply $conn->bin2perl($oid, $bin), $p_out;
+            is $conn->bin2text($oid, $bin), $s_out;
+            is $conn->text2bin($oid, $s_out), $bin if $type ne 'jsonb'; # jsonb pretty-prints for some reason
+        }
+    }
 }
 sub f($type, $p_in) {
     my $test = "$type $p_in" =~ s/\n/\\n/rg;
     utf8::encode($test);
-    ok !eval { $conn->q("SELECT \$1::$type", $p_in)->val; 1 }, "$test fail";
+    ok !eval { $conn->sql("SELECT \$1::$type", $p_in)->val; 1 }, "$test fail";
 }
 
-ok !defined $conn->q('SELECT pg_sleep(0)')->val; # void
+{ # void
+    my $array = $conn->sql('SELECT pg_sleep(0)')->flat;
+    ok !defined $array->[0];
+    $array->[0] = 0;
+}
 
-v bool => true,  undef, 1, 't';
+v bool => true,  true, 1, 't';
-v bool => false, undef, 0, 'f';
+v bool => \1,    true, 1, 't';
+v bool => 1,     true, 1, 't';
+v bool => 't',   true, 1, 't';
+v bool => false, false, 0, 'f';
+v bool => \0,    false, 0, 'f';
+v bool => 0,     false, 0, 'f';
+v bool => '',    false, 0, 'f';
+v bool => 'f',   false, 0, 'f';
+f bool => 2;
+f bool => [];
 
 v int2 => $_ for (1, -1, -32768, 32767, '12345', -12345, 123.0);
 f int2 => $_ for (-32769, 32768, [], '', 'a', 1.5);
@@ -160,10 +187,29 @@ f 'oidvector', [undef];
 
 # Example from https://www.postgresql.org/docs/17/arrays.html#ARRAYS-IO
 # Lower bounds are discarded.
-is_deeply $conn->q("SELECT '[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int[]")->val, [[[1,2,3],[4,5,6]]];
+is_deeply $conn->sql("SELECT '[1:1][-2:-1][3:5]={{{1,2,3},{4,5,6}}}'::int[]")->val, [[[1,2,3],[4,5,6]]];
 
-is $conn->q('SELECT ($1::int2[])[2]', [1,2,3,4])->val, 2;
+is $conn->sql('SELECT ($1::int2[])[2]', [1,2,3,4])->val, 2;
-is $conn->q('SELECT ($1::int2vector)[1]', [1,2,3,4])->val, 2;
+is $conn->sql('SELECT ($1::int2vector)[1]', [1,2,3,4])->val, 2;
-is $conn->q('SELECT ($1::oidvector)[1]', [1,2,3,4])->val, 2;
+is $conn->sql('SELECT ($1::oidvector)[1]', [1,2,3,4])->val, 2;
+
+is_deeply [$conn->bin2text(
+    16, $conn->perl2bin(16, 1),
+    25, 'Hello',
+    1007, $conn->perl2bin(1007, [-3,1,undef])
+)], ['t', 'Hello', '{-3,1,NULL}'];
+
+{
+    my($b,$s,$a) = $conn->text2bin(16, 't', 25, 'Hello', 1007, '{-3,1,NULL}');
+    is $conn->bin2perl(16, $b), 1;
+    is $conn->bin2perl(25, $s), 'Hello';
+    is_deeply $conn->bin2perl(1007, $a), [-3,1,undef];
+}
+
+{
+    my $v = $conn->sql("SELECT '{t,f,NULL}'::bool[]")->val;
+    is_deeply $v, [true, false, undef];
+    $_ = 0 for @$v;
+}
 
 done_testing;

t/query.t

@@ -7,8 +7,10 @@ is_deeply
     query_decode('a&a&%c3%be=%26%3d%c3%be&a=3'),
     { a => [ builtin::true, builtin::true, 3 ], "\xfe" => "&=\xfe" };
 
-ok !eval { query_decode('%10'); 1 };
+ok !eval { query_decode('a=%fe%83%bf%bf%bf%bf%bf%0a'); 1 };
-like $@, qr/Invalid control character/;
+like $@, qr/does not map to Unicode/;
+
+is_deeply query_decode('&&&a=b'), { a => 'b' };
 
 is query_encode
     { a => builtin::true, b => undef, c => builtin::false, d => 'string', e => "&=\xfe" },

t/sql.t

@@ -9,11 +9,15 @@ sub t($obj, $sql, $params, @opt) {
     is_deeply $gotparams, $params;
 }
 
+my @q_ident = (quote_identifier => sub($x) { $x =~ s/"/_/rg });
+
 my $x;
 t P '', '?', [''];
 t P '', '$1', [''], placeholder_style => 'pg';
 t P undef, '?', [undef];
 t RAW '', '', [];
+t IDENT '"hello"', '"hello"', [];
+t IDENT '"hello"', '_hello_', [], @q_ident;
 t SQL('select', '1'), 'select 1', [];
 t SQL('select', P '1'), 'select ?', [1];
 t SQL('select', $x = '1'), 'select ?', [1];
@@ -41,6 +45,7 @@ t WHERE($x, '1 = 2', SQL('x = ', $x)),
 t WHERE({ col1 => RAW 'NOW()', col2 => 'a'}),
   'WHERE ( col1 = NOW() ) AND ( col2 = ? )', ['a'];
 t WHERE(), 'WHERE 1=1', [];
+t WHERE({ '"x' => 1 }), 'WHERE ( _x = ? )', [1], @q_ident;
 
 t WHERE(AND('true', $x), OR($y, 'y'), AND, OR),
   'WHERE ( ( true ) AND ( ? ) ) AND ( ( ? ) OR ( y ) ) AND ( 1=1 ) AND ( 1=0 )', [$x, $y];
@@ -52,9 +57,11 @@ t SQL(SELECT => COMMA(qw/a b c/), FROM => 'table', WHERE { x => 1, a => undef })
 
 t SET({ a => 1, c => RAW 'NOW()', d => undef }),
   'SET a = ? , c = NOW() , d = ?', [1, undef];
+t SET({ '"x' => 1 }), 'SET _x = ?', [1], @q_ident;
 
 t VALUES({ a => 1, c => RAW 'NOW()', d => undef }),
   '( a , c , d ) VALUES ( ? , NOW() , ? )', [1, undef];
+t VALUES({ '"x' => 1 }), '( _x ) VALUES ( ? )', [1], @q_ident;
 
 t VALUES(1, $x, 'NOW()', RAW 'NOW()'), 'VALUES ( ? , ? , NOW() , NOW() )', [1, $x];
 t VALUES([1, $x, 'NOW()', RAW 'NOW()']), 'VALUES ( ? , ? , ? , NOW() )', [1, $x, 'NOW()'];
@@ -86,4 +93,7 @@ Hash::Util::lock_keys(%hash);
 Hash::Util::lock_value(%hash, 'v');
 t SQL($hash{v}), 'value', [];
 
+ok !eval { SQL('')->compile(oops => 1); 1 };
+like $@, qr/Unknown flag: oops/;
+
 done_testing;

t/validate-util.t

@@ -0,0 +1,36 @@
+use v5.36;
+use Test::More;
+use FU::Validate;
+use FU::Util 'json_format';
+
+my $schema = FU::Validate->compile({ keys => {
+    bool => { anybool => 1 },
+    num => { num => 1 },
+    int => { int => 1 },
+    str => { default => 'x' },
+    intarray => { elems => { int => 1 } },
+    any => { type => 'any' },
+}});
+
+
+is json_format($schema->coerce(undef)), 'null';
+is json_format($schema->coerce("str")), '"str"';
+
+is json_format($schema->coerce({
+        bool => 'abc',
+        num => " 1.5 ",
+        int => 9.7,
+        str => !1,
+        intarray => [ 1.5, -10, undef, ' 0E0 ' ],
+        any => {},
+        whatsthis => undef,
+    }, unknown => 'remove'), canonical => 1),
+    '{"any":{},"bool":true,"int":9,"intarray":[1,-10,null,0],"num":1.5,"str":""}';
+
+is json_format($schema->coerce({uhm => 1}), canonical => 1),
+    '{"any":null,"bool":false,"int":0,"intarray":[],"num":0,"str":"x","uhm":1}';
+
+is json_format($schema->empty, canonical => 1),
+    '{"any":null,"bool":false,"int":0,"intarray":[],"num":0,"str":"x"}';
+
+done_testing;

t/validate.t

@@ -14,11 +14,12 @@ my %validations = (
     setundef => { func => sub { $_[0] = undef; 1 } },
     defaultsub1 => { default => sub { 2 } },
     defaultsub2 => { default => sub { defined $_[0] } },
-    onerrorsub => { onerror => sub { ref $_[0] } },
+    onerrorsub => { onerror => sub { ref $_[1] } },
-    collapsews => { rmwhitespace => 0, func => sub { $_[0] =~ s/\s+/ /g; 1 } },
+    collapsews => { trim => 0, func => sub { $_[0] =~ s/\s+/ /g; 1 } },
     neverfails => { onerror => 'err' },
+    doublefunc => [ func => sub { $_[0] == 0 }, func => sub { $_[0] = 2; 1; } ],
     revnum => { type => 'array', sort => sub($x,$y) { $y <=> $x } },
-    uniquelength => { type => 'array', values => { type => 'array' }, unique => sub { scalar @{$_[0]} } },
+    uniquelength => { elems => { type => 'array' }, unique => sub { scalar @{$_[0]} } },
     person => {
         type => 'hash',
         unknown => 'pass',
@@ -31,214 +32,261 @@ my %validations = (
 );
 
 
-sub t {
+sub t($schema, $input, $output) {
-    my($schema, $input, $output, $error) = @_;
     my $line = (caller)[2];
 
     my $schema_copy = dclone([$schema])->[0];
     my $input_copy = dclone([$input])->[0];
 
+    #diag explain FU::Validate->compile($schema, \%validations) if $line == 95;
     my $res = FU::Validate->compile($schema, \%validations)->validate($input);
-    #diag explain FU::Validate->compile($schema, \%validations) if $line == 139;
-    is !$error, !!$res, "boolean context $line";
     is_deeply $schema, $schema_copy, "schema modification $line";
     is_deeply $input,  $input_copy,  "input modification $line";
-    is_deeply $res->unsafe_data(), $output, "unsafe_data $line";
+    is_deeply $res, $output, "data ok $line";
-    is_deeply $res->data(), $output, "data ok $line" if !$error;
+}
-    ok !eval { $res->data; 1}, "data err $line" if $error;
+
-    is_deeply $res->err(), $error, "err $line";
+sub f($schema, $input, $error, @msg) {
+    my $line = (caller)[2];
+
+    my $schema_copy = dclone([$schema])->[0];
+    my $input_copy = dclone([$input])->[0];
+
+    #diag explain FU::Validate->compile($schema, \%validations) if $line == 176;
+    ok !eval { FU::Validate->compile($schema, \%validations)->validate($input); 1 }, "eval $line";
+    is_deeply $schema, $schema_copy, "schema modification $line";
+    is_deeply $input,  $input_copy,  "input modification $line";
+    delete $@->{longmess};
+    is_deeply { $@->%* }, $error, "err $line";
+    is_deeply [$@->errors], \@msg, "errmsg $line";
 }
 
 
 # default
-t {}, 0, 0, undef;
+t {}, 0, 0;
-t {}, '', '', { validation => 'required' };
+f {}, '', { validation => 'required' }, 'required value missing';
-t {}, undef, undef, { validation => 'required' };
+f {}, undef, { validation => 'required' }, 'required value missing';
-t { default => undef }, undef, undef, undef;
+t { default => undef }, undef, undef;
-t { default => undef }, '', undef, undef;
+t { default => undef }, '', undef;
-t { defaultsub1 => 1 }, undef, 2, undef;
+f { default => \'required' }, '', { validation => 'required' }, 'required value missing';
-t { defaultsub2 => 1 }, undef, '', undef;
+t { defaultsub1 => 1 }, undef, 2;
-t { defaultsub2 => 1 }, '', 1, undef;
+t { defaultsub2 => 1 }, undef, '';
-t { onerrorsub => 1 }, undef, 'FU::Validate::Result', undef;
+t { defaultsub2 => 1 }, '', 1;
+t { onerrorsub => 1 }, undef, 'FU::Validate::err';
 
-# rmwhitespace
+# trim
-t {}, " Va\rl id \n ", 'Val id', undef;
+t {}, " Va\rl id \n ", 'Val id';
-t { rmwhitespace => 0 }, " Va\rl id \n ", " Va\rl id \n ", undef;
+t { trim => 0 }, " Va\rl id \n ", " Va\rl id \n ";
-t {}, '  ', '', { validation => 'required' };
+f {}, '  ', { validation => 'required' }, 'required value missing';
-t { rmwhitespace => 0 }, '  ', '  ', undef;
+t { trim => 0 }, '  ', '  ';
+
+# allow_control
+f {}, "\b", { validation => 'allow_control' }, 'invalid control character';
+t { allow_control => 1 }, "\b", "\b";
+
+# accept_array
+t { default => undef, accept_array => 'first' }, [], undef;
+t { default => undef, accept_array => 'first' }, [' x '], 'x';
+t { accept_array => 'first' }, [1,2,3], 1;
+t { accept_array => 'last' }, [1,2,3], 3;
+f { accept_array => 'first' }, ['  ', 1], { validation => 'required' }, 'required value missing';
+f { accept_array => 'first' }, [], { validation => 'required' }, 'required value missing';
 
 # arrays
-t {}, [], [], { validation => 'type', expected => 'scalar', got => 'array' };
+f {}, [], { validation => 'type', expected => 'scalar', got => 'array' }, "invalid type, expected 'scalar' but got 'array'";
-t { type => 'array' }, 1, 1, { validation => 'type', expected => 'array', got => 'scalar' };
+f { type => 'array' }, 1, { validation => 'type', expected => 'array', got => 'scalar' }, "invalid type, expected 'array' but got 'scalar'";
-t { type => 'array' }, [], [], undef;
+t { type => 'array' }, [], [];
-t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}], undef;
+t { type => 'array' }, [undef,1,2,{}], [undef,1,2,{}];
-t { type => 'array', scalar => 1 }, 1, [1], undef;
+t { type => 'array', accept_scalar => 1 }, 1, [1];
-t { type => 'array', values => {} }, [undef], [undef], { validation => 'values', errors => [{ index => 0, validation => 'required' }] };
+f { type => 'array', elems => {} }, [undef], { validation => 'elems', errors => [{ index => 0, validation => 'required' }] }, "[0]: required value missing";
-t { type => 'array', values => {} }, [' a '], ['a'], undef;
+t { type => 'array', elems => {} }, [' a '], ['a'];
-t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/], undef;
+t { type => 'array', sort => 'str' }, [qw/20 100 3/], [qw/100 20 3/];
-t { type => 'array', sort => 'num' }, [qw/20 100 3/], [qw/3 20 100/], undef;
+t { type => 'array', sort => 'num' }, [qw/20 100 3/], [qw/3 20 100/];
-t { revnum => 1 },                    [qw/20 100 3/], [qw/100 20 3/], undef;
+t { revnum => 1 },                    [qw/20 100 3/], [qw/100 20 3/];
-t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 1/], [qw/1 2 3/], undef;
+t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 1/], [qw/1 2 3/];
-t { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 3/], [qw/2 3 3/], { validation => 'unique', index_a => 1, value_a => 3, index_b => 2, value_b => 3 };
+f { type => 'array', sort => 'num', unique => 1 }, [qw/3 2 3/], { validation => 'unique', index_a => 1, value_a => 3, index_b => 2, value_b => 3 }, q{[2] value '"3"' duplicated};
-t { type => 'array', unique => 1 }, [qw/3 1 2/], [qw/3 1 2/], undef;
+t { type => 'array', unique => 1 }, [qw/3 1 2/], [qw/3 1 2/];
-t { type => 'array', unique => 1 }, [qw/3 1 3/], [qw/3 1 3/], { validation => 'unique', index_a => 0, value_a => 3, index_b => 2, value_b => 3, key => 3 };
+f { type => 'array', unique => 1 }, [qw/3 1 3/], { validation => 'unique', index_a => 0, value_a => 3, index_b => 2, value_b => 3, key => 3 }, q{[2] value '"3"' duplicated};
-t { uniquelength => 1 }, [[],[1],[1,2]], [[],[1],[1,2]], undef;
+t { uniquelength => 1 }, [[],[1],[1,2]], [[],[1],[1,2]];
-t { uniquelength => 1 }, [[],[1],[2]], [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 };
+f { uniquelength => 1 }, [[],[1],[2]], { validation => 'unique', index_a => 1, value_a => [1], index_b => 2, value_b => [2], key => 1 }, q{[2] value '[1]' duplicated};
-t { type => 'array', setundef => 1 }, [], undef, undef;
+t { type => 'array', setundef => 1 }, [], undef;
-t { type => 'array', values => { type => 'any', setundef => 1 } }, [[]], [undef], undef;
+t { type => 'array', elems => { type => 'any', setundef => 1 } }, [[]], [undef];
 
 # hashes
-t { type => 'hash' }, [], [], { validation => 'type', expected => 'hash', got => 'array' };
+f { type => 'hash' }, [], { validation => 'type', expected => 'hash', got => 'array' }, "invalid type, expected 'hash' but got 'array'";
-t { type => 'hash' }, 'a', 'a', { validation => 'type', expected => 'hash', got => 'scalar' };
+f { type => 'hash' }, 'a', { validation => 'type', expected => 'hash', got => 'scalar' }, "invalid type, expected 'hash' but got 'scalar'";
-t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {}, undef;
+t { type => 'hash' }, {a=>[],b=>undef,c=>{}}, {a=>[],b=>undef,c=>{}};
-t { type => 'hash', keys => { a=>{} } }, {}, {a=>undef}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] }; # XXX: the key doesn't necessarily have to be created
+f { type => 'hash', keys => { a=>{} } }, {}, { validation => 'keys', errors => [{ key => 'a', validation => 'required' }] }, '.a: required value missing';
-t { type => 'hash', keys => { a=>{missing=>'ignore'} } }, {}, {}, undef;
+t { type => 'hash', keys => { a=>{missing=>'ignore'} } }, {}, {};
-t { type => 'hash', keys => { a=>{default=>undef} } }, {}, {a=>undef}, undef;
+t { type => 'hash', keys => { a=>{default=>undef} } }, {}, {a=>undef};
-t { type => 'hash', keys => { a=>{missing=>'create',default=>undef} } }, {}, {a=>undef}, undef;
+t { type => 'hash', keys => { a=>{missing=>'create',default=>undef} } }, {}, {a=>undef};
-t { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {}, {key => 'a', validation => 'missing'};
+f { type => 'hash', keys => { a=>{missing=>'reject'} } }, {}, {key => 'a', validation => 'missing'}, '.a: required key missing';
 
-t { type => 'hash', keys => { a=>{} } }, {a=>' a '}, {a=>'a'}, undef; # Test against in-place modification
+t { type => 'hash', keys => { a=>{} } }, {a=>' a '}, {a=>'a'}; # Test against in-place modification
-t { type => 'hash', keys => { a=>{} }, unknown => 'remove' }, { a=>1,b=>1 }, { a=>1 }, undef;
+t { type => 'hash', keys => { a=>{} }, unknown => 'remove' }, { a=>1,b=>1 }, { a=>1 };
-t { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] };
+f { type => 'hash', keys => { a=>{} }, unknown => 'reject' }, { a=>1,b=>1 }, { validation => 'unknown', keys => ['b'], expected => ['a'] }, "unknown key: b";
-t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>1,b=>1 }, undef;
+t { type => 'hash', keys => { a=>{} }, unknown => 'pass' }, { a=>1,b=>1 }, { a=>1,b=>1 };
-t { type => 'hash', setundef => 1 }, {}, undef, undef;
+t { type => 'hash', setundef => 1 }, {}, undef;
-t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef}, undef;
+t { type => 'hash', unknown => 'reject', keys => { a=>{ type => 'any', setundef => 1}} }, {a=>[]}, {a=>undef};
+
+t [ keys => { a => {} }, keys => { b => {} } ], {a=>1, b=>2}, {a=>1, b=>2};
+f [ keys => { a => {} }, keys => { b => {} } ], {a=>1}, { validation => 'keys', errors => [{ key => 'b', validation => 'required' }] }, '.b: required value missing';
+f [ keys => { a => {} }, keys => { a => { int => 1 } } ], {a=>'abc'}, { validation => 'keys', errors => [{ key => 'a', validation => 'int', got => 'abc' }] }, ".a: failed validation 'int'";
+
+t { values => { int => 1 } }, { a => -1, b => 1 }, { a => -1, b => 1 };
+f { values => { int => 1 } }, { a => undef }, { validation => 'values', errors => [{ key => 'a', validation => 'required' }] }, '.a: required value missing';
 
 # default validations
-t { minlength => 3 }, 'ab', 'ab', { validation => 'minlength', expected => 3, got => 2 };
+f { minlength => 3 }, 'ab', { validation => 'minlength', expected => 3, got => 2 }, "input too short, expected minimum of 3 but got 2";
-t { minlength => 3 }, 'abc', 'abc', undef;
+t { minlength => 3 }, 'abc', 'abc';
-t { maxlength => 3 }, 'abcd', 'abcd', { validation => 'maxlength', expected => 3, got => 4 };
+f { maxlength => 3 }, 'abcd', { validation => 'maxlength', expected => 3, got => 4 }, "input too long, expected maximum of 3 but got 4";
-t { maxlength => 3 }, 'abc', 'abc', undef;
+t { maxlength => 3 }, 'abc', 'abc';
-t { minlength => 3, maxlength => 3 }, 'abc', 'abc', undef;
+t { minlength => 3, maxlength => 3 }, 'abc', 'abc';
-t { length => 3 }, 'ab',   'ab',   { validation => 'length', expected => 3, got => 2 };
+f { length => 3 }, 'ab',   { validation => 'length', expected => 3, got => 2 }, 'invalid input length, expected 3 but got 2';
-t { length => 3 }, 'abcd', 'abcd', { validation => 'length', expected => 3, got => 4 };
+f { length => 3 }, 'abcd', { validation => 'length', expected => 3, got => 4 }, 'invalid input length, expected 3 but got 4';
-t { length => 3 }, 'abc',  'abc',  undef;
+t { length => 3 }, 'abc',  'abc';
-t { length => [1,3] }, 'abc',  'abc', undef;
+t { length => [1,3] }, 'abc',  'abc';
-t { length => [1,3] }, 'abcd', 'abcd', { validation => 'length', expected => [1,3], got => 4 };;
+f { length => [1,3] }, 'abcd', { validation => 'length', expected => [1,3], got => 4 }, "invalid input length, expected between 1 and 3 but got 4";
-t { type => 'array', length => 0 }, [], [], undef;
+t { type => 'array', length => 0 }, [], [];
-t { type => 'array', length => 1 }, [1,2], [1,2], { validation => 'length', expected => 1, got => 2 };
+f { type => 'array', length => 1 }, [1,2], { validation => 'length', expected => 1, got => 2 }, "invalid input length, expected 1 but got 2";
-t { type => 'hash', length => 0 }, {}, {}, undef;
+t { type => 'hash', length => 0 }, {}, {};
-t { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 };
+f { type => 'hash', length => 1, unknown => 'pass' }, {qw/1 a 2 b/}, { validation => 'length', expected => 1, got => 2 }, "invalid input length, expected 1 but got 2";
-t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1}, undef;
+t { type => 'hash', length => 1, keys => {a => {missing=>'ignore'}, b => {missing=>'ignore'}} }, {a=>1}, {a=>1};
-t { regex => '^a' }, 'abc', 'abc', undef;  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
+t { regex => '^a' }, 'abc', 'abc';  # XXX: Can't use qr// here because t() does dclone(). The 'hex' test covers that case anyway.
-t { regex => '^a' }, 'cba', 'cba', { validation => 'regex', regex => '^a', got => 'cba' };
+f { regex => '^a' }, 'cba', { validation => 'regex', regex => '^a', got => 'cba' }, "failed validation 'regex'";
-t { enum => [1,2] }, 1, 1, undef;
+t [ regex => '^a', regex => 'z$' ], 'abcxyz', 'abcxyz';
-t { enum => [1,2] }, 2, 2, undef;
+f [ regex => '^a', regex => 'z$' ], 'bcxyz', { validation => 'regex', regex => '^a', got => 'bcxyz' }, "failed validation 'regex'";
-t { enum => [1,2] }, 3, 3, { validation => 'enum', expected => [1,2], got => 3 };
+f [ regex => '^a', regex => 'z$' ], 'abcxy', { validation => 'regex', regex => 'z$', got => 'abcxy' }, "failed validation 'regex'";
-t { enum => 1 }, 1, 1, undef;
+t { enum => [1,2] }, 1, 1;
-t { enum => 1 }, 2, 2, { validation => 'enum', expected => [1], got => 2 };
+t { enum => [1,2] }, 2, 2;
-t { enum => {a=>1,b=>2} }, 'a', 'a', undef;
+f { enum => [1,2] }, 3, { validation => 'enum', expected => [1,2], got => 3 }, "failed validation 'enum'";
-t { enum => {a=>1,b=>2} }, 'c', 'c', { validation => 'enum', expected => ['a','b'], got => 'c' };
+t { enum => 1 }, 1, 1;
-t { anybool => 1 }, 1, true, undef;
+f { enum => 1 }, 2, { validation => 'enum', expected => [1], got => 2 }, "failed validation 'enum'";
-t { anybool => 1 }, undef, false, undef;
+t { enum => {a=>1,b=>2} }, 'a', 'a';
-t { anybool => 1 }, '', false, undef;
+f { enum => {a=>1,b=>2} }, 'c', { validation => 'enum', expected => ['a','b'], got => 'c' }, "failed validation 'enum'";
-t { anybool => 1 }, {}, true, undef;
+t { anybool => 1 }, 1, true;
-t { anybool => 1 }, [], true, undef;
+t { anybool => 1 }, undef, false;
-t { anybool => 1 }, bless({}, 'test'), true, undef;
+t { anybool => 1 }, '', false;
-t { bool => 1 }, 1, 1, { validation => 'bool' };
+t { anybool => 1 }, {}, true;
-t { bool => 1 }, \1, true, undef;
+t { anybool => 1 }, [], true;
+t { anybool => 1 }, bless({}, 'test'), true;
+f { bool => 1 }, 1, { validation => 'bool' }, "failed validation 'bool'";
+t { bool => 1 }, \1, true;
 my($true, $false) = (1,0);
-t { bool => 1 }, bless(\$true, 'boolean'), true, undef;
+t { bool => 1 }, bless(\$true, 'boolean'), true;
-t { bool => 1 }, bless(\$false, 'boolean'), false, undef;
+t { bool => 1 }, bless(\$false, 'boolean'), false;
-t { bool => 1 }, bless(\$true, 'test'), bless(\$true, 'test'), { validation => 'bool' };
+f { bool => 1 }, bless(\$true, 'test'), { validation => 'bool' }, "failed validation 'bool'";
-t { ascii => 1 }, 'ab c', 'ab c', undef;
+t { ascii => 1 }, 'ab c', 'ab c';
-t { ascii => 1 }, "a\nb", "a\nb", { validation => 'ascii', got => "a\nb" };
+f { ascii => 1 }, "a\nb", { validation => 'ascii', got => "a\nb" }, "failed validation 'ascii'";
 
 # custom validations
-t { hex => 1 }, 'DeadBeef', 'DeadBeef', undef;
+t { hex => 1 }, 'DeadBeef', 'DeadBeef';
-t { hex => 1 }, 'x', 'x', { validation => 'hex', error => { validation => 'regex', regex => "$validations{hex}{regex}", got => 'x' } };
+f { hex => 1 }, 'x', { validation => 'hex', error => { validation => 'regex', regex => "$validations{hex}{regex}", got => 'x' } }, "validation 'hex': failed validation 'regex'";
-t { prefix => 'a' }, 'abc', 'abc', undef;
+t { prefix => 'a' }, 'abc', 'abc';
-t { prefix => 'a' }, 'cba', 'cba', { validation => 'prefix', error => { validation => 'func', result => '' } };
+f { prefix => 'a' }, 'cba', { validation => 'prefix', error => { validation => 'func', result => '' } }, "validation 'prefix': failed validation 'func'";
-t { mybool => 1 }, 'abc', 1, undef;
+t { mybool => 1 }, 'abc', 1;
-t { mybool => 1 }, undef, 0, undef;
+t { mybool => 1 }, undef, 0;
-t { mybool => 1 }, '', 0, undef;
+t { mybool => 1 }, '', 0;
-t { collapsews => 1 }, " \t\n ", ' ', undef;
+t { collapsews => 1 }, " \t\n ", ' ';
-t { collapsews => 1 }, '   x  ', ' x ', undef;
+t { collapsews => 1 }, '   x  ', ' x ';
-t { collapsews => 1, rmwhitespace => 1 }, '   x  ', 'x', undef;
+t { collapsews => 1, trim => 1 }, '   x  ', 'x';
-t { person => 1 }, 1, 1, { validation => 'type', expected => 'hash', got => 'scalar' };
+f { person => 1 }, 1, { validation => 'type', expected => 'hash', got => 'scalar' }, "invalid type, expected 'hash' but got 'scalar'";
-t { person => 1, default => 1 }, undef, 1, undef;
+t { person => 1, default => 1 }, undef, 1;
-t { person => 1 }, { sex => 1 }, { sex => 1, name => undef }, { validation => 'person', error => { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] } };
+f { person => 1 }, { sex => 1 }, { validation => 'keys', errors => [{ key => 'name', validation => 'required' }] }, ".name: required value missing";
-t { person => 1 }, { sex => undef, name => 'y' }, { sex => 1, name => 'y' }, undef;
+t { person => 1 }, { sex => undef, name => 'y' }, { sex => 1, name => 'y' };
-t { person => 1, keys => {age => {default => \'required'}} }, {name => 'x', sex => 'y'}, { name => 'x', sex => 'y', age => undef }, { validation => 'keys', errors => [{ key => 'age', validation => 'required' }] };
+f { person => 1, keys => {age => {missing => 'reject'}} }, {name => 'x', sex => 'y'}, { key => 'age', validation => 'missing' }, '.age: required key missing';
-t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => 1},  { name => 'x', sex => 'y', extra => 1 }, undef;
+t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => 1},  { name => 'x', sex => 'y', extra => 1 };
-t { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { name => 'x', sex => 'y', extra => '' }, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] };
+f { person => 1, keys => {extra => {}} }, {name => 'x', sex => 'y', extra => ''}, { validation => 'keys', errors => [{ key => 'extra', validation => 'required' }] }, '.extra: required value missing';
-t { person => 1 }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y', extra => 1}, undef;
+t { person => 1 }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y', extra => 1};
-t { person => 1, unknown => 'remove' }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y'}, undef;
+t { person => 1, unknown => 'remove' }, {name => 'x', sex => 'y', extra => 1}, {name => 'x', sex => 'y'};
-t { neverfails => 1, int => 1 }, undef, 'err', undef;
+t { neverfails => 1, int => 1 }, undef, 'err';
-t { neverfails => 1, int => 1 }, 'x', 'err', undef;
+t { neverfails => 1, int => 1 }, 'x', 'err';
-t { neverfails => 1, int => 1, onerror => undef }, 'x', undef, undef; # XXX: no way to 'unset' an inherited onerror clause, hmm.
+t { neverfails => 1, int => 1, onerror => undef }, 'x', undef; # XXX: no way to 'unset' an inherited onerror clause, hmm.
+t { doublefunc => 1 }, 0, 2;
+f { doublefunc => 1 }, 1, { validation => 'doublefunc', error => { validation => 'func', result => '' } }, "validation 'doublefunc': failed validation 'func'";
 
 # numbers
-sub nerr { +{ validation => 'num', got => $_[0] } }
+sub nerr { ({ validation => 'num', got => $_[0] }, "invalid number: \"$_[0]\"") }
-t { num => 1 }, 0, 0, undef;
+t { num => 1 }, 0, 0;
-t { num => 1 }, '-', '-', nerr '-';
+f { num => 1 }, '-', nerr '-';
-t { num => 1 }, '00', '00', nerr '00';
+f { num => 1 }, '00', nerr '00';
-t { num => 1 }, '1', '1', undef;
+t { num => 1 }, '1', '1';
-t { num => 1 }, '1.1.', '1.1.', nerr '1.1.';
+f { num => 1 }, '1.1.', nerr '1.1.';
-t { num => 1 }, '1.-1', '1.-1', nerr '1.-1';
+f { num => 1 }, '1.-1', nerr '1.-1';
-t { num => 1 }, '.1', '.1', nerr '.1';
+f { num => 1 }, '.1', nerr '.1';
-t { num => 1 }, '0.1e5', '0.1e5', undef;
+t { num => 1 }, '0.1e5', 10000;
-t { num => 1 }, '0.1e+5', '0.1e+5', undef;
+t { num => 1 }, '0.1e+5', 10000;
-t { num => 1 }, '0.1e5.1', '0.1e5.1', nerr '0.1e5.1';
+f { num => 1 }, '0.1e5.1', nerr '0.1e5.1';
-t { int => 1 }, 0, 0, undef;
+t { int => 1 }, 0, 0;
-t { int => 1 }, -123, -123, undef;
+t { int => 1 }, -123, -123;
-t { int => 1 }, -123.1, -123.1, { validation => 'int', got => -123.1 };
+f { int => 1 }, -123.1, { validation => 'int', got => -123.1 }, "failed validation 'int'";
-t { uint => 1 }, 0, 0, undef;
+t { uint => 1 }, 0, 0;
-t { uint => 1 }, 123, 123, undef;
+t { uint => 1 }, 123, 123;
-t { uint => 1 }, -123, -123, { validation => 'uint', got => -123 };
+f { uint => 1 }, -123, { validation => 'uint', got => -123 }, "failed validation 'uint'";
-t { min => 1 }, 1, 1, undef;
+t { min => 1 }, 1, 1;
-t { min => 1 }, 0.9, 0.9, { validation => 'min', expected => 1, got => 0.9 };
+f { min => 1 }, 0.9, { validation => 'min', expected => 1, got => 0.9 }, "expected minimum 1 but got 0.9";
-t { min => 1 }, 'a', 'a', { validation => 'min', error => nerr 'a' };
+f { min => 1 }, 'a', { validation => 'min', error => (nerr 'a')[0] }, 'invalid number: "a"';
-t { max => 1 }, 1, 1, undef;
+t { max => 1 }, 1, 1;
-t { max => 1 }, 1.1, 1.1, { validation => 'max', expected => 1, got => 1.1 };
+f { max => 1 }, 1.1, { validation => 'max', expected => 1, got => 1.1 }, "expected maximum 1 but got 1.1";
-t { max => 1 }, 'a', 'a', { validation => 'max', error => nerr 'a' };
+f { max => 1 }, 'a', { validation => 'max', error => (nerr 'a')[0] }, 'invalid number: "a"';
-t { range => [1,2] }, 1, 1, undef;
+t { range => [1,2] }, 1, 1;
-t { range => [1,2] }, 2, 2, undef;
+t { range => [1,2] }, 2, 2;
-t { range => [1,2] }, 0.9, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } };
+f { range => [1,2] }, 0.9, { validation => 'range', error => { validation => 'min', expected => 1, got => 0.9 } }, 'expected minimum 1 but got 0.9';
-t { range => [1,2] }, 2.1, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } };
+f { range => [1,2] }, 2.1, { validation => 'range', error => { validation => 'max', expected => 2, got => 2.1 } }, 'expected maximum 2 but got 2.1';
-#t { range => [1,2] }, 'a', 'a', { validation => 'range', error => { validation => 'max', error => nerr 'a' } }; # XXX: Error validation type depends on evaluation order
+f { range => [1,2] }, 'a', { validation => 'range', error => { validation => 'min', error => (nerr 'a')[0] } }, 'invalid number: "a"';
 
 # email template
 use utf8;
-t { email => 1 }, $_->[1], $_->[1], $_->[0] ? undef : { validation => 'email', got => $_->[1] } for (
+f { email => 1 }, $_, { validation => 'email', got => $_ }, "failed validation 'email'" for (
-  [ 0, 'abc.com' ],
+  'abc.com',
-  [ 0, 'abc@localhost' ],
+  'abc@localhost',
-  [ 0, 'abc@10.0.0.' ],
+  'abc@10.0.0.',
-  [ 0, 'abc@256.0.0.1' ],
+  'abc@256.0.0.1',
-  [ 0, '<whoami>@blicky.net' ],
+  '<whoami>@blicky.net',
-  [ 0, 'a @a.com' ],
+  'a @a.com',
-  [ 0, 'a"@a.com' ],
+  'a"@a.com',
-  [ 0, 'a@[:]' ],
+  'a@[:]',
-  [ 0, 'a@127.0.0.1' ],
+  'a@127.0.0.1',
-  [ 0, 'a@[::1]' ],
+  'a@[::1]',
-  [ 1, 'a@a.com' ],
+);
-  [ 1, 'a@a.com.' ],
+t { email => 1 }, $_, $_ for (
-  [ 1, 'é@yörhel.nl' ],
+  'a@a.com',
-  [ 1, 'a+_0-c@yorhel.nl' ],
+  'a@a.com.',
-  [ 1, 'é@x-y_z.example' ],
+  'é@yörhel.nl',
-  [ 1, 'abc@x-y_z.example' ],
+  'a+_0-c@yorhel.nl',
+  'é@x-y_z.example',
+  'abc@x-y_z.example',
 );
 my $long = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx@xxxxxxxxxxxxxxxxxxxx.xxxxxxxxxxxxxxxxxxxxxxxx.xxxxx';
-t { email => 1 }, $long, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } };
+f { email => 1 }, $long, { validation => 'email', error => { validation => 'maxlength', got => 255, expected => 254 } }, "validation 'email': input too long, expected maximum of 254 but got 255";
 
 # weburl template
-t { weburl => 1 }, $_->[1], $_->[1], $_->[0] ? undef : { validation => 'weburl', got => $_->[1] } for (
+f { weburl => 1 }, $_, { validation => 'weburl', got => $_ }, "failed validation 'weburl'" for (
-  [ 0, 'http' ],
+  'http',
-  [ 0, 'http://' ],
+  'http://',
-  [ 0, 'http:///' ],
+  'http:///',
-  [ 0, 'http://x/' ],
+  'http://x/',
-  [ 0, 'http://x/' ],
+  'http://x/',
-  [ 0, 'http://256.0.0.1/' ],
+  'http://256.0.0.1/',
-  [ 0, 'http://blicky.net:050/' ],
+  'http://blicky.net:050/',
-  [ 0, 'ftp//blicky.net/' ],
+  'ftp//blicky.net/',
-  [ 1, 'http://blicky.net/' ],
+);
-  [ 1, 'http://blicky.net:50/' ],
+t { weburl => 1}, $_, $_ for (
-  [ 1, 'https://blicky.net/' ],
+  'http://blicky.net/',
-  [ 1, 'https://[::1]:80/' ],
+  'http://blicky.net:50/',
-  [ 1, 'https://l-n.x_.example.com/' ],
+  'https://blicky.net/',
-  [ 1, 'https://blicky.net/?#Who\'d%20ever%22makeaurl_like-this/!idont.know' ],
+  'https://[::1]:80/',
+  'https://l-n.x_.example.com/',
+  'https://blicky.net/?#Who\'d%20ever%22makeaurl_like-this/!idont.know',
 );
 
+# Merging nested schemas
+my $pa = FU::Validate->compile({ regex => '^a' });
+my $pz = FU::Validate->compile({ regex => 'z$' });
+my $com = FU::Validate->compile([ elems => $pa, elems => $pz ]);
+is_deeply $com->validate(['axz']), ['axz'];
+ok !eval { $com->validate(['bz', 'axz', 'ax']) };
+is [$@->errors]->[0], "[0]: failed validation 'regex'";
+is [$@->errors]->[1], "[2]: failed validation 'regex'";
 
 # Things that should fail
 ok !eval { FU::Validate->compile({ recursive => 1 }, { recursive => { recursive => 1 } }); 1 }, 'recursive';

t/xmlwr.t

@@ -65,4 +65,21 @@ sub t {
 
 is fragment { t 'arg' }, '<div attr1="arg"><span>ab&quot; &lt; c &amp;&lt; d</span><span><ok🥳ay></span>🥳</div>';
 
+ok !eval { fragment { tag_ 'hi', \1 } };
+like $@, qr/Invalid attempt to output bare reference/;
+
+ok !eval { fragment { tag_ 'hi', {} } };
+like $@, qr/Invalid attempt to output bare reference/;
+
+is fragment { tag_ 'hi', bless {}, 'XTEST1' }, '<hi>string</hi>';
+like fragment { tag_ 'hi', bless {}, 'XTEST2' }, qr{<hi>HASH\(.*\)</hi>}; # Yeah, whatever.
+like fragment { tag_ 'hi', ''.{} }, qr{<hi>HASH\(.*\)</hi>};
+
 done_testing;
+
+
+package XTEST1;
+use overload '""' => sub { 'string' };
+
+package XTEST2;
+use overload '""' => sub { {} };