From 327fd9ea5051418bc86f85fcd7b6bddf30779a1d Mon Sep 17 00:00:00 2001 From: Yorhel Date: Thu, 27 Feb 2025 18:24:14 +0100 Subject: [PATCH] Pg: Support type override configuration --- FU.xs | 3 ++ FU/Pg.pm | 50 ++++++++++++++++++++++- c/pgconn.c | 96 +++++++++++++++++++++++++++++++++++++++++---- c/pgtypes.c | 24 +++++++++--- t/pgtypes-dynamic.t | 76 ++++++++++++++++++++++++++--------- 5 files changed, 215 insertions(+), 34 deletions(-) diff --git a/FU.xs b/FU.xs index cddf171..ffd3df6 100644 --- a/FU.xs +++ b/FU.xs @@ -229,6 +229,9 @@ void q(fupg_conn *c, SV *sv, ...) FUPG_CONN_COOKIE; ST(0) = fupg_q(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items); +void _set_type(fupg_conn *c, SV *name, SV *sendsv, SV *recvsv) + CODE: + fupg_set_type(c, name, sendsv, recvsv); MODULE = FU PACKAGE = FU::Pg::txn diff --git a/FU/Pg.pm b/FU/Pg.pm index 284cb72..59fa799 100644 --- a/FU/Pg.pm +++ b/FU/Pg.pm @@ -13,6 +13,13 @@ package FU::Pg::conn { my($sql, $params) = FU::SQL::SQL(@_)->compile(placeholder_style => 'pg', in_style => 'pg'); $s->q($sql, @$params); } + + sub set_type($s, $n, @arg) { + Carp::confess("Invalid number of arguments") if @arg == 0 || (@arg > 1 && @arg % 2); + return $s->_set_type($n, $arg[0], $arg[0]) if @arg == 1; + my %arg = @arg; + $s->_set_type($n, $arg{send}, $arg{recv}); + } }; *FU::Pg::txn::Q = \*FU::Pg::conn::Q; @@ -625,12 +632,51 @@ any of these. =back +=head3 Overriding types + +The default conversion for each type can be changed: + +=over + +=item $conn->set_type($affected_type, $type) + +=item $conn->set_type($affected_type, send => $type, recv => $type) + +Change how C<$affected_type> is being converted when used as a bind parameter +(I) or when received from query results (I). The two-argument +version is equivalent to setting I and I to the same C<$type>. + +Types can be specified either by their numeric I or by name. In the latter +case, the name must exactly match the internal type name used by PostgreSQL. +Note that this "internal type name" does not always match the names used in +documentation. For example, I, I and I should be +specified as I, I and I, respectively, and the I type +is internally called I. The full list of recognized types in your +database can be queried with: + + SELECT oid, typname FROM pg_type; + +The C<$affected_type> does not actually have to exist in the database, this +method only stores the type in its internal configuration, which is consulted +upon executing a query that takes the type as bind parameter or when it returns +a column of that type. + +The given C<$type> arguments must refer to a built-in type supported by this +module. Types can also be set to I to restore the conversion to its +default. + +=back + +I Type override examples and a warning about domain types. + +I Some handy special types for overriding common conversions. + +I Support for custom types through callbacks. + I Methods to convert between the various formats. I Methods to query type info. -I Custom per-type configuration. - =head2 Errors All methods can throw an exception on error. When possible, the error message diff --git a/c/pgconn.c b/c/pgconn.c index a2f1eec..c72e747 100644 --- a/c/pgconn.c +++ b/c/pgconn.c @@ -25,6 +25,15 @@ static void fupg_prep_destroy(fupg_prep *p) { safefree(p); } +typedef struct { + const fupg_type *send, *recv; +} fupg_override; + +#define fupg_name_hash(v) kh_hash_str((v).n) +#define fupg_name_eq(a,b) kh_eq_str((a).n, (b).n) +KHASHL_MAP_INIT(KH_LOCAL, fupg_oid_overrides, fupg_oid_overrides, Oid, fupg_override, kh_hash_uint32, kh_eq_generic); +KHASHL_MAP_INIT(KH_LOCAL, fupg_name_overrides, fupg_name_overrides, fupg_name, fupg_override, fupg_name_hash, fupg_name_eq); + typedef struct { SV *self; @@ -38,6 +47,8 @@ typedef struct { unsigned int prep_max; unsigned int prep_cur; /* Number of prepared statements not associated with an active $st object */ fupg_type *types; + fupg_oid_overrides *oidtypes; + fupg_name_overrides *nametypes; fupg_records *records; fupg_prepared *prep_map; fupg_prep *prep_head, *prep_tail; /* Inserted into head, removed at tail */ @@ -166,6 +177,8 @@ static SV *fupg_connect(pTHX_ const char *str) { c->ntypes = 0; c->types = NULL; c->records = fupg_records_init(); + c->oidtypes = fupg_oid_overrides_init(); + c->nametypes = fupg_name_overrides_init(); c->prep_cur = 0; c->prep_max = 256; c->prep_map = fupg_prepared_init(); @@ -196,6 +209,8 @@ static void fupg_conn_destroy(pTHX_ fupg_conn *c) { PQfinish(c->conn); if (c->buf.sv) SvREFCNT_dec(c->buf.sv); safefree(c->types); + fupg_oid_overrides_destroy(c->oidtypes); + fupg_name_overrides_destroy(c->nametypes); khint_t k; kh_foreach(c->records, k) safefree(kh_val(c->records, k)); fupg_records_destroy(c->records); @@ -352,6 +367,47 @@ static void fupg_prepared_unref(fupg_conn *c, fupg_prep *p) { /* Type handling */ +static const fupg_type *fupg_resolve_builtin(pTHX_ SV *name) { + SvGETMAGIC(name); + if (!SvOK(name)) return NULL; + UV uv; + const char *pv = SvPV_nomg_nolen(name); + const fupg_type *t = grok_atoUV(pv, &uv, NULL) && uv <= (UV)UINT_MAX + ? fupg_builtin_byoid((Oid)uv) + : fupg_builtin_byname(pv); + if (!t) fu_confess("No builtin type found with oid or name '%s'", pv); + return t; +} + +static void fupg_set_type(pTHX_ fupg_conn *c, SV *name, SV *sendsv, SV *recvsv) { + fupg_override o; + o.send = fupg_resolve_builtin(sendsv); + o.recv = fupg_resolve_builtin(recvsv); + if ((o.send && o.send->send == fupg_send_array) || (o.recv && o.recv->recv == fupg_recv_array)) + fu_confess("Cannot set a type to array, override the underlying element type instead"); + /* Can't currently happen since we have no records in the builtin type + * list, but catch this just in case that changes. */ + if ((o.send && o.send->send == fupg_send_record) || (o.recv && o.recv->recv == fupg_recv_record)) + fu_confess("Cannot set a type to record"); + + UV uv; + STRLEN len; + const char *pv = SvPV(name, len); + int k, i; + if (grok_atoUV(pv, &uv, NULL) && uv <= (UV)UINT_MAX) { + k = fupg_oid_overrides_put(c->oidtypes, (Oid)uv, &i); + kh_val(c->oidtypes, k) = o; + } else if (len < sizeof(fupg_name)) { + fupg_name n; + strcpy(n.n, pv); + k = fupg_name_overrides_put(c->nametypes, n, &i); + kh_val(c->nametypes, k) = o; + } else { + fu_confess("Invalid type oid or name '%s'", pv); + } +} + + /* XXX: It feels a bit wasteful to load *all* types; even on an empty database * that's ~55k of data, but it's easier and (potentially) faster than fetching * each type seperately as we encounter them. @@ -382,7 +438,7 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) { for (i=0; intypes; i++) { fupg_type *t = c->types + i; t->oid = fu_frombeU(32, PQgetvalue(r, i, 0)); - snprintf(t->name, sizeof(t->name), "%s", PQgetvalue(r, i, 1)); + snprintf(t->name.n, sizeof(t->name.n), "%s", PQgetvalue(r, i, 1)); char typ = *PQgetvalue(r, i, 2); t->elemoid = fu_frombeU(32, PQgetvalue(r, i, 3)); @@ -448,7 +504,7 @@ static const fupg_record *fupg_lookup_record(fupg_conn *c, Oid oid) { int i; for (i=0; inattrs; i++) { record->attrs[i].oid = fu_frombeU(32, PQgetvalue(r, i, 0)); - snprintf(record->attrs[i].name, sizeof(record->attrs->name), "%s", PQgetvalue(r, i, 1)); + snprintf(record->attrs[i].name.n, sizeof(record->attrs->name.n), "%s", PQgetvalue(r, i, 1)); } k = fupg_records_put(c->records, oid, &i); kh_val(c->records, k) = record; @@ -461,6 +517,21 @@ static const fupg_record *fupg_lookup_record(fupg_conn *c, Oid oid) { #define FUPGT_SEND 2 #define FUPGT_RECV 4 +static const fupg_type *fupg_override_get(fupg_conn *c, int flags, Oid oid, const fupg_name *name) { + khint_t k; + +#define R(t) if (k != kh_end(c->t)) return flags & FUPGT_SEND ? kh_val(c->t, k).send : kh_val(c->t, k).recv + if (name == NULL) { + k = fupg_oid_overrides_get(c->oidtypes, oid); + R(oidtypes); + } else { + k = fupg_name_overrides_get(c->nametypes, *name); + R(nametypes); + } +#undef R + return NULL; +} + static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid oid, int *refresh_done) { tio->oid = oid; if (flags & FUPGT_TEXT) { @@ -470,14 +541,25 @@ static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid return; } - const fupg_type *e, *t = fupg_lookup_type(aTHX_ conn, refresh_done, oid); + /* Minor wart? When the type is overridden by oid, the name & oid in error + * messages will be that of the builtin type. When overridden by name, the + * name will be correct but the oid is still of the builtin type. + * Some send/recv functions have slightly different behavior based on oid, + * in those cases this behavior is useful. */ + + const fupg_type *e, *t; + e = t = fupg_override_get(conn, flags, oid, NULL); + if (!t) t = fupg_lookup_type(aTHX_ conn, refresh_done, oid); if (!t) fu_confess("No type found with oid %u", oid); - if (!t->send || !t->recv) fu_confess("Unable to send or receive type '%s' (oid %u)", t->name, oid); - tio->name = t->name; + tio->name = t->name.n; + if (!e && (e = fupg_override_get(conn, flags, 0, &t->name))) t = e; + + if (flags & FUPGT_SEND && !t->send) fu_confess("Unable to send type '%s' (oid %u)", tio->name, oid); + if (flags & FUPGT_RECV && !t->recv) fu_confess("Unable to receive type '%s' (oid %u)", tio->name, oid); if (flags & FUPGT_SEND ? t->send == fupg_send_domain : t->recv == fupg_recv_domain) { e = fupg_lookup_type(aTHX_ conn, refresh_done, t->elemoid); - if (!e) fu_confess("Base type %u not found for domain '%s' (oid %u)", t->elemoid, t->name, t->oid); + if (!e) fu_confess("Base type %u not found for domain '%s' (oid %u)", t->elemoid, tio->name, t->oid); t = e; } @@ -488,7 +570,7 @@ static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid fupg_tio_setup(aTHX_ conn, tio->arrayelem, flags, t->elemoid, refresh_done); } else if (flags & FUPGT_SEND ? tio->send == fupg_send_record : tio->recv == fupg_recv_record) { tio->record.info = fupg_lookup_record(conn, t->elemoid); - if (!tio->record.info) fu_confess("Unable to find attributes for record type '%s' (oid %u, relid %u)", t->name, t->oid, t->elemoid); + if (!tio->record.info) fu_confess("Unable to find attributes for record type '%s' (oid %u, relid %u)", tio->name, t->oid, t->elemoid); tio->record.tio = safecalloc(tio->record.info->nattrs, sizeof(*tio->record.tio)); int i; for (i=0; irecord.info->nattrs; i++) diff --git a/c/pgtypes.c b/c/pgtypes.c index cb4eb41..8a44471 100644 --- a/c/pgtypes.c +++ b/c/pgtypes.c @@ -7,12 +7,16 @@ typedef void (*fupg_send_fn)(pTHX_ const fupg_tio *, SV *, fustr *); /* Receive function, takes a binary string and should return a Perl value. */ typedef SV *(*fupg_recv_fn)(pTHX_ const fupg_tio *, const char *, int); +typedef struct { + char n[64]; +} fupg_name; + /* Record/composite type definition */ typedef struct { int nattrs; struct { Oid oid; - char name[64]; + fupg_name name; } attrs[]; } fupg_record; @@ -34,7 +38,7 @@ struct fupg_tio { typedef struct { Oid oid; Oid elemoid; /* For arrays & domain types; relid for records */ - char name[64]; + fupg_name name; fupg_send_fn send; fupg_recv_fn recv; } fupg_type; @@ -377,7 +381,7 @@ RECVFN(record) { r = ctx->record.tio[i].recv(aTHX_ ctx->record.tio+i, buf, vlen); buf += vlen; len -= vlen; } - hv_store(hv, ctx->record.info->attrs[i].name, -strlen(ctx->record.info->attrs[i].name), r, 0); + hv_store(hv, ctx->record.info->attrs[i].name.n, -strlen(ctx->record.info->attrs[i].name.n), r, 0); } return SvREFCNT_inc(sv); } @@ -393,7 +397,7 @@ SENDFN(record) { I32 i; for (i=0; irecord.info->nattrs; i++) { fustr_writebeI(32, out, ctx->record.info->attrs[i].oid); - SV **rsv = hv_fetch(hv, ctx->record.info->attrs[i].name, -strlen(ctx->record.info->attrs[i].name), 0); + SV **rsv = hv_fetch(hv, ctx->record.info->attrs[i].name.n, -strlen(ctx->record.info->attrs[i].name.n), 0); if (!rsv || !*rsv) { fustr_writebeI(32, out, -1); continue; @@ -711,8 +715,8 @@ SENDFN(date) { B( 5069, "xid8", uint8 ) static const fupg_type fupg_builtin[] = { -#define B(oid, name, fun) { oid, 0, name"\0", fupg_send_##fun, fupg_recv_##fun }, -#define A(oid, name, eoid) { oid, eoid, name"\0", fupg_send_array, fupg_recv_array }, +#define B(oid, name, fun) { oid, 0, {name"\0"}, fupg_send_##fun, fupg_recv_##fun }, +#define A(oid, name, eoid) { oid, eoid, {name"\0"}, fupg_send_array, fupg_recv_array }, BUILTINS #undef B #undef A @@ -737,3 +741,11 @@ static const fupg_type *fupg_type_byoid(const fupg_type *list, int len, Oid oid) static const fupg_type *fupg_builtin_byoid(Oid oid) { return fupg_type_byoid(fupg_builtin, FUPG_BUILTIN, oid); } + +static const fupg_type *fupg_builtin_byname(const char *name) { + size_t i; + for (i=0; iQ('SELECT 1', IN([1,2,3]))->param_types, [1007]; is $conn->Q('SELECT 1', IN([1,2,3]))->val, 1; ok !eval { $conn->q('SELECT $1::aclitem', '')->exec; 1 }; -like $@, qr/Unable to send or receive/; +like $@, qr/Unable to send type/; + + +$conn->set_type(int4 => recv => 'bytea'); +is $conn->q('SELECT 5::int4')->val, "\0\0\0\5"; +is_deeply $conn->q('SELECT ARRAY[5::int4]')->val, ["\0\0\0\5"]; + +$conn->set_type(int4 => send => 'bytea'); +is $conn->q('SELECT $1::int4', "\0\0\0\5")->val, 5; +is_deeply $conn->q('SELECT $1::int4[]', ["\0\0\0\5"])->val, [5]; + +$conn->set_type(int4 => 'int2'); +ok !eval { $conn->q('SELECT 5::int4')->val }; +like $@, qr/Error parsing value/; +ok !eval { $conn->q('SELECT $1::int4', 5)->val }; +like $@, qr/insufficient data left in message/; + +$conn->set_type(int4 => undef); +is $conn->q('SELECT 5::int4')->val, 5; + +ok !eval { $conn->set_type(int4 => 1007); }; +like $@, qr/Cannot set a type to array/; + +ok !eval { $conn->set_type(int4 => 1); }; +like $@, qr/No builtin type found/; { my $txn = $conn->txn; @@ -22,29 +46,29 @@ like $@, qr/Unable to send or receive/; is $txn->Q('SELECT 1', IN([1,2,3]))->val, 1; $txn->exec(<<~_); - CREATE TYPE fupg_test_enum AS ENUM('a', 'b', 'ccccccccccccccccccc'); - CREATE DOMAIN fupg_test_domain AS fupg_test_enum CHECK(value IN('a','b')); + CREATE TYPE fupg_test_enum AS ENUM('aa', 'bb', 'ccccccccccccccccccc'); + CREATE DOMAIN fupg_test_domain AS fupg_test_enum CHECK(value IN('aa','bb')); CREATE TYPE fupg_test_record AS ( a int, aenum fupg_test_enum[], domain fupg_test_domain ); _ - is $txn->q("SELECT 'a'::fupg_test_enum")->val, 'a'; + is $txn->q("SELECT 'aa'::fupg_test_enum")->val, 'aa'; is $txn->q('SELECT $1::fupg_test_enum', 'ccccccccccccccccccc')->val, 'ccccccccccccccccccc'; - is_deeply $txn->q("SELECT '{a,b,null}'::fupg_test_enum[]")->val, ['a','b',undef]; - is $txn->q('SELECT $1::fupg_test_enum[]', ['a','b',undef])->text_results->val, '{a,b,NULL}'; + is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_enum[]")->val, ['aa','bb',undef]; + is $txn->q('SELECT $1::fupg_test_enum[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}'; - is $txn->q("SELECT 'a'::fupg_test_domain")->val, 'a'; - is $txn->q('SELECT $1::fupg_test_domain', 'b')->val, 'b'; + is $txn->q("SELECT 'aa'::fupg_test_domain")->val, 'aa'; + is $txn->q('SELECT $1::fupg_test_domain', 'bb')->val, 'bb'; - is_deeply $txn->q("SELECT '{a,b,null}'::fupg_test_domain[]")->val, ['a','b',undef]; - is $txn->q('SELECT $1::fupg_test_domain[]', ['a','b',undef])->text_results->val, '{a,b,NULL}'; + is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_domain[]")->val, ['aa','bb',undef]; + is $txn->q('SELECT $1::fupg_test_domain[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}'; - my $val = { a => undef, aenum => ['a','b'], domain => 'a' }; - is_deeply $txn->q("SELECT '(,\"{a,b}\",a)'::fupg_test_record")->val, $val; - is $txn->q('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{a,b}",a)'; + my $val = { a => undef, aenum => ['aa','bb'], domain => 'aa' }; + is_deeply $txn->q("SELECT '(,\"{aa,bb}\",aa)'::fupg_test_record")->val, $val; + is $txn->q('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{aa,bb}",aa)'; $txn->exec(<<~_); CREATE TEMPORARY TABLE fupg_test_table ( @@ -53,15 +77,29 @@ like $@, qr/Unable to send or receive/; ); _ - is_deeply $txn->q(q{SELECT '{"(\"(2,{},b)\",)","(\"(,,)\",b)"}'::fupg_test_table[]})->val, [ - { rec => { a => 2, aenum => [], domain => 'b' }, dom => undef }, - { rec => { a => undef, aenum => undef, domain => undef }, dom => 'b' }, + is_deeply $txn->q(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val, [ + { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef }, + { rec => { a => undef, aenum => undef, domain => undef }, dom => 'bb' }, ]; is $txn->q('SELECT $1::fupg_test_table[]', [ - { rec => { a => 2, aenum => [], domain => 'b' }, dom => undef }, - { rec => {}, dom => 'b', extra => 1 }, - ])->text_results->val, '{"(\"(2,{},b)\",)","(\"(,,)\",b)"}'; + { rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef }, + { rec => {}, dom => 'bb', extra => 1 }, + ])->text_results->val, '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'; + + # Wonky Postgres behavior: selecting a domain directly actually returns the + # underlying type, but going through an array does work. + $conn->set_type(fupg_test_domain => 21); + is_deeply $txn->q("SELECT ARRAY['aa'::fupg_test_domain]")->val, [0x6161]; + + # Bind param type doesn't match column type, argh. + is $txn->q('SELECT $1::fupg_test_domain', 0x6161)->val, 'aa'; + + # Same for selecting from a table :( + $txn->exec("INSERT INTO fupg_test_table VALUES (NULL, 'bb')"); + is $txn->q("SELECT dom FROM fupg_test_table")->val, 'bb'; + $conn->set_type(fupg_test_enum => 21); + is $txn->q("SELECT dom FROM fupg_test_table")->val, 0x6262; } done_testing;