Pg: Support type override configuration

This commit is contained in:
Yorhel 2025-02-27 18:24:14 +01:00
parent 3662931fc2
commit 327fd9ea50
5 changed files with 215 additions and 34 deletions

3
FU.xs
View file

@ -229,6 +229,9 @@ void q(fupg_conn *c, SV *sv, ...)
FUPG_CONN_COOKIE;
ST(0) = fupg_q(aTHX_ c, c->stflags, SvPVutf8_nolen(sv), ax, items);
void _set_type(fupg_conn *c, SV *name, SV *sendsv, SV *recvsv)
CODE:
fupg_set_type(c, name, sendsv, recvsv);
MODULE = FU PACKAGE = FU::Pg::txn

View file

@ -13,6 +13,13 @@ package FU::Pg::conn {
my($sql, $params) = FU::SQL::SQL(@_)->compile(placeholder_style => 'pg', in_style => 'pg');
$s->q($sql, @$params);
}
sub set_type($s, $n, @arg) {
Carp::confess("Invalid number of arguments") if @arg == 0 || (@arg > 1 && @arg % 2);
return $s->_set_type($n, $arg[0], $arg[0]) if @arg == 1;
my %arg = @arg;
$s->_set_type($n, $arg{send}, $arg{recv});
}
};
*FU::Pg::txn::Q = \*FU::Pg::conn::Q;
@ -625,12 +632,51 @@ any of these.
=back
=head3 Overriding types
The default conversion for each type can be changed:
=over
=item $conn->set_type($affected_type, $type)
=item $conn->set_type($affected_type, send => $type, recv => $type)
Change how C<$affected_type> is being converted when used as a bind parameter
(I<send>) or when received from query results (I<recv>). The two-argument
version is equivalent to setting I<send> and I<recv> to the same C<$type>.
Types can be specified either by their numeric I<Oid> or by name. In the latter
case, the name must exactly match the internal type name used by PostgreSQL.
Note that this "internal type name" does not always match the names used in
documentation. For example, I<smallint>, I<integer> and I<bigint> should be
specified as I<int2>, I<int4> and I<int8>, respectively, and the I<char> type
is internally called I<bpchar>. The full list of recognized types in your
database can be queried with:
SELECT oid, typname FROM pg_type;
The C<$affected_type> does not actually have to exist in the database, this
method only stores the type in its internal configuration, which is consulted
upon executing a query that takes the type as bind parameter or when it returns
a column of that type.
The given C<$type> arguments must refer to a built-in type supported by this
module. Types can also be set to I<undef> to restore the conversion to its
default.
=back
I<TODO:> Type override examples and a warning about domain types.
I<TODO:> Some handy special types for overriding common conversions.
I<TODO:> Support for custom types through callbacks.
I<TODO:> Methods to convert between the various formats.
I<TODO:> Methods to query type info.
I<TODO:> Custom per-type configuration.
=head2 Errors
All methods can throw an exception on error. When possible, the error message

View file

@ -25,6 +25,15 @@ static void fupg_prep_destroy(fupg_prep *p) {
safefree(p);
}
typedef struct {
const fupg_type *send, *recv;
} fupg_override;
#define fupg_name_hash(v) kh_hash_str((v).n)
#define fupg_name_eq(a,b) kh_eq_str((a).n, (b).n)
KHASHL_MAP_INIT(KH_LOCAL, fupg_oid_overrides, fupg_oid_overrides, Oid, fupg_override, kh_hash_uint32, kh_eq_generic);
KHASHL_MAP_INIT(KH_LOCAL, fupg_name_overrides, fupg_name_overrides, fupg_name, fupg_override, fupg_name_hash, fupg_name_eq);
typedef struct {
SV *self;
@ -38,6 +47,8 @@ typedef struct {
unsigned int prep_max;
unsigned int prep_cur; /* Number of prepared statements not associated with an active $st object */
fupg_type *types;
fupg_oid_overrides *oidtypes;
fupg_name_overrides *nametypes;
fupg_records *records;
fupg_prepared *prep_map;
fupg_prep *prep_head, *prep_tail; /* Inserted into head, removed at tail */
@ -166,6 +177,8 @@ static SV *fupg_connect(pTHX_ const char *str) {
c->ntypes = 0;
c->types = NULL;
c->records = fupg_records_init();
c->oidtypes = fupg_oid_overrides_init();
c->nametypes = fupg_name_overrides_init();
c->prep_cur = 0;
c->prep_max = 256;
c->prep_map = fupg_prepared_init();
@ -196,6 +209,8 @@ static void fupg_conn_destroy(pTHX_ fupg_conn *c) {
PQfinish(c->conn);
if (c->buf.sv) SvREFCNT_dec(c->buf.sv);
safefree(c->types);
fupg_oid_overrides_destroy(c->oidtypes);
fupg_name_overrides_destroy(c->nametypes);
khint_t k;
kh_foreach(c->records, k) safefree(kh_val(c->records, k));
fupg_records_destroy(c->records);
@ -352,6 +367,47 @@ static void fupg_prepared_unref(fupg_conn *c, fupg_prep *p) {
/* Type handling */
static const fupg_type *fupg_resolve_builtin(pTHX_ SV *name) {
SvGETMAGIC(name);
if (!SvOK(name)) return NULL;
UV uv;
const char *pv = SvPV_nomg_nolen(name);
const fupg_type *t = grok_atoUV(pv, &uv, NULL) && uv <= (UV)UINT_MAX
? fupg_builtin_byoid((Oid)uv)
: fupg_builtin_byname(pv);
if (!t) fu_confess("No builtin type found with oid or name '%s'", pv);
return t;
}
static void fupg_set_type(pTHX_ fupg_conn *c, SV *name, SV *sendsv, SV *recvsv) {
fupg_override o;
o.send = fupg_resolve_builtin(sendsv);
o.recv = fupg_resolve_builtin(recvsv);
if ((o.send && o.send->send == fupg_send_array) || (o.recv && o.recv->recv == fupg_recv_array))
fu_confess("Cannot set a type to array, override the underlying element type instead");
/* Can't currently happen since we have no records in the builtin type
* list, but catch this just in case that changes. */
if ((o.send && o.send->send == fupg_send_record) || (o.recv && o.recv->recv == fupg_recv_record))
fu_confess("Cannot set a type to record");
UV uv;
STRLEN len;
const char *pv = SvPV(name, len);
int k, i;
if (grok_atoUV(pv, &uv, NULL) && uv <= (UV)UINT_MAX) {
k = fupg_oid_overrides_put(c->oidtypes, (Oid)uv, &i);
kh_val(c->oidtypes, k) = o;
} else if (len < sizeof(fupg_name)) {
fupg_name n;
strcpy(n.n, pv);
k = fupg_name_overrides_put(c->nametypes, n, &i);
kh_val(c->nametypes, k) = o;
} else {
fu_confess("Invalid type oid or name '%s'", pv);
}
}
/* XXX: It feels a bit wasteful to load *all* types; even on an empty database
* that's ~55k of data, but it's easier and (potentially) faster than fetching
* each type seperately as we encounter them.
@ -382,7 +438,7 @@ static void fupg_refresh_types(pTHX_ fupg_conn *c) {
for (i=0; i<c->ntypes; i++) {
fupg_type *t = c->types + i;
t->oid = fu_frombeU(32, PQgetvalue(r, i, 0));
snprintf(t->name, sizeof(t->name), "%s", PQgetvalue(r, i, 1));
snprintf(t->name.n, sizeof(t->name.n), "%s", PQgetvalue(r, i, 1));
char typ = *PQgetvalue(r, i, 2);
t->elemoid = fu_frombeU(32, PQgetvalue(r, i, 3));
@ -448,7 +504,7 @@ static const fupg_record *fupg_lookup_record(fupg_conn *c, Oid oid) {
int i;
for (i=0; i<record->nattrs; i++) {
record->attrs[i].oid = fu_frombeU(32, PQgetvalue(r, i, 0));
snprintf(record->attrs[i].name, sizeof(record->attrs->name), "%s", PQgetvalue(r, i, 1));
snprintf(record->attrs[i].name.n, sizeof(record->attrs->name.n), "%s", PQgetvalue(r, i, 1));
}
k = fupg_records_put(c->records, oid, &i);
kh_val(c->records, k) = record;
@ -461,6 +517,21 @@ static const fupg_record *fupg_lookup_record(fupg_conn *c, Oid oid) {
#define FUPGT_SEND 2
#define FUPGT_RECV 4
static const fupg_type *fupg_override_get(fupg_conn *c, int flags, Oid oid, const fupg_name *name) {
khint_t k;
#define R(t) if (k != kh_end(c->t)) return flags & FUPGT_SEND ? kh_val(c->t, k).send : kh_val(c->t, k).recv
if (name == NULL) {
k = fupg_oid_overrides_get(c->oidtypes, oid);
R(oidtypes);
} else {
k = fupg_name_overrides_get(c->nametypes, *name);
R(nametypes);
}
#undef R
return NULL;
}
static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid oid, int *refresh_done) {
tio->oid = oid;
if (flags & FUPGT_TEXT) {
@ -470,14 +541,25 @@ static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid
return;
}
const fupg_type *e, *t = fupg_lookup_type(aTHX_ conn, refresh_done, oid);
/* Minor wart? When the type is overridden by oid, the name & oid in error
* messages will be that of the builtin type. When overridden by name, the
* name will be correct but the oid is still of the builtin type.
* Some send/recv functions have slightly different behavior based on oid,
* in those cases this behavior is useful. */
const fupg_type *e, *t;
e = t = fupg_override_get(conn, flags, oid, NULL);
if (!t) t = fupg_lookup_type(aTHX_ conn, refresh_done, oid);
if (!t) fu_confess("No type found with oid %u", oid);
if (!t->send || !t->recv) fu_confess("Unable to send or receive type '%s' (oid %u)", t->name, oid);
tio->name = t->name;
tio->name = t->name.n;
if (!e && (e = fupg_override_get(conn, flags, 0, &t->name))) t = e;
if (flags & FUPGT_SEND && !t->send) fu_confess("Unable to send type '%s' (oid %u)", tio->name, oid);
if (flags & FUPGT_RECV && !t->recv) fu_confess("Unable to receive type '%s' (oid %u)", tio->name, oid);
if (flags & FUPGT_SEND ? t->send == fupg_send_domain : t->recv == fupg_recv_domain) {
e = fupg_lookup_type(aTHX_ conn, refresh_done, t->elemoid);
if (!e) fu_confess("Base type %u not found for domain '%s' (oid %u)", t->elemoid, t->name, t->oid);
if (!e) fu_confess("Base type %u not found for domain '%s' (oid %u)", t->elemoid, tio->name, t->oid);
t = e;
}
@ -488,7 +570,7 @@ static void fupg_tio_setup(pTHX_ fupg_conn *conn, fupg_tio *tio, int flags, Oid
fupg_tio_setup(aTHX_ conn, tio->arrayelem, flags, t->elemoid, refresh_done);
} else if (flags & FUPGT_SEND ? tio->send == fupg_send_record : tio->recv == fupg_recv_record) {
tio->record.info = fupg_lookup_record(conn, t->elemoid);
if (!tio->record.info) fu_confess("Unable to find attributes for record type '%s' (oid %u, relid %u)", t->name, t->oid, t->elemoid);
if (!tio->record.info) fu_confess("Unable to find attributes for record type '%s' (oid %u, relid %u)", tio->name, t->oid, t->elemoid);
tio->record.tio = safecalloc(tio->record.info->nattrs, sizeof(*tio->record.tio));
int i;
for (i=0; i<tio->record.info->nattrs; i++)

View file

@ -7,12 +7,16 @@ typedef void (*fupg_send_fn)(pTHX_ const fupg_tio *, SV *, fustr *);
/* Receive function, takes a binary string and should return a Perl value. */
typedef SV *(*fupg_recv_fn)(pTHX_ const fupg_tio *, const char *, int);
typedef struct {
char n[64];
} fupg_name;
/* Record/composite type definition */
typedef struct {
int nattrs;
struct {
Oid oid;
char name[64];
fupg_name name;
} attrs[];
} fupg_record;
@ -34,7 +38,7 @@ struct fupg_tio {
typedef struct {
Oid oid;
Oid elemoid; /* For arrays & domain types; relid for records */
char name[64];
fupg_name name;
fupg_send_fn send;
fupg_recv_fn recv;
} fupg_type;
@ -377,7 +381,7 @@ RECVFN(record) {
r = ctx->record.tio[i].recv(aTHX_ ctx->record.tio+i, buf, vlen);
buf += vlen; len -= vlen;
}
hv_store(hv, ctx->record.info->attrs[i].name, -strlen(ctx->record.info->attrs[i].name), r, 0);
hv_store(hv, ctx->record.info->attrs[i].name.n, -strlen(ctx->record.info->attrs[i].name.n), r, 0);
}
return SvREFCNT_inc(sv);
}
@ -393,7 +397,7 @@ SENDFN(record) {
I32 i;
for (i=0; i<ctx->record.info->nattrs; i++) {
fustr_writebeI(32, out, ctx->record.info->attrs[i].oid);
SV **rsv = hv_fetch(hv, ctx->record.info->attrs[i].name, -strlen(ctx->record.info->attrs[i].name), 0);
SV **rsv = hv_fetch(hv, ctx->record.info->attrs[i].name.n, -strlen(ctx->record.info->attrs[i].name.n), 0);
if (!rsv || !*rsv) {
fustr_writebeI(32, out, -1);
continue;
@ -711,8 +715,8 @@ SENDFN(date) {
B( 5069, "xid8", uint8 )
static const fupg_type fupg_builtin[] = {
#define B(oid, name, fun) { oid, 0, name"\0", fupg_send_##fun, fupg_recv_##fun },
#define A(oid, name, eoid) { oid, eoid, name"\0", fupg_send_array, fupg_recv_array },
#define B(oid, name, fun) { oid, 0, {name"\0"}, fupg_send_##fun, fupg_recv_##fun },
#define A(oid, name, eoid) { oid, eoid, {name"\0"}, fupg_send_array, fupg_recv_array },
BUILTINS
#undef B
#undef A
@ -737,3 +741,11 @@ static const fupg_type *fupg_type_byoid(const fupg_type *list, int len, Oid oid)
static const fupg_type *fupg_builtin_byoid(Oid oid) {
return fupg_type_byoid(fupg_builtin, FUPG_BUILTIN, oid);
}
static const fupg_type *fupg_builtin_byname(const char *name) {
size_t i;
for (i=0; i<FUPG_BUILTIN; i++)
if (strcmp(fupg_builtin[i].name.n, name) == 0)
return fupg_builtin+i;
return NULL;
}

View file

@ -14,7 +14,31 @@ is_deeply $conn->Q('SELECT 1', IN([1,2,3]))->param_types, [1007];
is $conn->Q('SELECT 1', IN([1,2,3]))->val, 1;
ok !eval { $conn->q('SELECT $1::aclitem', '')->exec; 1 };
like $@, qr/Unable to send or receive/;
like $@, qr/Unable to send type/;
$conn->set_type(int4 => recv => 'bytea');
is $conn->q('SELECT 5::int4')->val, "\0\0\0\5";
is_deeply $conn->q('SELECT ARRAY[5::int4]')->val, ["\0\0\0\5"];
$conn->set_type(int4 => send => 'bytea');
is $conn->q('SELECT $1::int4', "\0\0\0\5")->val, 5;
is_deeply $conn->q('SELECT $1::int4[]', ["\0\0\0\5"])->val, [5];
$conn->set_type(int4 => 'int2');
ok !eval { $conn->q('SELECT 5::int4')->val };
like $@, qr/Error parsing value/;
ok !eval { $conn->q('SELECT $1::int4', 5)->val };
like $@, qr/insufficient data left in message/;
$conn->set_type(int4 => undef);
is $conn->q('SELECT 5::int4')->val, 5;
ok !eval { $conn->set_type(int4 => 1007); };
like $@, qr/Cannot set a type to array/;
ok !eval { $conn->set_type(int4 => 1); };
like $@, qr/No builtin type found/;
{
my $txn = $conn->txn;
@ -22,29 +46,29 @@ like $@, qr/Unable to send or receive/;
is $txn->Q('SELECT 1', IN([1,2,3]))->val, 1;
$txn->exec(<<~_);
CREATE TYPE fupg_test_enum AS ENUM('a', 'b', 'ccccccccccccccccccc');
CREATE DOMAIN fupg_test_domain AS fupg_test_enum CHECK(value IN('a','b'));
CREATE TYPE fupg_test_enum AS ENUM('aa', 'bb', 'ccccccccccccccccccc');
CREATE DOMAIN fupg_test_domain AS fupg_test_enum CHECK(value IN('aa','bb'));
CREATE TYPE fupg_test_record AS (
a int,
aenum fupg_test_enum[],
domain fupg_test_domain
);
_
is $txn->q("SELECT 'a'::fupg_test_enum")->val, 'a';
is $txn->q("SELECT 'aa'::fupg_test_enum")->val, 'aa';
is $txn->q('SELECT $1::fupg_test_enum', 'ccccccccccccccccccc')->val, 'ccccccccccccccccccc';
is_deeply $txn->q("SELECT '{a,b,null}'::fupg_test_enum[]")->val, ['a','b',undef];
is $txn->q('SELECT $1::fupg_test_enum[]', ['a','b',undef])->text_results->val, '{a,b,NULL}';
is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_enum[]")->val, ['aa','bb',undef];
is $txn->q('SELECT $1::fupg_test_enum[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
is $txn->q("SELECT 'a'::fupg_test_domain")->val, 'a';
is $txn->q('SELECT $1::fupg_test_domain', 'b')->val, 'b';
is $txn->q("SELECT 'aa'::fupg_test_domain")->val, 'aa';
is $txn->q('SELECT $1::fupg_test_domain', 'bb')->val, 'bb';
is_deeply $txn->q("SELECT '{a,b,null}'::fupg_test_domain[]")->val, ['a','b',undef];
is $txn->q('SELECT $1::fupg_test_domain[]', ['a','b',undef])->text_results->val, '{a,b,NULL}';
is_deeply $txn->q("SELECT '{aa,bb,null}'::fupg_test_domain[]")->val, ['aa','bb',undef];
is $txn->q('SELECT $1::fupg_test_domain[]', ['aa','bb',undef])->text_results->val, '{aa,bb,NULL}';
my $val = { a => undef, aenum => ['a','b'], domain => 'a' };
is_deeply $txn->q("SELECT '(,\"{a,b}\",a)'::fupg_test_record")->val, $val;
is $txn->q('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{a,b}",a)';
my $val = { a => undef, aenum => ['aa','bb'], domain => 'aa' };
is_deeply $txn->q("SELECT '(,\"{aa,bb}\",aa)'::fupg_test_record")->val, $val;
is $txn->q('SELECT $1::fupg_test_record', $val)->text_results->val, '(,"{aa,bb}",aa)';
$txn->exec(<<~_);
CREATE TEMPORARY TABLE fupg_test_table (
@ -53,15 +77,29 @@ like $@, qr/Unable to send or receive/;
);
_
is_deeply $txn->q(q{SELECT '{"(\"(2,{},b)\",)","(\"(,,)\",b)"}'::fupg_test_table[]})->val, [
{ rec => { a => 2, aenum => [], domain => 'b' }, dom => undef },
{ rec => { a => undef, aenum => undef, domain => undef }, dom => 'b' },
is_deeply $txn->q(q{SELECT '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}'::fupg_test_table[]})->val, [
{ rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
{ rec => { a => undef, aenum => undef, domain => undef }, dom => 'bb' },
];
is $txn->q('SELECT $1::fupg_test_table[]', [
{ rec => { a => 2, aenum => [], domain => 'b' }, dom => undef },
{ rec => {}, dom => 'b', extra => 1 },
])->text_results->val, '{"(\"(2,{},b)\",)","(\"(,,)\",b)"}';
{ rec => { a => 2, aenum => [], domain => 'bb' }, dom => undef },
{ rec => {}, dom => 'bb', extra => 1 },
])->text_results->val, '{"(\"(2,{},bb)\",)","(\"(,,)\",bb)"}';
# Wonky Postgres behavior: selecting a domain directly actually returns the
# underlying type, but going through an array does work.
$conn->set_type(fupg_test_domain => 21);
is_deeply $txn->q("SELECT ARRAY['aa'::fupg_test_domain]")->val, [0x6161];
# Bind param type doesn't match column type, argh.
is $txn->q('SELECT $1::fupg_test_domain', 0x6161)->val, 'aa';
# Same for selecting from a table :(
$txn->exec("INSERT INTO fupg_test_table VALUES (NULL, 'bb')");
is $txn->q("SELECT dom FROM fupg_test_table")->val, 'bb';
$conn->set_type(fupg_test_enum => 21);
is $txn->q("SELECT dom FROM fupg_test_table")->val, 0x6262;
}
done_testing;