diff --git a/Bug.pm b/Bug.pm index c05419a..8e39459 100644 --- a/Bug.pm +++ b/Bug.pm @@ -2,34 +2,25 @@ =head1 SQL Schema CREATE TABLE $p ( - id SERIAL PRIMARY KEY, - issue integer NOT NULL, - date timestamptz NOT NULL DEFAULT NOW(), - summary varchar(200) NOT NULL, - type varchar NOT NULL DEFAULT '', - status varchar NOT NULL DEFAULT '', - closed boolean NOT NULL DEFAULT false, - email varchar NOT NULL DEFAULT '', - message varchar NOT NULL DEFAULT '', - admin boolean NOT NULL DEFAULT false, - name varchar(200) NOT NULL DEFAULT '' + id integer PRIMARY KEY AUTOINCREMENT NOT NULL, + issue integer NOT NULL, + date integer NOT NULL, + name text DEFAULT '' NOT NULL, + email text DEFAULT '' NOT NULL, + admin integer DEFAULT 0 NOT NULL, + closed integer DEFAULT 0 NOT NULL, + type text DEFAULT '' NOT NULL, + status text DEFAULT '' NOT NULL, + summary text NOT NULL, + message text DEFAULT '' NOT NULL ); -=head2 Update queries - - ALTER TABLE ${p}messages RENAME TO $p; - DROP TABLE ${p}issues; - - ALTER TABLE ${p}messages ADD COLUMN admin boolean NOT NULL DEFAULT false; - ALTER TABLE ${p}messages ADD COLUMN name varchar(200) NOT NULL DEFAULT ''; - =cut -# TODO: Atom feed? - package TUWF::Bug; -use TUWF ':html', 'html_escape'; +use TUWF ':xml', ':html', 'html_escape'; +use POSIX 'strftime'; sub new { my $class = shift; @@ -58,7 +49,7 @@ sub dbListing { my %where = ( 'NOT EXISTS(SELECT 1 FROM !s im WHERE im.id > m.id AND im.issue = m.issue)' => $self->{table}, $o{id} ? ('issue = ?' => $o{id}) : (), - $o{closed} != 2 ? ('!s closed' => !$o{closed} ? 'NOT' : '') : (), + $o{closed} != 2 ? ('closed = ?' => $o{closed}?1:0) : (), ); my $order = sprintf { @@ -66,7 +57,7 @@ sub dbListing { }->{$o{sort}||'date'}, $o{reverse} ? 'DESC' : 'ASC'; my($r, $np) = $TUWF::OBJ->dbPage(\%o, q{ - SELECT issue, summary, to_char(date, 'YYYY-MM-DD') AS date, type, status, closed + SELECT issue, summary, date, type, status, closed FROM !s m !W ORDER BY !s}, $self->{table}, \%where, $order @@ -78,7 +69,7 @@ sub dbListing { sub dbItem { my($self, $id) = @_; return $TUWF::OBJ->dbAll(q{ - SELECT issue, summary, to_char(date, 'YYYY-MM-DD HH24:MI:SS (tz)') AS date, type, status, closed, name, admin, message + SELECT issue, summary, date, type, status, closed, name, admin, message FROM !s WHERE issue = ? ORDER BY id}, $self->{table}, $id @@ -86,6 +77,16 @@ sub dbItem { } +sub dbRecent { + my $self = shift; + return $TUWF::OBJ->dbAll(q{ + SELECT issue, summary, date, name + FROM !s + ORDER BY id DESC LIMIT 10}, $self->{table} + ); +} + + sub dbEmails { my($self, $id) = @_; return [ map $_->{email}, @{$TUWF::OBJ->dbAll(q|SELECT DISTINCT email FROM !s WHERE issue = ? AND email <> ''|, $self->{table}, $id)} ]; @@ -96,14 +97,13 @@ sub dbSave { my($self, $id, $closed, @a) = @_; # TODO: Issue ID allocation may currently cause two bug reports created at - # the same time to get the same id. It'd be better to use a PostgreSQL - # sequence... + # the same time to get the same id. my $issue = $id ? '?' : '(SELECT COALESCE(MAX(issue)+1, 1) FROM !s)'; - return $TUWF::OBJ->dbRow( - "INSERT INTO !s (issue, closed, summary, name, email, type, status, message, admin) VALUES ($issue, ?, !l) RETURNING issue", - $self->{table}, $id || $self->{table}, $closed?1:0, \@a - )->{issue} + $TUWF::OBJ->dbExec( + "INSERT INTO !s (issue, date, closed, summary, name, email, type, status, message, admin) VALUES ($issue, ?, ?, !l)", + $self->{table}, $id || $self->{table}, time(), $closed?1:0, \@a); + return $TUWF::OBJ->dbRow('SELECT MAX(issue) AS issue FROM !s', $self->{table})->{issue}; } @@ -127,7 +127,7 @@ sub htmlListing { td class => 'bug_col_id', $_->{issue}; td class => 'bug_col_type', $_->{type}; td class => 'bug_col_status', $_->{status}; - td class => 'bug_col_date', $_->{date}; + td class => 'bug_col_date', strftime '%Y-%m-%d', gmtime $_->{date}; td class => 'bug_col_summary'; a href => $lnk->($_->{issue}), $_->{summary}; end; @@ -166,7 +166,8 @@ sub htmlItem { dl; dt !$num ? 'Created' : 'Added'; dd; - txt "$m->{date} by "; + txt strftime '%Y-%m-%d %T GMT', gmtime $m->{date}; + txt ' by '; txt $m->{name}||'Anonymous' if !$m->{admin}; b $m->{name}||'Admin' if $m->{admin}; end; @@ -251,15 +252,16 @@ sub htmlForm { sub handleForm { my($s, $url) = @_; my $f = $TUWF::OBJ->formValidate( - { post => 'bug_id', min => 0 }, + { post => 'bug_id', template => 'uint' }, { post => 'bug_summary', maxlength => 200, minlength => 2 }, { post => 'bug_name', required => 0, default => '', maxlength => 200 }, - { post => 'bug_email', required => 0, regex => qr/^[^@<>]+@[^@.<>]+(?:\.[^@.<>]+)+$/ }, + { post => 'bug_email', required => 0, template => 'email' }, { post => 'bug_code', required => 0, default => '' }, { post => 'bug_message', maxlength => 256*1024, minlength => 1 }, ); return($f, undef) if $f->{_err}; return({ _err => [['bug_summary']], %$f}, undef) if $f->{bug_summary} =~ qr{http://}; # SPAM + return({ _err => [['bug_summary']], %$f}, undef) if $f->{bug_message} =~ qr{(?:{bug_code} = '' if $f->{bug_code} eq 'code'; my $admin = grep($_ eq $f->{bug_code}, @{$s->{admins}}) ? 1 : 0; @@ -326,4 +328,36 @@ sub handleForm { } +sub atomFeed { + my($s, $lnk) = @_; + + my $r = $s->dbRecent(); + my $t = $r->[0]{date}||1463296545; + + $TUWF::OBJ->resHeader('Last-Modified' => strftime '%a, %d %b %Y %H:%M:%S GMT', gmtime $t); + $TUWF::OBJ->resHeader('Content-Type' => 'application/atom+xml'); + xml; + tag feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => 'https://dev.yorhel.nl/'; + tag title => "\u$s->{table} Recent Comments"; + tag updated => strftime('%Y-%m-%dT%H:%M:%SZ', gmtime $t); + tag id => $lnk->('feed.atom'); + tag link => rel => 'self', type => 'application/atom+xml', href => $lnk->('feed.atom'), undef; + tag link => rel => 'alternate', type => 'text/html', href => $lnk->(''), undef; + + for(@$r) { + my $d = strftime('%Y-%m-%dT%H:%M:%SZ', gmtime $_->{date}); + tag 'entry'; + tag id => $lnk->($_->{issue})."#$d"; + tag title => $_->{summary}; + tag updated => $d; + tag published => $d; + tag 'author'; + tag name => $_->{name}; + end; + tag link => rel => 'alternate', type => 'text/html', href => $lnk->($_->{issue}), undef; + end 'entry'; + } + end 'feed'; +} + 1; diff --git a/dat/doc b/dat/doc index a707fb3..2dfbc0d 100644 --- a/dat/doc +++ b/dat/doc @@ -8,17 +8,21 @@ rare occasions are published on this page. =over -=item C<2014-01-09 > - L +=item C<2014-07-29 > - L + +A long rant about IPC systems. + +=item C<2014-01-09 > - L A short measurement study on the file lists obtained from a Direct Connect hub. Lots of graphs! -=item C<2012-02-15 > - L +=item C<2012-02-15 > - L In this article I explain a vision of mine, and the results of a small research project aimed at realizing that vision. -=item C<2011-11-26 > - L +=item C<2011-11-26 > - L So you have a single database and some threads. How do you combine these in a program? @@ -29,15 +33,15 @@ program? =over -=item C<2014-06-10 > - L (PDF) +=item C<2014-06-10 > - L (PDF) My masters thesis. -=item C<2013-04-05 > - L (PDF) +=item C<2013-04-05 > - L (PDF) The rather long-ish literature study that precluded my masters thesis. -=item C<2010-06-02 > - L (PDF) +=item C<2010-06-02 > - L (PDF) The report for the final project of my professional (HBO) bachelor of Electrical Engineering. I was very liberal with some terminology in this diff --git a/dat/doc-commvis b/dat/doc-commvis index 5638bff..beaee3b 100644 --- a/dat/doc-commvis +++ b/dat/doc-commvis @@ -2,7 +2,7 @@ A Distributed Communication System for Modular Applications =pod -(Published on B<2012-02-15>. Also available in L.) +(Published on B<2012-02-15>. Also available in L.) =head1 Introduction diff --git a/dat/doc-dcstats b/dat/doc-dcstats index 16072ca..548336d 100644 --- a/dat/doc-dcstats +++ b/dat/doc-dcstats @@ -7,8 +7,8 @@ Some Measurements on Direct Connect File Lists =head1 Introduction I've been working on Direct Connect related projects for a while now. This -includes maintaining L and -L, and doing a bit of research into +includes maintaining L and +L, and doing a bit of research into improving the downloading performance and scalability (to be published at some later date). Whether I'm writing code or trying to setup experiments for research, there's one thing that helps a lot in making decisions. Measurements @@ -83,7 +83,7 @@ others will throw extra hardware at it, and I did what I do best: Optimize away the constants. That is, I rewrote the data analysis program in C. Using the excellent L hash table library to keep track of the file information and the equally awesome -L library (a little bit of self-promotion +L library (a little bit of self-promotion doesn't hurt, right?) to do the XML parsing, I was able to do all the necessary processing in 30 minutes using at most 3.6GB of RAM. diff --git a/dat/doc-easyipc b/dat/doc-easyipc new file mode 100644 index 0000000..f4af4a5 --- /dev/null +++ b/dat/doc-easyipc @@ -0,0 +1,683 @@ +=pod + +(Published on B<2014-07-29>.) + +=head1 The Problem + +How do you implement communication between two or more processes? This is a +question that has been haunting me for at least 6 years now. Of course, this +question is very broad and has many possible answers, depending on your +scenario. So let me get more specific by describing the problem I want to +solve. + +What I want is to write a daemon process that runs in the background and can be +controlled from other programs or libraries. The intention is that people can +easily write custom interfaces or quick scripts to control the daemon. The +service that the daemon offers over this communication channel can be thought +of as its primary API, in this way you can think of the daemon as a persistent +programming library. This concept is similar to existing programs such as +L, L, +L and +L - I'll get back to these +later. + +More specifically, the most recent project I've been working on that follows +this pattern is L, a remotely +controllable Direct Connect client (if you're not familiar with Direct Connect, +think of it as IRC with some additional file sharing capabilities built in). +While the problem I describe is not specific to Globster, it still serves as an +important use case. I see many other projects with similar IPC requirements. + +The IPC mechanism should support two messaging patterns: Request/response and +asynchronous notifications. The request/response pattern is what you typically +get in RPC systems - the client requests something of the daemon and the daemon +then replies with a response. Asynchronous notifications are useful in allowing +the daemon to send asynchronous status updates to the client, such as incoming +chat messages or file transfer status. Lack of support for such notifications +would mean that a client needs to continuously poll for updates, which is +inefficient. + +So what I'm looking for is a high-level IPC mechanism that handles this +communication. Solutions are evaluated by the following criteria, in no +particular order. + +=over + +=item B + +And with I I refer to I. As mentioned above, other people +should be able to write applications and scripts to control the daemon. Not +many people are willing to invest days of work just to figure out how to +communicate with the daemon. + +=item B + +Simplicity refers to the actual protocol and the complexity of the code +necessary to implement it. Complex protocols require complex code, and complex +code is hard to maintain and will inevitably contain bugs. Note that I +and I are very different things and often even conflict with each other. + +=item B + +The IPC implementation shouldn't be too large, and shouldn't depend on huge +libraries. If you need several megabytes worth of libraries just to send a few +messages over a socket, you're doing it wrong. + +=item B + +Control the daemon with whatever programming language you're familiar with. + +=item B + +A good solution should be accessible from both the local system (daemon running +on the same machine as the client) and from the network (daemon and client +running different machines). + +=item B + +There's three parts in having a secure IPC mechanism. One part is to realize +that IPC operates at a I; The daemon can't blindly trust +everything the client says and vice versa, so message validation and other +mechanisms to prevent DoS or information disclosure on either part are +necessary. + +Then there the matter of I. On a local system, UNIX sockets +will provide all the confidentiality you can get, so that's trivial. Networked +access, on the other hand, requires some form of transport layer security. + +And finally, we need some form of I. There should be some +mechanism to prevent just about anyone to connect to the daemon. A +coarse-grained solution such as file permissions on a local UNIX socket or a +password-based approach for networked access will do just fine for most +purposes. Really, just keep it simple. + +=item B + +Although performance isn't really a primary goal, the communication between the +daemon and the clients shouldn't be too slow or heavyweight. For my purposes, +anything that supports about a hundred messages a second on average hardware +will do perfectly fine. And that shouldn't be particularly hard to achieve. + +=item B + +This isn't really a hard requirement either, but it would be nice to allow +other processes (say, plugins of the daemon, or clients connecting to the +daemon) to export services over the same IPC channel as the main daemon. This +is especially useful in implementing a cross-language plugin architecture. But +again, not a hard requirement, because even if the IPC mechanism doesn't +directly support proxying, it's always possible for the daemon to implement +some custom APIs to achieve the same effect. This, however, requires extra work +and may not be as elegant as a built-in solution. + +=back + +Now let's discuss some existing solutions... + + +=head1 Custom Protocol + +Why use an existing IPC mechanism in the first place when all you need is +UNIX/TCP sockets? This is the approach taken by +L, L +(L) and older +versions of Transmission (see their L<1.2x +spec|https://trac.transmissionbt.com/browser/branches/1.2x/doc/ipcproto.txt>). +Brpd hasn't taken the time to documented the protocol format, suggesting it's +not really intended to be used as a convenient API (other than through their +btcli), and Transmission has since changed to a different protocol. I'll mainly +focus on MPD here. + +MPD uses a text-based request/response mechanism, where each request is a +simple one-line command and a response consists of one or more lines, ending +with an C or C line. There's no support for asynchronous +notifications, although that could obviously have been implemented, too. Let's +grade this protocol... + +=over + +=item B Not really. + +Although MPD has conventions for how messages are formatted, each individual +message still requires custom parsing and validation. This can be automated by +designing an +L and +accompanying code generator, but writing one specific for a single project +doesn't seem like a particularly fun task. + +The protocol, despite its apparent simplicity, is apparently painful enough to +use that there is a special I library to abstract away the +communication with MPD, and interfaces to this library are available in many +programming languages. If you have access to such an application-specific +library for your language of choice, then sure, using the IPC mechanism is easy +enough. But that applies to literally any IPC mechanism. + +Ideally, such a library needs to be written only once for the IPC mechanism in +use, and after that no additional code is needed to communicate with +services/daemons using that particular IPC mechanism. Code re-use among +different projects is great, yo. It also doesn't scale very well when extending +the services offered by daemon, any addition to the API will require +modifications to all implementations. + +=item B Definitely. + +I only needed a quick glance at the MPD protocol reference and I was able to +play a bit with telnet and control my MPD. Writing an implementation doesn't +seem like a complex task. Of course, this doesn't necessarily apply to all +custom protocols, but you can make it as simple or complex as you want it to +be. + +=item B Sure. + +This obviously depends on how elaborate you design your protocol. If you have a +large or complex API, the size of a generic message parser and validator can +easily compensate for the custom parser and validator needed for each custom +message. But for a simple APIs, it's hard to beat a custom protocol in terms of +size. + +=item B Depends. + +Of course, a socket library is available to most programming languages, and in +that sense any IPC mechanism built on sockets is language independent. This is, +as such, more of an argument as to how convenient it is to communicate with the +protocol directly rather than with a library that abstracts the protocol away. +In the case of MPD, the text-based protocol seems easy enough to use directly +from most languages, yet for some reason most people prefer language-specific +libraries for MPD. + +If you design a binary protocol or anything more complex than simple +request/response message types, using your protocol directly is going to be a +pain in certain languages, and people will definitely want a library specific +to your daemon for their favourite programming language. Something you'll want +to avoid, I suppose. + +=item B Sure enough. + +Just a switch between UNIX sockets and TCP sockets. Whether a simple solution +like that is a good idea, however, depends on the next point... + +=item B Ugh. + +Security is hard to get right, so having an existing infrastructure that takes +care of most security sensitive features will help a lot. Implementing your own +protocol means that you also have to implement your own security, to some +extent at least. + +Writing code to parse and validate custom messages is error-prone, and a bug in +this code could make both the daemon and the client vulnerable to crashes and +buffer overflows. A statically-typed abstraction that handles parsing and +validation would help a lot. + +For networked communication, you'll need some form of confidentiality. MPD does +not seem to support this, so any networked access to an MPD server is +vulnerable to passive observers and MITM attacks. This may be fine for a local +network (presumably what it is intended to be used for), but certainly doesn't +work for exposing your MPD control interface to the wider internet. Existing +protocols such as TLS or SSH can be used to create a secure channel, but these +libraries tend to be large and hard to use securely. This is especially true +for TLS, but at least there's L to simplify +the implementation - at the cost of less convenient deployment. + +In terms of authentication, you again need to implement this yourself. MPD +supports authentication using a plain-text password. This is fine for a trusted +network, but on an untrusted network you certainly want confidentiality to +prevent a random observer from reading your password. + +=item B Sure. + +Existing protocols may have put more effort into profiling and implementing +various optimizations than one would typically do with a custom and +quickly-hacked-together protocol, but still, it probably takes effort to design +a protocol that isn't fast enough. + +=item B Depends... + +Really depends on how elaborate you want to be. It can be very simple if all +you want is to route some messages, it can get very complex if you want to +ensure that these messages follow some format or if you want to reserve certain +interfaces or namespaces to certain clients. What surprised me about the MPD +protocol is that it actually has L. But considering the +ad-hoc nature of the MPD protocol, the primitiveness and simplicity of this +proxy support wasn't too surprising. Gets the job done, I suppose. + +=back + +Overall, and as a rather obvious conclusion, a custom protocol really is what +you make of it. In general, though, it's a lot of work, not always easy to use, +and a challenge to get the security part right. + + + +=head1 D-Bus + +D-Bus is being used in L and is +what I used for L. + +On a quick glance, D-Bus looks I. It is high-level, has the messaging +patterns I described, the L does not +seem I complex (though certainly could be simplified), it has +implementations for a number of programming languages, has support for +networking, proxying is part of normal operation, and it seems fast enough for +most purposes. When you actually give it a closer look, however, reality isn't +as rose-colored. + +D-Bus is designed for two very specific use-cases. One is to allow local +applications to securely interact with system-level daemons such as +L (now long dead) and +L, and the other +use-case is to allow communication between different applications inside one +login session. As such, on a typical Linux system there are two D-Bus daemons +where applications can export interfaces and where messages can be routed +through. These are called the I and the I. + +=over + +=item B Almost. + +The basic ideas behind D-Bus seem easy enough to use. The fact that is has +type-safe messages, interface descriptions and introspection really help in +making D-Bus a convenient IPC mechanism. + +The main reasons why I think D-Bus isn't all that easy to use in practice is +due to the lack of good introductionary documentation and the crappy state of +the various D-Bus implementations. There is a L providing a +high-level overview to D-Bus, but there isn't a lot of material that covers how +to actually use D-Bus to interact with applications or to implement a service. + +On the implementations, I have had rather bad experiences with the actual +libraries. I've personally used the official libdbus-1, which markets itself a +"low-level" library designed to facilitate writing bindings for other +languages. In practice, the functionality that it offers appears to be too +high-level for writing bindings (L +doesn't use it for this reason), and it is indeed missing a lot of +functionality to make it convenient to use directly. I've also played around +with Perl's L and was +highly disappointed. Not only is the documentation rather incomplete, the +actual implementation has more bugs than features. And instead of building on +top of one of the many good event loops for Perl (such as +L), it chooses to implement +L. The existence +of several different libraries for Python doesn't incite much confidence, +either. + +I was also disappointed in terms of the available tooling to help in the +development, testing and debugging of services. The L tool is useful +for monitoring messages and scripting some things, but is not all that +convenient because D-Bus has too many namespaces and the terrible Java-like +naming conventions make typing everything out a rather painful experience. +L offers a great way to explore services, +but lacks functionality for quick debugging sessions. I L to write a convenient command-line +shell, but lost interest halfway. :-( + +D-Bus has the potential to be an easy and convenient IPC mechanism, but the +lack of any centralized organization to offer good implementations, +documentation and tooling makes using D-Bus a pain to use. + +=item B Not quite. + +D-Bus is conceptually easy and the message protocol is alright, too. Some +aspects of D-Bus, however, are rather more complex than they need to be. + +I have once made an attempt to fully understand how D-Bus discovers and +connects to the session bus, but I gave up halfway because there are too many +special cases. To quickly summarize what I found, there's the +C environment variable which could point to the +(filesystem or abstract) path of a UNIX socket or a TCP address. If that +variable isn't set, D-Bus will try to connect to your X server and get the +address from that. In order to avoid linking everything against X libraries, a +separate L utility is spawned instead. Then the bus address could +also be obtained from a file in your C<$HOME/.dbus/> directory, with added +complexity to still support a different session bus for each X session. I've no +idea how exactly connection initiation to the system bus works, but my +impression is that a bunch of special cases exist there, too, depending on +which init system your OS happens to use. + +As if all the options in connection initiation aren't annoying enough, there's +also work on L, a Linux kernel +implementation to get better performance. Not only will kdbus use a different +underlying communication mechanism, it will also switch to a completely +different serialization format. If/when this becomes widespread you will have +to implement and support two completely different protocols and pray that your +application works with both. + +On the design aspect there is, in my opinion, needless complexity with regards +to naming and namespaces. First there is a global namespace for I, +which are probably better called I, because that's usually +what they represent. Then, there is a separate I namespace local to +each bus name. Each object has methods and properties, and these are +associated with an I, in a namespace specific to the particular +object. Despite these different namespaces, the convention is to use a full and +globally unique path for everything that has a name. For example, to list the +IM protocols that Telepathy supports, you call the C method in +the C interface on the +C object at the +C bus. Fun times indeed. I can understand the +reasoning behind most of these choices, but in my opinion they found the wrong +trade-off. + +Another point of complexity that annoys me is the fact that an XML format is +used to describe interfaces. Supporting XML as an IDL format is alright, but +requiring a separate format for an introspection interface gives me the +impression that the message format wasn't powerful enough for such a simple +purpose. The direct effect of this is that any application wishing to use +introspection data will have to link against an XML parser, and almost all +conforming XML parser implementations are as large as the D-Bus implementation +itself. + +=item B Kind of. + +C on my system is about 240 KiB. It doesn't cover parsing +interface descriptions or implementing a D-Bus daemon, but still covers most of +what is needed to interact with services and to offer services over D-Bus. +It's not I small, but then again, libdbus-1 was not really written with +small size in mind. There's room for optimization. + +=item B Sure. + +D-Bus libraries exist for a number of programming languages. + +=item B Half-assed. + +D-Bus I supports networked connections to a D-Bus daemon. Actually +using this, however, is painful. Convincing L to accept +connections on a TCP socket involves disabling all authentication (it expects +UNIX credential passing, normally) and requires adding an undocumented C<< + >> tag in the configuration (I only figured this out from +reading the source code). + +Even when you've gotten that to work, there is the problem that D-Bus isn't +totally agnostic to the underlying socket protocol. D-Bus has support for +passing UNIX file descriptors over the connection, and this of course doesn't +work over TCP. While this feature is optional and easily avoided, some services +(I can't find one now) use UNIX fds in order to keep track of processes that +listen to a certain event. Obviously, those services can't be accessed over the +network. + +=item B Only locally. + +D-Bus has statically typed messages that can be validated automatically, so +that's a plus. + +For local authentication, there is support for standard UNIX permissions and +credential passing for more fine-grained authorization. For remote +authentication, I think there is support for a shared secret cookie, but I +haven't tried to use this yet. + +There is, as with MPD, no support at all for confidentiality, so using +networked D-Bus over an untrusted network would be a very bad idea anyway. + +=item B Mostly. + +The messaging protocol is fairly lightweight, so no problems there. I do have +to mention two potential performance issues, however. + +The first issue is that the normal mode of operation in D-Bus is to proxy all +messages through an intermediate D-Bus daemon. This involves extra context +switches and message parsing passes in order to get one message from +application A to application B. I believe it is I supported to +bypass this daemon and to communicate directly between two processes, but after +my experience with networking I am wary of trying anything that isn't part of +how D-Bus is I to be used. This particular performance issue is what +kdbus addresses, so I suppose it won't apply to future Linux systems. + +The other issue is that a daemon that provides a service over D-Bus does not +know whether there exists an application that is interested receiving its +notifications. This means that the daemon always has to spend resources to send +out notification messages, even if no application is actually interested in +receiving them. In practice this means that the notification mechanism is +avoided for events that may occur fairly often, and an equally inefficient +polling approach has to be used instead. It is possible for a service provider +to keep track of interested applications, but this is not part of the D-Bus +protocol and not something you would want to implement for each possible event. +I've no idea if kdbus addresses this issue, but it would be stupid not to. + +=item B Yup. + +It's part of normal operation, even. + +=back + +D-Bus has many faults, some of them are by design, but many are fixable. I +would have contributed to improving the situation, but I get the feeling that +the goals of the D-Bus maintainers are not at all aligned with mine. My +impression is that the D-Bus maintainers are far too focussed on their own +specific needs and care little about projects with slightly different needs. +Especially with the introduction of kdbus, I consider D-Bus too complex now to +consider it worth the effort to improve. Starting from scratch seems less work. + + +=head1 JSON/XML RPC + +While I haven't extensively used JSON-RPC or XML-RPC myself, it's still an +interesting alternative to study. +L uses JSON-RPC +(L) as +its primary IPC mechanism, and L +has support for an optional XML-RPC interface. (Why do I keep referencing +torrent clients? Surely there are other interesting applications? Oh well.) + +The main selling point of HTTP-based IPC is that it is accessible from +browser-based applications, assuming everything has been setup correctly. This +is a nice advantage, but lack of this support is not really a deal-breaker for +me. Browser-based applications can still use any other IPC mechanism, as long +as there are browser plugins or some form of proxy server that converts the +messages of the IPC mechanism to something that is usable over HTTP. For +example, both solutions exist for D-Bus, in the form of the L and +L. Of course, such solutions +typically aren't as convenient as native HTTP support. + +Since HTTP is, by design, purely request-response, JSON-RPC and XML-RPC don't +generally support asynchronous notifications. It's possible to still get +asynchronous notifications by using +L (Ugh, opaque stream +sockets, time to go back to our L) or by +having the client implement a HTTP server itself and send its URL to the +service provider (This is known as a +L in the +L world. I have a lot of respect for +developers who can put up with that crap). As I already hinted, neither +solution is simple or easy. + +Let's move on to the usual grading... + +=over + +=item B Sure. + +The ubiquity of HTTP, JSON and XML on the internet means that most developers +are already familiar with using it. And even if you aren't, there are so many +easy-to-use and well-documented libraries available that you're ready to go in +a matter of minutes. + +Although interface description languages/formats exist for XML-RPC (and +possibly for JSON-RPC, too), I get the impression these are not often used +outside of the SOAP world. As a result, interacting with such a service tends +be weakly/stringly typed, which, I imagine, is not as convenient in strongly +typed programming languages. + +=item B Not really. + +Many people have the impression that HTTP is somehow a simple protocol. Sure, +it may look simple on the wire, but in reality it is a hugely bloated and +complex protocol. I strongly encourage everyone to read through L at least once to get an idea of its +size and complexity. To make things worse, there's a lot of recent activity to +standardize on a next generation HTTP +(L and L), but I suppose we can ignore these +developments for the foreseeable future for the use case of IPC. + +Of course, a lot of the functionality specified for HTTP is optional and can be +ignored for the purpose of IPC, but that doesn't mean that these options don't +exist. When implementing a client, it would be useful to know exactly which +HTTP options the server supports. It would be wasteful to implement compression +support if the server doesn't support it, or keep-alive, or content +negotiation, or ranged requests, or authentication, or correct handling for all +response codes when the server will only ever send 'OK'. What also commonly +happens is that server implementors want to support as much as possible, to the +point that you can have JSON or XML output, depending on what the client +requested. + +XML faces a similar problem. The format looks simple, but the specification has +a bunch of features that hardly anyone uses. In contrast to HTTP, however, a +correct XML parser can't just decide to not parse C<< >> stuff, +so it I to implement some of this complexity. + +On the upside, JSON is a really simple serialization format, and if you're +careful enough to only implement the functionality necessary for basic HTTP, a +JSON-RPC implementation I be somewhat simple. + +=item B Not really. + +What typically happens is that implementors take an existing HTTP library and +build on top of that. A generic HTTP library likely implements a lot more than +necessary for IPC, so that's not going to be very small. RTorrent, for example, +makes use of the not-very-small L, +which in turn uses L (400 KiB, excluding TLS +library) and either the bloated L (1.5 MiB) or +L (170 KiB). In any case, expect your +programs to grow by a megabyte or more if you go this route. + +Transmission seems rather less bloated. It uses the HTTP library that is built +into L (totalling ~500 KiB, but libevent is also +used for other networking parts), and a simple JSON parser can't be that large +either. I'm sure that if you reimplement everything from scratch for the +purpose of building an API, you could get something much smaller. Then again, +even if you manage to shrink the size of the server that way, you can't expect +all your users to do the same. + +If HTTPS is to be supported, add ~500 KiB more. TLS isn't the simplest +protocol, either. + +=item B Yes. + +Almost every language has libraries for web stuff. + +=item B Definitely. + +In fact, I've never seen anyone use XML/JSON RPC over UNIX sockets. + +=item B Alright. + +HTTP has built-in support for authentication, but it also isn't uncommon to use +some other mechanism (based on cookies, I guess?). + +Confidentiality can be achieved with HTTPS. There is the problem of verifying +the certificate, since I doubt anyone is going to have certificates of their +local applications signed by a certificate authority, but there's always the +option of trust-on-first use. Custom applications can also include a +fingerprint of the server certificate in the URL for verification, but this +won't work for web apps. + +=item B No. + +JSON/XML RPC messages add significant overhead to the network and requires more +parsing than a simple custom solution or D-Bus. I wouldn't really call it +I, but admittedly, it might still be I for most purposes. + +=item B Sure. + +HTTP has native support for proxying, and it's always possible to proxy some +URI on the main server to another server, assuming the libraries you use +support that. It's not necessarily simple to implement, however. + +=back + +The lack of asynchronous notifications and the overhead and complexity of +JSON/XML RPC make me stay away from it, but it certainly is a solution that +many client developers will like because of its ease of use. + + +=head1 Other Systems + +There are a more alternatives out there than I have described so far. Most of +those were options I dismissed early on because they're either incomplete +solutions or specific to a single framework or language. I'll still mention a +few here. + +=head2 Message Queues + +In the context of IPC I see that message queues such as +L and L are +quite popular. I can't say I have much experience with any of these, but these +MQs don't seem to offer a solution to the problem I described in the +introduction. My impression of MQs is that they offer a higher-level and more +powerful alternative to TCP and UDP. That is, they route messages from one +endpoint to another. The contents of the messages are still completely up to +the application, so you're still on your own in implementing an RPC mechanism +on top of that. And for the purpose of building a simple RPC mechanism, I'm +convinced that plain old UNIX sockets or TCP will do just fine. + +=head2 Cap'n Proto + +I probably should be spending a full chapter on L instead of this tiny little section, +but I'm simply not familiar enough with it to offer any deep insights. I can +still offer my blatantly uninformed impression of it: It looks very promising, +but puts, in my opinion, too much emphasis on performance and too little +emphasis on ease of use. It lacks introspection and requires that clients have +already obtained the schema of the service in order to interact with it. It +also uses a capability system to handle authorization, which, despite being +elegant and powerful, increases complexity and cognitive load (though I +obviously need more experience to quantify this). It still lacks +confidentiality for networked access and the number of bindings to other +programming languages is limited, but these problems can be addressed. + +Cap'n Proto seems like the ideal IPC mechanism for internal communication +within a single (distributed) application and offers a bunch of unique features +not found in other RPC systems. But it doesn't feel quite right as an easy API +for others to use. + +=head2 CORBA + +CORBA has been used by the GNOME project in the past, and was later abandoned +in favour of D-Bus, primarily (I think) because CORBA was deemed too L. A system +that is deemed more complex than D-Bus is an immediate red flag. The L also makes +me want to avoid it, if only because that makes it very hard to judge the +quality and modernness of existing implementations. + +=head2 Project Tanja + +A bit over two years ago I was researching the same problem, but from a much +more generic angle. The result of that was a project that I called Tanja. I +described its concepts L, and wrote an incomplete +L along with implementations in +L, L +and L. I consider project Tanja a +failure, primarily because of its genericity. It supported too many +communication models and the lack of a specification as to which model was +used, and the lack of any guarantee that this model was actually followed, made +Tanja hard to use in practice. It was a very interesting experiment, but not +something I would actually use. I learned the hard way that you sometimes have +to move some complexity down into a lower abstraction layer in order to keep +the complexity in check at higher layers of abstraction. + + +=head1 Conclusions + +This must be the longest rant I've written so far. + +In any case, there isn't really a perfect IPC mechanism for my use case. A +custom protocol involves reimplementing a lot of stuff, D-Bus is a pain, and +JSON/XML RPC are bloat. + +I am still undecided on what to do. I have a lot of ideas as to what a perfect +IPC solution would look like, both in terms of features and in how to implement +it, and I feel like I have enough experience by now to actually develop a +proper solution. Unfortunately, writing a complete IPC system with the required +utilities and language bindings takes B of time and effort. It's not +really worth it if I am the only one using it. + +So here is my plea to you, dear reader: If you know of any existing solutions +I've missed, please tell me. If you empathize with me and want a better +solution to this problem, please get in touch as well! I'd love to hear about +projects which face similar problems and have similar requirements. + diff --git a/dat/dump b/dat/dump index b01d3f0..05b8605 100644 --- a/dat/dump +++ b/dat/dump @@ -5,7 +5,7 @@ for the purpose of getting something done. This page is a listing of those I thought might be of useful to others as well. I also maintain a collection of miscellaneous C micro-libraries. Those are -listed under the collective name of L. +listed under the collective name of L. =head2 maildir.pl @@ -25,9 +25,9 @@ Download: L<0.3|http://p.blicky.net/h25z8> =head2 ncdc-transfer-stats -September 2011. L gained transfer logging +September 2011. L gained transfer logging features, and I wrote a quick Perl script to fetch some simple statistics from -it. L +it. L (L<0.1|http://p.blicky.net/agolr>). =head2 json.mll @@ -43,7 +43,7 @@ November 2009. The L was designed to be easy to use even from low level languages. I wrote this simple program to see how much work it would be to use the API in C, and as example code for anyone wishing to use the API for something more useful. Read the comments for more -info. L +info. L =head2 Microdc2 log file parser @@ -51,25 +51,25 @@ June 2007. Simple perl script that parses log files created by L and outputs a simple and ugly html file with all uploaded files. It correctly merges chunked uploads, calculates average upload speed per file and total bandwidth used -for uploads. L +for uploads. L B for those of you who still use microdc2, please have a look at -L, a modern alternative. +L, a modern alternative. =head2 yapong.c Feburary 2006. Yet Another Pong, and yet another program written just for testing/ learning purposes. Tested to work with the ncurses or pdcurses -libraries. L (L). +libraries. L (L). =head2 echoserv.c February 2006. A simple non-blocking single-threaded TCP echo server, displaying how the select() system call can be used to handle multiple -connections. L +connections. L =head2 bbcode.c January 2006. Simple BBCode to HTML converter written in plain C, for learning -puroses. L +puroses. L diff --git a/dat/dump-awshrink b/dat/dump-awshrink index 0fe15dd..2703167 100644 --- a/dat/dump-awshrink +++ b/dat/dump-awshrink @@ -6,7 +6,7 @@ page generation times for the AWStats pages. I wrote a small script that analyzes these data files and can remove any information you think is unnecessary. -B L (copy to +B L (copy to /usr/bin to install). diff --git a/dat/dump-demo b/dat/dump-demo index 26514eb..6051bcf 100644 --- a/dat/dump-demo +++ b/dat/dump-demo @@ -22,7 +22,7 @@ only 3 actual entries, but oh well), best non-windows 64k intro (it was the only one in the competition), and the Digitale Kultur newcomer award, which actually is something to be proud of, I guess. -L - +L - L (includes linux binaries, windows port, and sources) - L. diff --git a/dat/dump-grenamr b/dat/dump-grenamr index 409a4ee..0a8f75f 100644 --- a/dat/dump-grenamr +++ b/dat/dump-grenamr @@ -14,7 +14,7 @@ renaming of the file. There's one other variable that the rename command does not have: $i, which reflects the file number (starting from 0) in the current list. This allows expressions such as as C<$_=sprintf'%03d.txt',$i>. -B L +B L (copy to /usr/bin/ to install) Requires the Gtk2 Perl module. Most distributions have a perl-gtk2 package. diff --git a/dat/dump-insbench b/dat/dump-insbench index 4b3d744..3e987ae 100644 --- a/dat/dump-insbench +++ b/dat/dump-insbench @@ -20,7 +20,7 @@ precise). My goal wasn't to benchmark the performance of different implementations, so I simply chose two implementations that I suspect are among the fastest. The vector implementation in the benchmarks is my own creation: L from -the L code base. +the L code base. B L @@ -39,7 +39,7 @@ search. Actual performance will be thus be a bit worse, depending on whether the final application needs that binary search or whether it can assume its input to be already sorted. -L<[img graph insbench-bench-thumb.png ]|http://dev.yorhel.nl/img/insbench-bench.png> +L<[img graph insbench-bench-thumb.png ]|https://dev.yorhel.nl/img/insbench-bench.png> Gnuplot script: (The awk(ward) part can likely be done natively in gnuplot as well, but I was too lazy to figure out how) @@ -63,7 +63,7 @@ be a more accurate simulation of some real-world applications. This time I'm not cheating with the vector implementation, a binary search is performed in order to insert the items in the correct location. -L<[img graph insbench-rand-thumb.png ]|http://dev.yorhel.nl/img/insbench-rand.png> +L<[img graph insbench-rand-thumb.png ]|https://dev.yorhel.nl/img/insbench-rand.png> set terminal png size 1000, 1500 set output "bench-rand.png" diff --git a/dat/dump-nccolour b/dat/dump-nccolour index fd442db..416a7fb 100644 --- a/dat/dump-nccolour +++ b/dat/dump-nccolour @@ -8,7 +8,7 @@ different terminals and different configurations. Note that only the 8 basic NCurses colours are tested, the more flexible init_color() function is not used. -B L +B L (L) =head2 Notes / observations @@ -97,3 +97,5 @@ program, which also explains what each column means. =item CentOS 6.4 [img scr nccol-centos64.png ] + +=back diff --git a/dat/globster b/dat/globster index 7efc7b0..341d42d 100644 --- a/dat/globster +++ b/dat/globster @@ -1,6 +1,15 @@ =pod -[html] +[html]

Project Abandoned
+I've stopped development on Globster. I still believe the overall idea and +architecture of Globster are good, and the DC community would definitely +benefit from a remotely controllable client, but Globster in its current form +wasn't going into the direction I wanted it to. I might restart the project +from scratch (yet again) in the future, but for now... it's as dead as a cute +zombie whale.

+

+ +
The Globster What?

@@ -77,13 +86,13 @@ Globster isn't particularly hard to use, but usage documentation is currently a bit lacking. I have every intention to fix that, but for now, you're encouraged to join the development hub and bug me for help: C. I did already write some -L. +L. There are at this point not many scripts or interfaces available for Globster: =over -=item * L - A control script for the daemon, included in the git repo. +=item * L - A control script for the daemon, included in the git repo. =item * L - An RSS / Atom notification script. @@ -107,7 +116,7 @@ Globster incorporates code from L, L, L and -L. +L. Additionally, L is used to keep the configure script fast. diff --git a/dat/globster-api b/dat/globster-api index ca00dbc..9d8fe63 120000 --- a/dat/globster-api +++ b/dat/globster-api @@ -1 +1 @@ -/home/yhdev/globster/doc/api.pod \ No newline at end of file +../../globster/doc/api.pod \ No newline at end of file diff --git a/dat/ncdc b/dat/ncdc index 333616c..7a33560 100644 --- a/dat/ncdc +++ b/dat/ncdc @@ -11,14 +11,13 @@ ncurses interface. =item Latest version 1.19.1 ([dllink ncdc-1.19.1.tar.gz download] -- L -- L) +- L) Convenient static binaries for Linux: -L<64-bit|http://dev.yorhel.nl/download/ncdc-linux-x86_64-1.19.1-3-g8e3a7.tar.gz> - -L<32-bit|http://dev.yorhel.nl/download/ncdc-linux-i486-1.19.1-3-g8e3a7.tar.gz> - -L. Check the -L for more info. +L<64-bit|https://dev.yorhel.nl/download/ncdc-linux-x86_64-1.19.1-12-g55616.tar.gz> - +L<32-bit|https://dev.yorhel.nl/download/ncdc-linux-i486-1.19.1-12-g55616.tar.gz> - +L. Check the +L for more info. =item Development version @@ -36,9 +35,9 @@ Ncdc is entirely written in C and available under a liberal MIT license. =item Community [html] -L - For bugs reports, feature requests and patches.
+L - For bugs reports, feature requests and patches.
L - For general discussions, questions and feedback.
-C - If the mailing list is too slow for you. +C - If the mailing list is too slow for you. É =item Packages and ports diff --git a/dat/ncdc-faq b/dat/ncdc-faq index 6f1ff99..e1fb804 100644 --- a/dat/ncdc-faq +++ b/dat/ncdc-faq @@ -34,13 +34,19 @@ reimplementing everything: I get to choose the library dependencies and the memory/CPU efficiency trade-offs, and I am not limited by an existing implementation that needs quite a few modifications to achieve what I want. Most of the "special features not commonly found in other clients" mentioned on -the L are a direct result of this. +the L are a direct result of this. B These are simple: I rather dislike C++ and working with other people's code. Working with other people's C++ code isn't exactly something I wish to spend my free time on. +=head2 Does ncdc support TLS 1.2? + +Yes, but you need a recent version of GnuTLS. Nobody knows what counts as +"recent", exactly, but I'm guessing any 3.0+ version will do. + + =head2 What protocol features does ncdc support? For ADC: BASE, RF, TIGR, BZIP, BLOM, ADCS, KEYP and SUDP. @@ -96,7 +102,7 @@ a crash, please report a bug. =head2 Ncdc uses too much disk space! First, look where this disk space goes to (hint: use -L). If it's the log files: you can safely +L). If it's the log files: you can safely delete or rotate them (see next question). The I file can also grow quite large in certain situations. If you @@ -141,7 +147,7 @@ myself, though. I just run ncdc directly on my router. :-) =head2 Are there any programs available for analyzing the transfers.log file? Nothing like that is included in the release yet, but there is a simple Perl -script available: L, and a short +script available: L, and a short Go program: L. diff --git a/dat/ncdc-install b/dat/ncdc-install index 7b6d8af..df15bdc 100644 --- a/dat/ncdc-install +++ b/dat/ncdc-install @@ -9,7 +9,7 @@ In theory, the following instructions should work everywhere: =item * Install the required dependencies: ncurses, bzip2, zlib, sqlite3, glib2 and gnutls, -=item * Download and extract the source tarball from the L, +=item * Download and extract the source tarball from the L, =item * C<./configure> @@ -38,11 +38,11 @@ compiling and/or installing it, I also offer statically linked binaries: =over -=item * L +=item * L -=item * L +=item * L -=item * L +=item * L =back @@ -130,7 +130,7 @@ First install some required packages (as root): Then, fetch the ncdc source tarball, extract and build as follows: - wget http://dev.yorhel.nl/download/ncdc-1.19.1.tar.gz + wget https://dev.yorhel.nl/download/ncdc-1.19.1.tar.gz tar -xf ncdc-1.19.1.tar.gz cd ncdc-1.19.1 export PATH="$PATH:/usr/perl5/5.10.0/bin" @@ -180,7 +180,7 @@ required libraries: Then run the following commands to download and install ncdc: - wget http://dev.yorhel.nl/download/ncdc-1.19.1.tar.gz + wget https://dev.yorhel.nl/download/ncdc-1.19.1.tar.gz tar -xf ncdc-1.19.1.tar.gz cd ncdc-1.19.1 ./configure --prefix=/usr @@ -224,7 +224,7 @@ website|http://cygwin.com/> and use it to install the following packages: Then open a Cygwin terminal and run the following commands to download, compile, and install ncdc: - wget http://dev.yorhel.nl/download/ncdc-1.19.1.tar.gz + wget https://dev.yorhel.nl/download/ncdc-1.19.1.tar.gz tar -xf ncdc-1.19.1.tar.gz cd ncdc-1.19.1 ./configure --prefix=/usr diff --git a/dat/ncdc-man b/dat/ncdc-man index 018391b..7bb929a 100644 --- a/dat/ncdc-man +++ b/dat/ncdc-man @@ -907,7 +907,7 @@ more information. =head1 BUGS Please report bugs or feature requests to the bug tracker or the mailing list. -Both can be found on the ncdc homepage at L. There +Both can be found on the ncdc homepage at L. There is also an ADC hub available at C for general support and discussions. @@ -915,4 +915,4 @@ support and discussions. ncdc is written by Yoran Heling -Web: L +Web: L diff --git a/dat/ncdu b/dat/ncdu index 1b0c692..6372a6e 100644 --- a/dat/ncdu +++ b/dat/ncdu @@ -1,11 +1,10 @@ =pod -Not quite happy with the available disk usage analyzers and looking for a fun -project to get used to C programming, I started working on ncdu: A disk usage -analyzer with an ncurses interface, aimed to be run on a remote server where -you don't have an entire gaphical setup, but have to do with a simple SSH -connection. ncdu aims to be fast, simple and easy to use, and should be able -to run in any minimal POSIX-like environment with ncurses installed. +Ncdu is a disk usage analyzer with an ncurses interface. It is designed to find +space hogs on a remote server where you don't have an entire graphical setup +available, but it is a useful tool even on regular desktop systems. Ncdu aims +to be fast, simple and easy to use, and should be able to run in any minimal +POSIX-like environment with ncurses installed. =head2 Download @@ -14,13 +13,12 @@ to run in any minimal POSIX-like environment with ncurses installed. =item Latest version -1.10 ([dllink ncdu-1.10.tar.gz download] - - L - - L) +1.11 ([dllink ncdu-1.11.tar.gz download] + - L) I also have convenient static binaries for Linux -L and -L. Download, +L and +L. Download, extract and run; no compilation or installation necessary (uses L). @@ -34,9 +32,6 @@ L. Ncdu is entirely written in C and available under a liberal MIT license. -Subscribe to L to receive -notifications for new releases. - =head2 Packages and ports @@ -45,26 +40,25 @@ Ncdu has been packaged for quite a few systems, here's a list of the ones I am a L - L - -L - -L - +L - +L - L - L - L - L - -L - +L - L - L - L - L - L - -L - +L - Mac OS X (L - L - L) - -L - L - L - Slackware (L - L) - -L - -L +L - +L Packages for CentOS, RHEL and (open)SUSE can be found on the L. diff --git a/dat/ncdu-changelog b/dat/ncdu-changelog index e4b99e5..fec291c 100644 --- a/dat/ncdu-changelog +++ b/dat/ncdu-changelog @@ -1,3 +1,9 @@ +1.11 - 2015-04-05 + - Added 'b' key to spawn shell in the current directory + - Support scanning (and refreshing) of empty directories + - Added --si flag for base 10 prefixes + - Fix toggle dirs before files + 1.10 - 2013-05-09 - Added 'c' key to display item counts - Added 'C' key to order by item counts diff --git a/dat/ncdu-jsonfmt b/dat/ncdu-jsonfmt index 4de72bc..54f62ca 100644 --- a/dat/ncdu-jsonfmt +++ b/dat/ncdu-jsonfmt @@ -1,8 +1,8 @@ =pod -This document describes the file format that ncdu 1.9 and 1.10 uses for its +This document describes the file format that ncdu 1.9 and later use for the export/import feature (the C<-o> and C<-f> options). Check the L for a description on how to use that +manual|https://dev.yorhel.nl/ncdu/man> for a description on how to use that feature. =head2 Top-level object @@ -29,7 +29,7 @@ the existing format. =head2 Metadata The C<< >> element is a JSON object holding whatever (short) -metadata you'd want. This block is currently (1.9-1.10) ignored by ncdu when +metadata you'd want. This block is currently (1.9-1.11) ignored by ncdu when importing, but it writes out the following keys when exporting: =over @@ -109,10 +109,10 @@ C<< 0 <= dev < 2^64 >>. Number. Inode number as reported by C. Together with the Device ID this uniquely identifies a file in this dump. In the case of hard links, two objects may appear with the same (C,C) combination. A value of 0 is -assumed if this field is absent. This is currently (ncdu 1.9) not a problem as -long as the C field is false, otherwise it will consider everything with -the same C and empty C values as a single hardlinked file. Accepted -values are in the range of C<< 0 <= ino < 2^64 >>. +assumed if this field is absent. This is currently (ncdu 1.9-1.11) not a +problem as long as the C field is false, otherwise it will consider +everything with the same C and empty C values as a single hardlinked +file. Accepted values are in the range of C<< 0 <= ino < 2^64 >>. =item hlnkc diff --git a/dat/ncdu-man b/dat/ncdu-man index 89ecf82..6b429f3 100644 --- a/dat/ncdu-man +++ b/dat/ncdu-man @@ -97,6 +97,12 @@ option has no effect when C<-o> is used, because there will not be a browser interface in that case. It has no effect when C<-f> is used, either, because the deletion feature is disabled in that case anyway. +=item --si + +List sizes using base 10 prefixes, that is, powers of 1000 (KB, MB, etc), as +defined in the International System of Units (SI), instead of the usual base 2 +prefixes, that is, powers of 1024 (KiB, MiB, etc). + =back =head2 Scan Options @@ -204,6 +210,14 @@ Show information about the current selected item. Refresh/recalculate the current directory. +=item b + +Spawn shell in current directory. + +We first check the $SHELL environment variable of the user for the preferred +shell interpreter. If it's not set, we fall back to the compile time +configured default shell (usually /bin/bash). + =item q Quit @@ -296,7 +310,7 @@ directory larger than 8 EiB minus one byte, ncdu will clip its size to 8 EiB minus one byte. Please report any other bugs you may find at the bug tracker, which can be -found on the web site at http://dev.yorhel.nl/ncdu +found on the web site at https://dev.yorhel.nl/ncdu =head1 AUTHOR diff --git a/dat/ncdu-scr b/dat/ncdu-scr index f22541f..e65e393 100644 --- a/dat/ncdu-scr +++ b/dat/ncdu-scr @@ -1,7 +1,7 @@ =pod Note: While these screenshots are from version 1.7, the latest version has only -little visible changes. Let me also apologise for the crappy formatting, I +little visible changes. Let me also apologize for the crappy formatting, I should take some smaller shots next time... =head2 Scanning... diff --git a/dat/sqlaccess b/dat/sqlaccess index bb18877..3f0c61e 100644 --- a/dat/sqlaccess +++ b/dat/sqlaccess @@ -2,15 +2,15 @@ Multi-threaded Access to an SQLite3 Database =pod -(Published on B<2011-11-26>. Also available in L.) +(Published on B<2011-11-26>. Also available in L.) (Minor 2013-04-06 update: I abstracted my message passing solution from ncdc and implemented it in a POSIX C library for general use. It's called -I and is part of my L.) +I and is part of my L.) =head1 Introduction -As I was porting L over to use SQLite3 as +As I was porting L over to use SQLite3 as storage backend, I stumbled on a problem: The program uses a few threads for background jobs, and it would be nice to give these threads access to the database. diff --git a/dat/tuwf b/dat/tuwf index 986d49b..fdc5e8c 100644 --- a/dat/tuwf +++ b/dat/tuwf @@ -39,7 +39,7 @@ information and details. =head2 Download -B 0.2 ([dllink TUWF-0.2.tar.gz download] +B 1.0 ([dllink TUWF-1.0.tar.gz download] - L) TUWF is also available on a git repository at L. @@ -55,11 +55,11 @@ TUWF is also available on a git repository at L. =item * L (L) -=item * L (L) +=item * L (L) =item * L (L) -=item * The website embedded in the L mixing console. +=item * The website embedded in the L mixing console. =item * L<333networks|http://333networks.com/> diff --git a/dat/tuwf-changelog b/dat/tuwf-changelog index b614397..999709a 100644 --- a/dat/tuwf-changelog +++ b/dat/tuwf-changelog @@ -1,3 +1,22 @@ +1.0 - 2015-09-17 + - !! Some backwards-imcompatible changes, marked * !! + - kv_validate() improvements: + - Fix maxcount option + - Fix non-array argument to 'func' + - Added some default templates: num, int, uint, ascii, email, weburl + - * Removed 'min' and 'max' options, these now require the num template + - Add 'inherit' option for template definitions + - Allow templates to provide default values for 'required', 'default', + 'rmwhitespace', 'multi', 'mincount' and 'maxcount' + - Add tests + - * reqPath() now includes the leading slash + - * reqGet(), reqPost(), reqParam(), reqUploadMIME() and reqUploadRaw() + now only work in scalar context. + - * Add plural versions of the above methods (reqGets() etc) that only + work in list context. + - Add reqQuery() + - Fix warning with Perl 5.22 + 0.2 - 2012-01-19 - Fixed bug with in-place utf8_decode() in recent Perls - Lowered minimum Perl version to 5.8.0 diff --git a/dat/yxml b/dat/yxml index e39484c..8db17b6 100644 --- a/dat/yxml +++ b/dat/yxml @@ -12,7 +12,7 @@ is available under a permissive MIT license. The only two files you need are L and L, which can easily be included and compiled as part of your project. Complete API documentation is -available in L. +available in L. The API follows a simple and mostly buffer-less design, and only consists of three functions: diff --git a/download/TUWF-1.0.tar.gz.asc b/download/TUWF-1.0.tar.gz.asc new file mode 100644 index 0000000..8917114 --- /dev/null +++ b/download/TUWF-1.0.tar.gz.asc @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2 + +iQIcBAABCgAGBQJWB5ibAAoJEGI5TGmMJzn6mrQQAOt844yvmllGTuOIV6RLfLHP +nk5n7F0i2NglktCeN3DQXeIDIXPx2IgE1XYu7n0OFCQxiUjpMHx7HdSz53E4XHt6 +HR8tkfsQgrhX8bg0eP7ehb9wHfQh/4rL/UtHwSg94IacfPPffrjDwH9P1BhfMrfc +K78VmzdIxVbTIH+TDvQWPBY9ae/ZmoC6y7A9pDwTs3OaL1n1l9qSmnRnEtGF+7te +w8lIPJyMlD8Gfg9g7+5eZBCeyfwEZXPfHOHYFaFYU5juqdc7M7sAmsixgJ7WEV2G +G2+jNEgZXOjpQu81auVZk6NaQJbP5vdeu5yQUyhlYiqqOS94o+ljNEn+5wXMi67D +23m3hf91rHQqdmsb72tzly44/rGZmLBEZkNsdqlZNsFomgKLS+T/D4n6UqfONuDH +y0DfgaFtm0AxAmlFyUVM83aJFXkttGU+59G3LPO5PeNAjDA1UZamAqJyac+IeLj2 +Ikaa73MjTPllQftgD0HOnKPV/a2QfBMw3jajEzOA/Vd0fLsWz90hwwYYgCww2Tfe +KhhzW917S+VfOsRVvnI//JeSfEbZ99OUt/9z4o98TOt2mKUN00epLTej7qcvFvAy +oul4H/6AxDKOj2IGDHoC9JxKxa5RbXbhkCDMu2qdg8YSe3/EKFn05Ohzw53Il5o9 +8Ag6HUAyvdopGu0Wjo4H +=Gq1J +-----END PGP SIGNATURE----- diff --git a/download/TUWF-1.0.tar.gz.md5 b/download/TUWF-1.0.tar.gz.md5 new file mode 100644 index 0000000..0b8bce1 --- /dev/null +++ b/download/TUWF-1.0.tar.gz.md5 @@ -0,0 +1 @@ +fe6ed1c21b43bd8b57cee447c3093d28 TUWF-1.0.tar.gz diff --git a/download/TUWF-1.0.tar.gz.sha1 b/download/TUWF-1.0.tar.gz.sha1 new file mode 100644 index 0000000..87d7ded --- /dev/null +++ b/download/TUWF-1.0.tar.gz.sha1 @@ -0,0 +1 @@ +c4be0e1b08896053bbccec0194583b7ab03e1cbc TUWF-1.0.tar.gz diff --git a/download/ncdu-1.11.tar.gz.asc b/download/ncdu-1.11.tar.gz.asc new file mode 100644 index 0000000..ee70177 --- /dev/null +++ b/download/ncdu-1.11.tar.gz.asc @@ -0,0 +1,17 @@ +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2 + +iQIcBAABCgAGBQJVIO1TAAoJEGI5TGmMJzn6iOoQAL3UlfW62oEhg6PxhxpXaVi9 ++o1NK0mWlrg+0qXfCUX7yZK6g7TZUtBjXZd9REafjQU+RIj/UhadFHMS4Dps4SJ1 +4E+qeciZBdGs5H6vCwgpRX6bLJNit4CZc/2vPa8NlMmwT/d03bIQqIcS1QKiUB7L +hTTmngpZDjtrLgCspYd+HolQFl8xnTb+VJA9zC+1U3rD02gH1/iRs7WsK4krGOD6 +iVBPeiNtMEaHiIfnme43l13il3MUZ9yAatQpDG3oYVNKIvTaKkR6i1FOzsAkUhgn +TW+7PsZl6MZkebMsBuS4X8A+8euLX+z/VAa6/BcTY6vxrr+MKZogAx2MItELHtwp +vYrOKoNQtLYPQ8RFtmCAo3cnt/zL5KaKHdgSAHyzSX88vauqp4pKAT/Xdpj/hRe3 +YvI0DXFHix9Pyc52h58vRtldQ4qNKp41Jp5frbtEgweq+jJ9Q5kkm2w/5NiyXiat +/L+UIySKmSIHgkJoQDhwHPIOO1S4vvP4W8s6d1r+jMLaLgLDmoIK9Eicdx9OAJ7s +G9r3WP43mq1gjW+xf2ujLDKolUNiaSev6cno18E+dTRioWAj497MfSRpEXO1eoL4 +5GDP5PaQbfXsdKEJrs9bmrJz4pRbTrDAzpKo66ysFUpr3v5V5k0r2AWqgO3pPqaQ +rOT5YYg6Lu6yiOmrbnO6 +=alZd +-----END PGP SIGNATURE----- diff --git a/download/ncdu-1.11.tar.gz.md5 b/download/ncdu-1.11.tar.gz.md5 new file mode 100644 index 0000000..2ebc6f7 --- /dev/null +++ b/download/ncdu-1.11.tar.gz.md5 @@ -0,0 +1 @@ +9e44240a5356b029f05f0e70a63c4d12 ncdu-1.11.tar.gz diff --git a/download/ncdu-1.11.tar.gz.sha1 b/download/ncdu-1.11.tar.gz.sha1 new file mode 100644 index 0000000..efa3e4d --- /dev/null +++ b/download/ncdu-1.11.tar.gz.sha1 @@ -0,0 +1 @@ +8f22d713e6d2b28e1f501eee2bd1e03b55550c8d ncdu-1.11.tar.gz diff --git a/index.cgi b/index.cgi index 0001040..95f515e 100755 --- a/index.cgi +++ b/index.cgi @@ -12,6 +12,9 @@ BEGIN { ($ROOT = abs_path $0) =~ s{index\.cgi$}{}; } my @changes = ( + [ '2015-09-27', '/tuwf', 'TUWF 1.0 released' ], + [ '2015-04-05', '/ncdu', 'ncdu 1.11 released' ], + [ '2014-07-29', '/doc/easyipc', 'New article: The Sorry State of Convenient IPC' ], [ '2014-06-25', '/doc', 'Uploaded my masters thesis' ], [ '2014-04-23', '/ncdc', 'ncdc 1.19.1 released' ], [ '2014-02-11', '/ncdc', 'ncdc 1.19 released' ], @@ -113,6 +116,7 @@ TUWF::register( qr{doc/sqlaccess} => sub { podpage(shift, 'sqlaccess', 'doc', '', 'Multi-threaded Access to an SQLite3 Database', 1) }, qr{doc/commvis} => sub { podpage(shift, 'doc-commvis', 'doc', '', 'A Distributed Communication System for Modular Applications', 1) }, qr{doc/dcstats} => sub { podpage(shift, 'doc-dcstats', 'doc', '', 'Some Measurements on Direct Connect File Lists', 1) }, + qr{doc/easyipc} => sub { podpage(shift, 'doc-easyipc', 'doc', '', 'The Sorry State of Convenient IPC', 1) }, qr{dump} => sub { podpage(shift, 'dump', 'dump', '', 'Code dump') }, qr{demo} => sub { podpage(shift, 'dump-demo', 'dump', 'demo', 'Demos') }, qr{dump/awshrink} => sub { podpage(shift, 'dump-awshrink', 'dump', 'awshrink', 'AWStats Data File Shrinker') }, @@ -121,18 +125,20 @@ TUWF::register( qr{dump/insbench} => sub { podpage(shift, 'dump-insbench', 'dump', 'insbench', 'Insertion Performance Benchmarks') }, qr{(?:($feedreg)/)?feed\.atom} => \&atom, qr{(ncdc|ncdu|globster|yxml)/bug} => \&bug_list, + qr{(ncdc|ncdu|globster|yxml)/bug/feed\.atom} => \&bug_atom, qr{(ncdc|ncdu|globster|yxml)/bug/post} => \&bug_post, qr{(ncdc|ncdu|globster|yxml)/bug/new} => \&bug_new, qr{(ncdc|ncdu|globster|yxml)/bug/([1-9][0-9]*)} => \&bug_item, ); TUWF::set( - logfile => '/home/yorhel/tuwf.log', + logfile => '/var/log/apache2/tuwf.log', error_404_handler => \¬found, mail_from => 'Yorhels Bug Tracker ', # this is a fairly static site, allow some aggressive caching pre_request_handler => sub { $_[0]->resHeader('Cache-Control', 's-max-age=86400, max-age=3600'); 1; }, - cookie_defaults => { domain => 'dev.yorhel.nl', path => '/' }, + cookie_defaults => { domain => 'dev.yorhel.nl', path => '/', secure => 1 }, + db_login => [ undef, undef, undef ], ); TUWF::run(); @@ -194,28 +200,28 @@ sub atom { $s->resHeader('Last-Modified' => strftime '%a, %d %b %Y %H:%M:%S GMT', gmtime $t); $s->resHeader('Content-Type' => 'application/atom+xml'); xml; - tag feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => 'http://dev.yorhel.nl/'; + tag feed => xmlns => 'http://www.w3.org/2005/Atom', 'xml:lang' => 'en', 'xml:base' => 'https://dev.yorhel.nl/'; tag title => $sub ? "\u$sub Project Announcements" : "Yorhel's Projects"; tag updated => strftime('%Y-%m-%dT%H:%M:%SZ', gmtime $t); - tag id => "http://dev.yorhel.nl/feed.atom"; - tag link => rel => 'self', type => 'application/atom+xml', href => "http://dev.yorhel.nl/feed.atom", undef; - tag link => rel => 'alternate', type => 'text/html', href => 'http://dev.yorhel.nl/', undef; + tag id => "https://dev.yorhel.nl/feed.atom"; + tag link => rel => 'self', type => 'application/atom+xml', href => "https://dev.yorhel.nl/feed.atom", undef; + tag link => rel => 'alternate', type => 'text/html', href => 'https://dev.yorhel.nl/', undef; my $n = 0; for(@changes) { next if $sub && (!$_->[1] || $_->[1] !~ /^\/\Q$sub/); last if $n++ >= 10; tag 'entry'; - tag id => 'http://dev.yorhel.nl'.($_->[1]||'/').'#'.$_->[0]; + tag id => 'https://dev.yorhel.nl'.($_->[1]||'/').'#'.$_->[0]; tag title => $_->[2]; tag updated => $_->[0].'T12:00:00Z'; tag published => $_->[0].'T12:00:00Z'; tag 'author'; tag name => 'Yoran Heling'; - tag uri => 'http://dev.yorhel.nl/'; + tag uri => 'https://dev.yorhel.nl/'; tag email => 'projects@yorhel.nl'; end; - tag link => rel => 'alternate', type => 'text/html', href => 'http://dev.yorhel.nl'.($_->[1]||'/'), undef; + tag link => rel => 'alternate', type => 'text/html', href => 'https://dev.yorhel.nl'.($_->[1]||'/'), undef; end 'entry'; } end 'feed'; @@ -226,18 +232,18 @@ sub notfound { my $s = shift; my $u = lc $s->reqPath; ($_->[0] eq $u || $_->[0] eq "$u/") && return $s->resRedirect($_->[1], 'perm') for ( - [ 'bluecubes', '/demo' ], - [ 'ncdc/guide', '/ncdc/man' ], - [ 'dump', '/dump' ], - [ 'dump/index2', '/dump' ], - [ 'dump/pmdc2-parse', '/dump' ], - [ 'dump/cbbcode', '/dump' ], - [ 'dump/cechoserv', '/dump' ], - [ 'dump/cyapong', '/dump' ], - [ 'dump/awshrink', '/dump/awshrink' ], - [ 'dump/grenamr', '/dump/grenamr' ], + [ '/bluecubes', '/demo' ], + [ '/ncdc/guide', '/ncdc/man' ], + [ '/dump', '/dump' ], + [ '/dump/index2', '/dump' ], + [ '/dump/pmdc2-parse', '/dump' ], + [ '/dump/cbbcode', '/dump' ], + [ '/dump/cechoserv', '/dump' ], + [ '/dump/cyapong', '/dump' ], + [ '/dump/awshrink', '/dump/awshrink' ], + [ '/dump/grenamr', '/dump/grenamr' ], ); - return $s->resRedirect("/$u", 'perm') if $u =~ s/\/$//; + return $s->resRedirect("$u", 'perm') if $u =~ s/\/$//; return $s->resRedirect("/$1/bug$2", 'perm') if $u =~ /^(ncd[uc])\/issue(.*)$/; $s->resStatus(404); $s->htmlHeader(title => '404', page => '404'); @@ -254,9 +260,9 @@ sub _bug_init { require "$ROOT/Bug.pm"; my($s, $p) = @_; $s->resHeader('Cache-Control', 'no-cache'); - $s->_load_module('TUWF::DB'); - $s->{_TUWF}{db_login} = [ undef, undef, undef ]; - $s->dbInit; + #$s->_load_module('TUWF::DB'); + #$s->{_TUWF}{db_login} = [ undef, undef, undef ]; + #$s->dbInit; return TUWF::Bug->new(table => $p, admins => [ $ENV{ISSUE_CODE} ]); } @@ -287,6 +293,13 @@ sub bug_list { } +sub bug_atom { + my($s, $p) = @_; + my $is = _bug_init(@_); + $is->atomFeed(sub { "https://dev.yorhel.nl/$p/bug/".shift }); +} + + sub bug_new { my($s, $p) = @_; my $is = _bug_init(@_); @@ -312,13 +325,13 @@ sub bug_post { } # Announce this report to the ncdc hub, through the globster bot - eval { - $ENV{DBUS_SESSION_BUS_ADDRESS} = 'unix:path=/tmp/dbus-globster'; - require Net::DBus; - my $msg = "Bug activity for $p: $l->{summary} -> http://dev.yorhel.nl/$p/bug/$l->{issue}"; - Net::DBus->find->get_service("net.blicky.Globster")->get_object("/net/blicky/Globster/Hub/1")->SendChat(-1, $msg, 0); - 1; - } || warn $@; + #eval { + # $ENV{DBUS_SESSION_BUS_ADDRESS} = 'unix:path=/tmp/dbus-globster'; + # require Net::DBus; + # my $msg = "Bug activity for $p: $l->{summary} -> https://dev.yorhel.nl/$p/bug/$l->{issue}"; + # Net::DBus->find->get_service("net.blicky.Globster")->get_object("/net/blicky/Globster/Hub/1")->SendChat(-1, $msg, 0); + # 1; + #} || warn $@; } @@ -380,6 +393,7 @@ sub htmlPOD { $p->html_header_before_title(''); $p->html_footer(''); + $p->parse_characters(1); $p->output_string(\$html); $p->{podhtml_LOT} = { 'TUWF' => '/tuwf/man', @@ -398,8 +412,6 @@ sub htmlPOD { $html =~ s{\[html\](.*)É}{(my $h = $1) =~ s/\>/>/g; $h =~ s/\</genChanges()/e; $html =~ s/

( +(?:method|signal|property)) /
$1 /g;
-  #$html =~ s/  (method|signal|property [a-z]) ([a-zA-Z0-9]+)/  $1 $2<\/b>/g;
-  #$html =~ s/  (method|signal|property) /  $1<\/b> /g;
   lit $html;
 }
 
@@ -448,13 +460,12 @@ sub htmlHeader {
       div class => 'notes';
        txt 'Yoran Heling'; br;
        a href => 'mailto:projects@yorhel.nl', 'projects@yorhel.nl';
-       br;        a href => 'http://yorhel.nl', 'home';
+       br;        a href => 'https://yorhel.nl', 'home';
        txt ' - '; a href => 'http://g.blicky.net', 'git repos';
        br; b '= donate =';
        a href => 'https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=BBF8LGT2LLNFN&lc=US¤cy_code=EUR&bn=PP%2dDonationsBF%3abtn_donate_SM%2egif%3aNonHosted', 'paypal';
-       txt ' - '; a href => 'bitcoin:1PhXZaKbPFhuz4KbRcfUL9VveB58psa8R', 'bitcoin';
        br; b '= pgp =';
-       a href => 'http://yorhel.nl/key.asc', 'key';
+       a href => 'https://yorhel.nl/key.asc', 'key';
        txt ' - '; a href => 'http://pgp.mit.edu:11371/pks/lookup?search=0x8c2739fa', 'mit';
        br; i '7446 0D32 B808 10EB A9AF A2E9 6239 4C69 8C27 39FA';
       end;