From 963f5277762e632f3c3ee5123799bef6e6ce290f Mon Sep 17 00:00:00 2001 From: ascottDI Date: Thu, 13 Aug 2026 09:44:23 +0100 Subject: [PATCH 1/6] Initial implimentation --- di/tickerplant/VERSION | 1 + di/tickerplant/init.q | 11 ++ di/tickerplant/tickerplant.q | 200 +++++++++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 di/tickerplant/VERSION create mode 100644 di/tickerplant/init.q create mode 100644 di/tickerplant/tickerplant.q diff --git a/di/tickerplant/VERSION b/di/tickerplant/VERSION new file mode 100644 index 00000000..6e8bf73a --- /dev/null +++ b/di/tickerplant/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/di/tickerplant/init.q b/di/tickerplant/init.q new file mode 100644 index 00000000..4a796242 --- /dev/null +++ b/di/tickerplant/init.q @@ -0,0 +1,11 @@ +/ di.tickerplant - tick-capture: log and publish incoming updates, roll at end of day +/ hard dependencies - imported here as module-local handles (before the impl loads), used by tickerplant.q +pubsub:use`di.pubsub +eodtime:use`di.eodtime +tplog:use`di.tplog +\l ::tickerplant.q +/ module version, read from the VERSION file (one plain-text file to bump per release). read +/ module-relative at load (`:::` resolves to di/tickerplant) and BEFORE export, since export:([...]) +/ evaluates each name; version stays in the export so di.depcheck reads it from the export dict +version:trim first read0`:::VERSION +export:([init;upd;subscribe;endofday;getcounts;gettables;getapimeta;version]) diff --git a/di/tickerplant/tickerplant.q b/di/tickerplant/tickerplant.q new file mode 100644 index 00000000..2aa3de0f --- /dev/null +++ b/di/tickerplant/tickerplant.q @@ -0,0 +1,200 @@ +/ core tick-capture for the modular torq world: receive updates from feeds, write them to a +/ tickerplant log for recovery, and publish them to subscribers, rolling the log at end of day. +/ orchestrates three hard deps - di.pubsub (subscribe/publish), di.eodtime (roll timing), and +/ di.tplog (log check/repair on recovery) - with an injected log and timer. the pubsub / eodtime / +/ tplog module handles are imported in init.q (module-local) and used here. +/ . +/ NB the tickerplant's data tables live at ROOT, not in .z.m. a tickerplant process owns its tables +/ (feeds insert into them, subscribers replay them), and di.pubsub reads them by name via `value`, +/ so they cannot be module-local. this is the one deliberate root-state exception for this process +/ module; all other mutable state is module-local in .z.m. +/ . +/ NB subscriber-disconnect cleanup is handled by di.pubsub's own .z.pc (it self-assigns it). this +/ module therefore takes no di.handlers dependency. FLAG: di.pubsub should migrate to di.handlers so +/ .z.pc is not assigned outside the central registry - out of scope here, tracked separately. + +/ ============================================================ +/ constants (load-time) +/ ============================================================ + +/ default batch publish interval (matches TorQ's 1s system timer); di.timer periods are whole seconds +defaultbatchperiod:0D00:00:01; + +/ optional config keys forwarded verbatim to di.eodtime.init +eodtimekeys:`rolltimezone`datatimezone`rolltimeoffset; + +/ ============================================================ +/ internal helpers +/ ============================================================ + +raiseerror:{[ctx;msg] + / internal - log an error under ctx then signal it, so failures are observable as well as thrown + .z.m.logerr[ctx;msg]; + '"di.tickerplant: ",string[ctx],": ",msg; + }; + +stamp:{[x] + / prepend a data-timestamp column unless the update already carries one (first value is a timestamp) + if[-12h=type first first x;:x]; + a:.z.p+eodtime.getdailyadj[]; + :$[0>type first x;a,x;(enlist(count first x)#a),x]; + }; + +openlog:{[date] + / open the tp log for `date`, checking/repairing a pre-existing log via di.tplog; 0i if logging off + if[0=count .z.m.logdir;:0i]; + l:hsym `$ .z.m.logdir,"/",.z.m.logname,string date; + if[count key l;l:tplog.check[l;0]]; + if[not count key l;l set ()]; + .z.m.logfile:l; + h:hopen l; + .z.m.loginfo[`openlog;"logging to ",string l]; + :h; + }; + +publishbuffer:{[] + / batch mode: publish each root table's buffered rows to subscribers then clear them; i catches j + pubsub.pubclear[.z.m.tabs]; + .z.m.i:.z.m.j; + }; + +publishrows:{[t;x] + / zero-latency mode: publish one stamped update immediately, as a table keyed by t's columns + f:cols t; + pubsub.publish[t;$[0>type first x;enlist f!x;flip f!x]]; + }; + +writelog:{[t;x] + / append the update to the tp log (if enabled) and bump the total message count + if[.z.m.logfile>0i;.z.m.logfile enlist (`upd;t;x);.z.m.j+:1]; + }; + +rollcheck:{[now] + / trigger end-of-day if we have passed the next scheduled roll timestamp + if[eodtime.getnextroll[]0i;hclose .z.m.logfile]; + .z.m.i:.z.m.j:0; + .z.m.logfile:openlog .z.m.d; + }; + +/ ============================================================ +/ public api +/ ============================================================ + +init:{[deps] + / wire the injected log + timer, initialise the dep modules, materialise the tables at root, open + / the tp log and schedule the batch/roll timer job. + / deps: a dict with `log (required), `timer (required), `schemas (required, tablename!schema) and + / optional `batch (1b), `batchperiod (timespan), `logdir (string, "" disables logging), + / `logname (string), `subtables (symbol list), plus di.eodtime keys (rolltimezone/datatimezone/ + / rolltimeoffset) forwarded verbatim. + / example: tp.init[`log`timer`schemas!(logdep;timerdep;`trade`quote!(tradeschema;quoteschema))] + if[99h<>type deps; + '"di.tickerplant: deps must be a dict with `log, `timer and `schemas keys"]; + if[not `log in key deps; + '"di.tickerplant: log dependency is required; pass `info`warn`error functions keyed on `log"]; + if[99h<>type deps`log; + '"di.tickerplant: log value must be a dict; pass `info`warn`error functions"]; + if[not all `info`warn`error in key deps`log; + '"di.tickerplant: log dict must have `info`warn`error keys; got: ",(", " sv string key deps`log)]; + if[not `timer in key deps; + '"di.tickerplant: timer dependency is required; pass di.timer's exports keyed on `timer"]; + if[99h<>type deps`timer; + '"di.tickerplant: timer value must be a dict exposing addjob"]; + if[not `addjob in key deps`timer; + '"di.tickerplant: timer dict must expose addjob"]; + if[not `custom in key deps[`timer]`addjob; + '"di.tickerplant: timer addjob must expose the custom variant"]; + if[not `schemas in key deps; + '"di.tickerplant: schemas is required; pass a tablename!schema dict keyed on `schemas"]; + if[99h<>type deps`schemas; + '"di.tickerplant: schemas must be a dict of tablename!schema"]; + .z.m.loginfo:(deps`log)`info; + .z.m.logwarn:(deps`log)`warn; + .z.m.logerr:(deps`log)`error; + .z.m.timer:deps`timer; + / optional config with defaults + .z.m.batch:$[`batch in key deps;deps`batch;1b]; + .z.m.batchperiod:$[`batchperiod in key deps;deps`batchperiod;defaultbatchperiod]; + .z.m.logdir:$[`logdir in key deps;deps`logdir;""]; + .z.m.logname:$[`logname in key deps;deps`logname;"tp"]; + / materialise the schemas as root tables (see the header note on root state), applying `g# to sym + .z.m.schemas:deps`schemas; + .z.m.tabs:key deps`schemas; + {[nm;s] (nm) set $[`sym in cols s;@[s;`sym;`g#];s]}'[.z.m.tabs;value deps`schemas]; + / initialise the dependency modules: eodtime with the injected log + tz passthrough, pubsub over + / the subscribable tables + eodtime.init[(enlist[`log]!enlist deps`log),(key[deps] inter eodtimekeys)#deps]; + pubsub.setsubtables[$[`subtables in key deps;deps`subtables;.z.m.tabs]]; + pubsub.init[]; + / date, counts, and the tp log for today + .z.m.d:eodtime.getd[]; + .z.m.i:.z.m.j:0; + .z.m.logfile:openlog .z.m.d; + / schedule the timer job that flushes the buffer (batch) and checks the roll; mode 1 = fixed period. + / guarded so a re-init does not re-add (di.timer.addjob throws on a duplicate id); tick reads + / .z.m.batch live, so the one job serves both modes across re-inits + if[not `scheduled in key .z.m; + .z.m.timer[`addjob][`custom][`tickerplant;tick;();`int$.z.m.batchperiod%0D00:00:01;1h;()!()]; + .z.m.scheduled:1b]; + .z.m.loginfo[`init;"di.tickerplant initialised (",$[.z.m.batch;"batch";"zero-latency"]," mode)"]; + }; + +upd:{[t;x] + / feed entry point: stamp the update, then buffer+log (batch) or publish+log (zero-latency). + / t is the table name, x the column data. wired to root `upd` by di.torq so feeds can call it. + if[not -11h=type t;raiseerror[`upd;"table must be a symbol"]]; + if[not t in .z.m.tabs;raiseerror[`upd;"unknown table ",string t]]; + rollcheck .z.p; + x:stamp x; + if[.z.m.batch;t insert x;writelog[t;x]]; + if[not .z.m.batch;publishrows[t;x];writelog[t;x]]; + }; + +subscribe:{[tabs;filters] + / register a subscriber (delegates to di.pubsub); called by downstream processes over IPC + :pubsub.subscribe[tabs;filters]; + }; + +endofday:{[] + / flush any buffer, notify subscribers, roll the tp log, advance eodtime state and reset counts + if[.z.m.batch;publishbuffer[]]; + pubsub.callendofday[.z.m.d]; + .z.m.d+:1; + rolllog[]; + eodtime.setnextroll eodtime.getroll[.z.p]; + eodtime.setdailyadj eodtime.getdailyadjustment[]; + .z.m.loginfo[`endofday;"rolled to ",string .z.m.d]; + }; + +getcounts:{[] + / current message counts and trading date - i (in the log), j (log plus buffered), d (date) + :`i`j`d!(.z.m.i;.z.m.j;.z.m.d); + }; + +gettables:{[] + / the tables this tickerplant captures + :.z.m.tabs; + }; + +getapimeta:{[] + / this module's api metadata, one row per CALLABLE api function (NOT init/getapimeta/version - those + / are plumbing/metadata di.torq handles by convention), for di.torq to collect and register with + / di.api. names are bare; di.torq applies process-wide qualification. + :flip `name`public`descrip`params`return!flip( + (`upd; 1b; "feed entry point - stamp, log and publish (or buffer) an update"; "[symbol: table; list: data]"; "null"); + (`subscribe; 1b; "register a subscriber for tables/syms (delegates to di.pubsub)"; "[symbol|list: tables; filters]"; "subscription result"); + (`endofday; 1b; "flush, notify subscribers, roll the log and advance eod state"; "[]"; "null"); + (`getcounts; 1b; "current log/buffer message counts and trading date"; "[]"; "dict: `i`j`d"); + (`gettables; 1b; "the tables this tickerplant captures"; "[]"; "symbol list: table names")); + }; From 5339964d097c488a738750e0b0bc24c89d2016fe Mon Sep 17 00:00:00 2001 From: ascottDI Date: Tue, 18 Aug 2026 10:23:53 +0100 Subject: [PATCH 2/6] connecting tplog and tickerplant --- di/tickerplant/deps.q | 4 ++++ di/tickerplant/tickerplant.q | 24 +++++++++++------------- 2 files changed, 15 insertions(+), 13 deletions(-) create mode 100644 di/tickerplant/deps.q diff --git a/di/tickerplant/deps.q b/di/tickerplant/deps.q new file mode 100644 index 00000000..736b8306 --- /dev/null +++ b/di/tickerplant/deps.q @@ -0,0 +1,4 @@ +/ hard module dependencies and their minimum versions, validated by di.depcheck. +/ these are the modules di.tickerplant `use`s in init.q. the injected log and timer are not declared +/ here - di.depcheck validates them through its core-contract check. +deps:`di.pubsub`di.eodtime`di.tplog!("0.1.0";"0.1.0";"0.1.0"); diff --git a/di/tickerplant/tickerplant.q b/di/tickerplant/tickerplant.q index 2aa3de0f..28be2683 100644 --- a/di/tickerplant/tickerplant.q +++ b/di/tickerplant/tickerplant.q @@ -44,7 +44,7 @@ openlog:{[date] / open the tp log for `date`, checking/repairing a pre-existing log via di.tplog; 0i if logging off if[0=count .z.m.logdir;:0i]; l:hsym `$ .z.m.logdir,"/",.z.m.logname,string date; - if[count key l;l:tplog.check[l;0]]; + if[count key l;l:tplog.check l]; if[not count key l;l set ()]; .z.m.logfile:l; h:hopen l; @@ -98,7 +98,6 @@ init:{[deps] / optional `batch (1b), `batchperiod (timespan), `logdir (string, "" disables logging), / `logname (string), `subtables (symbol list), plus di.eodtime keys (rolltimezone/datatimezone/ / rolltimeoffset) forwarded verbatim. - / example: tp.init[`log`timer`schemas!(logdep;timerdep;`trade`quote!(tradeschema;quoteschema))] if[99h<>type deps; '"di.tickerplant: deps must be a dict with `log, `timer and `schemas keys"]; if[not `log in key deps; @@ -131,10 +130,11 @@ init:{[deps] / materialise the schemas as root tables (see the header note on root state), applying `g# to sym .z.m.schemas:deps`schemas; .z.m.tabs:key deps`schemas; - {[nm;s] (nm) set $[`sym in cols s;@[s;`sym;`g#];s]}'[.z.m.tabs;value deps`schemas]; - / initialise the dependency modules: eodtime with the injected log + tz passthrough, pubsub over - / the subscribable tables + {[nm;s] nm set $[`sym in cols s;@[s;`sym;`g#];s]}'[.z.m.tabs;value deps`schemas]; + / initialise the dependency modules: eodtime (log + tz passthrough), tplog (log), pubsub (over the + / subscribable tables). tplog now takes an injected log and must be init'd before check is called. eodtime.init[(enlist[`log]!enlist deps`log),(key[deps] inter eodtimekeys)#deps]; + tplog.init[enlist[`log]!enlist deps`log]; pubsub.setsubtables[$[`subtables in key deps;deps`subtables;.z.m.tabs]]; pubsub.init[]; / date, counts, and the tp log for today @@ -188,13 +188,11 @@ gettables:{[] }; getapimeta:{[] - / this module's api metadata, one row per CALLABLE api function (NOT init/getapimeta/version - those - / are plumbing/metadata di.torq handles by convention), for di.torq to collect and register with - / di.api. names are bare; di.torq applies process-wide qualification. + / callable api for di.torq to register with di.api (init/getapimeta/version are plumbing, omitted) :flip `name`public`descrip`params`return!flip( - (`upd; 1b; "feed entry point - stamp, log and publish (or buffer) an update"; "[symbol: table; list: data]"; "null"); - (`subscribe; 1b; "register a subscriber for tables/syms (delegates to di.pubsub)"; "[symbol|list: tables; filters]"; "subscription result"); - (`endofday; 1b; "flush, notify subscribers, roll the log and advance eod state"; "[]"; "null"); - (`getcounts; 1b; "current log/buffer message counts and trading date"; "[]"; "dict: `i`j`d"); - (`gettables; 1b; "the tables this tickerplant captures"; "[]"; "symbol list: table names")); + (`upd;1b;"feed entry point - stamp, log and publish (or buffer) an update";"[symbol table; list data]";"null"); + (`subscribe;1b;"register a subscriber for tables/syms (delegates to di.pubsub)";"[symbol|list tables; filters]";"subscription result"); + (`endofday;1b;"flush, notify subscribers, roll the log and advance eod state";"[]";"null"); + (`getcounts;1b;"current log/buffer message counts and trading date";"[]";"dict `i`j`d"); + (`gettables;1b;"the tables this tickerplant captures";"[]";"symbol list of table names")); }; From a5ac34d442909d2ccfa2a7e708a3bbfe312b38f8 Mon Sep 17 00:00:00 2001 From: ascottDI Date: Tue, 18 Aug 2026 11:48:52 +0100 Subject: [PATCH 3/6] adding dependent tests that will only pass when both TPlog and tickerplant are merged to the same branch --- di/tickerplant/test.csv | 31 ++++++++++ di/tickerplant/test.q | 105 ++++++++++++++++++++++++++++++++++ di/tickerplant/tickerplant.md | 96 +++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 di/tickerplant/test.csv create mode 100644 di/tickerplant/test.q create mode 100644 di/tickerplant/tickerplant.md diff --git a/di/tickerplant/test.csv b/di/tickerplant/test.csv new file mode 100644 index 00000000..cc3e8f1b --- /dev/null +++ b/di/tickerplant/test.csv @@ -0,0 +1,31 @@ +action,ms,bytes,lang,code,repeat,minver,comment +before,0,0,q,tp:use`di.tickerplant,1,,load the module under test +before,0,0,q,timer:use`di.timer,1,,real di.timer (addjob works without timer.init - no live .z.ts) +before,0,0,q,ntp:use`di.tplog,1,,di.tplog, to replay a tickerplant-written log +before,0,0,q,os:use`di.os,1,,os module for portable path resolution +before,0,0,q,"system ""l "",os.abspath[""di/tickerplant/test.q""]",1,,load fixture helpers +before,0,0,q,setupfixture[],1,,create the temp fixture root +comment,,,,,,,module metadata - version and getapimeta +true,0,0,q,10h=type tp`version,1,1,version is a string +true,0,0,q,0 upd -> roll cycle. the data tables live at ROOT (the tickerplant owns them and di.pubsub +/ reads them by name), and replay runs the ROOT upd, so a schema + recorder upd live here at root. +/ tp (di.tickerplant), timer (di.timer) and ntp (di.tplog) are bound by test.csv's before rows. + +base:"/tmp/di_tickerplant_k4unit"; +trade:([]time:`timestamp$();sym:`symbol$();price:`float$();size:`long$()); +upd:{[t;x] t insert x}; + +/ capturing logger shared by tickerplant and its dep modules, so any module's output is assertable +logcap:([]lvl:`symbol$();ctx:`symbol$();msg:()); +caplog:{`info`warn`error!( + {[c;m]`logcap insert(`info;c;m);}; + {[c;m]`logcap insert(`warn;c;m);}; + {[c;m]`logcap insert(`error;c;m);})}; + +freshdir:{[sub] dd:base,"/",sub; system"rm -rf ",dd; system"mkdir -p ",dd; dd}; + +/ deps for init - real timer, capturing log, one trade table, logging into dd, batch flag as given +mkdeps:{[dd;batch] `log`timer`schemas`logdir`logname`batch!(caplog[];timer;enlist[`trade]!enlist trade;dd;"tp";batch)}; + +/ fresh init into a clean per-test dir; clears the root table and the log capture first +freshinit:{[sub;batch] + dd:freshdir sub; + `trade set 0#trade; + `logcap set 0#logcap; + tp.init mkdeps[dd;batch]; + dd}; + +/ a feed update: sym, price, size - no time, so the tickerplant stamps it +row:{[s] (s;1.0;100)}; + +setupfixture:{system"rm -rf ",base; system"mkdir -p ",base;}; +teardownfixture:{system"rm -rf ",base;}; + +/ deps builders for the init validation fail rows +depsnolog:{(enlist`x)!enlist 1}; +depsonlylog:{enlist[`log]!enlist caplog[]}; +depsnoschemas:{`log`timer!(caplog[];timer)}; +depsbadtimer:{`log`timer`schemas!(caplog[];(enlist`x)!enlist 1;enlist[`trade]!enlist trade)}; + +/ ============================================================================= +/ tests (each returns 1b on success) +/ ============================================================================= + +/ init materialises the tables at root (g# on sym), schedules the timer job, zeroes the counts +testinit:{[] + freshinit["init";1b]; + c:tp[`getcounts][]; + (`g=attr exec sym from trade) and (enlist[`trade]~tp[`gettables][]) and (0=c`i) and (0=c`j) + and (-14h=type c`d) and `tickerplant in exec id from timer.getalljobs[]}; + +/ batch mode: upd stamps, buffers into the root table, and logs (bumping j; i unchanged) +testupdbatch:{[] + freshinit["updb";1b]; + tp[`upd][`trade;row`AAPL]; + tp[`upd][`trade;row`MSFT]; + c:tp[`getcounts][]; + (2=count trade) and (`AAPL`MSFT~exec sym from trade) and (not any null exec time from trade) + and (2=c`j) and 0=c`i}; + +/ zero-latency mode: upd publishes immediately, does NOT buffer into the root table, still logs +testzerolatency:{[] + freshinit["zl";0b]; + tp[`upd][`trade;row`AAPL]; + c:tp[`getcounts][]; + (0=count trade) and 1=c`j}; + +/ the tp log tickerplant writes replays through di.tplog - the two modules agree on the log format +testlogroundtrip:{[] + dd:freshinit["rt";1b]; + tp[`upd][`trade;row`AAPL]; + tp[`upd][`trade;row`MSFT]; + lf:hsym`$dd,"/tp",string first tp[`getcounts][]`d; + `trade set 0#trade; `rcv set 0; + `upd set {[t;x] `rcv set rcv+1; t insert x;}; + n:ntp[`replay] lf; + `upd set {[t;x] t insert x;}; + (2=n) and (2=rcv) and 2=count trade}; + +/ endofday flushes the buffer, rolls to the next day's log, and resets the counts +testendofday:{[] + dd:freshinit["eod";1b]; + tp[`upd][`trade;row`AAPL]; + oldd:first tp[`getcounts][]`d; + tp[`endofday][]; + c:tp[`getcounts][]; + ((oldd+1)=c`d) and (0=c`i) and (0=c`j) and (0=count trade) + and not ()~key hsym`$dd,"/tp",string oldd+1}; + +/ rolling into a pre-existing CORRUPT log makes openlog repair it via di.tplog.check +testcheckrepaironroll:{[] + dd:freshinit["rep";1b]; + oldd:first tp[`getcounts][]`d; + nl:hsym`$dd,"/tp",string oldd+1; + h:hopen nl; + h enlist (`upd;`trade;(enlist 2026.08.13D10:00;enlist`AAPL;enlist 1.0;enlist 100)); + h enlist (`upd;`trade;(enlist 2026.08.13D10:01;enlist`IBM;enlist 2.0;enlist 200)); + hclose h; + nl set (-8)_read1 nl; + `logcap set 0#logcap; + tp[`endofday][]; + (not ()~key hsym`$dd,"/tp",(string oldd+1),".good") and `warn in exec lvl from logcap where ctx=`check}; diff --git a/di/tickerplant/tickerplant.md b/di/tickerplant/tickerplant.md new file mode 100644 index 00000000..e3353c8c --- /dev/null +++ b/di/tickerplant/tickerplant.md @@ -0,0 +1,96 @@ +# di.tickerplant + +Core tick-capture for the modular TorQ world: receive updates from feeds, stamp them, write them to a +tickerplant log for recovery, and publish them to subscribers, rolling the log at end of day. It is +the modular replacement for TorQ's `code/processes/tickerplant.q`. + +It orchestrates three hard dependencies — `di.pubsub` (subscribe/publish), `di.eodtime` (roll timing) +and `di.tplog` (log check/repair) — with an injected logger and timer. + +## Import and init + +```q +tp:use`di.tickerplant + +trade:([]time:`timestamp$();sym:`symbol$();price:`float$();size:`long$()) +tp.init[`log`timer`schemas!(logdep;timerdep;enlist[`trade]!enlist trade)] +``` + +`init` takes a single deps dict: + +| Key | Required | Description | +|---|---|---| +| `log` | yes | `` `info`warn`error `` dict of `{[ctx;msg]}` functions | +| `timer` | yes | `di.timer`'s exports (must expose `addjob`) | +| `schemas` | yes | `tablename!schema` dict; the tables to capture | +| `batch` | no | `1b` (default) buffers and publishes on a timer; `0b` publishes each update immediately | +| `batchperiod` | no | batch publish interval (timespan, whole seconds; default `0D00:00:01`) | +| `logdir` | no | directory for the tp log; `""` (default) disables logging | +| `logname` | no | log filename prefix (default `"tp"`; file is `/`) | +| `subtables` | no | tables offered for subscription (default: all captured tables) | +| `rolltimezone` / `datatimezone` / `rolltimeoffset` | no | forwarded to `di.eodtime` | + +`init` initialises the dependency modules (`di.eodtime`, `di.tplog`, `di.pubsub`), materialises the +schemas as root tables (applying `` `g# `` to any `sym` column), opens today's log, and schedules a +single timer job that flushes the buffer (batch mode) and checks for the end-of-day roll. It is +idempotent — a re-init does not re-add the timer job. + +## Root tables and the upd contract + +The captured tables live at **root**, not in `.z.m`: a tickerplant owns its tables, feeds insert into +them, and `di.pubsub` reads them by name, so they cannot be module-local. This is the one deliberate +root-state exception; all other mutable state is module-local. `di.torq` wires the process's root +`upd` to `tickerplant.upd` so feeds can publish to it. + +## Exported functions + +| Function | Signature | Description | +|---|---|---| +| `upd` | `[table;data]` | Feed entry point: stamp the update, then buffer+log (batch) or publish+log (zero-latency). | +| `subscribe` | `[tables;filters]` | Register a subscriber (delegates to `di.pubsub`); called by downstream processes over IPC. | +| `endofday` | `[]` | Flush the buffer, notify subscribers, roll the tp log, and advance the end-of-day state. | +| `getcounts` | `[]` | `` `i`j`d `` — messages published (`i`), messages logged (`j`), and the trading date. | +| `gettables` | `[]` | The tables this tickerplant captures. | + +`getapimeta[]` and `version` are also exported, as metadata for `di.torq` / `di.depcheck`. + +## Dependencies + +Hard (imported via `use` in `init.q`, declared in `deps.q`): `di.pubsub`, `di.eodtime`, `di.tplog`. +Injected via `init`: `log` and `timer` (both required; validated by `di.depcheck`'s contract check, +not declared in `deps.q`). + +`di.tplog` is used only for check/repair on recovery — when `openlog` finds a pre-existing log it runs +it through `tplog.check`, repairing a corrupt one. The tickerplant appends to and rolls the log itself +(opening for append, not replaying), since `di.tplog`'s `open`/`roll` replay through `upd`, which a +tickerplant must not do to its own log. + +## Design notes + +- **Batch vs zero-latency.** In batch mode `upd` inserts into the root table and the timer job + publishes the accumulated rows every `batchperiod`, then clears them. In zero-latency mode `upd` + publishes each update immediately and does not buffer. Both modes log every message. +- **End of day.** The roll fires when the current time passes `di.eodtime`'s next roll timestamp, + checked on every `upd` and on every timer tick. `endofday` flushes, notifies subscribers, rolls the + log to the next day, and refreshes the roll time and data-timestamp offset from `di.eodtime`. +- **No `di.handlers` dependency.** Subscriber-disconnect cleanup is handled by `di.pubsub`'s own + `.z.pc`. (`di.pubsub` should migrate to `di.handlers` so `.z.*` is not assigned outside the central + registry — tracked separately, out of scope here.) + +## Testing + +`test.csv` / `test.q` (k4unit) run against the **real** `di.pubsub`, `di.eodtime`, `di.tplog` and +`di.timer` — no dependencies are mocked (the timer is used without `init`, so no live `.z.ts`, and its +job is exercised through `endofday`). A capturing logger is shared across the modules so their output +is assertable. + +Coverage: the metadata/version contract; strict `init` dependency validation (a `fail` row per guard); +init materialising the root tables and scheduling the timer job; batch and zero-latency `upd`; +`endofday` flushing and rolling; `upd` input validation; and the two `di.tplog` integration points — a +tickerplant-written log replaying through `di.tplog`, and rolling into a corrupt log repairing it via +`tplog.check`. + +```q +k4unit:use`di.k4unit +k4unit.moduletest`di.tickerplant +``` From a07ac5ccc7a62a0245178bc17b86a39c764e8a60 Mon Sep 17 00:00:00 2001 From: ascottDI Date: Tue, 18 Aug 2026 12:12:56 +0100 Subject: [PATCH 4/6] updates following automated reviewer comments --- di/tickerplant/test.csv | 2 ++ di/tickerplant/test.q | 18 ++++++++++++++++++ di/tickerplant/tickerplant.q | 5 ++++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/di/tickerplant/test.csv b/di/tickerplant/test.csv index cc3e8f1b..86ef50f6 100644 --- a/di/tickerplant/test.csv +++ b/di/tickerplant/test.csv @@ -20,8 +20,10 @@ fail,0,0,q,tp.init depsnoschemas[],1,1,init rejects deps without schemas comment,,,,,,,init / lifecycle true,0,0,q,testinit[],1,1,init materialises root tables (g# sym) and schedules the timer job true,0,0,q,testupdbatch[],1,1,batch upd stamps, buffers and logs +true,0,0,q,testemptyupd[],1,1,an empty update is a no-op (no throw) true,0,0,q,testzerolatency[],1,1,zero-latency upd publishes without buffering true,0,0,q,testendofday[],1,1,endofday flushes, rolls the log and resets counts +true,0,0,q,testreinitnoleak[],1,1,re-init closes the old log handle (no fd leak) comment,,,,,,,di.tplog integration true,0,0,q,testlogroundtrip[],1,1,a tickerplant-written log replays through di.tplog true,0,0,q,testcheckrepaironroll[],1,1,rolling into a corrupt log repairs it via di.tplog.check diff --git a/di/tickerplant/test.q b/di/tickerplant/test.q index cd618d6e..7dd87fd0 100644 --- a/di/tickerplant/test.q +++ b/di/tickerplant/test.q @@ -61,6 +61,13 @@ testupdbatch:{[] (2=count trade) and (`AAPL`MSFT~exec sym from trade) and (not any null exec time from trade) and (2=c`j) and 0=c`i}; +/ an empty update is a no-op: no throw, nothing buffered, nothing logged +testemptyupd:{[] + freshinit["empty";1b]; + tp[`upd][`trade;()]; + c:tp[`getcounts][]; + (0=count trade) and 0=c`j}; + / zero-latency mode: upd publishes immediately, does NOT buffer into the root table, still logs testzerolatency:{[] freshinit["zl";0b]; @@ -68,6 +75,17 @@ testzerolatency:{[] c:tp[`getcounts][]; (0=count trade) and 1=c`j}; +/ re-init is safe: it closes the previous log handle instead of leaking the descriptor, and logging +/ keeps working. fd count (linux /proc, as the suite is already unix-coupled) must not grow. +testreinitnoleak:{[] + fddir:"/proc/",(string .z.i),"/fd"; + freshinit["reinit";1b]; + b:"J"$first system"ls ",fddir," | wc -l"; + freshinit["reinit";1b]; freshinit["reinit";1b]; freshinit["reinit";1b]; + a:"J"$first system"ls ",fddir," | wc -l"; + tp[`upd][`trade;row`AAPL]; + (a=b) and 1=tp[`getcounts][]`j}; + / the tp log tickerplant writes replays through di.tplog - the two modules agree on the log format testlogroundtrip:{[] dd:freshinit["rt";1b]; diff --git a/di/tickerplant/tickerplant.q b/di/tickerplant/tickerplant.q index 28be2683..f976f197 100644 --- a/di/tickerplant/tickerplant.q +++ b/di/tickerplant/tickerplant.q @@ -137,9 +137,11 @@ init:{[deps] tplog.init[enlist[`log]!enlist deps`log]; pubsub.setsubtables[$[`subtables in key deps;deps`subtables;.z.m.tabs]]; pubsub.init[]; - / date, counts, and the tp log for today + / date, counts, and the tp log for today. close a handle held from a previous init before + / reopening, so a re-init does not leak the old file descriptor (rolllog closes on its own path) .z.m.d:eodtime.getd[]; .z.m.i:.z.m.j:0; + if[`logfile in key .z.m;if[.z.m.logfile>0i;hclose .z.m.logfile]]; .z.m.logfile:openlog .z.m.d; / schedule the timer job that flushes the buffer (batch) and checks the roll; mode 1 = fixed period. / guarded so a re-init does not re-add (di.timer.addjob throws on a duplicate id); tick reads @@ -155,6 +157,7 @@ upd:{[t;x] / t is the table name, x the column data. wired to root `upd` by di.torq so feeds can call it. if[not -11h=type t;raiseerror[`upd;"table must be a symbol"]]; if[not t in .z.m.tabs;raiseerror[`upd;"unknown table ",string t]]; + if[not count x;:()]; rollcheck .z.p; x:stamp x; if[.z.m.batch;t insert x;writelog[t;x]]; From 87dbd12f8f3a428b18e5a10888458146d1cc471d Mon Sep 17 00:00:00 2001 From: ascottDI Date: Tue, 18 Aug 2026 14:46:26 +0100 Subject: [PATCH 5/6] changes following auto reviewer comments --- di/tickerplant/test.csv | 2 ++ di/tickerplant/test.q | 12 ++++++++++-- di/tickerplant/tickerplant.q | 3 ++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/di/tickerplant/test.csv b/di/tickerplant/test.csv index 86ef50f6..c580f70a 100644 --- a/di/tickerplant/test.csv +++ b/di/tickerplant/test.csv @@ -2,6 +2,7 @@ action,ms,bytes,lang,code,repeat,minver,comment before,0,0,q,tp:use`di.tickerplant,1,,load the module under test before,0,0,q,timer:use`di.timer,1,,real di.timer (addjob works without timer.init - no live .z.ts) before,0,0,q,ntp:use`di.tplog,1,,di.tplog, to replay a tickerplant-written log +before,0,0,q,eod:use`di.eodtime,1,,di.eodtime, to force a roll due in a test before,0,0,q,os:use`di.os,1,,os module for portable path resolution before,0,0,q,"system ""l "",os.abspath[""di/tickerplant/test.q""]",1,,load fixture helpers before,0,0,q,setupfixture[],1,,create the temp fixture root @@ -21,6 +22,7 @@ comment,,,,,,,init / lifecycle true,0,0,q,testinit[],1,1,init materialises root tables (g# sym) and schedules the timer job true,0,0,q,testupdbatch[],1,1,batch upd stamps, buffers and logs true,0,0,q,testemptyupd[],1,1,an empty update is a no-op (no throw) +true,0,0,q,testemptyupdtriggersroll[],1,1,an empty update still triggers an overdue roll true,0,0,q,testzerolatency[],1,1,zero-latency upd publishes without buffering true,0,0,q,testendofday[],1,1,endofday flushes, rolls the log and resets counts true,0,0,q,testreinitnoleak[],1,1,re-init closes the old log handle (no fd leak) diff --git a/di/tickerplant/test.q b/di/tickerplant/test.q index 7dd87fd0..d44501e0 100644 --- a/di/tickerplant/test.q +++ b/di/tickerplant/test.q @@ -68,6 +68,14 @@ testemptyupd:{[] c:tp[`getcounts][]; (0=count trade) and 0=c`j}; +/ an empty update still triggers an overdue roll (the roll check runs before the empty-data skip) +testemptyupdtriggersroll:{[] + freshinit["emptyroll";1b]; + oldd:first tp[`getcounts][]`d; + eod.setnextroll .z.p-0D01:00:00; + tp[`upd][`trade;()]; + (oldd+1)=first tp[`getcounts][]`d}; + / zero-latency mode: upd publishes immediately, does NOT buffer into the root table, still logs testzerolatency:{[] freshinit["zl";0b]; @@ -80,8 +88,8 @@ testzerolatency:{[] testreinitnoleak:{[] fddir:"/proc/",(string .z.i),"/fd"; freshinit["reinit";1b]; - b:"J"$first system"ls ",fddir," | wc -l"; - freshinit["reinit";1b]; freshinit["reinit";1b]; freshinit["reinit";1b]; + b:"J"$first system"ls ",fddir," | wc -l"; / baseline AFTER the first init - one log handle is open + freshinit["reinit";1b]; freshinit["reinit";1b]; freshinit["reinit";1b]; / re-init must not add fds a:"J"$first system"ls ",fddir," | wc -l"; tp[`upd][`trade;row`AAPL]; (a=b) and 1=tp[`getcounts][]`j}; diff --git a/di/tickerplant/tickerplant.q b/di/tickerplant/tickerplant.q index f976f197..27f28b7e 100644 --- a/di/tickerplant/tickerplant.q +++ b/di/tickerplant/tickerplant.q @@ -157,8 +157,9 @@ upd:{[t;x] / t is the table name, x the column data. wired to root `upd` by di.torq so feeds can call it. if[not -11h=type t;raiseerror[`upd;"table must be a symbol"]]; if[not t in .z.m.tabs;raiseerror[`upd;"unknown table ",string t]]; - if[not count x;:()]; rollcheck .z.p; + / empty update carries no data - nothing to stamp/log, but the roll check above still runs + if[not count x;:()]; x:stamp x; if[.z.m.batch;t insert x;writelog[t;x]]; if[not .z.m.batch;publishrows[t;x];writelog[t;x]]; From 44ec616d58fae72996ddbeeb8254a089b0fd48ee Mon Sep 17 00:00:00 2001 From: alowrydi Date: Wed, 19 Aug 2026 16:29:02 +0100 Subject: [PATCH 6/6] di.tickerplant: speak the subdetails protocol, fix log path, counters and re-init Publish subdetails and tablelist at root so di.subscriptions can attach. A standard TorQ tickerplant never had these - only chainedtp.q and segmentedtickerplant.q define subdetails - so this is a deliberate capability addition rather than restored parity, recorded in the design-divergences section of tickerplant.md. - keep the tp log path in .z.m.logpath; .z.m.logfile stays the handle, so the path is no longer overwritten and can be reported to subscribers - advance .z.m.i in zero-latency mode, as chainedtp.q's tickpub does; without it every subscriber was told to replay nothing - report the PUBLISHED watermark i in logfilelist, not the logged total j: in batch mode a row logged but not yet flushed is still buffered and goes out at the next tick, so j would replay it AND deliver it again. TorQ sends .u.i for the same reason (chainedtp.q) and kdb+tick's r.q replays with .u`i - seed runtime state only on a fresh init, so a re-init cannot rewind the trading date, zero the counts a subscriber replays against, or reopen the log; table materialisation follows the same rule, since re-running the schema over a live tickerplant discarded buffered rows - track per-table published rowcounts for the protocol's rowcounts field - fail loud on a missing, unreadable or empty VERSION - guard the cold-path exports with requireinit; upd stays unguarded as the per-message hot path, and a test pins which side each is on Adds test_integration.csv: a real di.subscriptions driven against a real di.tickerplant over IPC, covering both batch and zero-latency modes plus the VERSION guards. It needs di.tplog with init (feature-tplog) and di.subscriptions on the same branch, as a5ac34d notes for the existing dependent tests. --- di/tickerplant/init.q | 12 +- di/tickerplant/test.csv | 24 ++- di/tickerplant/test.q | 167 +++++++++++++++-- di/tickerplant/test_integration.csv | 137 ++++++++++++++ di/tickerplant/tickerplant.md | 194 ++++++++++++++++++-- di/tickerplant/tickerplant.q | 268 ++++++++++++++++++++++++---- 6 files changed, 728 insertions(+), 74 deletions(-) create mode 100644 di/tickerplant/test_integration.csv diff --git a/di/tickerplant/init.q b/di/tickerplant/init.q index 4a796242..b373f068 100644 --- a/di/tickerplant/init.q +++ b/di/tickerplant/init.q @@ -6,6 +6,12 @@ tplog:use`di.tplog \l ::tickerplant.q / module version, read from the VERSION file (one plain-text file to bump per release). read / module-relative at load (`:::` resolves to di/tickerplant) and BEFORE export, since export:([...]) -/ evaluates each name; version stays in the export so di.depcheck reads it from the export dict -version:trim first read0`:::VERSION -export:([init;upd;subscribe;endofday;getcounts;gettables;getapimeta;version]) +/ evaluates each name; version stays in the export so di.depcheck reads it from the export dict. +/ trim, and fail LOUD on a missing/unreadable/empty VERSION, rather than a bare `first read0`: +/ a raw OS error names no module, and an empty or whitespace-padded value is worse than an error - +/ di.depcheck compares versions as STRINGS, so padding silently breaks the comparison and an empty +/ value reads as "exports no version", failing every dependent module's check for a reason that +/ points nowhere near the real cause. same shape as di.rdb, di.dbwrite and di.eodtime +version:@[{trim first read0 x};`:::VERSION;{'"di.tickerplant: VERSION file missing or unreadable"}]; +if[0=count version;'"di.tickerplant: VERSION file is empty"]; +export:([init;teardown;upd;subscribe;subdetails;tablelist;endofday;getcounts;gettables;getapimeta;version]) diff --git a/di/tickerplant/test.csv b/di/tickerplant/test.csv index c580f70a..bdd67a9e 100644 --- a/di/tickerplant/test.csv +++ b/di/tickerplant/test.csv @@ -1,8 +1,9 @@ action,ms,bytes,lang,code,repeat,minver,comment before,0,0,q,tp:use`di.tickerplant,1,,load the module under test before,0,0,q,timer:use`di.timer,1,,real di.timer (addjob works without timer.init - no live .z.ts) -before,0,0,q,ntp:use`di.tplog,1,,di.tplog, to replay a tickerplant-written log -before,0,0,q,eod:use`di.eodtime,1,,di.eodtime, to force a roll due in a test +before,0,0,q,ntp:use`di.tplog,1,,"di.tplog, to replay a tickerplant-written log" +before,0,0,q,eod:use`di.eodtime,1,,"di.eodtime, to force a roll due in a test" +before,0,0,q,ps:use`di.pubsub,1,,"di.pubsub, to release the self-subscription a subdetails test leaves on handle 0" before,0,0,q,os:use`di.os,1,,os module for portable path resolution before,0,0,q,"system ""l "",os.abspath[""di/tickerplant/test.q""]",1,,load fixture helpers before,0,0,q,setupfixture[],1,,create the temp fixture root @@ -17,18 +18,29 @@ fail,0,0,q,tp.init[(::)],1,1,init rejects a non-dict deps fail,0,0,q,tp.init depsnolog[],1,1,init rejects deps without a log key fail,0,0,q,tp.init depsonlylog[],1,1,init rejects deps without a timer fail,0,0,q,tp.init depsbadtimer[],1,1,init rejects a timer that does not expose addjob +fail,0,0,q,tp.init depsnodeletejobs[],1,1,init rejects a timer without deletejobs - teardown needs it fail,0,0,q,tp.init depsnoschemas[],1,1,init rejects deps without schemas +true,0,0,q,"0 upd -> roll cycle. the data tables live at ROOT (the tickerplant owns them and di.pubsub / reads them by name), and replay runs the ROOT upd, so a schema + recorder upd live here at root. -/ tp (di.tickerplant), timer (di.timer) and ntp (di.tplog) are bound by test.csv's before rows. +/ tp (di.tickerplant), timer (di.timer), ntp (di.tplog), eod (di.eodtime) and ps (di.pubsub) are +/ bound by test.csv's before rows. base:"/tmp/di_tickerplant_k4unit"; trade:([]time:`timestamp$();sym:`symbol$();price:`float$();size:`long$()); @@ -21,45 +22,71 @@ freshdir:{[sub] dd:base,"/",sub; system"rm -rf ",dd; system"mkdir -p ",dd; dd}; / deps for init - real timer, capturing log, one trade table, logging into dd, batch flag as given mkdeps:{[dd;batch] `log`timer`schemas`logdir`logname`batch!(caplog[];timer;enlist[`trade]!enlist trade;dd;"tp";batch)}; +/ force the module back to a pre-init state, so the NEXT init is a fresh one. +/ init deliberately seeds runtime state (date, counts, open log) only on the first call - see +/ tickerplant.md - so a suite that wants a genuinely clean tickerplant per test has to clear the name +/ initialised[] probes. done here rather than by exporting a reset verb: a state-wiping function that +/ exists only for tests does not belong in the public api. also drops the self-subscription a +/ subdetails test leaves behind on handle 0, so a later flush does not publish into it +resetmodule:{[] + if[`schemas in key `.m.di.0tickerplant; + tp[`teardown][]; + @[{if[.m.di.0tickerplant.logfile>0i;hclose .m.di.0tickerplant.logfile]};::;{[e] :(::)}]; + ![`.m.di.0tickerplant;();0b;`schemas`scheduled]]; + @[ps`closesub;0i;{[e] :(::)}]; + }; + / fresh init into a clean per-test dir; clears the root table and the log capture first freshinit:{[sub;batch] + resetmodule[]; dd:freshdir sub; `trade set 0#trade; `logcap set 0#logcap; tp.init mkdeps[dd;batch]; dd}; +/ drive one batch flush without the roll check tick[] also does - the timer is never started here +flushbuffer:{[] .m.di.0tickerplant.publishbuffer[]}; + / a feed update: sym, price, size - no time, so the tickerplant stamps it row:{[s] (s;1.0;100)}; setupfixture:{system"rm -rf ",base; system"mkdir -p ",base;}; -teardownfixture:{system"rm -rf ",base;}; +teardownfixture:{resetmodule[]; system"rm -rf ",base;}; / deps builders for the init validation fail rows depsnolog:{(enlist`x)!enlist 1}; depsonlylog:{enlist[`log]!enlist caplog[]}; depsnoschemas:{`log`timer!(caplog[];timer)}; depsbadtimer:{`log`timer`schemas!(caplog[];(enlist`x)!enlist 1;enlist[`trade]!enlist trade)}; +depsnodeletejobs:{`log`timer`schemas!(caplog[];`deletejobs _ timer;enlist[`trade]!enlist trade)}; / ============================================================================= / tests (each returns 1b on success) / ============================================================================= -/ init materialises the tables at root (g# on sym), schedules the timer job, zeroes the counts +/ init materialises the tables at root (g# on sym), publishes the subscription protocol at root, +/ schedules the timer job and zeroes the counts testinit:{[] freshinit["init";1b]; c:tp[`getcounts][]; (`g=attr exec sym from trade) and (enlist[`trade]~tp[`gettables][]) and (0=c`i) and (0=c`j) - and (-14h=type c`d) and `tickerplant in exec id from timer.getalljobs[]}; + and (-14h=type c`d) and (`tickerplant in exec id from timer.getalljobs[]) + and all `subdetails`tablelist in key `.}; -/ batch mode: upd stamps, buffers into the root table, and logs (bumping j; i unchanged) +/ batch mode: upd stamps, buffers into the root table and logs (bumping j). i is the PUBLISHED +/ watermark and must NOT move while the rows are still buffered - it catches up at the flush testupdbatch:{[] freshinit["updb";1b]; tp[`upd][`trade;row`AAPL]; tp[`upd][`trade;row`MSFT]; c:tp[`getcounts][]; - (2=count trade) and (`AAPL`MSFT~exec sym from trade) and (not any null exec time from trade) - and (2=c`j) and 0=c`i}; + n:count trade; + s:exec sym from trade; + stamped:not any null exec time from trade; + flushbuffer[]; + a:tp[`getcounts][]; + (2=n) and (`AAPL`MSFT~s) and stamped and (2=c`j) and (0=c`i) and (2=a`i) and 2=a`j}; / an empty update is a no-op: no throw, nothing buffered, nothing logged testemptyupd:{[] @@ -76,23 +103,33 @@ testemptyupdtriggersroll:{[] tp[`upd][`trade;()]; (oldd+1)=first tp[`getcounts][]`d}; -/ zero-latency mode: upd publishes immediately, does NOT buffer into the root table, still logs +/ zero-latency mode: upd publishes immediately, does NOT buffer into the root table, still logs - and +/ i tracks j, because every logged message has already gone out. without that a zero-latency +/ tickerplant tells every subscriber to replay nothing testzerolatency:{[] freshinit["zl";0b]; tp[`upd][`trade;row`AAPL]; + tp[`upd][`trade;row`MSFT]; c:tp[`getcounts][]; - (0=count trade) and 1=c`j}; + (0=count trade) and (2=c`j) and 2=c`i}; -/ re-init is safe: it closes the previous log handle instead of leaking the descriptor, and logging -/ keeps working. fd count (linux /proc, as the suite is already unix-coupled) must not grow. -testreinitnoleak:{[] +/ a re-init refreshes config but must NOT rewind runtime state: the date, the message counts a +/ subscriber replays against, the buffered rows, or the open log handle. it must not leak a +/ descriptor either, which it cannot now that it no longer reopens the log +testreinitpreservesstate:{[] fddir:"/proc/",(string .z.i),"/fd"; - freshinit["reinit";1b]; - b:"J"$first system"ls ",fddir," | wc -l"; / baseline AFTER the first init - one log handle is open - freshinit["reinit";1b]; freshinit["reinit";1b]; freshinit["reinit";1b]; / re-init must not add fds - a:"J"$first system"ls ",fddir," | wc -l"; + dd:freshinit["reinit";1b]; tp[`upd][`trade;row`AAPL]; - (a=b) and 1=tp[`getcounts][]`j}; + tp[`upd][`trade;row`MSFT]; + c0:tp[`getcounts][]; + lp0:.m.di.0tickerplant.logpath; + b:"J"$first system"ls ",fddir," | wc -l"; + tp.init mkdeps[dd;1b]; tp.init mkdeps[dd;1b]; tp.init mkdeps[dd;1b]; + a:"J"$first system"ls ",fddir," | wc -l"; + c1:tp[`getcounts][]; + tp[`upd][`trade;row`IBM]; + (c0~c1) and (a=b) and (lp0~.m.di.0tickerplant.logpath) and (3=count trade) + and 3=tp[`getcounts][]`j}; / the tp log tickerplant writes replays through di.tplog - the two modules agree on the log format testlogroundtrip:{[] @@ -129,3 +166,99 @@ testcheckrepaironroll:{[] `logcap set 0#logcap; tp[`endofday][]; (not ()~key hsym`$dd,"/tp",(string oldd+1),".good") and `warn in exec lvl from logcap where ctx=`check}; + +/ ============================================================================= +/ the subscription protocol di.subscriptions speaks +/ ============================================================================= + +/ subdetails carries all four required keys, in the shapes di.subscriptions' guards check: +/ schemalist as (tablename;schema) pairs, logfilelist as (integer count;symbol file) pairs, +/ rowcounts as a dict keyed by table, date as a date +testsubdetails:{[] + freshinit["subd";1b]; + tp[`upd][`trade;row`AAPL]; + flushbuffer[]; + d:tp[`subdetails][`;`]; + sl:d`schemalist; + lfl:d`logfilelist; + (all `schemalist`logfilelist`rowcounts`date in key d) + and (1=count sl) and (`trade~first first sl) and (.Q.qt last first sl) + and (1=count lfl) and ((type first first lfl) in -7 -6h) and (-11h=type last first lfl) + and (99h=type d`rowcounts) and (1=(d`rowcounts)`trade) and (-14h=type d`date)}; + +/ the message count is the PUBLISHED watermark i, not the logged total j: rows logged but not yet +/ flushed are still in the buffer and go out to a NEW subscriber at the next flush, so reporting j +/ would have it replay them from the log AND receive them again +testsubdetailscountispublished:{[] + freshinit["subdi";1b]; + tp[`upd][`trade;row`AAPL]; + tp[`upd][`trade;row`MSFT]; + flushbuffer[]; + tp[`upd][`trade;row`IBM]; + c:tp[`getcounts][]; + n:first first tp[`subdetails][`;`]`logfilelist; + (2=c`i) and (3=c`j) and 2=n}; + +/ with logging disabled there is no log to replay, so logfilelist is EMPTY rather than naming a file +testsubdetailsnolog:{[] + resetmodule[]; + `trade set 0#trade; + `logcap set 0#logcap; + tp.init[`log`timer`schemas`logdir!(caplog[];timer;enlist[`trade]!enlist trade;"")]; + d:tp[`subdetails][`;`]; + (()~d`logfilelist) and 0=tp[`getcounts][]`j}; + +/ subdetails for a table this tickerplant does not publish fails with a message that NAMES the table +/ and logs it. a bare `fail` row would pass on any throw at all, including an unrelated one +testsubdetailsunknown:{[] + freshinit["subdunk";1b]; + e:string @[{tp[`subdetails][x;`];`NOTHROW};`nope;{[e] `$e}]; + (0type first x;flip c!enlist each x;flip c!x]]}",1,1,"a root upd, which the -11! replay drives. di.subscriptions refuses to replay without one. it normalises the same payload shapes di.subscriptions' own payloadtable does - a table, a column dict, a list of atoms, or a list of column vectors" +before,0,0,q,"tmpdir:{[nm] b:getenv`TMPDIR; if[0=count b;b:""/tmp""]; :b,""/"",nm,string .z.i}",1,1,honours TMPDIR and appends the pid so two runs cannot collide +before,0,0,q,"BASE:tmpdir""ditickerplantintegration""",1,1,fixture: BASE +before,0,0,q,"system""rm -rf "",BASE",1,1,clean slate +before,0,0,q,"system""mkdir -p "",BASE",1,1,create the working directory +before,0,0,q,"@[system;""pkill -f '"",BASE,""' 2>/dev/null || true"";{[e] :(::)}]",1,1,kill any peer left behind by an aborted earlier run +comment,,,,,,,NO PORT NUMBER APPEARS IN THIS SUITE. each peer is started with -p 0W so the OS assigns a free +comment,,,,,,,port and the peer writes it to a file we read back. scanning from a hardcoded base is not portable +comment,,,,,,,and still races - a port held by a blocked non-q process reads as free +before,0,0,q,"readport:{[f;timeoutms] deadline:.z.p+`timespan$1000000*timeoutms; p:0N; while[(.z.p0;exit 0]};""",1,1,"self-exit backstop, CHAINED onto di.pubsub's handler. GUARDED on w>0: a child started with /dev/null 2>&1 &""; p:readport[d,""/port"";8000]; if[null p;'""integration: peer "",nm,"" never reported a port""]; h:hopen (`$"":localhost:"",string p;5000); if[not tok~@[h;""TOKEN"";{[e] :`}];hclose h;'""integration: the peer that answered is not this run's""]; :h}",1,1,spawn one tickerplant peer and return a handle to it. NB the script path comes BEFORE the flags - q silently ignores a script that follows them +before,0,0,q,"settle:{[h] system""sleep 0.4""; h""1+1""; system""sleep 0.2""; h""1+1"";}",1,1,let an async publish arrive and our own inbound queue drain +comment,,,,,,,SCENARIO 1 - batch mode (the default). we subscribe with rows already published AND rows still +comment,,,,,,,buffered on the tickerplant. that is the case which separates the two candidate message counts: +comment,,,,,,,"subdetails must report the PUBLISHED watermark i (4), so we replay 4 and then receive the 2" +comment,,,,,,,buffered rows at the next flush. reporting the logged total j would replay 6 and deliver 2 again +before,0,0,q,"H1:spawnpeer[""tp1"";`tp1tok;1b]",1,1,"spawn a batch-mode tickerplant. this ONE handle carries the control calls, the subscription and the live feed - exactly as a real subscriber's does" +before,0,0,q,"PID1:H1"".z.i""",1,1,"its pid, for the leak check" +before,0,0,q,H1 (`feed;til 4),1,1,"four updates: logged, and buffered in the peer's root table" +before,0,0,q,"PRE:H1""counts[]""",1,1,capture: counts before anything is published +before,0,0,q,"H1""flushtp[]""",1,1,publish the buffer to nobody - this is what moves i up to j +before,0,0,q,"MID:H1""counts[]""",1,1,capture: counts after the flush +before,0,0,q,H1 (`feed;4 5),1,1,"two more updates: logged, still buffered, NOT yet published" +before,0,0,q,"ATSUB:H1""counts[]""",1,1,capture: the counts a subscriber arriving now is answered from +before,0,0,q,TPD:ATSUB`d,1,1,capture: the trading date the tickerplant reports +before,0,0,q,R1:sub.subscribe[H1;`;`;1b;1b],1,1,"THE CALL UNDER TEST - ` for all tables, so tablelist is exercised too; define the schemas at root and replay the log" +before,0,0,q,REPLAYED:count trade,1,1,"capture: rows recovered from the tp log, before any live delivery" +before,0,0,q,ATT1:exec first a from meta trade where c=`sym,1,1,capture: the attribute the tickerplant's schema carried across +before,0,0,q,SUBSCRIBED1:sub.subscribed[],1,1,capture: di.subscriptions reports a live subscription +before,0,0,q,"H1""flushtp[]""",1,1,"publish the two buffered rows - we are a registered subscriber now, so they come to us" +before,0,0,q,settle[H1],1,1,let them arrive +before,0,0,q,FINAL:count trade,1,1,capture: total rows here +before,0,0,q,SYMS:exec sym from trade,1,1,"capture: the syms, in order - proves no gap and no duplicate" +before,0,0,q,"PERR1:H1""errs[]""",1,1,capture: anything the tickerplant logged at error +before,0,0,q,"TABS1:H1""tablelist[`]""",1,1,"capture: the tablelist reply, in the arity di.subscriptions actually sends" +comment,,,,,,,"SCENARIO 2 - zero-latency mode. every logged message has already gone out, so i must equal j and" +comment,,,,,,,the subscriber must replay all three. with i stuck at its seeded 0 this replayed nothing +before,0,0,q,sub.unsubscribe[H1],1,1,release the first subscription so the duplicate guard does not reject the second +before,0,0,q,@[hclose;H1;{[e] :(::)}],1,1,close the peer - the backstop makes it exit +before,0,0,q,![`.;();0b;enlist`trade],1,1,"drop the table, so scenario 2 starts from nothing and its row counts stand alone" +before,0,0,q,"system""sleep 0.4""",1,1,let the peer exit +before,0,0,q,"H2:spawnpeer[""tp2"";`tp2tok;0b]",1,1,spawn a ZERO-LATENCY tickerplant +before,0,0,q,"PID2:H2"".z.i""",1,1,"its pid, for the leak check" +before,0,0,q,H2 (`feed;til 3),1,1,"three updates: published immediately to nobody, and logged" +before,0,0,q,"PRE2:H2""counts[]""",1,1,capture: i must already equal j - nothing is ever buffered in this mode +before,0,0,q,"BUF2:H2""count trade""",1,1,capture: zero-latency does not buffer into the peer's root table +before,0,0,q,R2:sub.subscribe[H2;`trade;`;1b;1b],1,1,"subscribe against the zero-latency tickerplant, naming the table explicitly this time" +before,0,0,q,REPLAYED2:count trade,1,1,capture: rows recovered from the log +before,0,0,q,H2 (`feed;3 4),1,1,"two more updates, published live this time" +before,0,0,q,settle[H2],1,1,let them arrive +before,0,0,q,FINAL2:count trade,1,1,capture: total rows here +before,0,0,q,SYMS2:exec sym from trade,1,1,"capture: the syms, in order" +before,0,0,q,"PERR2:H2""errs[]""",1,1,capture: anything the tickerplant logged at error +comment,,,,,,,teardown - close the peer and prove neither leaked +before,0,0,q,@[sub.unsubscribe;H2;{[e] :(::)}],1,1,release the second subscription +before,0,0,q,@[hclose;H2;{[e] :(::)}],1,1,close the peer - the backstop makes it exit +before,0,0,q,"system""sleep 0.5""",1,1,let it exit +before,0,0,q,"gone:{[pid] :0=count @[{system"""" sv (""ps -p "";string x;"" -o pid="")};pid;{[e] :()}]}",1,1,"ps throws 'os when the pid is absent, so the probe has to be wrapped" +before,0,0,q,GONE1:gone PID1,1,1,capture: no leaked first peer +before,0,0,q,GONE2:gone PID2,1,1,capture: no leaked second peer +before,0,0,q,SERR:exec ctx from caprows where lvl=`error,1,1,capture: anything di.subscriptions logged at error on this side +before,0,0,q,"system""rm -rf "",BASE",1,1,remove the working directory +comment,,,,,,,scenario 1 - batch mode +true,0,0,q,(0=PRE`i) and 4=PRE`j,1,1,"i does not move while the rows are only buffered - four logged, none published" +true,0,0,q,4=MID`i,1,1,the flush brings i up to j +true,0,0,q,(4=ATSUB`i) and 6=ATSUB`j,1,1,at subscribe time two more messages are logged but not yet published +true,0,0,q,4=REPLAYED,1,1,"THE TEST - di.subscriptions replayed exactly the PUBLISHED watermark, not the logged total" +true,0,0,q,6=FINAL,1,1,and then received the two buffered rows once each: no gap and no duplicate. reporting j would give 8 +true,0,0,q,"SYMS~`$""S"",/:string til 6",1,1,"every row arrived exactly once, in order - a row count alone could hide an offsetting error" +true,0,0,q,`g=ATT1,1,1,the g# attribute reached this side through schemalist +true,0,0,q,(enlist`trade)~R1`subtables,1,1,di.subscriptions resolved the ` request through tablelist and subscribed to trade +true,0,0,q,(enlist`trade)~TABS1,1,1,tablelist answers the (`tablelist;`) call di.subscriptions sends - a niladic form would throw 'rank and be silently downgraded +true,0,0,q,TPD~R1`tplogdate,1,1,tplogdate matches the date the tickerplant reported and named its log after +true,0,0,q,(99h=type R1`rowcounts) and 4=(R1`rowcounts)`trade,1,1,"rowcounts reports rows PUBLISHED for the table, so it agrees with the message count in the same reply" +true,0,0,q,SUBSCRIBED1,1,1,di.subscriptions holds a live subscription on the handle +true,0,0,q,0=count PERR1,1,1,the tickerplant logged nothing at error during the whole exchange +comment,,,,,,,scenario 2 - zero-latency mode +true,0,0,q,(3=PRE2`i) and 3=PRE2`j,1,1,i tracks j when every message is published as it is logged +true,0,0,q,0=BUF2,1,1,zero-latency does not buffer into the tickerplant's root table +true,0,0,q,3=REPLAYED2,1,1,all three pre-subscription messages were replayed - with i stuck at 0 this would replay none +true,0,0,q,5=FINAL2,1,1,"the two live updates arrived on top, once each" +true,0,0,q,"SYMS2~`$""S"",/:string til 5",1,1,"every row arrived exactly once, in order" +true,0,0,q,(enlist`trade)~R2`subtables,1,1,the explicitly-named table was subscribed +true,0,0,q,0=count PERR2,1,1,the tickerplant logged nothing at error +comment,,,,,,,teardown +true,0,0,q,GONE1 and GONE2,1,1,neither tickerplant peer leaked +true,0,0,q,0=count SERR,1,1,di.subscriptions logged nothing at error during the entire run +comment,,,,,,,"VERSION guards. these are LOAD-time, so they can only be observed in a CHILD process - the module" +comment,,,,,,,is already loaded here and cannot be reloaded with a different VERSION file. a temp QPATH tree +comment,,,,,,,symlinks every other module and copies di/tickerplant so only its VERSION is altered +before,0,0,q,"VBASE:BASE,""/vtree""",1,1,root for the VERSION-guard trees +before,0,0,q,"mkvtree:{[nm;mut] d:VBASE,""/"",nm; realdi:""/"" sv -1_ ""/"" vs .Q.m.mp`di.tickerplant; system""rm -rf "",d; system""mkdir -p "",d,""/di""; system""ln -s "",realdi,""/* "",d,""/di/ 2>/dev/null""; system""rm -f "",d,""/di/tickerplant""; system""cp -r "",realdi,""/tickerplant "",d,""/di/tickerplant""; mut d; :d}",1,1,build a temp module tree whose di/tickerplant VERSION can be mutated. links EVERY module and overwrites only tickerplant - hand-listing deps would rot as the graph changes +before,0,0,q,"loadin:{[d] f:d,""/probe.q""; (hsym`$f) 0: (""r:@[{use`di.tickerplant; :`OK};::;{[e] `$\""THREW: \"",e}];"";""if[r~`OK; tpx:use`di.tickerplant; r:`$\""OK [\"",tpx.version,\""]\""];"";""(hsym`$\"""",d,""/out\"") 0: enlist string r;"";""exit 0;""); system ""QPATH="",d,"" "",(getenv[`QHOME]),""/bin/q "",f,"" -q /dev/null 2>&1""; :$[count key hsym`$d,""/out"";first read0 hsym`$d,""/out"";""NO OUTPUT""]}",1,1,load di.tickerplant in a child q against that tree. QPATH MUST be set for the child - inheriting ours makes it load the REAL module and every case passes vacuously +before,0,0,q,"VGOOD:loadin mkvtree[""good"";{[d] }]",1,1,control: an untouched VERSION must load cleanly +before,0,0,q,"VMISSING:loadin mkvtree[""missing"";{[d] system""rm -f "",d,""/di/tickerplant/VERSION""}]",1,1,a missing VERSION must fail loudly and name the module +before,0,0,q,"VEMPTY:loadin mkvtree[""empty"";{[d] (hsym`$d,""/di/tickerplant/VERSION"") 0: enlist """"}]",1,1,an empty VERSION must fail loudly and name the module +before,0,0,q,"VPAD:loadin mkvtree[""pad"";{[d] (hsym`$d,""/di/tickerplant/VERSION"") 0: enlist ""0.1.0 ""}]",1,1,"a padded VERSION must load and be TRIMMED. written from q, not shell printf - the quoting does not survive system[] and silently produces an empty file, which tests the wrong guard" +before,0,0,q,"system""rm -rf "",VBASE",1,1,remove the VERSION-guard trees +true,0,0,q,"""OK [0.1.0]""~VGOOD",1,1,"control - a clean VERSION loads and reports the exact version. BRACKETED because writing a bare string through 0:/read0 strips trailing spaces, which would destroy the padding the last assert looks for" +true,0,0,q,"00i`); `.z.m.logpath` holds the path, which `logfilelist` reports. `openlog` sets + the path and returns the handle for the caller to store. They were one name, and the path never + survived — every caller overwrote it with the handle. - **End of day.** The roll fires when the current time passes `di.eodtime`'s next roll timestamp, checked on every `upd` and on every timer tick. `endofday` flushes, notifies subscribers, rolls the - log to the next day, and refreshes the roll time and data-timestamp offset from `di.eodtime`. + log to the next day, resets the message and row counts, and refreshes the roll time and + data-timestamp offset from `di.eodtime`. +- **A re-init preserves runtime state.** Dependencies and config are refreshed on every `init`, but + the trading date, the message and row counts, and the open log are seeded only on the first. A + re-init — `di.torq` re-applying config, a config reload, a second wiring — must not rewind the date, + zero the counts a subscriber replays against, or reopen (and so leak) the log already being written + to. Table materialisation follows the same rule: a fresh `init` defines every captured table from + its schema, attributes included, while a re-init defines only names not already at root, because + re-running `nm set schema` over a live tickerplant discards every buffered row that has been logged + but not yet published — leaving the counts describing data that had just been thrown away. Same + precedent as `di.rdb` and `di.subscriptions`, which seed their runtime state only when fresh. + **Consequence:** a `logdir` or `logname` change on a re-init takes effect at the next roll, not + immediately. +- **`init` must be called first, and every callable export says so — except `upd`.** `teardown`, + `subscribe`, `subdetails`, `tablelist`, `endofday`, `getcounts` and `gettables` each guard with + `requireinit` and report `di.tickerplant: : init must be called before any other function`. + `upd` deliberately does not: it runs once per feed message, and the `initialised[]` probe is a + protected apply costing ~0.7µs a call (measured) — a permanent per-message tax to catch a wiring + mistake that can only happen at startup and surfaces immediately when it does. Called before `init` + it throws a bare `'.m.di.0tickerplant.tabs` instead. `di.rdb`'s `updfn` is unguarded for the same + reason. A test pins which functions are on each side of that line, so the split cannot drift. +- **The `VERSION` read fails loud.** `init.q` signals `di.tickerplant: VERSION file missing or + unreadable` / `... is empty` rather than doing a bare `first read0`. `di.depcheck` compares versions + as **strings**, so a whitespace-padded value silently breaks every dependent's check and an empty + one reads to it as "exports no version" — both failing far from the real cause. The value is + trimmed. Same shape as `di.rdb`, `di.dbwrite` and `di.eodtime`. +- **`teardown` is idempotent and narrow.** It withdraws the process-global bindings — the root + protocol and the timer job — and deliberately leaves module state and the captured tables intact, + so a shutdown path can still inspect or save what is buffered. A re-init after a teardown + re-publishes the protocol and re-schedules the job. - **No `di.handlers` dependency.** Subscriber-disconnect cleanup is handled by `di.pubsub`'s own `.z.pc`. (`di.pubsub` should migrate to `di.handlers` so `.z.*` is not assigned outside the central registry — tracked separately, out of scope here.) +## Design divergences from TorQ + +- **The `subdetails` protocol is a deliberate capability addition, not restored parity.** TorQ's + standard `code/processes/tickerplant.q` defines **no** `subdetails` — a grep over the TorQ tree + finds it only in `code/processes/chainedtp.q` and `code/processes/segmentedtickerplant.q`, which + `di.subscriptions`' own source comment also attributes it to. A standard TorQ tickerplant never + spoke this protocol. + + It is implemented here anyway because there is no `di.chainedtp` or `di.segmentedtp` for `di.rdb` + to subscribe to instead, so without it no modular subscriber can attach to a modular tickerplant at + all. Both existing integration harnesses (`di/rdb/test_integration.csv`, + `di/subscriptions/test_integration.csv`) already hand-roll exactly this adapter in their spawned + peer, and both label it *"the subdetails adapter a modular tickerplant would own"*. This module now + owns it. Nobody should read its presence as evidence that a standard TorQ tickerplant had one. +- **One log, not one per table.** `logfilelist` is a list because the protocol also serves a + segmented tickerplant, which writes one log per table. This module writes a single log, so it + reports at most one entry. +- **Removed entirely:** all `.finspace.*` / `.aws.*` code. FinSpace is end-of-life. + ## Testing `test.csv` / `test.q` (k4unit) run against the **real** `di.pubsub`, `di.eodtime`, `di.tplog` and @@ -84,13 +215,40 @@ tickerplant must not do to its own log. job is exercised through `endofday`). A capturing logger is shared across the modules so their output is assertable. -Coverage: the metadata/version contract; strict `init` dependency validation (a `fail` row per guard); -init materialising the root tables and scheduling the timer job; batch and zero-latency `upd`; -`endofday` flushing and rolling; `upd` input validation; and the two `di.tplog` integration points — a -tickerplant-written log replaying through `di.tplog`, and rolling into a corrupt log repairing it via -`tplog.check`. +Coverage: the metadata/version contract; strict `init` dependency validation (a `fail` row per guard, +plus a message assertion so a guard cannot pass on an unrelated throw); init materialising the root +tables and scheduling the timer job; batch and zero-latency `upd`, including that `i` lags `j` while +rows are buffered and tracks it when they are not; a re-init preserving the counts, the buffer and the +log handle; `endofday` flushing and rolling; the `subdetails`/`tablelist` shapes, the published-not- +logged message count, an empty `logfilelist` when logging is off, and the error naming an unpublished +table; root publication and `teardown`; `upd` input validation; and the two `di.tplog` integration +points. ```q k4unit:use`di.k4unit k4unit.moduletest`di.tickerplant ``` + +It also carries the **`VERSION` guard** checks. Those fire at *load* time, so they can only be +observed in a child process: the suite builds a temp `QPATH` tree that symlinks every other module +and copies `di/tickerplant`, mutates only its `VERSION`, and loads the module in a child q — covering +an untouched file, a missing one, an empty one, and a whitespace-padded one. + +`test_integration.csv` drives a **real `di.subscriptions`** against a **real `di.tickerplant`** over +genuine IPC — the only test that actually proves the protocol gap is closed. The tickerplant is the +spawned peer and `di.subscriptions` runs in the test process, not the other way round: a q process +blocked in a sync call does not accept a new inbound connection, so the side that *dials* has to be +the side that drives. Two scenarios, on OS-assigned ports with no port number anywhere in the file: + +- **batch mode** — subscribe with four rows already published and two still buffered, then flush. + The subscriber must replay 4 and end with 6 rows in order. Reporting `j` gives 8. +- **zero-latency mode** — subscribe with three rows published-and-logged, then two more live. The + subscriber must replay 3 and end with 5. With `i` not advancing it replays 0. + +`moduletest` only ever loads `test.csv`, so load and run this suite directly, in a fresh session: + +```q +k4unit:use`di.k4unit +.m.di.0k4unit.KUltf .Q.dd[hsym`$.Q.m.mp`di.tickerplant;`test_integration.csv] +.m.di.0k4unit.KUrt[] +``` diff --git a/di/tickerplant/tickerplant.q b/di/tickerplant/tickerplant.q index 27f28b7e..e9fc09e7 100644 --- a/di/tickerplant/tickerplant.q +++ b/di/tickerplant/tickerplant.q @@ -9,6 +9,10 @@ / so they cannot be module-local. this is the one deliberate root-state exception for this process / module; all other mutable state is module-local in .z.m. / . +/ NB init also publishes subdetails and tablelist at ROOT - see installroot. they are the subscription +/ protocol di.subscriptions speaks, and an IPC caller reaches them through the default .z.pg/.z.ps, +/ so a module-local binding would be invisible. teardown gives them back. +/ . / NB subscriber-disconnect cleanup is handled by di.pubsub's own .z.pc (it self-assigns it). this / module therefore takes no di.handlers dependency. FLAG: di.pubsub should migrate to di.handlers so / .z.pc is not assigned outside the central registry - out of scope here, tracked separately. @@ -23,10 +27,28 @@ defaultbatchperiod:0D00:00:01; / optional config keys forwarded verbatim to di.eodtime.init eodtimekeys:`rolltimezone`datatimezone`rolltimeoffset; +/ the root names init publishes and teardown gives back - the subscription protocol di.subscriptions +/ calls over IPC. see installroot +rootnames:`subdetails`tablelist; + / ============================================================ / internal helpers / ============================================================ +initialised:{[] + / has init run? a direct (module-rewritten) reference detects prior setup without touching root. + / schemas has no module-level default - its only value comes from init, so this probe cannot be + / fooled by a load-time constant of the same name + :@[{.z.m.schemas;1b};::;{[e] :0b}]; + }; + +requireinit:{[ctx] + / every exported function except init depends on init having wired the logger. there is no default + / logger, so without this an early call dies with a bare 'type instead of a usable message + if[not initialised[]; + '"di.tickerplant: ",string[ctx],": init must be called before any other function"]; + }; + raiseerror:{[ctx;msg] / internal - log an error under ctx then signal it, so failures are observable as well as thrown .z.m.logerr[ctx;msg]; @@ -41,27 +63,40 @@ stamp:{[x] }; openlog:{[date] - / open the tp log for `date`, checking/repairing a pre-existing log via di.tplog; 0i if logging off - if[0=count .z.m.logdir;:0i]; + / open the tp log for `date`, checking/repairing a pre-existing log via di.tplog; 0i if logging off. + / records the PATH in .z.m.logpath and RETURNS the handle, which the caller stores in .z.m.logfile. + / the two are deliberately separate names: writelog tests the handle with >0i, and subdetails has to + / report the path to subscribers, so one name cannot carry both - it used to try, and the path + / assignment was dead because every caller overwrote it with the handle + if[0=count .z.m.logdir;.z.m.logpath:`;:0i]; l:hsym `$ .z.m.logdir,"/",.z.m.logname,string date; if[count key l;l:tplog.check l]; if[not count key l;l set ()]; - .z.m.logfile:l; + .z.m.logpath:l; h:hopen l; .z.m.loginfo[`openlog;"logging to ",string l]; :h; }; publishbuffer:{[] - / batch mode: publish each root table's buffered rows to subscribers then clear them; i catches j + / batch mode: publish each root table's buffered rows to subscribers then clear them; i catches j. + / the row counts are taken BEFORE pubclear, which publishes and then empties each table. + / `. is the explicit ROOT namespace - the captured tables live there, not in .z.m + .z.m.rowcounts:.z.m.rowcounts+.z.m.tabs!count each `. .z.m.tabs; pubsub.pubclear[.z.m.tabs]; .z.m.i:.z.m.j; }; publishrows:{[t;x] - / zero-latency mode: publish one stamped update immediately, as a table keyed by t's columns + / zero-latency mode: publish one stamped update immediately, as a table keyed by t's columns, then + / mark it published. i:j is what makes i the PUBLISHED-message watermark in this mode - without it + / i never leaves its seeded 0 and every subscriber is told to replay nothing. TorQ does exactly this + / in chainedtp.q's tickpub (:96-99), and upd calls writelog FIRST so j is already bumped here f:cols t; - pubsub.publish[t;$[0>type first x;enlist f!x;flip f!x]]; + d:$[0>type first x;enlist f!x;flip f!x]; + pubsub.publish[t;d]; + .z.m.rowcounts:@[.z.m.rowcounts;t;+;count d]; + .z.m.i:.z.m.j; }; writelog:{[t;x] @@ -69,6 +104,21 @@ writelog:{[t;x] if[.z.m.logfile>0i;.z.m.logfile enlist (`upd;t;x);.z.m.j+:1]; }; +logfilelist:{[] + / internal - the (messagecount;logfile) pairs subdetails reports. a LIST because the protocol also + / serves a segmented tickerplant, which writes one log per table; this one writes a single log, so + / there is at most one entry. EMPTY when logging is disabled - di.subscriptions accepts that + / (subscriptions.q:307) and replays nothing. + / the count is .z.m.i, the PUBLISHED watermark, NOT .z.m.j. in batch mode a row that has been logged + / but not yet flushed is still sitting in the buffer, and pubclear publishes the WHOLE table to every + / registered handle - including one that registered after the row was buffered. so reporting j would + / have the subscriber replay those rows from the log AND receive them again at the next tick. + / TorQ sends the same field for the same reason: chainedtp.q:169 puts .u.i in the reply, and + / kdb+tick's r.q replays with .u`i + if[null .z.m.logpath;:()]; + :enlist (.z.m.i;.z.m.logpath); + }; + rollcheck:{[now] / trigger end-of-day if we have passed the next scheduled roll timestamp if[eodtime.getnextroll[]0i;hclose .z.m.logfile]; .z.m.i:.z.m.j:0; + .z.m.rowcounts:.z.m.tabs!(count .z.m.tabs)#0; .z.m.logfile:openlog .z.m.d; }; +createtables:{[schemas;fresh] + / internal - materialise the captured tables at ROOT, applying `g# to any sym column. + / a FRESH init owns the tables outright and defines every one of them from its schema, attributes + / included. a RE-INIT defines only names not already at root - a schema key added since the first + / call - because re-running `nm set schema` over a live tickerplant would discard every buffered row + / that had been logged but not yet published, leaving the message counts describing data that had + / just been thrown away + nms:$[fresh;key schemas;(key schemas) where not (key schemas) in tables[`.]]; + if[0=count nms;:()]; + {[nm;s] nm set $[`sym in cols s;@[s;`sym;`g#];s]}'[nms;schemas nms]; + }; + +publishroot:{[nm;f] + / internal - publish ONE root entry point, warning first when the name already holds something that + / is neither the function about to be installed nor the one this module installed last time. + / uninstallroot deliberately refuses to delete a binding that is not ours; installing over one + / silently is that same asymmetry in reverse, and it is how a co-hosted process loses its own + / bindings without a word. same shape as di.rdb's publishroot + if[nm in key `.; + cur:`. nm; + if[not any (f;.z.m.rootinstalled nm)~\:cur; + .z.m.logwarn[`installroot;"root ",(string nm)," was already bound to something di.tickerplant ", + "did not install - replacing it. teardown will not give the previous binding back"]]]; + @[`.;nm;:;f]; + }; + +installroot:{[] + / publish the subscription protocol at ROOT, where an IPC caller reaches it through the default + / .z.pg/.z.ps. a bare module-level assignment lands in this module's private namespace and would + / never be found: + / subdetails - di.subscriptions sends (`subdetails;tabs;syms) and requires the reply keys + / tablelist - di.subscriptions sends (`tablelist;`) to resolve a ` (all tables) request + / NB the root `upd` is deliberately NOT published here - di.torq wires the process's feed entry + / point, which may legitimately be a caller-supplied wrapper around this module's upd + fs:(subdetails;tablelist); + publishroot'[rootnames;fs]; + / record what was published, so the NEXT install can tell "someone else took this name" apart from + / a legitimate re-init. written as ONE dict rather than amended per name + .z.m.rootinstalled:rootnames!fs; + }; + +dropifours:{[nm;f] + / internal - delete a root name only if it still holds the function we installed there + if[not nm in key `.;:()]; + if[not f~`. nm;:()]; + ![`.;();0b;enlist nm]; + }; + +uninstallroot:{[] + / internal - give back exactly what installroot published. only the names still bound to THIS + / module's functions are removed: a later module that has taken over one of these root names owns it + / now, and silently deleting its binding would be a worse outcome than leaving ours behind + dropifours'[rootnames;(subdetails;tablelist)]; + }; + / ============================================================ / public api / ============================================================ init:{[deps] / wire the injected log + timer, initialise the dep modules, materialise the tables at root, open - / the tp log and schedule the batch/roll timer job. + / the tp log, publish the subscription protocol at root and schedule the batch/roll timer job. / deps: a dict with `log (required), `timer (required), `schemas (required, tablename!schema) and / optional `batch (1b), `batchperiod (timespan), `logdir (string, "" disables logging), / `logname (string), `subtables (symbol list), plus di.eodtime keys (rolltimezone/datatimezone/ / rolltimeoffset) forwarded verbatim. + / dependencies and config are re-applied on EVERY init; RUNTIME state - the date, the message and + / row counts, and the open log - is seeded only on the FIRST. see the fresh block below if[99h<>type deps; '"di.tickerplant: deps must be a dict with `log, `timer and `schemas keys"]; if[not `log in key deps; @@ -114,47 +223,89 @@ init:{[deps] '"di.tickerplant: timer dict must expose addjob"]; if[not `custom in key deps[`timer]`addjob; '"di.tickerplant: timer addjob must expose the custom variant"]; + / deletejobs is required because teardown deletes the job init schedules. a dict-valued dep returns + / a null-shaped value for an absent key rather than erroring, so a missing deletejobs would fail + / silently inside teardown's protected apply - same reasoning as di.rdb's timer validation + if[not `deletejobs in key deps`timer; + '"di.tickerplant: timer dict must expose deletejobs - teardown needs it"]; + if[not (type deps[`timer]`deletejobs) within 100 112h; + '"di.tickerplant: timer deletejobs must be a function [ids]"]; if[not `schemas in key deps; '"di.tickerplant: schemas is required; pass a tablename!schema dict keyed on `schemas"]; if[99h<>type deps`schemas; '"di.tickerplant: schemas must be a dict of tablename!schema"]; + / is this the FIRST init in this process? read it BEFORE any write, because initialised[] probes + / schemas and this must reflect the state on entry + fresh:not initialised[]; .z.m.loginfo:(deps`log)`info; .z.m.logwarn:(deps`log)`warn; .z.m.logerr:(deps`log)`error; .z.m.timer:deps`timer; - / optional config with defaults + / optional config with defaults - one explicit write per key, so a reader can see at a glance which + / keys reach module state .z.m.batch:$[`batch in key deps;deps`batch;1b]; .z.m.batchperiod:$[`batchperiod in key deps;deps`batchperiod;defaultbatchperiod]; .z.m.logdir:$[`logdir in key deps;deps`logdir;""]; .z.m.logname:$[`logname in key deps;deps`logname;"tp"]; - / materialise the schemas as root tables (see the header note on root state), applying `g# to sym .z.m.schemas:deps`schemas; .z.m.tabs:key deps`schemas; - {[nm;s] nm set $[`sym in cols s;@[s;`sym;`g#];s]}'[.z.m.tabs;value deps`schemas]; + / materialise the schemas as root tables (see the header note on root state), applying `g# to sym + createtables[deps`schemas;fresh]; / initialise the dependency modules: eodtime (log + tz passthrough), tplog (log), pubsub (over the - / subscribable tables). tplog now takes an injected log and must be init'd before check is called. + / subscribable tables). tplog now takes an injected log and must be init'd before check is called eodtime.init[(enlist[`log]!enlist deps`log),(key[deps] inter eodtimekeys)#deps]; tplog.init[enlist[`log]!enlist deps`log]; pubsub.setsubtables[$[`subtables in key deps;deps`subtables;.z.m.tabs]]; pubsub.init[]; - / date, counts, and the tp log for today. close a handle held from a previous init before - / reopening, so a re-init does not leak the old file descriptor (rolllog closes on its own path) - .z.m.d:eodtime.getd[]; - .z.m.i:.z.m.j:0; - if[`logfile in key .z.m;if[.z.m.logfile>0i;hclose .z.m.logfile]]; - .z.m.logfile:openlog .z.m.d; + / RUNTIME state is seeded only on a FRESH init. a re-init - di.torq re-applying config, a config + / reload, a second wiring - must not rewind a live tickerplant's date, zero the message counts a + / subscriber replays against, or reopen (and so leak) the log it is already writing to. di.rdb and + / di.subscriptions set the same precedent; the dependency and config writes above are refreshed + / unconditionally. consequence, documented in tickerplant.md: a logdir/logname change applies at the + / next roll rather than immediately + if[fresh; + .z.m.d:eodtime.getd[]; + .z.m.i:.z.m.j:0; + .z.m.rowcounts:.z.m.tabs!(count .z.m.tabs)#0; + / what installroot last published at root, keyed by name. seeded here rather than at module load + / because publishroot reads it on the FIRST install, before installroot has written it + .z.m.rootinstalled:(`$())!(); + .z.m.scheduled:0b; + .z.m.logfile:openlog .z.m.d]; + installroot[]; / schedule the timer job that flushes the buffer (batch) and checks the roll; mode 1 = fixed period. - / guarded so a re-init does not re-add (di.timer.addjob throws on a duplicate id); tick reads - / .z.m.batch live, so the one job serves both modes across re-inits - if[not `scheduled in key .z.m; + / guarded on the flag rather than on `fresh` so a teardown-then-init pair re-schedules it - di.timer + / throws on a duplicate id, and teardown has deleted it. tick reads .z.m.batch live, so the one job + / serves both modes across re-inits + if[not .z.m.scheduled; .z.m.timer[`addjob][`custom][`tickerplant;tick;();`int$.z.m.batchperiod%0D00:00:01;1h;()!()]; .z.m.scheduled:1b]; .z.m.loginfo[`init;"di.tickerplant initialised (",$[.z.m.batch;"batch";"zero-latency"]," mode)"]; }; +teardown:{[] + / release everything init installed process-wide: the root subscription protocol and the timer job. + / paired with init's side effects, the way di.rdb's teardown is paired with its root entry points. + / module state and the CAPTURED TABLES are deliberately left intact - a shutdown path may still need + / to inspect or save what is buffered; only the process-global bindings are withdrawn + requireinit[`teardown]; + uninstallroot[]; + / deleting a job that was never scheduled is a no-op in di.timer (a delete-where over the jobs + / table), so the id does not have to exist. protected only because a timer whose deletejobs throws + / must not take down a shutdown path - init has already established that it is a function + @[.z.m.timer[`deletejobs];enlist`tickerplant;{[e] :(::)}]; + .z.m.scheduled:0b; + .z.m.loginfo[`teardown;"di.tickerplant root entry points and timer job removed"]; + }; + upd:{[t;x] - / feed entry point: stamp the update, then buffer+log (batch) or publish+log (zero-latency). + / feed entry point: stamp the update, then buffer+log (batch) or log+publish (zero-latency). / t is the table name, x the column data. wired to root `upd` by di.torq so feeds can call it. + / NB the ONLY callable export without a requireinit guard, deliberately: this runs once per feed + / message, and initialised[] is a protected apply costing ~0.7us a call (measured) - a permanent + / per-message tax to catch a wiring mistake that can only happen at startup and shows up instantly + / when it does. calling it before init throws a bare '.m.di.0tickerplant.tabs instead of this + / module's own message; that is the accepted trade. di.rdb's updfn is unguarded for the same reason if[not -11h=type t;raiseerror[`upd;"table must be a symbol"]]; if[not t in .z.m.tabs;raiseerror[`upd;"unknown table ",string t]]; rollcheck .z.p; @@ -162,16 +313,57 @@ upd:{[t;x] if[not count x;:()]; x:stamp x; if[.z.m.batch;t insert x;writelog[t;x]]; - if[not .z.m.batch;publishrows[t;x];writelog[t;x]]; + / log BEFORE publishing, so publishrows can copy the bumped j into i, and so a message that fails to + / publish is at least recoverable from the log. TorQ orders it the same way - chainedtp.q:159-161 + if[not .z.m.batch;writelog[t;x];publishrows[t;x]]; }; subscribe:{[tabs;filters] - / register a subscriber (delegates to di.pubsub); called by downstream processes over IPC + / register a subscriber (delegates to di.pubsub); the kdb+tick .u.sub entry point, for a caller that + / wants the raw (tables;schemas) reply rather than the subdetails protocol + requireinit[`subscribe]; :pubsub.subscribe[tabs;filters]; }; +subdetails:{[tabs;instruments] + / TorQ's subdetails protocol, published at ROOT by init. di.subscriptions sends + / (`subdetails;tabs;syms) and requires `schemalist`logfilelist`rowcounts`date + / (subscriptions.q:20,266,272); asking for the schemas IS the subscription, so this registers the + / calling handle for live delivery as a side effect. + / di.pubsub.subscribe returns THREE shapes (pubsub.q:126-137): (tables;schemas) when every requested + / table exists, (errmsg;(tables;schemas)) when only some do, and a bare errmsg SYMBOL when none do. + / flip turns the pair into the (tablename;schema) rows schemalist wants + requireinit[`subdetails]; + r:pubsub.subscribe[tabs;instruments]; + / nothing matched: di.pubsub registered NOTHING (pubsub.q:28,44), so failing here leaves no half + / subscription behind. raiseerror rather than passing the symbol back - di.subscriptions wraps this + / call in a protected apply and reports the message verbatim, whereas a bare symbol reaches it as + / the far less useful "subdetails must return a dictionary, got type -11h" + if[-11h=type r;raiseerror[`subdetails;"no requested table is published: ",string r]]; + / a PARTIAL match still succeeds: the tables that did match are already registered, so signalling + / would tell the caller it failed while leaving it subscribed. say so at warn instead + partial:-11h=type first r; + if[partial;.z.m.logwarn[`subdetails;string first r]]; + pairs:flip $[partial;last r;r]; + nms:pairs[;0]; + :`schemalist`logfilelist`rowcounts`date! + (pairs;logfilelist[];nms!0^.z.m.rowcounts nms;.z.m.d); + }; + +tablelist:{[x] + / the tables offered for subscription, so di.subscriptions can resolve a ` (all tables) request + / (subscriptions.q:31,365). published at ROOT by init. + / UNARY on purpose - di.subscriptions sends (`tablelist;`), so a niladic {[] ...} would throw 'rank, + / which it CATCHES and downgrades to asking for ` (subscriptions.q:366-369): a silent degradation + / that a segmented tickerplant cannot answer. TorQ's own tablelist:{.stpps.t} is unary for the same + / reason. the argument is accepted and ignored, exactly as legacy does + requireinit[`tablelist]; + :pubsub.getsubtables[]; + }; + endofday:{[] / flush any buffer, notify subscribers, roll the tp log, advance eodtime state and reset counts + requireinit[`endofday]; if[.z.m.batch;publishbuffer[]]; pubsub.callendofday[.z.m.d]; .z.m.d+:1; @@ -182,21 +374,37 @@ endofday:{[] }; getcounts:{[] - / current message counts and trading date - i (in the log), j (log plus buffered), d (date) + / message counts and trading date - i (published to subscribers), j (written to the log), d (date). + / i is the watermark subdetails reports for replay; in batch mode it lags j by whatever is still + / buffered, and in zero-latency mode the two move together + requireinit[`getcounts]; :`i`j`d!(.z.m.i;.z.m.j;.z.m.d); }; gettables:{[] / the tables this tickerplant captures + requireinit[`gettables]; :.z.m.tabs; }; getapimeta:{[] / callable api for di.torq to register with di.api (init/getapimeta/version are plumbing, omitted) :flip `name`public`descrip`params`return!flip( - (`upd;1b;"feed entry point - stamp, log and publish (or buffer) an update";"[symbol table; list data]";"null"); - (`subscribe;1b;"register a subscriber for tables/syms (delegates to di.pubsub)";"[symbol|list tables; filters]";"subscription result"); - (`endofday;1b;"flush, notify subscribers, roll the log and advance eod state";"[]";"null"); - (`getcounts;1b;"current log/buffer message counts and trading date";"[]";"dict `i`j`d"); - (`gettables;1b;"the tables this tickerplant captures";"[]";"symbol list of table names")); + (`upd; 1b; "feed entry point - stamp, log and publish (or buffer) an update"; + "[symbol: table; list: column data]"; "null"); + (`subscribe; 1b; "register a subscriber for tables/syms - the kdb+tick .u.sub entry point"; + "[symbol(list): tables (` for all); filters]"; "list: (tables;schemas), or an error symbol"); + (`subdetails; 1b; "subscribe and return the schemas, log details and counts di.subscriptions needs"; + "[symbol(list): tables (` for all); symbol(list)|dict: syms (` for all)]"; + "dict: schemalist, logfilelist, rowcounts and date"); + (`tablelist; 1b; "the tables offered for subscription, so a ` (all tables) request can be resolved"; + "[ignored - unary because di.subscriptions sends (`tablelist;`)]"; "symbol list: table names"); + (`endofday; 1b; "flush, notify subscribers, roll the log and advance eod state"; + "[]"; "null"); + (`teardown; 1b; "remove the root subscription protocol and the timer job installed by init"; + "[]"; "null"); + (`getcounts; 1b; "published (i) and logged (j) message counts, and the trading date"; + "[]"; "dict: i, j and d"); + (`gettables; 1b; "the tables this tickerplant captures"; + "[]"; "symbol list: table names")); };