From 865bb7343241c9905b19331a5317c30aadbe6d27 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 7 Aug 2026 11:02:41 -0400 Subject: [PATCH 1/5] Avoid non-termination that results from in-band progress tracking --- dogsdogsdogs/examples/dogsdogsdogs.rs | 14 +++---- dogsdogsdogs/src/lib.rs | 60 ++++++++++----------------- 2 files changed, 30 insertions(+), 44 deletions(-) diff --git a/dogsdogsdogs/examples/dogsdogsdogs.rs b/dogsdogsdogs/examples/dogsdogsdogs.rs index 110ec3922..9e3a542eb 100644 --- a/dogsdogsdogs/examples/dogsdogsdogs.rs +++ b/dogsdogsdogs/examples/dogsdogsdogs.rs @@ -31,17 +31,17 @@ fn main() { println!("loaded {} nodes, {} edges", nodes, edges.len()); - let index = worker.dataflow::(|scope| { - CollectionIndex::index(Collection::new(edges.to_stream(scope))) - }); - - let mut index_xz = index.extend_using(|&(ref x, ref _y)| *x); - let mut index_yz = index.extend_using(|&(ref _x, ref y)| *y); - let mut probe = Handle::new(); + // The index and its readers must share a dataflow: the extenders hold scope-bound + // arrangements rather than exported traces. let mut edges = worker.dataflow::(|scope| { + let index = CollectionIndex::index(Collection::new(edges.to_stream(scope))); + + let mut index_xz = index.extend_using(|&(ref x, ref _y)| *x); + let mut index_yz = index.extend_using(|&(ref _x, ref y)| *y); + let (edges_input, edges) = scope.new_collection(); // determine stream of (prefix, count, index) indicating relation with fewest extensions. diff --git a/dogsdogsdogs/src/lib.rs b/dogsdogsdogs/src/lib.rs index ce484e3b7..b370d0d6d 100644 --- a/dogsdogsdogs/src/lib.rs +++ b/dogsdogsdogs/src/lib.rs @@ -91,11 +91,18 @@ impl<'scope, T: Timestamp, R: Monoid+Multiply, P, E> ValidateExtensi } // These are all defined here so that users can be assured a common layout. +use differential_dataflow::operators::arrange::Arranged; use differential_dataflow::trace::implementations::{KeySpine, ValSpine}; type TraceValHandle = TraceAgent>; type TraceKeyHandle = TraceAgent>; -pub struct CollectionIndex +/// The three arrangements a relation must present to extend prefixes. +/// +/// The arrangements are scope-bound rather than exported traces, so that the operators +/// reading them observe timely's own progress tracking. An imported trace instead reports +/// its frontier in-band, and in a cycle those statements circulate without ever settling. +#[derive(Clone)] +pub struct CollectionIndex<'scope, K, V, T, R> where K: ExchangeData, V: ExchangeData, @@ -103,40 +110,23 @@ where R: Monoid+Multiply+ExchangeData, { /// A trace of type (K, ()), used to count extensions for each prefix. - count_trace: TraceKeyHandle, + count_trace: Arranged<'scope, TraceKeyHandle>, /// A trace of type (K, V), used to propose extensions for each prefix. - propose_trace: TraceValHandle, + propose_trace: Arranged<'scope, TraceValHandle>, /// A trace of type ((K, V), ()), used to validate proposed extensions. - validate_trace: TraceKeyHandle<(K, V), T, R>, + validate_trace: Arranged<'scope, TraceKeyHandle<(K, V), T, R>>, } -impl Clone for CollectionIndex +impl<'scope, K, V, T, R> CollectionIndex<'scope, K, V, T, R> where K: ExchangeData+Hash, V: ExchangeData+Hash, T: Lattice+ExchangeData+Timestamp, R: Monoid+Multiply+ExchangeData, { - fn clone(&self) -> Self { - CollectionIndex { - count_trace: self.count_trace.clone(), - propose_trace: self.propose_trace.clone(), - validate_trace: self.validate_trace.clone(), - } - } -} - -impl CollectionIndex -where - K: ExchangeData+Hash, - V: ExchangeData+Hash, - T: Lattice+ExchangeData+Timestamp, - R: Monoid+Multiply+ExchangeData, -{ - - pub fn index<'scope>(collection: VecCollection<'scope, T, (K, V), R>) -> Self { + pub fn index(collection: VecCollection<'scope, T, (K, V), R>) -> Self { // We need to count the number of (k, v) pairs and not rely on the given Monoid R and its binary addition operation. // counts and validate can share the base arrangement let arranged = collection.clone().arrange_by_self(); @@ -146,10 +136,9 @@ where .as_collection(|k,_v| k.clone()) .distinct() .map(|(k, _v)| k) - .arrange_by_self() - .trace; - let propose = collection.arrange_by_key().trace; - let validate = arranged.trace; + .arrange_by_self(); + let propose = collection.arrange_by_key(); + let validate = arranged; CollectionIndex { count_trace: counts, @@ -157,7 +146,7 @@ where validate_trace: validate, } } - pub fn extend_usingK+Clone>(&self, logic: F) -> CollectionExtender { + pub fn extend_usingK+Clone>(&self, logic: F) -> CollectionExtender<'scope, K, V, T, R, P, F> { CollectionExtender { phantom: std::marker::PhantomData, indices: self.clone(), @@ -166,7 +155,7 @@ where } } -pub struct CollectionExtender +pub struct CollectionExtender<'scope, K, V, T, R, P, F> where K: ExchangeData, V: ExchangeData, @@ -175,11 +164,11 @@ where F: Fn(&P)->K+Clone, { phantom: std::marker::PhantomData

, - indices: CollectionIndex, + indices: CollectionIndex<'scope, K, V, T, R>, key_selector: F, } -impl<'scope, T, K, V, R, P, F> PrefixExtender<'scope, T, R> for CollectionExtender +impl<'scope, T, K, V, R, P, F> PrefixExtender<'scope, T, R> for CollectionExtender<'scope, K, V, T, R, P, F> where T: Timestamp + Lattice + ExchangeData + Hash, K: ExchangeData+Hash+Default, @@ -192,17 +181,14 @@ where type Extension = V; fn count(&mut self, prefixes: VecCollection<'scope, T, (P, usize, usize), R>, index: usize) -> VecCollection<'scope, T, (P, usize, usize), R> { - let counts = self.indices.count_trace.import(prefixes.scope()); - operators::count::count(prefixes, counts, self.key_selector.clone(), index) + operators::count::count(prefixes, self.indices.count_trace.clone(), self.key_selector.clone(), index) } fn propose(&mut self, prefixes: VecCollection<'scope, T, P, R>) -> VecCollection<'scope, T, (P, V), R> { - let propose = self.indices.propose_trace.import(prefixes.scope()); - operators::propose::propose(prefixes, propose, self.key_selector.clone()) + operators::propose::propose(prefixes, self.indices.propose_trace.clone(), self.key_selector.clone()) } fn validate(&mut self, extensions: VecCollection<'scope, T, (P, V), R>) -> VecCollection<'scope, T, (P, V), R> { - let validate = self.indices.validate_trace.import(extensions.scope()); - operators::validate::validate(extensions, validate, self.key_selector.clone()) + operators::validate::validate(extensions, self.indices.validate_trace.clone(), self.key_selector.clone()) } } From 148223607b2b257d2ac781ed5af401316702ff70 Mon Sep 17 00:00:00 2001 From: Frank McSherry Date: Fri, 7 Aug 2026 15:17:10 -0400 Subject: [PATCH 2/5] Retarget count/propose/validate onto half_join The three shared `lookup_map`, which matched arranged updates at times less-or-equal in the *partial* order, and emitted at the update's own time. That is a correct lookup but not a correct delta join. Pairs at incomparable times never met, and a chained stage compared against the previous stage's time rather than the original. They now sit on half_join, which compares in a caller-supplied total order and carries the original time beside a payload time advanced by lattice join. `propose` and `validate` are wrappers over the safe entry point. `count` uses the unsafe one, to sum a scalar and annotate the prefix without joining times. `lookup_map` and `propose_distinct` are deleted; the latter proposed presence without maintaining an indicator to test it against. Differences are pinned to `isize` and the `Multiply` bounds dropped. BiGJoin (Ammar et al., VLDB 2018) is stated over sets: its extension indices are set-valued and its intersection step is an existence test. Above arity two an extension index is a projection, whose multiplicity counts completions rather than the record's own annotation, so carrying annotations would need indicator projections and a contributes-once rule. That is InsideOut, a different algorithm with different indices. The set expectation is documented rather than enforced, so callers whose data are already distinct pay nothing. `count` routes a prefix with a zero count rather than dropping it, because the sum spans an interval and zero there does not mean empty at every time within it. It sums absolute values, since magnitude carries the extension count and polarity only churn. Co-Authored-By: Claude Opus 5 (1M context) --- dogsdogsdogs/examples/delta_query.rs | 38 ++-- dogsdogsdogs/examples/delta_query_wcoj.rs | 41 +++-- dogsdogsdogs/examples/dogsdogsdogs.rs | 22 ++- dogsdogsdogs/src/lib.rs | 183 ++++++++++++++----- dogsdogsdogs/src/operators/count.rs | 90 ++++++--- dogsdogsdogs/src/operators/lookup_map.rs | 145 --------------- dogsdogsdogs/src/operators/mod.rs | 4 +- dogsdogsdogs/src/operators/propose.rs | 92 ++++------ dogsdogsdogs/src/operators/validate.rs | 62 ++++--- dogsdogsdogs/tests/lookup_map_regression.rs | 108 ----------- dogsdogsdogs/tests/wcoj_triangle.rs | 193 ++++++++++++++++++++ 11 files changed, 538 insertions(+), 440 deletions(-) delete mode 100644 dogsdogsdogs/src/operators/lookup_map.rs delete mode 100644 dogsdogsdogs/tests/lookup_map_regression.rs create mode 100644 dogsdogsdogs/tests/wcoj_triangle.rs diff --git a/dogsdogsdogs/examples/delta_query.rs b/dogsdogsdogs/examples/delta_query.rs index bf5388fce..d35b18713 100644 --- a/dogsdogsdogs/examples/delta_query.rs +++ b/dogsdogsdogs/examples/delta_query.rs @@ -1,4 +1,6 @@ use timely::dataflow::operators::probe::Handle; +use timely::dataflow::operators::vec::Map; +use differential_dataflow::AsCollection; use differential_dataflow::input::Input; use graph_map::GraphMMap; @@ -73,23 +75,37 @@ fn main() { use differential_dogs3::operators::propose; use differential_dogs3::operators::validate; + // Hold compaction back one base-time step, so the alt/neu distinction survives, + // and compare in the total order on `AltNeu` (lexicographic on `(time, neu)`). + let frontier_func = |time: &AltNeu, antichain: &mut timely::progress::Antichain>| { + antichain.insert(AltNeu::alt(time.time.saturating_sub(1))); + }; + let comparison = |t1: &AltNeu, t2: &AltNeu| t1 <= t2; + + // Stash each delta's own time as its payload, to be advanced by the times of + // the records it matches, and delayed to once we leave the delta region. + let deltas = changes.inner.map(|(d, t, r)| ((d, t.clone()), t, r)).as_collection(); + // Prior technology // dQ/dE1 := dE1(a,b), E2(b,c), E3(a,c) - let changes1 = propose(changes.clone(), forward_key_neu.clone(), key2.clone()); - let changes1 = validate(changes1, forward_self_neu.clone(), key1.clone()); - let changes1 = changes1.map(|((a,b),c)| (a,b,c)); + let changes1 = propose(deltas.clone(), forward_key_neu.clone(), key2.clone(), frontier_func, comparison); + let changes1 = validate(changes1, forward_self_neu.clone(), key1.clone(), frontier_func, comparison); + let changes1 = changes1.map(|(((a,b),c), payload)| ((a,b,c), payload)); // dQ/dE2 := dE2(b,c), E1(a,b), E3(a,c) - let changes2 = propose(changes.clone(), reverse_key_alt.clone(), key1.clone()); - let changes2 = validate(changes2, reverse_self_neu.clone(), key2.clone()); - let changes2 = changes2.map(|((b,c),a)| (a,b,c)); + let changes2 = propose(deltas.clone(), reverse_key_alt.clone(), key1.clone(), frontier_func, comparison); + let changes2 = validate(changes2, reverse_self_neu.clone(), key2.clone(), frontier_func, comparison); + let changes2 = changes2.map(|(((b,c),a), payload)| ((a,b,c), payload)); // dQ/dE3 := dE3(a,c), E1(a,b), E2(b,c) - let changes3 = propose(changes, forward_key_alt.clone(), key1.clone()); - let changes3 = validate(changes3, reverse_self_alt.clone(), key2.clone()); - let changes3 = changes3.map(|((a,c),b)| (a,b,c)); - - let prev_changes = changes1.concat(changes2).concat(changes3).leave(scope); + let changes3 = propose(deltas, forward_key_alt.clone(), key1.clone(), frontier_func, comparison); + let changes3 = validate(changes3, reverse_self_alt.clone(), key2.clone(), frontier_func, comparison); + let changes3 = changes3.map(|(((a,c),b), payload)| ((a,b,c), payload)); + + // Delay updates to the payload time worked out while extending. + let prev_changes = changes1.concat(changes2).concat(changes3) + .inner.map(|((d, payload), _time, r)| (d, payload, r)).as_collection() + .leave(scope); // New ideas let d_edges = edges.differentiate(inner); diff --git a/dogsdogsdogs/examples/delta_query_wcoj.rs b/dogsdogsdogs/examples/delta_query_wcoj.rs index cba34a71f..6539f8ce6 100644 --- a/dogsdogsdogs/examples/delta_query_wcoj.rs +++ b/dogsdogsdogs/examples/delta_query_wcoj.rs @@ -1,4 +1,6 @@ use timely::dataflow::operators::probe::Handle; +use timely::dataflow::operators::vec::Map; +use differential_dataflow::AsCollection; use differential_dataflow::input::Input; use graph_map::GraphMMap; @@ -36,11 +38,25 @@ fn main() { let forward = forward.enter(inner); let reverse = reverse.enter(inner); + // Hold compaction back one base-time step, so that the alt/neu distinction + // survives, and compare in the total order on `AltNeu` (lexicographic on + // `(time, neu)`). Delta streams enter at `alt`, so an `alt` arrangement is + // seen at the delta's own time and a `neu` one only strictly after it: the + // tag is what makes the comparison strict or not. + let frontier_func = |time: &AltNeu, antichain: &mut timely::progress::Antichain>| { + antichain.insert(AltNeu::alt(time.time.saturating_sub(1))); + }; + let comparison = |t1: &AltNeu, t2: &AltNeu| t1 <= t2; + // Without using wrappers yet, maintain an "old" and a "new" copy of edges. - let alt_forward = CollectionIndex::index(forward.clone()); - let alt_reverse = CollectionIndex::index(reverse.clone()); - let neu_forward = CollectionIndex::index(forward.clone().delay(|time| AltNeu::neu(time.time.clone()))); - let neu_reverse = CollectionIndex::index(reverse.clone().delay(|time| AltNeu::neu(time.time.clone()))); + let alt_forward = CollectionIndex::index(forward.clone(), frontier_func, comparison); + let alt_reverse = CollectionIndex::index(reverse.clone(), frontier_func, comparison); + let neu_forward = CollectionIndex::index(forward.clone().delay(|time| AltNeu::neu(time.time.clone())), frontier_func, comparison); + let neu_reverse = CollectionIndex::index(reverse.clone().delay(|time| AltNeu::neu(time.time.clone())), frontier_func, comparison); + + // Stash each delta's own time as its payload, to be advanced by the times of + // the records it matches, and delayed to once we leave the delta region. + let deltas = forward.inner.map(|(d, t, r)| ((d, t.clone()), t, r)).as_collection(); // For each relation, we form a delta query driven by changes to that relation. // @@ -53,33 +69,36 @@ fn main() { // dQ/dE1 := dE1(a,b), E2(b,c), E3(a,c) let changes1 = - forward + deltas .clone() .extend(&mut [ &mut neu_forward.extend_using(|(_a,b)| *b), &mut neu_forward.extend_using(|(a,_b)| *a), ]) - .map(|((a,b),c)| (a,b,c)); + .map(|(((a,b),c), payload)| ((a,b,c), payload)); // dQ/dE2 := dE2(b,c), E1(a,b), E3(a,c) let changes2 = - forward + deltas .clone() .extend(&mut [ &mut alt_reverse.extend_using(|(b,_c)| *b), &mut neu_reverse.extend_using(|(_b,c)| *c), ]) - .map(|((b,c),a)| (a,b,c)); + .map(|(((b,c),a), payload)| ((a,b,c), payload)); // dQ/dE3 := dE3(a,c), E1(a,b), E2(b,c) - let changes3 = forward + let changes3 = deltas .extend(&mut [ &mut alt_forward.extend_using(|(a,_c)| *a), &mut alt_reverse.extend_using(|(_a,c)| *c), ]) - .map(|((a,c),b)| (a,b,c)); + .map(|(((a,c),b), payload)| ((a,b,c), payload)); - changes1.concat(changes2).concat(changes3).leave(scope) + // Delay updates to the payload time worked out while extending. + changes1.concat(changes2).concat(changes3) + .inner.map(|((d, payload), _time, r)| (d, payload, r)).as_collection() + .leave(scope) }); triangles diff --git a/dogsdogsdogs/examples/dogsdogsdogs.rs b/dogsdogsdogs/examples/dogsdogsdogs.rs index 9e3a542eb..58822152b 100644 --- a/dogsdogsdogs/examples/dogsdogsdogs.rs +++ b/dogsdogsdogs/examples/dogsdogsdogs.rs @@ -1,4 +1,4 @@ -use timely::dataflow::operators::{ToStream, vec::{Partition, count::Accumulate}, Inspect, Probe}; +use timely::dataflow::operators::{ToStream, vec::{Map, Partition, count::Accumulate}, Inspect, Probe}; use timely::dataflow::operators::probe::Handle; use differential_dataflow::{Collection, AsCollection}; use differential_dataflow::input::Input; @@ -37,20 +37,31 @@ fn main() { // arrangements rather than exported traces. let mut edges = worker.dataflow::(|scope| { - let index = CollectionIndex::index(Collection::new(edges.to_stream(scope))); + // Hold compaction back one step, and let a prefix see arranged updates at its own + // time and earlier. The index is static here, so every prefix sees all of it. + let frontier_func = |time: &usize, antichain: &mut timely::progress::Antichain| { + antichain.insert(time.saturating_sub(1)); + }; + let comparison = |t1: &usize, t2: &usize| t1 <= t2; + + let index = CollectionIndex::index(Collection::new(edges.to_stream(scope)), frontier_func, comparison); let mut index_xz = index.extend_using(|&(ref x, ref _y)| *x); let mut index_yz = index.extend_using(|&(ref _x, ref y)| *y); let (edges_input, edges) = scope.new_collection(); + // Stash each prefix's own time as its payload, to be advanced by the times of the + // records it matches, and delayed to once we are done extending. + let prefixes = edges.inner.map(|(p, t, r): ((u32, u32), usize, isize)| ((p, t.clone()), t, r)).as_collection(); + // determine stream of (prefix, count, index) indicating relation with fewest extensions. - let counts = edges.map(|p| (p, usize::MAX, usize::MAX)); + let counts = prefixes.map(|(p, payload)| ((p, usize::MAX, usize::MAX), payload)); let counts0 = index_xz.count(counts, 0); let counts1 = index_yz.count(counts0, 1); // partition by index. - let parts = counts1.inner.partition(2, |((p, _c, i),t,d)| (i as u64,(p,t,d))); + let parts = counts1.inner.partition(2, |(((p, _c, i), payload),t,d)| (i as u64,((p, payload),t,d))); // propose extensions using relation based on index. let propose0 = index_xz.propose(parts[0].clone().as_collection()); @@ -62,7 +73,8 @@ fn main() { validate0 .concat(validate1) - .inner + // Delay updates to the payload time worked out while extending. + .inner.map(|((extended, payload), _time, r)| (extended, payload, r)) .count() .inspect(move |x| println!("{:?}", x)) // .inspect(move |x| println!("{:?}:\t{:?}", timer.elapsed(), x)) diff --git a/dogsdogsdogs/src/lib.rs b/dogsdogsdogs/src/lib.rs index b370d0d6d..afedd39cd 100644 --- a/dogsdogsdogs/src/lib.rs +++ b/dogsdogsdogs/src/lib.rs @@ -1,11 +1,41 @@ +//! Worst-case optimal joins as differential dataflows. +//! +//! This crate implements the BiGJoin / Delta-GJ algorithms of Ammar, McSherry, Salihoglu and +//! Joglekar, "Distributed Evaluation of Subgraph Queries Using Worst-case Optimal and Low-Memory +//! Dataflows" (VLDB 2018). Prefixes are extended one attribute at a time: each relation binding +//! the next attribute reports how many extensions it would propose, the smallest proposes them, +//! and the others intersect against their own extensions. +//! +//! # Set semantics +//! +//! The algorithm is stated over *sets*. Its extension indices are set-valued, its intersection +//! step is an existence test, and its count minimization ranks by set cardinality. Accordingly, +//! the differences here are `isize` and the relations are **expected to be sets**: each record +//! present with multiplicity one. +//! +//! This expectation is documented rather than enforced, because a caller whose data are already +//! distinct should not pay for a `distinct()` that does nothing. A caller who is unsure should +//! apply `distinct()` to the collection before handing it to [`CollectionIndex::index`]. +//! +//! Feeding a multiset in does not produce the multiset join. `propose` and `validate` multiply +//! the matched record's multiplicity into the output, so multiplicities scale rather than filter, +//! and `count` reports set cardinalities that no longer describe the proposals. Nor is this +//! repairable by adjusting the operators: with relations of arity above two, an extension index +//! is a *projection* of its relation, and a projection's multiplicity is a count of completions +//! rather than the record's own annotation. Carrying annotations correctly through a worst-case +//! optimal join requires indicator projections and a rule that each relation contributes its +//! annotation exactly once, when its last attribute binds — that is InsideOut (Abo Khamis, Ngo, +//! Rudra, "FAQ: Questions Asked Frequently", PODS 2016), a different algorithm with different +//! indices, not a tuning of this one. + use std::hash::Hash; +use std::rc::Rc; -use timely::progress::Timestamp; +use timely::progress::{Antichain, Timestamp}; use timely::dataflow::operators::vec::Partition; use timely::dataflow::operators::Concatenate; use differential_dataflow::{ExchangeData, VecCollection, AsCollection}; -use differential_dataflow::difference::{Monoid, Multiply}; use differential_dataflow::lattice::Lattice; use differential_dataflow::operators::arrange::TraceAgent; @@ -13,43 +43,60 @@ pub mod altneu; pub mod calculus; pub mod operators; +/// Holds back logical compaction so that total-order time comparisons stay meaningful. +/// +/// Conventional compaction collapses unequal times to the frontier, which would lose the +/// distinction between "strictly before" and "at the same time" that the delta discipline +/// rests on. See [`crate::operators::half_join`]. +pub type FrontierFunc = Rc)>; + +/// Compares an arranged record's time against a prefix's own time, in the *total order*. +/// +/// The two useful choices are strictly-less and less-or-equal, which is how a delta query +/// decides whether a stage sees updates concurrent with the delta it is responding to. +pub type Comparison = Rc bool>; + /// A type capable of extending a stream of prefixes. /// /** Implementors of `PrefixExtension` provide types and methods for extending a differential dataflow collection, via the three methods `count`, `propose`, and `validate`. + + Each prefix travels with a payload time alongside it. The payload starts as the prefix's own + time, accumulates the times of the records it matches, and is delayed to once the delta region + is left. The update itself stays at the time it entered on, which is what lets the total-order + comparison decide exactly once which stage produces each output. **/ -pub trait PrefixExtender<'scope, T: Timestamp, R: Monoid+Multiply> { +pub trait PrefixExtender<'scope, T: Timestamp> { /// The required type of prefix to extend. type Prefix; /// The type to be produced as extension. type Extension; /// Annotates prefixes with the number of extensions the relation would propose. - fn count(&mut self, prefixes: VecCollection<'scope, T, (Self::Prefix, usize, usize), R>, index: usize) -> VecCollection<'scope, T, (Self::Prefix, usize, usize), R>; + fn count(&mut self, prefixes: VecCollection<'scope, T, ((Self::Prefix, usize, usize), T), isize>, index: usize) -> VecCollection<'scope, T, ((Self::Prefix, usize, usize), T), isize>; /// Extends each prefix with corresponding extensions. - fn propose(&mut self, prefixes: VecCollection<'scope, T, Self::Prefix, R>) -> VecCollection<'scope, T, (Self::Prefix, Self::Extension), R>; + fn propose(&mut self, prefixes: VecCollection<'scope, T, (Self::Prefix, T), isize>) -> VecCollection<'scope, T, ((Self::Prefix, Self::Extension), T), isize>; /// Restricts proposed extensions by those the extender would have proposed. - fn validate(&mut self, extensions: VecCollection<'scope, T, (Self::Prefix, Self::Extension), R>) -> VecCollection<'scope, T, (Self::Prefix, Self::Extension), R>; + fn validate(&mut self, extensions: VecCollection<'scope, T, ((Self::Prefix, Self::Extension), T), isize>) -> VecCollection<'scope, T, ((Self::Prefix, Self::Extension), T), isize>; } -pub trait ProposeExtensionMethod<'scope, T: Timestamp, P: ExchangeData+Ord, R: Monoid+Multiply> { - fn propose_using>(self, extender: &mut PE) -> VecCollection<'scope, T, (P, PE::Extension), R>; - fn extend(self, extenders: &mut [&mut dyn PrefixExtender<'scope, T,R,Prefix=P,Extension=E>]) -> VecCollection<'scope, T, (P, E), R>; +pub trait ProposeExtensionMethod<'scope, T: Timestamp, P: ExchangeData+Ord> { + fn propose_using>(self, extender: &mut PE) -> VecCollection<'scope, T, ((P, PE::Extension), T), isize>; + fn extend(self, extenders: &mut [&mut dyn PrefixExtender<'scope, T, Prefix=P, Extension=E>]) -> VecCollection<'scope, T, ((P, E), T), isize>; } -impl<'scope, T, P, R> ProposeExtensionMethod<'scope, T, P, R> for VecCollection<'scope, T, P, R> +impl<'scope, T, P> ProposeExtensionMethod<'scope, T, P> for VecCollection<'scope, T, (P, T), isize> where T: Timestamp, P: ExchangeData+Ord, - R: Monoid+Multiply+'static, { - fn propose_using(self, extender: &mut PE) -> VecCollection<'scope, T, (P, PE::Extension), R> + fn propose_using(self, extender: &mut PE) -> VecCollection<'scope, T, ((P, PE::Extension), T), isize> where - PE: PrefixExtender<'scope, T, R, Prefix=P> + PE: PrefixExtender<'scope, T, Prefix=P> { extender.propose(self) } - fn extend(self, extenders: &mut [&mut dyn PrefixExtender<'scope, T,R,Prefix=P,Extension=E>]) -> VecCollection<'scope, T, (P, E), R> + fn extend(self, extenders: &mut [&mut dyn PrefixExtender<'scope, T, Prefix=P, Extension=E>]) -> VecCollection<'scope, T, ((P, E), T), isize> where E: ExchangeData+Ord { @@ -58,12 +105,12 @@ where extenders[0].propose(self) } else { - let mut counts = self.clone().map(|p| (p, 1 << 31, 0)); + let mut counts = self.clone().map(|(p, payload)| ((p, 1 << 31, 0), payload)); for (index,extender) in extenders.iter_mut().enumerate() { counts = extender.count(counts, index); } - let parts = counts.inner.partition(extenders.len() as u64, |((p, _, i),t,d)| (i as u64, (p,t,d))); + let parts = counts.inner.partition(extenders.len() as u64, |(((p, _, i), payload),t,d)| (i as u64, ((p, payload),t,d))); let mut results = Vec::new(); for (index, nominations) in parts.into_iter().enumerate() { @@ -80,12 +127,12 @@ where } } -pub trait ValidateExtensionMethod<'scope, T: Timestamp, R: Monoid+Multiply, P, E> { - fn validate_using>(self, extender: &mut PE) -> VecCollection<'scope, T, (P, E), R>; +pub trait ValidateExtensionMethod<'scope, T: Timestamp, P, E> { + fn validate_using>(self, extender: &mut PE) -> VecCollection<'scope, T, ((P, E), T), isize>; } -impl<'scope, T: Timestamp, R: Monoid+Multiply, P, E> ValidateExtensionMethod<'scope, T, R, P, E> for VecCollection<'scope, T, (P, E), R> { - fn validate_using>(self, extender: &mut PE) -> VecCollection<'scope, T, (P, E), R> { +impl<'scope, T: Timestamp, P, E> ValidateExtensionMethod<'scope, T, P, E> for VecCollection<'scope, T, ((P, E), T), isize> { + fn validate_using>(self, extender: &mut PE) -> VecCollection<'scope, T, ((P, E), T), isize> { extender.validate(self) } } @@ -93,40 +140,67 @@ impl<'scope, T: Timestamp, R: Monoid+Multiply, P, E> ValidateExtensi // These are all defined here so that users can be assured a common layout. use differential_dataflow::operators::arrange::Arranged; use differential_dataflow::trace::implementations::{KeySpine, ValSpine}; -type TraceValHandle = TraceAgent>; -type TraceKeyHandle = TraceAgent>; +type TraceValHandle = TraceAgent>; +type TraceKeyHandle = TraceAgent>; /// The three arrangements a relation must present to extend prefixes. /// /// The arrangements are scope-bound rather than exported traces, so that the operators /// reading them observe timely's own progress tracking. An imported trace instead reports /// its frontier in-band, and in a cycle those statements circulate without ever settling. -#[derive(Clone)] -pub struct CollectionIndex<'scope, K, V, T, R> +/// +/// The indexed collection is expected to be a set; see the note on set semantics in [`crate`]. +pub struct CollectionIndex<'scope, K, V, T> where K: ExchangeData, V: ExchangeData, T: Lattice+ExchangeData+Timestamp, - R: Monoid+Multiply+ExchangeData, { /// A trace of type (K, ()), used to count extensions for each prefix. - count_trace: Arranged<'scope, TraceKeyHandle>, + count_trace: Arranged<'scope, TraceKeyHandle>, /// A trace of type (K, V), used to propose extensions for each prefix. - propose_trace: Arranged<'scope, TraceValHandle>, + propose_trace: Arranged<'scope, TraceValHandle>, /// A trace of type ((K, V), ()), used to validate proposed extensions. - validate_trace: Arranged<'scope, TraceKeyHandle<(K, V), T, R>>, + validate_trace: Arranged<'scope, TraceKeyHandle<(K, V), T>>, + + /// Holds back compaction; see [`FrontierFunc`]. + frontier_func: FrontierFunc, + + /// Decides which arranged times a prefix sees; see [`Comparison`]. + comparison: Comparison, } -impl<'scope, K, V, T, R> CollectionIndex<'scope, K, V, T, R> +impl<'scope, K, V, T> Clone for CollectionIndex<'scope, K, V, T> +where + K: ExchangeData, + V: ExchangeData, + T: Lattice+ExchangeData+Timestamp, +{ + fn clone(&self) -> Self { + CollectionIndex { + count_trace: self.count_trace.clone(), + propose_trace: self.propose_trace.clone(), + validate_trace: self.validate_trace.clone(), + frontier_func: Rc::clone(&self.frontier_func), + comparison: Rc::clone(&self.comparison), + } + } +} + +impl<'scope, K, V, T> CollectionIndex<'scope, K, V, T> where K: ExchangeData+Hash, V: ExchangeData+Hash, T: Lattice+ExchangeData+Timestamp, - R: Monoid+Multiply+ExchangeData, { - pub fn index(collection: VecCollection<'scope, T, (K, V), R>) -> Self { + + pub fn index(collection: VecCollection<'scope, T, (K, V), isize>, frontier_func: FF, comparison: CF) -> Self + where + FF: Fn(&T, &mut Antichain) + 'static, + CF: Fn(&T, &T) -> bool + 'static, + { // We need to count the number of (k, v) pairs and not rely on the given Monoid R and its binary addition operation. // counts and validate can share the base arrangement let arranged = collection.clone().arrange_by_self(); @@ -144,9 +218,11 @@ where count_trace: counts, propose_trace: propose, validate_trace: validate, + frontier_func: Rc::new(frontier_func), + comparison: Rc::new(comparison), } } - pub fn extend_usingK+Clone>(&self, logic: F) -> CollectionExtender<'scope, K, V, T, R, P, F> { + pub fn extend_usingK+Clone>(&self, logic: F) -> CollectionExtender<'scope, K, V, T, P, F> { CollectionExtender { phantom: std::marker::PhantomData, indices: self.clone(), @@ -155,40 +231,57 @@ where } } -pub struct CollectionExtender<'scope, K, V, T, R, P, F> +pub struct CollectionExtender<'scope, K, V, T, P, F> where K: ExchangeData, V: ExchangeData, T: Lattice+ExchangeData+Timestamp, - R: Monoid+Multiply+ExchangeData, F: Fn(&P)->K+Clone, { phantom: std::marker::PhantomData

, - indices: CollectionIndex<'scope, K, V, T, R>, + indices: CollectionIndex<'scope, K, V, T>, key_selector: F, } -impl<'scope, T, K, V, R, P, F> PrefixExtender<'scope, T, R> for CollectionExtender<'scope, K, V, T, R, P, F> +impl<'scope, K, V, T, P, F> CollectionExtender<'scope, K, V, T, P, F> +where + K: ExchangeData, + V: ExchangeData, + T: Lattice+ExchangeData+Timestamp, + F: Fn(&P)->K+Clone, +{ + /// The index's time closures, as plain callables the operators can accept. + fn time_logic(&self) -> (impl Fn(&T, &mut Antichain) + 'static, impl Fn(&T, &T) -> bool + 'static) { + let frontier_func = Rc::clone(&self.indices.frontier_func); + let comparison = Rc::clone(&self.indices.comparison); + (move |t: &T, a: &mut Antichain| frontier_func(t, a), move |t1: &T, t2: &T| comparison(t1, t2)) + } +} + +impl<'scope, T, K, V, P, F> PrefixExtender<'scope, T> for CollectionExtender<'scope, K, V, T, P, F> where T: Timestamp + Lattice + ExchangeData + Hash, - K: ExchangeData+Hash+Default, - V: ExchangeData+Hash+Default, + K: ExchangeData+Hash, + V: ExchangeData+Hash, P: ExchangeData, - R: Monoid+Multiply+ExchangeData, F: Fn(&P)->K+Clone+'static, { type Prefix = P; type Extension = V; - fn count(&mut self, prefixes: VecCollection<'scope, T, (P, usize, usize), R>, index: usize) -> VecCollection<'scope, T, (P, usize, usize), R> { - operators::count::count(prefixes, self.indices.count_trace.clone(), self.key_selector.clone(), index) + fn count(&mut self, prefixes: VecCollection<'scope, T, ((P, usize, usize), T), isize>, index: usize) -> VecCollection<'scope, T, ((P, usize, usize), T), isize> { + let (frontier_func, comparison) = self.time_logic(); + operators::count::count(prefixes, self.indices.count_trace.clone(), self.key_selector.clone(), index, frontier_func, comparison) } - fn propose(&mut self, prefixes: VecCollection<'scope, T, P, R>) -> VecCollection<'scope, T, (P, V), R> { - operators::propose::propose(prefixes, self.indices.propose_trace.clone(), self.key_selector.clone()) + fn propose(&mut self, prefixes: VecCollection<'scope, T, (P, T), isize>) -> VecCollection<'scope, T, ((P, V), T), isize> { + let (frontier_func, comparison) = self.time_logic(); + operators::propose::propose(prefixes, self.indices.propose_trace.clone(), self.key_selector.clone(), frontier_func, comparison) } - fn validate(&mut self, extensions: VecCollection<'scope, T, (P, V), R>) -> VecCollection<'scope, T, (P, V), R> { - operators::validate::validate(extensions, self.indices.validate_trace.clone(), self.key_selector.clone()) + fn validate(&mut self, extensions: VecCollection<'scope, T, ((P, V), T), isize>) -> VecCollection<'scope, T, ((P, V), T), isize> { + let (frontier_func, comparison) = self.time_logic(); + let key_selector = self.key_selector.clone(); + operators::validate::validate(extensions, self.indices.validate_trace.clone(), key_selector, frontier_func, comparison) } } diff --git a/dogsdogsdogs/src/operators/count.rs b/dogsdogsdogs/src/operators/count.rs index c46be8c25..44756dd67 100644 --- a/dogsdogsdogs/src/operators/count.rs +++ b/dogsdogsdogs/src/operators/count.rs @@ -1,41 +1,75 @@ -use differential_dataflow::{ExchangeData, VecCollection, Hashable}; -use differential_dataflow::difference::{Semigroup, Monoid, Multiply}; +use timely::container::CapacityContainerBuilder; +use timely::container::PushInto; +use timely::progress::Antichain; + +use differential_dataflow::{AsCollection, ExchangeData, VecCollection, Hashable}; +use differential_dataflow::difference::Monoid; use differential_dataflow::operators::arrange::Arranged; -use differential_dataflow::trace::{BatchCursor, BatchDiff, BatchDiffGat, Cursor, Navigable, TraceReader}; +use differential_dataflow::trace::{BatchCursor, BatchTimeGat, BatchVal, Cursor, Navigable, TraceReader}; +use differential_dataflow::trace::implementations::BatchContainer; -/// Reports a number of extensions to a stream of prefixes. +/// Updates a stream of prefix routing judgements based on approximate counts. +/// +/// Each prefix observes the changes in distinct values over time, and treats this as a lower +/// bound on the count that will be experienced. When the lower bound improves on the routing +/// judgement's current count, it is overwritten and the `index` argument is substituted in. /// -/// This method takes as input a stream of `(prefix, count, index)` triples. -/// For each triple, it extracts a key using `key_selector`, and finds the -/// associated count in `arrangement`. If the found count is less than `count`, -/// the `count` and `index` fields are overwritten with their new values. -pub fn count<'scope, Tr, K, R, F, P>( - prefixes: VecCollection<'scope, Tr::Time, (P, usize, usize), R>, +/// A prefix is dropped only when its key is absent from `arrangement` entirely, which is only +/// expected to happen when there have never been counts (they are meant to be non-negative). +pub fn count<'scope, Tr, K, F, P, R, FF, CF>( + prefixes: VecCollection<'scope, Tr::Time, ((P, usize, usize), Tr::Time), R>, arrangement: Arranged<'scope, Tr>, key_selector: F, index: usize, -) -> VecCollection<'scope, Tr::Time, (P, usize, usize), R> + frontier_func: FF, + comparison: CF, +) -> VecCollection<'scope, Tr::Time, ((P, usize, usize), Tr::Time), R> where Tr: TraceReader+Clone+'static, - BatchCursor: Cursor

, indices: CollectionIndex<'scope, K, V, T>, key_selector: F, + strict: bool, } impl<'scope, K, V, T, P, F> CollectionExtender<'scope, K, V, T, P, F> @@ -250,11 +246,10 @@ where T: Lattice+ExchangeData+Timestamp, F: Fn(&P)->K+Clone, { - /// The index's time closures, as plain callables the operators can accept. - fn time_logic(&self) -> (impl Fn(&T, &mut Antichain) + 'static, impl Fn(&T, &T) -> bool + 'static) { + /// The index's compaction closure, as a plain callable the operators can accept. + fn frontier_func(&self) -> impl Fn(&T, &mut Antichain) + 'static { let frontier_func = Rc::clone(&self.indices.frontier_func); - let comparison = Rc::clone(&self.indices.comparison); - (move |t: &T, a: &mut Antichain| frontier_func(t, a), move |t1: &T, t2: &T| comparison(t1, t2)) + move |t: &T, a: &mut Antichain| frontier_func(t, a) } } @@ -270,18 +265,14 @@ where type Extension = V; fn count(&mut self, prefixes: VecCollection<'scope, T, ((P, usize, usize), T), isize>, index: usize) -> VecCollection<'scope, T, ((P, usize, usize), T), isize> { - let (frontier_func, comparison) = self.time_logic(); - operators::count::count(prefixes, self.indices.count_trace.clone(), self.key_selector.clone(), index, frontier_func, comparison) + operators::count::count(prefixes, self.indices.count_trace.clone(), self.key_selector.clone(), index, self.frontier_func(), self.strict) } fn propose(&mut self, prefixes: VecCollection<'scope, T, (P, T), isize>) -> VecCollection<'scope, T, ((P, V), T), isize> { - let (frontier_func, comparison) = self.time_logic(); - operators::propose::propose(prefixes, self.indices.propose_trace.clone(), self.key_selector.clone(), frontier_func, comparison) + operators::propose::propose(prefixes, self.indices.propose_trace.clone(), self.key_selector.clone(), self.frontier_func(), self.strict) } fn validate(&mut self, extensions: VecCollection<'scope, T, ((P, V), T), isize>) -> VecCollection<'scope, T, ((P, V), T), isize> { - let (frontier_func, comparison) = self.time_logic(); - let key_selector = self.key_selector.clone(); - operators::validate::validate(extensions, self.indices.validate_trace.clone(), key_selector, frontier_func, comparison) + operators::validate::validate(extensions, self.indices.validate_trace.clone(), self.key_selector.clone(), self.frontier_func(), self.strict) } } diff --git a/dogsdogsdogs/src/operators/count.rs b/dogsdogsdogs/src/operators/count.rs index 44756dd67..8aa0eebc7 100644 --- a/dogsdogsdogs/src/operators/count.rs +++ b/dogsdogsdogs/src/operators/count.rs @@ -16,13 +16,13 @@ use differential_dataflow::trace::implementations::BatchContainer; /// /// A prefix is dropped only when its key is absent from `arrangement` entirely, which is only /// expected to happen when there have never been counts (they are meant to be non-negative). -pub fn count<'scope, Tr, K, F, P, R, FF, CF>( +pub fn count<'scope, Tr, K, F, P, R, FF>( prefixes: VecCollection<'scope, Tr::Time, ((P, usize, usize), Tr::Time), R>, arrangement: Arranged<'scope, Tr>, key_selector: F, index: usize, frontier_func: FF, - comparison: CF, + strict: bool, ) -> VecCollection<'scope, Tr::Time, ((P, usize, usize), Tr::Time), R> where Tr: TraceReader+Clone+'static, @@ -33,7 +33,7 @@ where F: Fn(&P)->K+'static, P: ExchangeData, FF: Fn(&Tr::Time, &mut Antichain) + 'static, - CF: Fn(BatchTimeGat<'_, Tr>, &Tr::Time) -> bool + 'static, + for<'a, 'b> BatchTimeGat<'a, Tr>: PartialOrd<&'b Tr::Time>, { // The payload time is carried in the record as well as in the half-join's own payload // slot, because the output closure is handed the joined times rather than the payload. @@ -63,13 +63,16 @@ where builder.push_into(((triple, payload.clone()), initial.clone(), diff1.clone())); }; - crate::operators::half_join::half_join_internal_unsafe::<_, _, _, _, _, _, _, _, Output>( - requests, - arrangement, - frontier_func, - comparison, - |_timer, _count| false, - output_func, - ) + use crate::operators::half_join::half_join_internal_unsafe as half_join_unsafe; + // Branch once here, so that each comparison monomorphizes rather than testing `strict` at + // every timestamp. The cost is instantiating `half_join` twice. + if strict { + half_join_unsafe::<_, _, _, _, _, _, _, _, Output>( + requests, arrangement, frontier_func, |t1, t2| t1 < t2, |_timer, _count| false, output_func) + } + else { + half_join_unsafe::<_, _, _, _, _, _, _, _, Output>( + requests, arrangement, frontier_func, |t1, t2| t1 <= t2, |_timer, _count| false, output_func) + } .as_collection() } diff --git a/dogsdogsdogs/src/operators/propose.rs b/dogsdogsdogs/src/operators/propose.rs index 008576184..f6db7c1a6 100644 --- a/dogsdogsdogs/src/operators/propose.rs +++ b/dogsdogsdogs/src/operators/propose.rs @@ -12,19 +12,19 @@ use differential_dataflow::trace::implementations::BatchContainer; /// /// This operator matches streamed updates with arranged updates, and pairs the streamed updates /// with arranged updates whose times are less or equal under the *total order* on timestamps. -/// This inequality is allowed to either be strict or non-strict, as determined by `comparison`. +/// This inequality is allowed to either be strict or non-strict, as determined by `strict`. /// The total order allows the caller to ensure that each pair of updates match exactly once. /// The streamed updates also carry a time as data, and that time is advanced (by lattice join) /// by the time of the arranged update. The time of the streamed update cannot be advanced, as /// it needs to stay put to ensure the total order math works out. /// /// The arrangement is expected to hold a *set*: see the note on set semantics in [`crate`]. -pub fn propose<'scope, Tr, K, F, P, V, R, FF, CF>( +pub fn propose<'scope, Tr, K, F, P, V, R, FF>( prefixes: VecCollection<'scope, Tr::Time, (P, Tr::Time), R>, arrangement: Arranged<'scope, Tr>, key_selector: F, frontier_func: FF, - comparison: CF, + strict: bool, ) -> VecCollection<'scope, Tr::Time, ((P, V), Tr::Time), >>::Output> where Tr: TraceReader+Clone+'static, @@ -36,14 +36,17 @@ where P: ExchangeData, V: Clone + 'static, FF: Fn(&Tr::Time, &mut Antichain) + 'static, - CF: Fn(BatchTimeGat<'_, Tr>, &Tr::Time) -> bool + 'static, + for<'a, 'b> BatchTimeGat<'a, Tr>: PartialOrd<&'b Tr::Time>, { let requests = prefixes.map(move |(prefix, payload)| (key_selector(&prefix), prefix, payload)); - crate::operators::half_join( - requests, - arrangement, - frontier_func, - comparison, - |_key, prefix, value| (prefix.clone(), as Cursor>::owned_val(value)), - ) + // Branch once here, so that each comparison monomorphizes rather than testing `strict` at + // every timestamp. The cost is instantiating `half_join` twice. + if strict { + crate::operators::half_join(requests, arrangement, frontier_func, |t1, t2| t1 < t2, + |_key, prefix, value| (prefix.clone(), as Cursor>::owned_val(value))) + } + else { + crate::operators::half_join(requests, arrangement, frontier_func, |t1, t2| t1 <= t2, + |_key, prefix, value| (prefix.clone(), as Cursor>::owned_val(value))) + } } diff --git a/dogsdogsdogs/src/operators/validate.rs b/dogsdogsdogs/src/operators/validate.rs index bb5f40273..5e61f942f 100644 --- a/dogsdogsdogs/src/operators/validate.rs +++ b/dogsdogsdogs/src/operators/validate.rs @@ -12,19 +12,19 @@ use differential_dataflow::trace::implementations::BatchContainer; /// /// This operator matches streamed updates with arranged updates, and pairs the streamed updates /// with arranged updates whose times are less or equal under the *total order* on timestamps. -/// This inequality is allowed to either be strict or non-strict, as determined by `comparison`. +/// This inequality is allowed to either be strict or non-strict, as determined by `strict`. /// The total order allows the caller to ensure that each pair of updates match exactly once. /// The streamed updates also carry a time as data, and that time is advanced (by lattice join) /// by the time of the arranged update. The time of the streamed update cannot be advanced, as /// it needs to stay put to ensure the total order math works out. /// /// The arrangement is expected to hold a *set*: see the note on set semantics in [`crate`]. -pub fn validate<'scope, Tr, K, V, F, P, R, FF, CF>( +pub fn validate<'scope, Tr, K, V, F, P, R, FF>( extensions: VecCollection<'scope, Tr::Time, ((P, V), Tr::Time), R>, arrangement: Arranged<'scope, Tr>, key_selector: F, frontier_func: FF, - comparison: CF, + strict: bool, ) -> VecCollection<'scope, Tr::Time, ((P, V), Tr::Time), >>::Output> where Tr: TraceReader+Clone+'static, @@ -36,16 +36,19 @@ where F: Fn(&P)->K+'static, P: ExchangeData, FF: Fn(&Tr::Time, &mut Antichain) + 'static, - CF: Fn(BatchTimeGat<'_, Tr>, &Tr::Time) -> bool + 'static, + for<'a, 'b> BatchTimeGat<'a, Tr>: PartialOrd<&'b Tr::Time>, { let requests = extensions.map(move |((prefix, extension), payload)| { ((key_selector(&prefix), extension.clone()), (prefix, extension), payload) }); - crate::operators::half_join( - requests, - arrangement, - frontier_func, - comparison, - |_key, extended, _value| extended.clone(), - ) + // Branch once here, so that each comparison monomorphizes rather than testing `strict` at + // every timestamp. The cost is instantiating `half_join` twice. + if strict { + crate::operators::half_join(requests, arrangement, frontier_func, |t1, t2| t1 < t2, + |_key, extended, _value| extended.clone()) + } + else { + crate::operators::half_join(requests, arrangement, frontier_func, |t1, t2| t1 <= t2, + |_key, extended, _value| extended.clone()) + } } diff --git a/dogsdogsdogs/tests/wcoj_partial_order.rs b/dogsdogsdogs/tests/wcoj_partial_order.rs index 28b1c6310..5a001854d 100644 --- a/dogsdogsdogs/tests/wcoj_partial_order.rs +++ b/dogsdogsdogs/tests/wcoj_partial_order.rs @@ -1,6 +1,6 @@ //! The worst-case-optimal `extend` path over a *partially ordered* time. //! -//! This is the case the payload time exists for. `comparison` orders times totally, so a prefix +//! This is the case the payload time exists for. Times are compared totally, so a prefix //! at `initial` matches records at times incomparable to it, and `lub(t2, payload)` then differs //! from match to match. Two updates to the same extension therefore land at two *different* //! output times rather than collapsing onto one, and no longer cancel. @@ -18,7 +18,7 @@ use timely::progress::Antichain; use differential_dataflow::AsCollection; use differential_dataflow::lattice::Lattice; -use differential_dogs3::{CollectionIndex, altneu::AltNeu, ProposeExtensionMethod}; +use differential_dogs3::{CollectionIndex, ProposeExtensionMethod}; type Time = Product; @@ -42,54 +42,45 @@ fn triangles(deltas: &[((u32, u32), Time, isize)]) -> Vec<((u32, u32, u32), Time let forward = edges.clone(); let reverse = edges.map(|(x, y)| (y, x)); - let triangles = outer.scoped::, _, _>("Triangles", |inner| { - let forward = forward.enter(inner); - let reverse = reverse.enter(inner); - - // Hold compaction back one step in each coordinate, so that total-order - // comparisons are not misled, and compare in the total order on `AltNeu` - // (lexicographic on `(time, neu)`, and `time` lexicographic on `Product`). - let frontier_func = |time: &AltNeu