diff --git a/client/dive-common/components/BottomPanel.vue b/client/dive-common/components/BottomPanel.vue index a8588edd2..1a5a6c0d1 100644 --- a/client/dive-common/components/BottomPanel.vue +++ b/client/dive-common/components/BottomPanel.vue @@ -154,6 +154,7 @@ export default defineComponent({ :show-empty-types="clientSettings.typeSettings.showEmptyTypes" :height="130" :width="300" + :header-height="50" :style-manager="trackStyleManager" :filter-controls="trackFilters" :disabled="disableAnnotationFilters" diff --git a/client/dive-common/components/TypeSettingsPanel.vue b/client/dive-common/components/TypeSettingsPanel.vue index 79b3e6244..49db05311 100644 --- a/client/dive-common/components/TypeSettingsPanel.vue +++ b/client/dive-common/components/TypeSettingsPanel.vue @@ -25,6 +25,7 @@ export default defineComponent({ preventCascadeTypes: 'When a track has multiple types, this will prevent the type from displaying if the max type is not visible in the type list', filterTypesByFrame: 'Filters the type list by only types that are visible in the current frame', maxCountButton: 'Show a max count button that will jump to the frame with the max count for the type', + suppressionType: 'Detections lying 50% or more under an annotation of this type are hidden and excluded from counts. Leave empty to disable.', }); const importInstructions = ref('Please provide a list of types (separated by a new line) that you would like to import'); const importDialog = ref(false); @@ -279,6 +280,39 @@ export default defineComponent({ + + + + + + + + {{ help.suppressionType }} + + + diff --git a/client/dive-common/store/settings.ts b/client/dive-common/store/settings.ts index 47adc672e..badebebb1 100644 --- a/client/dive-common/store/settings.ts +++ b/client/dive-common/store/settings.ts @@ -22,6 +22,9 @@ interface AnnotationSettings { preventCascadeTypes?: boolean; filterTypesByFrame?: boolean; maxCountButton?: boolean; + // Detections whose geometry lies >=50% under a region of this type are + // hidden and excluded from counts. Empty string disables the feature. + suppressionType?: string; }; trackSettings: { newTrackSettings: { @@ -127,6 +130,7 @@ const defaultSettings: AnnotationSettings = { lockTypes: false, preventCascadeTypes: false, maxCountButton: false, + suppressionType: 'Suppressed', }, rowsPerPage: 20, annotationFPS: 10, diff --git a/client/src/components/FilterList.vue b/client/src/components/FilterList.vue index de3aa24aa..311f2cc94 100644 --- a/client/src/components/FilterList.vue +++ b/client/src/components/FilterList.vue @@ -9,6 +9,7 @@ import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import { clientSettings } from 'dive-common/store/settings'; import { useCameraStore, useHandler, useReadOnlyMode, useSelectedCamera, useTime, + usePendingSaveCount, } from '../provides'; import TooltipBtn from './TooltipButton.vue'; import TypeEditor from './TypeEditor.vue'; @@ -17,6 +18,7 @@ import BaseFilterControls from '../BaseFilterControls'; import Track from '../track'; import Group from '../Group'; import StyleManager from '../StyleManager'; +import { getSuppressedTrackIds } from '../use/suppression'; interface VirtualTypeItem { type: string; @@ -24,11 +26,9 @@ interface VirtualTypeItem { displayText: string; color: string; checked: boolean; + isSuppressionType: boolean; } -/* Magic numbers involved in height calculation */ -const TypeListHeaderHeight = 80; - export default defineComponent({ name: 'FilterList', @@ -47,6 +47,11 @@ export default defineComponent({ type: Number, default: 300, }, + /* Sidebar (~80) vs bottom bar (~50) header chrome */ + headerHeight: { + type: Number, + default: 80, + }, filterControls: { type: Object as PropType>, required: true, @@ -68,6 +73,10 @@ export default defineComponent({ const cameraStore = useCameraStore(); const selectedCamera = useSelectedCamera(); const { frame } = useTime(); + // Bumped on every annotation edit (add/move/delete); read in the frame + // counts so they re-evaluate suppression when a region is moved, since a + // track's geometry is not itself a reactive dependency. + const pendingSaveCount = usePendingSaveCount(); const trackStore = cameraStore.camMap.value.get(selectedCamera.value)?.trackStore; // Ordering of these lists should match const sortingMethods: ('a-z' | 'count' | 'frame count')[] = ['a-z', 'count', 'frame count']; @@ -137,20 +146,76 @@ export default defineComponent({ } } - const typeCounts = computed(() => filteredTracksRef.value.reduce((acc, filteredTrack) => { - const confidencePair = filteredTrack.annotation - .getType(filteredTrack.context.confidencePairIndex); - const trackType = confidencePair; - acc.set(trackType, (acc.get(trackType) || 0) + 1); + /** + * Ids of tracks whose every keyframe detection is suppressed (a region + * covers it on each frame it appears), across all cameras - these are + * excluded from the dataset-wide type totals. Reactive to region edits via + * the save counter, since track geometry is not itself a reactive value. + */ + const fullySuppressedIds = computed(() => { + const editRevision = pendingSaveCount.value; + const suppType = clientSettings.typeSettings.suppressionType; + const excluded = new Set(); + if (!suppType || editRevision < 0) { + return excluded; + } + cameraStore.camMap.value.forEach(({ trackStore: store }) => { + const perFrame = new Map>(); + const suppressedAt = (f: number) => { + let s = perFrame.get(f); + if (s === undefined) { + s = getSuppressedTrackIds(store, f, suppType); + perFrame.set(f, s); + } + return s; + }; + store.annotationMap.forEach((annotation) => { + const track = annotation as Track; + if (typeof track.getFeature !== 'function') { + return; + } + const keyframes = track.features.filter((f) => f && f.keyframe); + if (keyframes.length > 0 + && keyframes.every((f) => suppressedAt(f.frame).has(track.id))) { + excluded.add(track.id); + } + }); + }); + return excluded; + }); - return acc; - }, new Map())); + const typeCounts = computed(() => { + const excluded = fullySuppressedIds.value; + return filteredTracksRef.value.reduce((acc, filteredTrack) => { + if (excluded.has(filteredTrack.annotation.id)) { + return acc; + } + const confidencePair = filteredTrack.annotation + .getType(filteredTrack.context.confidencePairIndex); + const trackType = confidencePair; + acc.set(trackType, (acc.get(trackType) || 0) + 1); + + return acc; + }, new Map()); + }); const filteredTracksForFrame = computed(() => { + // Depend on the edit counter so moving/resizing a suppression region + // (which mutates geometry, not the reactive track set) re-runs the count. + // It is always >= 0, so this reads the dependency without changing logic. + const editRevision = pendingSaveCount.value; const trackIdsForFrame = trackStore?.intervalTree .search([frame.value, frame.value]) .map((str) => parseInt(str, 10)); + // Detections suppressed by a region on this frame are dropped so the + // per-frame type counts read off the interface exclude them. + const suppressedIds = (trackStore && editRevision >= 0) + ? getSuppressedTrackIds(trackStore, frame.value, clientSettings.typeSettings.suppressionType) + : new Set(); const filteredKeyFrameTracks = filteredTracksRef.value.filter((track) => { + if (suppressedIds.has(track.annotation.id)) { + return false; + } const keyframe = trackStore?.getPossible(track.annotation.id)?.getFeature(frame.value)[0]; return !!keyframe?.keyframe; }); @@ -206,12 +271,14 @@ export default defineComponent({ if (filterTypesByFrame.value) { filteredTypeList = filteredTypeList.filter((item) => frameTrackTypesDeRef.get(item)); } + const { suppressionType } = clientSettings.typeSettings; return filteredTypeList.map((item) => ({ type: item, confidenceFilterNum: confidenceFiltersDeRef[item] || 0, displayText: `${typeCountsDeRef.get(item) || 0}:${frameTrackTypesDeRef.get(item) || 0} ${item}`, color: typeStylingDeRef.color(item), checked: checkedTypesDeRef.includes(item), + isSuppressionType: !!suppressionType && item === suppressionType, })); }); const headCheckState = computed(() => { @@ -249,7 +316,7 @@ export default defineComponent({ } } - const virtualHeight = computed(() => props.height - TypeListHeaderHeight); + const virtualHeight = computed(() => props.height - props.headerHeight); const goToPeakTrackFrame = (trackType: string) => { const frameCounts = new Map(); @@ -425,6 +492,7 @@ export default defineComponent({ :width="width" :display-max-button="showMaxFrameButton" :disabled="disableAnnotationFilters" + :is-suppression-type="item.isSuppressionType" @setCheckedTypes="updateCheckedType($event, item.type)" @goToMaxFrame="goToPeakTrackFrame($event)" @clickEdit="clickEdit" diff --git a/client/src/components/LayerManager.vue b/client/src/components/LayerManager.vue index 83010be8e..12e7db7a0 100644 --- a/client/src/components/LayerManager.vue +++ b/client/src/components/LayerManager.vue @@ -21,6 +21,7 @@ import TextLayer, { FormatTextRow } from '../layers/AnnotationLayers/TextLayer'; import AttributeLayer from '../layers/AnnotationLayers/AttributeLayer'; import AttributeBoxLayer from '../layers/AnnotationLayers/AttributeBoxLayer'; import type { AnnotationId } from '../BaseAnnotation'; +import { getSuppressedTrackIds } from '../use/suppression'; import { VisibleAnnotationTypes } from '../layers'; import UILayer from '../layers/UILayers/UILayer'; import ToolTipWidget from '../layers/UILayers/ToolTipWidget.vue'; @@ -344,6 +345,11 @@ export default defineComponent({ if (currentFrameIds === undefined) { return; } + // Detections lying >=50% under a suppression region on this frame are + // hidden from every layer at once (and excluded from counts elsewhere). + const suppressedIds = trackStore + ? getSuppressedTrackIds(trackStore, frame, clientSettings.typeSettings.suppressionType) + : new Set(); currentFrameIds.forEach( (trackId: AnnotationId) => { const track = trackStore?.getPossible(trackId); @@ -352,6 +358,9 @@ export default defineComponent({ // TODO: Find a better way to represent tracks outside of cameras return; } + if (suppressedIds.has(trackId)) { + return; + } const enabledIndex = enabledTracks.findIndex( (trackWithContext) => trackWithContext.annotation.id === trackId, @@ -607,6 +616,8 @@ export default defineComponent({ toRef(props, 'colorBy'), selectedCamera, selectedKeyRef, + // re-render when the suppression-region type is changed + () => clientSettings.typeSettings.suppressionType, ], () => { refreshLayers(); diff --git a/client/src/components/Tracks/TrackList.vue b/client/src/components/Tracks/TrackList.vue index cbb7d89fd..49e755d87 100644 --- a/client/src/components/Tracks/TrackList.vue +++ b/client/src/components/Tracks/TrackList.vue @@ -18,8 +18,11 @@ import { useTrackStyleManager, useMultiSelectList, useCameraStore, + useSelectedCamera, + usePendingSaveCount, } from '../../provides'; import useVirtualScrollTo from '../../use/useVirtualScrollTo'; +import { getSuppressedTrackIds } from '../../use/suppression'; import SideBarTrackListView from './sidebar/SideBarTrackListView.vue'; import BottomBarTrackListView from './bottombar/BottomBarTrackListView.vue'; @@ -81,6 +84,8 @@ export default defineComponent({ const editingModeRef = useEditingMode(); const selectedTrackIdRef = useSelectedTrackId(); const cameraStore = useCameraStore(); + const selectedCamera = useSelectedCamera(); + const pendingSaveCount = usePendingSaveCount(); const filteredTracksRef = trackFilters.filteredAnnotations; const typeStylingRef = useTrackStyleManager().typeStyling; const { frame: frameRef, isPlaying } = useTime(); @@ -109,7 +114,18 @@ export default defineComponent({ const finalFilteredTracks = computed(() => { let tracks = filteredTracksRef.value; if (filterDetectionsByFrame.value && !isPlaying.value) { + // Depend on the edit counter so moving a suppression region re-runs the + // filter (geometry mutations are not reactive track-set changes). + const editRevision = pendingSaveCount.value; + const suppressCamStore = cameraStore.camMap.value.get(selectedCamera.value)?.trackStore; + const suppType = clientSettings.typeSettings.suppressionType; + const suppressedIds = (suppressCamStore && editRevision >= 0) + ? getSuppressedTrackIds(suppressCamStore, frameRef.value, suppType) + : new Set(); tracks = tracks.filter((track) => { + if (suppressedIds.has(track.annotation.id)) { + return false; + } const possibleTrack = cameraStore.getAnyPossibleTrack(track.annotation.id); if (possibleTrack) { const [feature] = possibleTrack.getFeature(frameRef.value); diff --git a/client/src/components/TypeItem.vue b/client/src/components/TypeItem.vue index 5281955d7..fadd8701f 100644 --- a/client/src/components/TypeItem.vue +++ b/client/src/components/TypeItem.vue @@ -40,6 +40,10 @@ export default defineComponent({ type: Boolean, default: false, }, + isSuppressionType: { + type: Boolean, + default: false, + }, }, setup(props, { emit }) { /* Horizontal padding is the width of checkbox, scrollbar, and edit button */ @@ -111,6 +115,28 @@ export default defineComponent({ Type has threshold set individually + + + + This type is used for suppression. + Detections lying 50% or more under its regions are hidden and excluded from counts. + + @@ -190,4 +216,9 @@ export default defineComponent({ padding: 0 5px; font-size: 12px; } + +.suppression-icon { + flex-shrink: 0; + align-self: center; +} diff --git a/client/src/use/suppression.ts b/client/src/use/suppression.ts new file mode 100644 index 000000000..a67871269 --- /dev/null +++ b/client/src/use/suppression.ts @@ -0,0 +1,134 @@ +/** + * Suppression regions. + * + * A "suppression region" is an annotation whose type matches the configured + * suppression type (clientSettings.typeSettings.suppressionType). Any detection + * whose geometry lies at least SUPPRESSION_THRESHOLD (50%) under one or more + * suppression regions on a given frame is treated as suppressed: it is hidden + * from the canvas and excluded from type/track/detection counts. + * + * Overlap uses whichever geometry each side actually has: the region's polygon + * if it has one, else its bounding box; likewise for the detection. The overlap + * fraction (area of the detection covered by the union of regions, divided by + * the detection's own area) is estimated by point sampling, which handles the + * concave, warped field-of-view polygons the sea-lion suppressor emits without + * a polygon-clipping dependency. + */ +import type { AnnotationId } from 'vue-media-annotator/BaseAnnotation'; +import type BaseAnnotationStore from 'vue-media-annotator/BaseAnnotationStore'; +import type Track from 'vue-media-annotator/track'; +import type { Feature } from 'vue-media-annotator/track'; + +export const SUPPRESSION_THRESHOLD = 0.5; + +type Pt = [number, number]; +type Rect = [number, number, number, number]; +interface Shape { poly?: Pt[]; bbox: Rect } + +// Sampling resolution used to estimate the covered-area fraction. 16x16 is +// plenty for a 50% threshold and keeps the per-frame cost low. +const GRID = 16; + +function bboxOfPoly(poly: Pt[]): Rect { + let x1 = Infinity; let y1 = Infinity; let x2 = -Infinity; let y2 = -Infinity; + poly.forEach(([x, y]) => { + x1 = Math.min(x1, x); y1 = Math.min(y1, y); + x2 = Math.max(x2, x); y2 = Math.max(y2, y); + }); + return [x1, y1, x2, y2]; +} + +/** Geometry of a detection/region on a frame: its polygon if present, else its box. */ +function featureShape(feature: Feature | null): Shape | null { + if (!feature) return null; + const polyFeat = feature.geometry?.features?.find( + (f) => f.geometry?.type === 'Polygon', + ); + if (polyFeat) { + const ring = (polyFeat.geometry as GeoJSON.Polygon).coordinates[0]; + if (ring && ring.length >= 3) { + const poly = ring.map((c) => [c[0], c[1]] as Pt); + return { poly, bbox: bboxOfPoly(poly) }; + } + } + if (feature.bounds) return { bbox: feature.bounds as Rect }; + return null; +} + +function pointInPoly(px: number, py: number, poly: Pt[]): boolean { + let inside = false; + for (let i = 0, j = poly.length - 1; i < poly.length; j = i, i += 1) { + const [xi, yi] = poly[i]; + const [xj, yj] = poly[j]; + if (((yi > py) !== (yj > py)) + && (px < ((xj - xi) * (py - yi)) / (yj - yi) + xi)) { + inside = !inside; + } + } + return inside; +} + +function pointInShape(px: number, py: number, shape: Shape): boolean { + if (shape.poly) return pointInPoly(px, py, shape.poly); + const [x1, y1, x2, y2] = shape.bbox; + return px >= x1 && px <= x2 && py >= y1 && py <= y2; +} + +/** Fraction of the detection's area covered by the union of the regions. */ +function overlapFraction(det: Shape, regions: Shape[]): number { + const [x1, y1, x2, y2] = det.bbox; + const w = x2 - x1; + const h = y2 - y1; + if (w <= 0 || h <= 0) return 0; + let inDet = 0; + let covered = 0; + for (let iy = 0; iy < GRID; iy += 1) { + const py = y1 + ((iy + 0.5) / GRID) * h; + for (let ix = 0; ix < GRID; ix += 1) { + const px = x1 + ((ix + 0.5) / GRID) * w; + if (pointInShape(px, py, det)) { + inDet += 1; + if (regions.some((r) => pointInShape(px, py, r))) covered += 1; + } + } + } + return inDet === 0 ? 0 : covered / inDet; +} + +/** + * Track ids whose detection on `frame` is suppressed by a region on that frame. + * Empty when suppressionType is falsy (feature disabled) or no regions exist. + */ +export function getSuppressedTrackIds( + trackStore: BaseAnnotationStore, + frame: number, + suppressionType: string | undefined, +): Set { + const result = new Set(); + if (!suppressionType) return result; + const ids = trackStore.intervalTree.search([frame, frame]) + .map((s: string) => parseInt(s, 10)); + if (ids.length === 0) return result; + + const regions: Shape[] = []; + const candidates: { id: AnnotationId; shape: Shape }[] = []; + ids.forEach((id) => { + const track = trackStore.getPossible(id); + if (!track) return; + const shape = featureShape(track.getFeature(frame)[0]); + if (!shape) return; + if (track.confidencePairs.some(([t]) => t === suppressionType)) { + regions.push(shape); + } else { + candidates.push({ id, shape }); + } + }); + if (regions.length === 0) return result; + + candidates.forEach(({ id, shape }) => { + if (overlapFraction(shape, regions) >= SUPPRESSION_THRESHOLD) { + result.add(id); + } + }); + return result; +}