diff --git a/skills-manifest.json b/skills-manifest.json index d430d372e7..77daa2239e 100644 --- a/skills-manifest.json +++ b/skills-manifest.json @@ -46,8 +46,8 @@ "files": 10 }, "media-use": { - "hash": "14ea99a7f4d750cd", - "files": 116 + "hash": "6c4aa8649e1eaf99", + "files": 121 }, "motion-graphics": { "hash": "96ed2f7d8051b009", diff --git a/skills/media-use/SKILL.md b/skills/media-use/SKILL.md index c00040dcfe..5d2066103d 100644 --- a/skills/media-use/SKILL.md +++ b/skills/media-use/SKILL.md @@ -135,6 +135,8 @@ node /scripts/resolve.mjs --type lut --intent "teal orange blockbuste | `--provider` | Force one generator (e.g. `codex`, `mflux`, `kokoro`, `heygen`) | | `--adopt` | Bulk-import existing assets/ into manifest | | `--doctor` | Check local CLI dependencies; no manifest changes | +| `--stats` | Print local usage stats from `.media/` and `~/.media`; no manifest changes | +| `--days N` | Limit `--stats` to timestamped records/misses from the last N days | | `--json` | Output JSON instead of one-line result | ## Reuse before you resolve @@ -316,11 +318,24 @@ Assets are cached automatically on resolve. Every resolved/ingested asset is aut For a _semantically_ similar (not identical) need in another project, the exact-match floor won't fire — use [Reuse before you resolve](#reuse-before-you-resolve): `--candidates` lists the global assets, and `--reuse ` imports the one you pick. This is how a track resolved in one project gets reused in the next when the wording differs. +## Usage stats + +Use `resolve --stats` for a local, shareable report over the current project's `.media/` manifest, the global `~/.media/` cache, and local resolve misses. Human output is compact; add `--json` for a single machine-readable object, and `--days N` to window timestamped records. + +```bash +node /scripts/resolve.mjs --stats --project . --days 7 +# media-use stats +# total resolves: 12 +# misses: 2 +# hit rate: 86% +``` + ## Files - `.media/manifest.jsonl`: machine SSOT, one JSON record per line - `.media/index.md`: agent-readable table (id, type, dur, dims, path, description) - `~/.media/`: global cross-project reuse cache (content-addressed, SHA-256) +- `~/.media/misses.jsonl`: local-only resolve misses, including intent text for `--stats` ## Audio engine: voiceover, music, SFX, captions, transcription @@ -389,3 +404,12 @@ resolve never waits on or fails from telemetry). Opt out with `DO_NOT_TRACK=1` or `HYPERFRAMES_NO_TELEMETRY=1` (also off in CI and dev). Same public PostHog project key and opt-outs as the `hyperframes` CLI. + +## Privacy + +media-use uses the same shared install id as the `hyperframes` CLI/studio +(`~/.hyperframes/config.json`). When you are signed in to HeyGen, usage is +linked to your account email, or username when email is unavailable, matching +the CLI behavior. The events stay coarse: media type, source, provider, and +small counts only; intent text and paths stay local. Disable telemetry with +`HYPERFRAMES_NO_TELEMETRY=1` or `DO_NOT_TRACK=1`. diff --git a/skills/media-use/references/telemetry-dashboard.md b/skills/media-use/references/telemetry-dashboard.md new file mode 100644 index 0000000000..11df43b378 --- /dev/null +++ b/skills/media-use/references/telemetry-dashboard.md @@ -0,0 +1,50 @@ +# media-use usage dashboard (PostHog) + +Reproducible definition of the media-use usage dashboard. The dashboard answers +"how much is media-use used, for what, is reuse working, and what can't it +satisfy" from the telemetry `scripts/lib/telemetry.mjs` already emits. Build it +in PostHog project **Hyperframes (356858)**; this doc is the source of truth so +it can be recreated. Local complement: `resolve --stats` (same questions, from +`.media/` + `~/.media`, no PostHog access needed). + +## Identity (see `scripts/lib/telemetry.mjs`) + +Events attribute to the **same PostHog person as the hyperframes CLI and studio** +— the shared install id in `~/.hyperframes/config.json` (`anonymousId`), stitched +to the HeyGen account (`$identify`, `distinct_id` = email/username) on sign-in. +Not fully anonymous by design; pseudonymous before sign-in, account-linked after. +`$ip:null`. Opt-out: `HYPERFRAMES_NO_TELEMETRY=1` / `DO_NOT_TRACK=1` (also CI, dev). + +## Event catalog (verified present in-project) + +Every event carries `surface: "media-use"`. Event **properties are coarse** — +never intent text, file names, or paths. + +| Event | Fires on | Key properties | +| ---------------------------------------------------------------------- | ----------------------------------------- | ---------------------------------------------------------------------- | +| `media_use_resolve` | a resolve that produced/returned an asset | `type`, `source`, `provider`, `via`, `local_only`, `provider_override` | +| `media_use_resolve_miss` | a resolve that found nothing | `type`, `local_only`, `provider_override` (no intent) | +| `media_use_candidates` | `--candidates` / `--dry-run` listing | `type`, counts | +| `media_use_doctor_run` | `--doctor` | `ok`, `checks_failed`, `failed[]` | +| `media_use_compare` | `grade-compare` / `compare` | `command`, `cells`, `truncated`, `total`, `render_ready_timed_out` | +| `media_use_transcribe` · `media_use_duck` · `media_use_transcript_cut` | audio-engine ops | op-specific | + +## Dashboard tiles + +1. **Invocation volume** — `query-trends`, count of `media_use_resolve` over time (daily). "How much." +2. **By media type** — `media_use_resolve` broken down by `type` (bgm/sfx/image/icon/logo/voice/grade/lut). "For what." +3. **Resolve hit-rate** — trends formula: `A / (A + B)` where A = `media_use_resolve`, B = `media_use_resolve_miss`. "Is the catalog covering needs." +4. **Provider mix** — `media_use_resolve` broken down by `provider`; a second tile by `via` (`url` / `params-fallback` / `params`) to catch CDN→params LUT downgrades. +5. **Top misses** — `media_use_resolve_miss` broken down by `type` (the tuning signal — pair with local `resolve --stats`, which also shows the missed _intents_ that telemetry deliberately omits). +6. **Doctor health** — `media_use_doctor_run` broken down by `failed[]` (which dependency check fails most) + `checks_failed` distribution. +7. **Compare cost** — `media_use_compare` by `command`, plus `truncated` / `render_ready_timed_out` rates (observe before lifting the 16-cell cap). +8. **Adoption (optional)** — if the `first_run` property ships (plan U5), segment `media_use_resolve` first-run vs repeat. + +## Recreate via the PostHog MCP + +For each tile: `read-data-schema` to confirm the event/property, then a +`query-*` tool (`query-trends` for 1–7), then `insight-create`, then +`dashboard-create` collecting the insights. Keep names prefixed `media-use:` so +the dashboard is greppable. Cross-surface note: because identity is shared with +CLI/studio, you can also break these down by the same person across `cli_command*` +and `studio:*` events. diff --git a/skills/media-use/scripts/lib/misses.mjs b/skills/media-use/scripts/lib/misses.mjs new file mode 100644 index 0000000000..585bb96384 --- /dev/null +++ b/skills/media-use/scripts/lib/misses.mjs @@ -0,0 +1,49 @@ +import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +const MISSES_FILE = "misses.jsonl"; + +function missesPath() { + return join(homedir(), ".media", MISSES_FILE); +} + +export function recordMiss({ type, intent, provider_override, local_only }) { + try { + const dir = join(homedir(), ".media"); + mkdirSync(dir, { recursive: true }); + appendFileSync( + join(dir, MISSES_FILE), + JSON.stringify({ + ts: new Date().toISOString(), + type, + intent, + provider_override: !!provider_override, + local_only: !!local_only, + }) + "\n", + ); + } catch { + // local miss logging is best-effort; never surface into resolve + } +} + +export function readMisses() { + const p = missesPath(); + try { + if (!existsSync(p)) return []; + const raw = readFileSync(p, "utf8"); + const records = []; + for (const line of raw.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed) continue; + try { + records.push(JSON.parse(trimmed)); + } catch { + // skip malformed local lines, don't crash stats + } + } + return records; + } catch { + return []; + } +} diff --git a/skills/media-use/scripts/lib/misses.test.mjs b/skills/media-use/scripts/lib/misses.test.mjs new file mode 100644 index 0000000000..d9b2585662 --- /dev/null +++ b/skills/media-use/scripts/lib/misses.test.mjs @@ -0,0 +1,78 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { readMisses, recordMiss } from "./misses.mjs"; + +function sandbox() { + const root = mkdtempSync(join(tmpdir(), "mu-misses-")); + const home = join(root, "home"); + mkdirSync(home, { recursive: true }); + process.env.HOME = home; + return { root, home }; +} + +function restoreEnv(saved) { + for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k]; + Object.assign(process.env, saved); +} + +test("recordMiss appends a well-formed local miss", () => { + const savedEnv = { ...process.env }; + const { root, home } = sandbox(); + try { + recordMiss({ type: "bgm", intent: "moody synth pulse", provider_override: true }); + const misses = readMisses(); + assert.equal(misses.length, 1); + assert.equal(misses[0].type, "bgm"); + assert.equal(misses[0].intent, "moody synth pulse"); + assert.equal(misses[0].provider_override, true); + assert.equal(misses[0].local_only, false); + assert.ok(!Number.isNaN(Date.parse(misses[0].ts))); + + const raw = readFileSync(join(home, ".media/misses.jsonl"), "utf8"); + assert.match(raw, /moody synth pulse/); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("recordMiss swallows filesystem failures", () => { + const savedEnv = { ...process.env }; + const { root, home } = sandbox(); + try { + writeFileSync(join(home, ".media"), "not a directory"); + assert.doesNotThrow(() => + recordMiss({ type: "image", intent: "unwritable", local_only: true }), + ); + assert.equal(existsSync(join(home, ".media/misses.jsonl")), false); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("readMisses skips corrupt lines", () => { + const savedEnv = { ...process.env }; + const { root, home } = sandbox(); + try { + mkdirSync(join(home, ".media"), { recursive: true }); + writeFileSync( + join(home, ".media/misses.jsonl"), + [ + JSON.stringify({ ts: "2026-07-09T00:00:00.000Z", type: "bgm", intent: "one" }), + "{not json", + JSON.stringify({ ts: "2026-07-09T00:00:01.000Z", type: "sfx", intent: "two" }), + ].join("\n"), + ); + assert.deepEqual( + readMisses().map((miss) => miss.intent), + ["one", "two"], + ); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/scripts/lib/stats.mjs b/skills/media-use/scripts/lib/stats.mjs new file mode 100644 index 0000000000..39611971cc --- /dev/null +++ b/skills/media-use/scripts/lib/stats.mjs @@ -0,0 +1,117 @@ +import { statSync } from "node:fs"; +import { readGlobalManifest } from "./cache.mjs"; +import { readManifest } from "./manifest.mjs"; +import { readMisses } from "./misses.mjs"; + +const TOP_MISSES = 5; + +function emptyReport() { + return { + total_resolves: 0, + by_type: {}, + by_source: {}, + by_provider: {}, + by_via: {}, + misses: 0, + hit_rate: null, + top_missed_intents: {}, + global_cache_assets: 0, + global_cache_disk_bytes: 0, + cross_project_reuse: 0, + }; +} + +function increment(map, key) { + if (!key) return; + map[key] = (map[key] || 0) + 1; +} + +function timestampOf(record) { + return record?.ts || record?.timestamp || record?.created_at || record?.createdAt || null; +} + +function inWindow(record, cutoff) { + if (!cutoff) return true; + const ts = timestampOf(record); + // Older manifest records may not carry a timestamp; keep them in the report + // because --days can only window records/misses that carry a ts/timestamp. + if (!ts) return true; + const time = Date.parse(ts); + return Number.isNaN(time) ? true : time >= cutoff; +} + +function sourceOf(record) { + return record?._source || record?.source || record?.provenance?.source || "unknown"; +} + +function normalizeIntent(intent) { + return String(intent ?? "") + .trim() + .toLowerCase() + .replace(/\s+/g, " "); +} + +function topMissedIntents(misses) { + const grouped = {}; + for (const miss of misses) { + const type = miss?.type || "unknown"; + const intent = normalizeIntent(miss?.intent); + if (!intent) continue; + grouped[type] ||= {}; + grouped[type][intent] = (grouped[type][intent] || 0) + 1; + } + const out = {}; + for (const [type, intents] of Object.entries(grouped)) { + out[type] = Object.entries(intents) + .map(([intent, count]) => ({ intent, count })) + .sort((a, b) => b.count - a.count || a.intent.localeCompare(b.intent)) + .slice(0, TOP_MISSES); + } + return out; +} + +function diskBytes(records) { + let total = 0; + for (const record of records) { + const p = record?.cached_path || record?.path; + if (!p) continue; + try { + total += statSync(p).size; + } catch { + // cache entries can outlive files; stats skips missing files + } + } + return total; +} + +export function buildStats({ projectDir, days, now = Date.now() } = {}) { + // Only a positive finite --days windows the report; null / NaN / <= 0 mean + // "all time" rather than silently excluding everything (a negative cutoff + // would land in the future and drop every record). The reads below are each + // best-effort (they return [] / skip on IO errors), so there is no top-level + // catch masking a real logic bug as an all-zero "no usage" report. + const n = Number(days); + const cutoff = Number.isFinite(n) && n > 0 ? Number(now) - n * 24 * 60 * 60 * 1000 : null; + const records = (projectDir ? readManifest(projectDir) : []).filter((r) => inWindow(r, cutoff)); + const misses = readMisses().filter((miss) => inWindow(miss, cutoff)); + const globalRecords = readGlobalManifest(); + const report = emptyReport(); + + report.total_resolves = records.length; + report.misses = misses.length; + for (const record of records) { + increment(report.by_type, record?.type || "unknown"); + increment(report.by_source, sourceOf(record)); + increment(report.by_provider, record?.provenance?.provider); + increment(report.by_via, record?.provenance?.via); + } + + const attempts = report.total_resolves + report.misses; + report.hit_rate = attempts === 0 ? null : report.total_resolves / attempts; + report.top_missed_intents = topMissedIntents(misses); + report.global_cache_assets = globalRecords.length; + report.global_cache_disk_bytes = diskBytes(globalRecords); + report.cross_project_reuse = globalRecords.filter((r) => r?.provenance?.reused_by).length; + + return report; +} diff --git a/skills/media-use/scripts/lib/stats.test.mjs b/skills/media-use/scripts/lib/stats.test.mjs new file mode 100644 index 0000000000..ff0b6b466e --- /dev/null +++ b/skills/media-use/scripts/lib/stats.test.mjs @@ -0,0 +1,165 @@ +import { strict as assert } from "node:assert"; +import { test } from "node:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { buildStats } from "./stats.mjs"; + +function sandbox() { + const root = mkdtempSync(join(tmpdir(), "mu-stats-")); + const home = join(root, "home"); + const projectDir = join(root, "project"); + mkdirSync(home, { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + process.env.HOME = home; + return { root, home, projectDir }; +} + +function restoreEnv(saved) { + for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k]; + Object.assign(process.env, saved); +} + +function seedManifest(dir, records) { + mkdirSync(join(dir, ".media"), { recursive: true }); + writeFileSync( + join(dir, ".media/manifest.jsonl"), + records.map((record) => JSON.stringify(record)).join("\n") + "\n", + ); +} + +function isoDaysAgo(days) { + return new Date(Date.now() - days * 24 * 60 * 60 * 1000).toISOString(); +} + +test("buildStats aggregates resolves, misses, providers, sources, and reuse", () => { + const savedEnv = { ...process.env }; + const { root, home, projectDir } = sandbox(); + try { + seedManifest(projectDir, [ + { + id: "bgm_001", + type: "bgm", + source: "search", + ts: isoDaysAgo(0), + provenance: { provider: "heygen.audio.sounds", prompt: "upbeat", via: "url" }, + }, + { + id: "image_001", + type: "image", + _source: "reused-explicit", + timestamp: isoDaysAgo(0), + provenance: { provider: "local", reused_by: "agent" }, + }, + { + id: "bgm_002", + type: "bgm", + source: "generated", + provenance: { provider: "codex" }, + }, + ]); + const cachedFile = join(home, ".media/mu-v1-cache/asset.wav"); + mkdirSync(join(cachedFile, ".."), { recursive: true }); + writeFileSync(cachedFile, "12345"); + seedManifest(home, [ + { + id: "bgm_009", + type: "bgm", + reusable: true, + cached_path: cachedFile, + provenance: { provider: "heygen.audio.sounds", reused_by: "agent" }, + }, + ]); + writeFileSync( + join(home, ".media/misses.jsonl"), + JSON.stringify({ + ts: isoDaysAgo(0), + type: "bgm", + intent: "dark cinematic riser", + }) + "\n", + { flag: "a" }, + ); + + const stats = buildStats({ projectDir }); + assert.equal(stats.total_resolves, 3); + assert.deepEqual(stats.by_type, { bgm: 2, image: 1 }); + assert.deepEqual(stats.by_source, { search: 1, "reused-explicit": 1, generated: 1 }); + assert.equal(stats.by_provider["heygen.audio.sounds"], 1); + assert.equal(stats.by_via.url, 1); + assert.equal(stats.misses, 1); + assert.equal(stats.hit_rate, 0.75); + assert.deepEqual(stats.top_missed_intents.bgm[0], { + intent: "dark cinematic riser", + count: 1, + }); + assert.equal(stats.global_cache_assets, 1); + assert.equal(stats.global_cache_disk_bytes, 5); + assert.equal(stats.cross_project_reuse, 1); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("buildStats returns a zeroed report when nothing exists", () => { + const savedEnv = { ...process.env }; + const { root, projectDir } = sandbox(); + try { + const stats = buildStats({ projectDir }); + assert.equal(stats.total_resolves, 0); + assert.deepEqual(stats.by_type, {}); + assert.deepEqual(stats.by_source, {}); + assert.deepEqual(stats.by_provider, {}); + assert.deepEqual(stats.by_via, {}); + assert.equal(stats.misses, 0); + assert.equal(stats.hit_rate, null); + assert.deepEqual(stats.top_missed_intents, {}); + assert.equal(stats.global_cache_assets, 0); + assert.equal(stats.global_cache_disk_bytes, 0); + assert.equal(stats.cross_project_reuse, 0); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("buildStats returns JSON-safe output", () => { + const savedEnv = { ...process.env }; + const { root, projectDir } = sandbox(); + try { + const stats = buildStats({ projectDir }); + assert.deepEqual(JSON.parse(JSON.stringify(stats)), stats); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); + +test("buildStats applies days window to timestamped records and misses", () => { + const savedEnv = { ...process.env }; + const { root, home, projectDir } = sandbox(); + try { + seedManifest(projectDir, [ + { id: "bgm_old", type: "bgm", ts: isoDaysAgo(100), provenance: { provider: "old" } }, + { id: "bgm_new", type: "bgm", ts: isoDaysAgo(1), provenance: { provider: "new" } }, + { id: "image_untimed", type: "image", provenance: { provider: "none" } }, + ]); + mkdirSync(join(home, ".media"), { recursive: true }); + writeFileSync( + join(home, ".media/misses.jsonl"), + [ + JSON.stringify({ ts: isoDaysAgo(100), type: "bgm", intent: "old miss" }), + JSON.stringify({ ts: isoDaysAgo(1), type: "bgm", intent: "new miss" }), + ].join("\n"), + ); + + const stats = buildStats({ projectDir, days: 7 }); + assert.equal(stats.total_resolves, 2); + assert.deepEqual(stats.by_type, { bgm: 1, image: 1 }); + assert.equal(stats.misses, 1); + assert.equal(stats.top_missed_intents.bgm[0].intent, "new miss"); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/skills/media-use/scripts/lib/telemetry.mjs b/skills/media-use/scripts/lib/telemetry.mjs index ed82491b28..eefcdc346a 100644 --- a/skills/media-use/scripts/lib/telemetry.mjs +++ b/skills/media-use/scripts/lib/telemetry.mjs @@ -1,11 +1,14 @@ -// Anonymous, opt-out usage tracking for media-use, mirroring the hyperframes CLI -// telemetry (packages/cli/src/telemetry). Answers "is this actually used, and -// which capabilities" without any PII: we send the media TYPE, the resolution -// SOURCE, and the winning PROVIDER: never the intent text, file names, or paths. +// Opt-out usage tracking for media-use, sharing the hyperframes CLI/studio +// identity (packages/cli/src/telemetry): the same install id from +// ~/.hyperframes/config.json, plus a $identify to the HeyGen account on sign-in, +// so a person is one PostHog profile across surfaces — not a fresh id per tool. +// Not fully anonymous by design (it must dedupe): pseudonymous before sign-in, +// account-linked after. Event PROPERTIES stay coarse — media TYPE, resolution +// SOURCE, winning PROVIDER — never the intent text, file names, or paths. // // Same public PostHog project key as the CLI (a write-only ingestion key, safe -// to ship), same opt-outs (DO_NOT_TRACK / HYPERFRAMES_NO_TELEMETRY / CI), and -// $ip:null so no IP is recorded. Fire-and-forget: telemetry never blocks a +// to ship), same opt-outs (DO_NOT_TRACK / HYPERFRAMES_NO_TELEMETRY / CI / dev), +// and $ip:null so no IP is recorded. Fire-and-forget: telemetry never blocks a // resolve and never throws into it. import { randomUUID } from "node:crypto"; @@ -16,6 +19,7 @@ import { join } from "node:path"; const POSTHOG_API_KEY = "phc_zjjbX0PnWxERXrMHhkEJWj9A9BhGVLRReICgsfTMmpx"; const POSTHOG_HOST = "https://us.i.posthog.com"; const TIMEOUT_MS = 1500; +let identifiedAccount = false; /** True when telemetry must NOT be sent (opt-out envs, CI, dev). */ export function optedOut() { @@ -28,47 +32,152 @@ export function optedOut() { ); } -// Stable per-machine anonymous id, persisted in the dir media-use already owns. +// CLI + studio share one install identity in ~/.hyperframes/config.json +// (packages/cli/src/telemetry/config.ts — same path, same `anonymousId` / +// `telemetryNoticeShown` fields). Read and write that same file so media-use is +// the same PostHog person and shows the notice once per person, not per tool. +// Computed per call (not a module const) so it honors HOME at runtime — tests +// sandbox HOME, and os.homedir() re-reads it each call. +function sharedConfigPath() { + return join(homedir(), ".hyperframes", "config.json"); +} + +function readSharedConfig() { + try { + const file = sharedConfigPath(); + if (existsSync(file)) { + const parsed = JSON.parse(readFileSync(file, "utf8")); + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return parsed; + } + } catch { + // unreadable config → treat as empty; never throw + } + return {}; +} + +function writeSharedConfig(config) { + const dir = join(homedir(), ".hyperframes"); + if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, "config.json"), JSON.stringify(config, null, 2) + "\n"); +} + +// Adopt a pre-existing media-use-only id (~/.media/anon-id from before this +// change) so upgraders keep their PostHog persona instead of resetting to a new +// one — otherwise cross-surface continuity would start over on upgrade. +function legacyMediaAnonId() { + try { + const file = join(homedir(), ".media", "anon-id"); + if (existsSync(file)) { + const id = readFileSync(file, "utf8").trim(); + if (id) return id; + } + } catch { + // ignore + } + return null; +} + +// Stable per-machine id from the shared config; seeds it (adopting a legacy +// media-use id when present) if absent. function anonymousId() { - const dir = join(homedir(), ".media"); - const file = join(dir, "anon-id"); try { - if (existsSync(file)) return readFileSync(file, "utf8").trim(); - if (!existsSync(dir)) mkdirSync(dir, { recursive: true }); - const id = randomUUID(); - writeFileSync(file, id); + const config = readSharedConfig(); + if (typeof config.anonymousId === "string" && config.anonymousId.trim()) { + return config.anonymousId.trim(); + } + const id = legacyMediaAnonId() || randomUUID(); + writeSharedConfig({ ...config, anonymousId: id }); return id; } catch { return "anon"; // best-effort; a shared bucket is fine if the fs is read-only } } -/** - * Fire-and-forget a single event to PostHog. Best-effort: awaited with a short - * timeout so a short-lived script flushes before exit, but any failure (offline, - * opted out) is swallowed. `properties` must be non-PII (no intent/paths). - */ -export async function track(event, properties = {}) { +function heygenAccountDistinctId() { + const file = join(process.env.HEYGEN_CONFIG_DIR || join(homedir(), ".heygen"), "credentials"); + try { + if (!existsSync(file)) return null; + const raw = readFileSync(file, "utf8").trim(); + if (!raw.startsWith("{")) return null; + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; + const user = parsed.user; + if (!user || typeof user !== "object" || Array.isArray(user)) return null; + const id = typeof user.email === "string" && user.email.trim() ? user.email : user.username; + return typeof id === "string" && id.trim() ? id.trim() : null; + } catch { + return null; + } +} + +function showTelemetryNotice() { if (optedOut()) return; - const body = JSON.stringify({ - api_key: POSTHOG_API_KEY, - batch: [ - { - event, - properties: { ...properties, surface: "media-use", $ip: null }, - distinct_id: anonymousId(), - timestamp: new Date().toISOString(), - }, - ], - }); + try { + const config = readSharedConfig(); + // Shared with the CLI (config.telemetryNoticeShown): shown once per person + // across surfaces, not once per tool. + if (config.telemetryNoticeShown === true) return; + console.error( + [ + "media-use sends usage telemetry: media type, resolution source, and provider; never intent text, file names, or paths.", + "If you sign in to HeyGen, usage links to your account email or username. Opt out with HYPERFRAMES_NO_TELEMETRY=1 or DO_NOT_TRACK=1.", + ].join("\n"), + ); + writeSharedConfig({ ...config, telemetryNoticeShown: true }); + } catch { + // notice is best-effort; never surface into the command + } +} + +async function postBatch(batch) { try { await fetch(`${POSTHOG_HOST}/batch/`, { method: "POST", headers: { "Content-Type": "application/json", Connection: "close" }, - body, + body: JSON.stringify({ api_key: POSTHOG_API_KEY, batch }), signal: AbortSignal.timeout(TIMEOUT_MS), }); } catch { // telemetry is best-effort; never surface into the command } } + +async function postEvent(event, properties, distinctId) { + await postBatch([ + { + event, + properties: { ...properties, surface: "media-use", $ip: null }, + distinct_id: distinctId, + timestamp: new Date().toISOString(), + }, + ]); +} + +async function identifyAccount(anonId) { + if (optedOut() || identifiedAccount) return; + const distinctId = heygenAccountDistinctId(); + if (!distinctId) return; + identifiedAccount = true; + await postEvent("$identify", { $anon_distinct_id: anonId }, distinctId); +} + +/** + * Fire-and-forget a single event to PostHog. Best-effort: awaited with a short + * timeout so a short-lived script flushes before exit, but any failure (offline, + * opted out) is swallowed. `properties` must be non-PII (no intent/paths). + */ +export async function track(event, properties = {}) { + if (optedOut()) return; + showTelemetryNotice(); + const anonId = anonymousId(); + await identifyAccount(anonId); + await postEvent(event, properties, anonId); +} + +export function __anonymousIdForTest() { + return anonymousId(); +} + +export function __resetTelemetryForTest() { + identifiedAccount = false; +} diff --git a/skills/media-use/scripts/lib/telemetry.test.mjs b/skills/media-use/scripts/lib/telemetry.test.mjs index ba8d24b68a..fa61997bb1 100644 --- a/skills/media-use/scripts/lib/telemetry.test.mjs +++ b/skills/media-use/scripts/lib/telemetry.test.mjs @@ -1,6 +1,39 @@ import { strict as assert } from "node:assert"; import { test } from "node:test"; -import { optedOut, track } from "./telemetry.mjs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, + chmodSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { __anonymousIdForTest, __resetTelemetryForTest, optedOut, track } from "./telemetry.mjs"; + +function sandbox() { + const root = mkdtempSync(join(tmpdir(), "mu-telemetry-")); + const home = join(root, "home"); + mkdirSync(home, { recursive: true }); + process.env.HOME = home; + return { root, home }; +} + +function restoreEnv(saved) { + for (const k of Object.keys(process.env)) if (!(k in saved)) delete process.env[k]; + Object.assign(process.env, saved); +} + +function withoutTelemetryOptOut() { + for (const k of ["DO_NOT_TRACK", "HYPERFRAMES_NO_TELEMETRY", "CI", "NODE_ENV"]) + delete process.env[k]; +} + +function parseFetchBodies(calls) { + return calls.flatMap((call) => JSON.parse(call.options.body).batch); +} test("optedOut respects DO_NOT_TRACK / HYPERFRAMES_NO_TELEMETRY / CI", () => { const saved = { ...process.env }; @@ -23,13 +56,220 @@ test("optedOut respects DO_NOT_TRACK / HYPERFRAMES_NO_TELEMETRY / CI", () => { }); test("track is a no-op (no network, resolves) when opted out", async () => { - const saved = process.env.DO_NOT_TRACK; + const savedEnv = { ...process.env }; + const originalFetch = globalThis.fetch; + const { root, home } = sandbox(); + const calls = []; + globalThis.fetch = async (...args) => { + calls.push(args); + return { ok: true }; + }; process.env.DO_NOT_TRACK = "1"; try { // must resolve immediately without throwing or hitting the network await track("media_use_resolve", { type: "bgm", source: "search" }); + assert.equal(calls.length, 0); + assert.equal(existsSync(join(home, ".hyperframes/config.json")), false); + assert.equal(existsSync(join(home, ".media/telemetry-notice-shown")), false); + } finally { + globalThis.fetch = originalFetch; + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("anonymous id uses the shared hyperframes config", () => { + const savedEnv = { ...process.env }; + const { root, home } = sandbox(); + try { + withoutTelemetryOptOut(); + mkdirSync(join(home, ".hyperframes"), { recursive: true }); + writeFileSync( + join(home, ".hyperframes/config.json"), + JSON.stringify({ anonymousId: "shared-install-id", keep: true }), + ); + mkdirSync(join(home, ".media"), { recursive: true }); + writeFileSync(join(home, ".media/anon-id"), "old-media-id"); + + assert.equal(__anonymousIdForTest(), "shared-install-id"); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("anonymous id seeds missing config once and reuses it", () => { + const savedEnv = { ...process.env }; + const { root, home } = sandbox(); + try { + withoutTelemetryOptOut(); + const first = __anonymousIdForTest(); + const configPath = join(home, ".hyperframes/config.json"); + assert.ok(existsSync(configPath)); + assert.match( + first, + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i, + ); + const second = __anonymousIdForTest(); + assert.equal(second, first); + assert.equal(JSON.parse(readFileSync(configPath, "utf8")).anonymousId, first); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("anonymous id adopts a legacy ~/.media/anon-id on upgrade (persona continuity)", () => { + const savedEnv = { ...process.env }; + const { root, home } = sandbox(); + try { + withoutTelemetryOptOut(); + mkdirSync(join(home, ".media"), { recursive: true }); + writeFileSync(join(home, ".media/anon-id"), "legacy-media-id"); + // no ~/.hyperframes/config.json yet — the old media-use-only id must carry over + assert.equal(__anonymousIdForTest(), "legacy-media-id"); + // and it is persisted into the shared config so CLI/studio see the same id + assert.equal( + JSON.parse(readFileSync(join(home, ".hyperframes/config.json"), "utf8")).anonymousId, + "legacy-media-id", + ); + } finally { + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("track identifies a signed-in HeyGen account once and still sends events", async () => { + const savedEnv = { ...process.env }; + const originalFetch = globalThis.fetch; + const { root, home } = sandbox(); + const calls = []; + globalThis.fetch = async (url, options) => { + calls.push({ url, options }); + return { ok: true }; + }; + try { + withoutTelemetryOptOut(); + mkdirSync(join(home, ".hyperframes"), { recursive: true }); + writeFileSync( + join(home, ".hyperframes/config.json"), + JSON.stringify({ anonymousId: "anon-1" }), + ); + mkdirSync(join(home, ".heygen"), { recursive: true }); + writeFileSync( + join(home, ".heygen/credentials"), + JSON.stringify({ user: { email: "alice@example.com", username: "alice" } }), + ); + + await track("media_use_resolve", { type: "bgm", source: "search" }); + await track("media_use_resolve", { type: "image", source: "generated" }); + + const batch = parseFetchBodies(calls); + const identify = batch.filter((item) => item.event === "$identify"); + assert.equal(identify.length, 1); + assert.equal(identify[0].distinct_id, "alice@example.com"); + assert.equal(identify[0].properties.$anon_distinct_id, "anon-1"); + assert.equal(batch.filter((item) => item.event === "media_use_resolve").length, 2); + } finally { + globalThis.fetch = originalFetch; + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("track does not identify when signed out", async () => { + const savedEnv = { ...process.env }; + const originalFetch = globalThis.fetch; + const { root, home } = sandbox(); + const calls = []; + globalThis.fetch = async (url, options) => { + calls.push({ url, options }); + return { ok: true }; + }; + try { + withoutTelemetryOptOut(); + mkdirSync(join(home, ".hyperframes"), { recursive: true }); + writeFileSync( + join(home, ".hyperframes/config.json"), + JSON.stringify({ anonymousId: "anon-2" }), + ); + + await track("media_use_resolve", { type: "bgm", source: "search" }); + + const batch = parseFetchBodies(calls); + assert.equal( + batch.some((item) => item.event === "$identify"), + false, + ); + assert.equal(batch[0].event, "media_use_resolve"); + assert.equal(batch[0].distinct_id, "anon-2"); + } finally { + globalThis.fetch = originalFetch; + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("first run notice prints to stderr once and never stdout", async () => { + const savedEnv = { ...process.env }; + const originalFetch = globalThis.fetch; + const originalError = console.error; + const originalLog = console.log; + const { root, home } = sandbox(); + const stderr = []; + const stdout = []; + globalThis.fetch = async () => ({ ok: true }); + console.error = (...args) => stderr.push(args.join(" ")); + console.log = (...args) => stdout.push(args.join(" ")); + try { + withoutTelemetryOptOut(); + await track("media_use_resolve", { type: "bgm" }); + await track("media_use_resolve", { type: "sfx" }); + + assert.equal(stderr.length, 1); + assert.match(stderr[0], /media-use sends usage telemetry/); + assert.equal(stdout.length, 0); + // notice-shown lives in the shared config (config.telemetryNoticeShown), so + // the CLI and media-use show it once per person — not a media-use-only marker. + assert.equal( + JSON.parse(readFileSync(join(home, ".hyperframes/config.json"), "utf8")).telemetryNoticeShown, + true, + ); + } finally { + globalThis.fetch = originalFetch; + console.error = originalError; + console.log = originalLog; + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); + } +}); + +test("read-only telemetry state degrades without throwing", async () => { + const savedEnv = { ...process.env }; + const originalFetch = globalThis.fetch; + const { root, home } = sandbox(); + globalThis.fetch = async () => ({ ok: true }); + try { + withoutTelemetryOptOut(); + writeFileSync(join(home, ".hyperframes"), "not a directory"); + writeFileSync(join(home, ".media"), "not a directory"); + await track("media_use_resolve", { type: "bgm" }); } finally { - if (saved === undefined) delete process.env.DO_NOT_TRACK; - else process.env.DO_NOT_TRACK = saved; + globalThis.fetch = originalFetch; + try { + chmodSync(join(home, ".hyperframes"), 0o600); + } catch { + // best effort for cleanup on platforms with different chmod behavior + } + restoreEnv(savedEnv); + rmSync(root, { recursive: true, force: true }); + __resetTelemetryForTest(); } }); diff --git a/skills/media-use/scripts/resolve.mjs b/skills/media-use/scripts/resolve.mjs index 3bf43d42af..1de70e7bf9 100644 --- a/skills/media-use/scripts/resolve.mjs +++ b/skills/media-use/scripts/resolve.mjs @@ -11,6 +11,8 @@ import { runCapability, listTypes, providerMatches, providerNamesFor } from "./l import { freezeUrl, freezeLocalFile, isDirectMediaUrl } from "./lib/freeze.mjs"; import { findExistingAsset } from "./lib/adopt.mjs"; import { track } from "./lib/telemetry.mjs"; +import { recordMiss } from "./lib/misses.mjs"; +import { buildStats } from "./lib/stats.mjs"; import { typesMatch } from "./lib/match.mjs"; import { listCandidates, formatCandidates, CANDIDATE_CAP } from "./lib/candidates.mjs"; import { findGlobalBySha } from "./lib/cache.mjs"; @@ -45,6 +47,8 @@ const { values: args } = parseArgs({ adopt: { type: "boolean", default: false }, candidates: { type: "boolean", default: false }, doctor: { type: "boolean", default: false }, + stats: { type: "boolean", default: false }, + days: { type: "string" }, "dry-run": { type: "boolean", default: false }, reuse: { type: "string" }, from: { type: "string" }, @@ -75,6 +79,9 @@ Options: --candidates List reusable assets (project + global cache) for --type; no download, no mutation. Read them and decide reuse yourself. --doctor Check local CLI dependencies; no manifest changes. + --stats Print local usage stats from .media and ~/.media; no mutation. + --days Limit --stats to records/misses from the last N days when + timestamps are available. --reuse Import a specific global-cache asset (by content sha/prefix, from --candidates) into this project --from Freeze a local file or direct public URL (ingest) @@ -134,6 +141,19 @@ if (args.doctor) { process.exit(doctor.ok ? 0 : 1); } +if (args.stats) { + const report = buildStats({ + projectDir, + days: args.days ? Number(args.days) : undefined, + }); + if (args.json) { + console.log(JSON.stringify(report)); + } else { + printStats(report); + } + process.exit(0); +} + // Reuse: import a specific global-cache asset (by content sha/prefix, taken // from --candidates) into this project. `!== undefined` so an empty --reuse "" // still routes here (and gets a clear empty-sha error) instead of falling @@ -326,6 +346,12 @@ async function run() { local_only: !!localOnly, provider_override: !!args.provider, }); + recordMiss({ + type, + intent, + provider_override: !!args.provider, + local_only: !!args["local-only"], + }); // brand stays local: no frame.md/design.md -> upsell the HyperFrames design // flow rather than reporting a generic miss (B5). const msg = @@ -513,6 +539,12 @@ async function colorMiss(type, intent) { local_only: !!args["local-only"], provider_override: !!args.provider, }); + recordMiss({ + type, + intent, + provider_override: !!args.provider, + local_only: !!args["local-only"], + }); const msg = `no local color grade could resolve ${type}: "${intent}"`; if (args.json) { console.log(JSON.stringify({ ok: false, error: msg })); @@ -898,6 +930,42 @@ function printDoctor(checks) { } } +function printStats(report) { + console.log("media-use stats"); + console.log(`total resolves: ${report.total_resolves}`); + console.log(`misses: ${report.misses}`); + console.log( + `hit rate: ${report.hit_rate == null ? "n/a" : `${Math.round(report.hit_rate * 100)}%`}`, + ); + printMap("by type", report.by_type); + printMap("by source", report.by_source); + printMap("by provider", report.by_provider); + printMap("by via", report.by_via); + console.log(`global cache assets: ${report.global_cache_assets}`); + console.log(`global cache disk: ${report.global_cache_disk_bytes} bytes`); + console.log(`cross-project reuse: ${report.cross_project_reuse}`); + console.log("top missed intents:"); + const entries = Object.entries(report.top_missed_intents); + if (entries.length === 0) { + console.log(" none"); + return; + } + for (const [type, misses] of entries) { + console.log(` ${type}:`); + for (const miss of misses) console.log(` ${miss.count} ${miss.intent}`); + } +} + +function printMap(label, values) { + const entries = Object.entries(values); + console.log(`${label}:`); + if (entries.length === 0) { + console.log(" none"); + return; + } + for (const [key, value] of entries) console.log(` ${key}: ${value}`); +} + function runCommand(bin, argv) { return spawnSync(bin, argv, { encoding: "utf8", diff --git a/skills/media-use/scripts/resolve.test.mjs b/skills/media-use/scripts/resolve.test.mjs index 770432792a..2c4ff924a5 100644 --- a/skills/media-use/scripts/resolve.test.mjs +++ b/skills/media-use/scripts/resolve.test.mjs @@ -57,18 +57,22 @@ function makeRecord(overrides = {}) { // a separate argv entry, so a value with spaces or shell metacharacters can't // break out — never build a command string and hand it to a shell. function runResolve(args, opts = {}) { + const { env, ...rest } = opts; return execFileSync(process.execPath, [RESOLVE_CLI, ...args], { cwd: REPO_ROOT, encoding: "utf8", - ...opts, + env: { ...process.env, DO_NOT_TRACK: "1", ...env }, + ...rest, }); } function spawnResolve(args, opts = {}) { + const { env, ...rest } = opts; return spawnSync(process.execPath, [RESOLVE_CLI, ...args], { cwd: REPO_ROOT, encoding: "utf8", - ...opts, + env: { ...process.env, DO_NOT_TRACK: "1", ...env }, + ...rest, }); } @@ -293,6 +297,7 @@ test("--help exits 0", () => { assert.ok(out.includes("--for")); assert.ok(out.includes("--from")); assert.ok(out.includes("--local-only")); + assert.ok(out.includes("--stats")); }); test("unknown type error lists grade and lut", () => {