Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 27 additions & 17 deletions Sources/FluidAudioCLI/DatasetParsers/DatasetDownloader.swift
Original file line number Diff line number Diff line change
Expand Up @@ -111,16 +111,22 @@ struct DatasetDownloader {
) async
-> Bool
{
// Try multiple URL patterns - the AMI corpus mirror structure has some variations
let baseURLs = [
"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus", // Double slash pattern (from user's working example)
"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror/amicorpus", // Single slash pattern
"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus", // Alternative with extra slash
// Prefer the HuggingFace mirror (hosts the official 16-meeting SDM test
// split), then fall back to the intermittently-available Edinburgh
// server. Meetings or variants absent from the mirror 404 and fall
// through to upstream.
let mirrorURL =
"https://huggingface.co/datasets/FluidInference/ami-corpus-mirror/resolve/main/\(variant.rawValue)/\(meetingId).\(variant.filePattern)"
// Both slash patterns of the upstream corpus mirror have been seen working.
let upstreamBases = [
"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror//amicorpus",
"https://groups.inf.ed.ac.uk/ami/AMICorpusMirror/amicorpus",
]
let candidateURLs =
[mirrorURL]
+ upstreamBases.map { "\($0)/\(meetingId)/audio/\(meetingId).\(variant.filePattern)" }

for (_, baseURL) in baseURLs.enumerated() {
let urlString = "\(baseURL)/\(meetingId)/audio/\(meetingId).\(variant.filePattern)"

for urlString in candidateURLs {
guard let url = URL(string: urlString) else {
continue
}
Expand Down Expand Up @@ -192,19 +198,23 @@ struct DatasetDownloader {
}

// Download and extract AMI manual annotations v1.6.2.
// The Edinburgh server is occasionally flaky, so retry with backoff —
// a single transient failure here once poisoned a CI benchmark run
// with placeholder ground truth (issue #752).
let zipURL =
"https://groups.inf.ed.ac.uk/ami/AMICorpusAnnotations/ami_public_manual_1.6.2.zip"
// Prefer the HuggingFace mirror — the upstream Edinburgh server is
// intermittently down, and a single transient failure here once
// poisoned a CI benchmark run with placeholder ground truth (#752).
let zipURLs = [
"https://huggingface.co/datasets/FluidInference/ami-corpus-mirror/resolve/main/annotations/ami_public_manual_1.6.2.zip",
"https://groups.inf.ed.ac.uk/ami/AMICorpusAnnotations/ami_public_manual_1.6.2.zip",
]
let zipFile = annotationsDir.appendingPathComponent("ami_public_manual_1.6.2.zip")

var zipSuccess = false
let maxAttempts = 3
for attempt in 1...maxAttempts {
zipSuccess = await downloadAnnotationFile(from: zipURL, to: zipFile)
if zipSuccess { break }
logger.warning("Annotation download attempt \(attempt)/\(maxAttempts) failed")
attempts: for attempt in 1...maxAttempts {
for zipURL in zipURLs {
zipSuccess = await downloadAnnotationFile(from: zipURL, to: zipFile)
if zipSuccess { break attempts }
}
logger.warning("Annotation download attempt \(attempt)/\(maxAttempts) failed for all sources")
if attempt < maxAttempts {
try? await Task.sleep(nanoseconds: UInt64(attempt) * 2_000_000_000)
}
Expand Down
Loading