diff --git a/IOSAccessAssessment.xcodeproj/project.pbxproj b/IOSAccessAssessment.xcodeproj/project.pbxproj index b7546d51..afb69376 100644 --- a/IOSAccessAssessment.xcodeproj/project.pbxproj +++ b/IOSAccessAssessment.xcodeproj/project.pbxproj @@ -24,7 +24,7 @@ A30801612EC09BB700B1BA3A /* CocoCustom53ClassConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = A30801592EC09BB700B1BA3A /* CocoCustom53ClassConfig.swift */; }; A308016C2EC15CC400B1BA3A /* AccessibilityFeatureAttributeExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = A308016B2EC15CBA00B1BA3A /* AccessibilityFeatureAttributeExtension.swift */; }; A312FF232FA430510044808E /* AccessibilityFeatureKindExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = A312FF222FA4304D0044808E /* AccessibilityFeatureKindExtension.swift */; }; - A312FF2B2FA436CB0044808E /* IsExistingExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = A312FF2A2FA436C60044808E /* IsExistingExtension.swift */; }; + A312FF2B2FA436CB0044808E /* AppBasedExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = A312FF2A2FA436C60044808E /* AppBasedExtension.swift */; }; A322B9042FA53B6F00F4D488 /* MappedEditableAccessibilityFeature.swift in Sources */ = {isa = PBXBuildFile; fileRef = A322B9032FA53B6800F4D488 /* MappedEditableAccessibilityFeature.swift */; }; A322B9052FA53B9B00F4D488 /* MappedAccessibilityFeature.swift in Sources */ = {isa = PBXBuildFile; fileRef = A322B8082FA5287300F4D488 /* MappedAccessibilityFeature.swift */; }; A329433C2EE7BEE100C4C1BC /* OSWPolicy.swift in Sources */ = {isa = PBXBuildFile; fileRef = A329433B2EE7BEDD00C4C1BC /* OSWPolicy.swift */; }; @@ -71,6 +71,8 @@ A3B61FCB2F79036A0052AE2C /* OSMMapDataResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3B61FCA2F7903660052AE2C /* OSMMapDataResponse.swift */; }; A3BCBC502EFBB92900D15E15 /* AccessibilityFeatureEncoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3BCBC4F2EFBB92500D15E15 /* AccessibilityFeatureEncoder.swift */; }; A3C22FD82CF2F0C300533BF7 /* DequeModule in Frameworks */ = {isa = PBXBuildFile; productRef = A3C22FD72CF2F0C300533BF7 /* DequeModule */; }; + A3CBCD53301E81B60019BB72 /* TestAnnotationView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3CBCD52301E81B60019BB72 /* TestAnnotationView.swift */; }; + A3CBCD59301E9B2D0019BB72 /* AmbiguityCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3CBCD58301E9B290019BB72 /* AmbiguityCase.swift */; }; A3D78D742E65108E003BFE78 /* WorkspaceViewModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3D78D732E65108A003BFE78 /* WorkspaceViewModel.swift */; }; A3D78D762E654F18003BFE78 /* ProfileView.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3D78D752E654F14003BFE78 /* ProfileView.swift */; }; A3E162782F3AFC66002D4D08 /* MeshCoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3E162772F3AFC63002D4D08 /* MeshCoder.swift */; }; @@ -195,7 +197,7 @@ A308016B2EC15CBA00B1BA3A /* AccessibilityFeatureAttributeExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccessibilityFeatureAttributeExtension.swift; sourceTree = ""; }; A312FD852FA3391C0044808E /* PointNMapSharedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = PointNMapSharedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; A312FF222FA4304D0044808E /* AccessibilityFeatureKindExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccessibilityFeatureKindExtension.swift; sourceTree = ""; }; - A312FF2A2FA436C60044808E /* IsExistingExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IsExistingExtension.swift; sourceTree = ""; }; + A312FF2A2FA436C60044808E /* AppBasedExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppBasedExtension.swift; sourceTree = ""; }; A322B8082FA5287300F4D488 /* MappedAccessibilityFeature.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MappedAccessibilityFeature.swift; sourceTree = ""; }; A322B9032FA53B6800F4D488 /* MappedEditableAccessibilityFeature.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MappedEditableAccessibilityFeature.swift; sourceTree = ""; }; A329433B2EE7BEDD00C4C1BC /* OSWPolicy.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OSWPolicy.swift; sourceTree = ""; }; @@ -240,6 +242,8 @@ A3B61FC42F7647FC0052AE2C /* EnvironmentService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EnvironmentService.swift; sourceTree = ""; }; A3B61FCA2F7903660052AE2C /* OSMMapDataResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OSMMapDataResponse.swift; sourceTree = ""; }; A3BCBC4F2EFBB92500D15E15 /* AccessibilityFeatureEncoder.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccessibilityFeatureEncoder.swift; sourceTree = ""; }; + A3CBCD52301E81B60019BB72 /* TestAnnotationView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TestAnnotationView.swift; sourceTree = ""; }; + A3CBCD58301E9B290019BB72 /* AmbiguityCase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AmbiguityCase.swift; sourceTree = ""; }; A3D78D732E65108A003BFE78 /* WorkspaceViewModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WorkspaceViewModel.swift; sourceTree = ""; }; A3D78D752E654F14003BFE78 /* ProfileView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileView.swift; sourceTree = ""; }; A3DA4DB42EBAE101005BB812 /* IOSAccessAssessment-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "IOSAccessAssessment-Bridging-Header.h"; sourceTree = ""; }; @@ -456,6 +460,7 @@ A30801662EC0AE6B00B1BA3A /* Components */ = { isa = PBXGroup; children = ( + A3CBCD58301E9B290019BB72 /* AmbiguityCase.swift */, A329433B2EE7BEDD00C4C1BC /* OSWPolicy.swift */, ); path = Components; @@ -479,7 +484,7 @@ A312FF292FA436390044808E /* Extensions */ = { isa = PBXGroup; children = ( - A312FF2A2FA436C60044808E /* IsExistingExtension.swift */, + A312FF2A2FA436C60044808E /* AppBasedExtension.swift */, ); path = Extensions; sourceTree = ""; @@ -779,6 +784,7 @@ A3EE6E442F57FE4400F515E6 /* TestMode */ = { isa = PBXGroup; children = ( + A3CBCD52301E81B60019BB72 /* TestAnnotationView.swift */, A3EE6E472F580D0A00F515E6 /* TestListView.swift */, A3EE6E492F580D6000F515E6 /* TestCameraView.swift */, ); @@ -1058,7 +1064,8 @@ A3B61FCB2F79036A0052AE2C /* OSMMapDataResponse.swift in Sources */, CAF812BC2CF78F8100D44B84 /* NetworkError.swift in Sources */, A305B06C2E18A85F00ECCF9B /* DepthCoder.swift in Sources */, - A312FF2B2FA436CB0044808E /* IsExistingExtension.swift in Sources */, + A312FF2B2FA436CB0044808E /* AppBasedExtension.swift in Sources */, + A3CBCD59301E9B2D0019BB72 /* AmbiguityCase.swift in Sources */, A308016C2EC15CC400B1BA3A /* AccessibilityFeatureAttributeExtension.swift in Sources */, A35E05162EDEA050003C26CF /* APIChangesetUploadController.swift in Sources */, A36C6E022E134CE600A86004 /* bisenetv2_35_640_640.mlpackage in Sources */, @@ -1070,6 +1077,7 @@ A32D66532F7C3F2F00DC4173 /* OSWMultiPolygon.swift in Sources */, A35A8BD12E5D0D1200CC8AA7 /* WorkspaceService.swift in Sources */, CAA947792CDE700A000C6918 /* AuthService.swift in Sources */, + A3CBCD53301E81B60019BB72 /* TestAnnotationView.swift in Sources */, A37E720E2ED5783600CFE4EF /* SharedAppContext.swift in Sources */, A3F38C4C2D38A2C700900547 /* DepthModel.swift in Sources */, 55659C102BB7863F0094DF01 /* SetupView.swift in Sources */, diff --git a/IOSAccessAssessment/AccessibilityFeature/AttributeEstimation/Extensions/AppBasedExtension.swift b/IOSAccessAssessment/AccessibilityFeature/AttributeEstimation/Extensions/AppBasedExtension.swift new file mode 100644 index 00000000..fcdc9b42 --- /dev/null +++ b/IOSAccessAssessment/AccessibilityFeature/AttributeEstimation/Extensions/AppBasedExtension.swift @@ -0,0 +1,255 @@ +// +// AppBasedExtension.swift +// IOSAccessAssessment +// +// Created by Himanshu on 4/30/26. +// +import PointNMapShared +import CoreLocation +import MapKit + +enum LocationType: String, Codable { + case captureLocation = "capture_location" + case correctedLocation = "corrected_location" +} + +extension AttributeEstimationPipeline { + func processIsExistingRequest( + deviceLocation: CLLocationCoordinate2D, + mappingData: CurrentMappingData, + accessibilityFeature: MappedEditableAccessibilityFeature, + locationType: LocationType = .captureLocation, + featureIndex: Int? = nil + ) { + /// Threshold needs to be in Map Units + let distanceThreshold = PointNMapConstants.WorkspaceConstants.fetchUpdateRadiusThresholdInMeters * MKMapPointsPerMeterAtLatitude(deviceLocation.latitude) + guard let locationDetails = locationType == .captureLocation ? accessibilityFeature.locationDetails : accessibilityFeature.correctedLocationDetails else { +// accessibilityFeature.setIsExisting(false) + locationType == .captureLocation ? accessibilityFeature.setIsExisting(false) : accessibilityFeature.setCorrectedIsExisting(false) + return + } + let matchedElement: (any OSWElement)? = mappingData.getMatchedFeature( + to: locationDetails, featureClass: accessibilityFeature.accessibilityFeatureClass, + captureId: self.captureImageData?.id, + distanceThreshold: distanceThreshold, +// featureIndex: featureIndex + ) + guard let matchedElement = matchedElement else { +// accessibilityFeature.setIsExisting(false) + locationType == .captureLocation ? accessibilityFeature.setIsExisting(false) : accessibilityFeature.setCorrectedIsExisting(false) + return + } + let isExisting = accessibilityFeature.accessibilityFeatureClass.kind.oswPolicy.isExistingFirst +// accessibilityFeature.setIsExisting(isExisting) +// accessibilityFeature.setOSWElement(oswElement: matchedElement) + locationType == .captureLocation ? accessibilityFeature.setIsExisting(isExisting) : accessibilityFeature.setCorrectedIsExisting(isExisting) + locationType == .captureLocation ? accessibilityFeature.setOSWElement(oswElement: matchedElement) : accessibilityFeature.setCorrectedOSWElement(matchedElement) + } + + func processNearestFeaturesRequest( + deviceLocation: CLLocationCoordinate2D, + mappingData: CurrentMappingData, + accessibilityFeature: MappedEditableAccessibilityFeature, + locationType: LocationType = .captureLocation, + featureIndex: Int? = nil + ) { + /// Threshold needs to be in Map Units + let distanceThreshold = PointNMapConstants.WorkspaceConstants.fetchUpdateRadiusThresholdInMeters * MKMapPointsPerMeterAtLatitude(deviceLocation.latitude) + guard let locationDetails = locationType == .captureLocation ? accessibilityFeature.locationDetails : accessibilityFeature.correctedLocationDetails else { + return + } + let nearestOSWElements: [(any OSWElement, CLLocationDistance)] = mappingData.getNearestFeatures( + to: locationDetails, featureClass: accessibilityFeature.accessibilityFeatureClass, + captureId: self.captureImageData?.id, + distanceThreshold: distanceThreshold, +// featureIndex: featureIndex + ) +// accessibilityFeature.setNearestOSWElements(nearestOSWElements: nearestOSWElements) + locationType == .captureLocation ? accessibilityFeature.setNearestOSWElements(nearestOSWElements: nearestOSWElements) : accessibilityFeature.setCorrectedNearestOSWElements(nearestOSWElements) + } + + func processLocationRequestTypeBased( + deviceLocation: CLLocationCoordinate2D, + accessibilityFeature: MappedEditableAccessibilityFeature, + locationType: LocationType = .captureLocation, + ) throws { + let locationRequestResult = try self.calculateLocation( + deviceLocation: deviceLocation, + accessibilityFeature: accessibilityFeature + ) +// accessibilityFeature.setLocationDetails(locationDetails: locationRequestResult.locationDetails) + locationType == .captureLocation ? accessibilityFeature.setLocationDetails(locationDetails: locationRequestResult.locationDetails) : accessibilityFeature.setCorrectedLocationDetails(locationRequestResult.locationDetails) + /// Set Lidar Depth as experimental attribute + if locationType == .captureLocation { + if let lidarDepthAttributeValue = AccessibilityFeatureAttribute.lidarDepth.value( + from: Double(locationRequestResult.lidarDepth) + ) { + do { + try accessibilityFeature.setExperimentalAttributeValue(lidarDepthAttributeValue, for: .lidarDepth) + } catch { + print("Error setting lidar depth attribute for feature \(accessibilityFeature.id): \(error.localizedDescription)") + } + } + if let latitudeDeltaAttributeValue = AccessibilityFeatureAttribute.latitudeDelta.value( + from: Double(locationRequestResult.locationDelta.x) + ) { + do { + try accessibilityFeature.setExperimentalAttributeValue(latitudeDeltaAttributeValue, for: .latitudeDelta) + } catch { + print("Error setting latitude delta attribute for feature " + + "\(accessibilityFeature.id): \(error.localizedDescription)") + } + } + if let longitudeDeltaAttributeValue = AccessibilityFeatureAttribute.longitudeDelta.value( + from: Double(locationRequestResult.locationDelta.y) + ) { + do { + try accessibilityFeature.setExperimentalAttributeValue(longitudeDeltaAttributeValue, for: .longitudeDelta) + } catch { + print("Error setting longitude delta attribute for feature " + + "\(accessibilityFeature.id): \(error.localizedDescription)") + } + } + } + } +} + +/** + This unique extension is used to process all the features of a specific class at once for the above multiple requests. + This is to ensure that each feature gets assigned to a unique OSW element and not multiple features getting assigned to the same OSW element, which can happen if each feature is processed independently. + */ +extension AttributeEstimationPipeline { + func processAllFeaturesForAssignments( +// deviceLocation: CLLocationCoordinate2D, + captureLocation: CLLocationCoordinate2D, + correctedLocation: CLLocationCoordinate2D?, + mappingData: CurrentMappingData, + accessibilityFeatures: [MappedEditableAccessibilityFeature] + ) throws { + let correctedLocation = correctedLocation ?? captureLocation + /// First, calculate all the feature locations + for feature in accessibilityFeatures { + try self.processLocationRequestTypeBased( + deviceLocation: captureLocation, + accessibilityFeature: feature, + locationType: .captureLocation + ) + try self.processLocationRequestTypeBased( + deviceLocation: correctedLocation, + accessibilityFeature: feature, + locationType: .correctedLocation + ) + } + /// Second, get all the nearest elements (existing features) for each feature + for feature in accessibilityFeatures { + self.processNearestFeaturesRequest( + deviceLocation: captureLocation, + mappingData: mappingData, + accessibilityFeature: feature, + locationType: .captureLocation + ) + self.processNearestFeaturesRequest( + deviceLocation: correctedLocation, + mappingData: mappingData, + accessibilityFeature: feature, + locationType: .correctedLocation + ) + } + /// Third, get all the nearest features (element, total distance, total feature matches) + var captureNearestFeatures: [(any OSWElement, Float, Int)] = [] + var correctedNearestFeatures: [(any OSWElement, Float, Int)] = [] + for feature in accessibilityFeatures { + if let nearestElements = feature.nearestOSWElements { + for (element, distance) in nearestElements { + if let index = captureNearestFeatures.firstIndex(where: { $0.0.id == element.id }) { + captureNearestFeatures[index].1 += Float(distance) + captureNearestFeatures[index].2 += 1 + } else { + captureNearestFeatures.append((element, Float(distance), 1)) + } + } + } + if let correctedNearestElements = feature.correctedNearestOSWElements { + for (element, distance) in correctedNearestElements { + if let index = correctedNearestFeatures.firstIndex(where: { $0.0.id == element.id }) { + correctedNearestFeatures[index].1 += Float(distance) + correctedNearestFeatures[index].2 += 1 + } else { + correctedNearestFeatures.append((element, Float(distance), 1)) + } + } + } + } + /// Now, sort all the nearest features with the following rules + /// If capture id matches, assign priority to that feature + /// Then, sort the matched features by average distance ascending, then sort the non-matched features by average distance ascending + /// Then, combine the two lists, with matched features first, then non-matched features + let captureId = self.captureImageData?.id.uuidString + let sortedCaptureNearestFeatures = captureNearestFeatures.sorted { + (feature1: (any OSWElement, Float, Int), feature2: (any OSWElement, Float, Int)) -> Bool in + if feature1.0.getCaptureId() == captureId && feature2.0.getCaptureId() != captureId { + return true + } else if feature1.0.getCaptureId() != captureId && feature2.0.getCaptureId() == captureId { + return false + } else { + let avgDistance1 = feature1.1 / Float(feature1.2) + let avgDistance2 = feature2.1 / Float(feature2.2) + return avgDistance1 < avgDistance2 + } + } + let sortedCorrectedNearestFeatures = correctedNearestFeatures.sorted { + (feature1: (any OSWElement, Float, Int), feature2: (any OSWElement, Float, Int)) -> Bool in + if feature1.0.getCaptureId() == captureId && feature2.0.getCaptureId() != captureId { + return true + } else if feature1.0.getCaptureId() != captureId && feature2.0.getCaptureId() == captureId { + return false + } else { + let avgDistance1 = feature1.1 / Float(feature1.2) + let avgDistance2 = feature2.1 / Float(feature2.2) + return avgDistance1 < avgDistance2 + } + } + + /// Now, do a first-come first-serve matching, where each existing feature is matched to the new features + /// We will iterate through the sorted nearest features and assign them to the new features, ensuring that each existing feature is only assigned to one new feature. + /// Both the oswElement and correctedOSWElement for now will be assigned using sortedCorrectedNearestFeatures + /// Also update the new feature with the isExisting and isCaptureMatched flag. + var unassignedFeatures = accessibilityFeatures + for sortedCorrectedNearestFeature in sortedCorrectedNearestFeatures { + print("Capture id comparison: \(sortedCorrectedNearestFeature.0.getCaptureId()) == \(captureId)") + let isCaptureMatched = sortedCorrectedNearestFeature.0.getCaptureId() == captureId + var nearestCandidateFeatures: [(MappedEditableAccessibilityFeature, Float)] = [] + for feature in unassignedFeatures { + if let nearestElements = feature.correctedNearestOSWElements { + /// Check if the current sortedCorrectedNearestFeature is in the nearestElements of the feature + /// and get the distance + if let distance = nearestElements.first(where: { $0.0.id == sortedCorrectedNearestFeature.0.id })?.1 { + nearestCandidateFeatures.append((feature, Float(distance))) + } + } + } + /// Now, sort the nearestCandidateFeatures by distance ascending + nearestCandidateFeatures.sort { $0.1 < $1.1 } + /// Now, assign the nearestCandidateFeatures to the sortedCorrectedNearestFeature, ensuring that each existing feature is only assigned to one new feature. + if let nearestCandidateFeature = nearestCandidateFeatures.first { + let feature = nearestCandidateFeature.0 + feature.isCaptureMatched = isCaptureMatched + feature.setIsExisting(true) + feature.setOSWElement(oswElement: sortedCorrectedNearestFeature.0) + feature.setCorrectedIsExisting(true) + feature.setCorrectedOSWElement(sortedCorrectedNearestFeature.0) + /// Now, remove the assigned feature from the unassignedFeatures list + if let index = unassignedFeatures.firstIndex(where: { $0.id == feature.id }) { + unassignedFeatures.remove(at: index) + } + } + } + /// For the remaining unassigned features, set them as non-existing and remove any existing oswElement or correctedOSWElement + for feature in unassignedFeatures { + feature.setIsExisting(false) + feature.setOSWElement(oswElement: nil) + feature.setCorrectedIsExisting(false) + feature.setCorrectedOSWElement(nil) + } + } +} diff --git a/IOSAccessAssessment/AccessibilityFeature/AttributeEstimation/Extensions/IsExistingExtension.swift b/IOSAccessAssessment/AccessibilityFeature/AttributeEstimation/Extensions/IsExistingExtension.swift deleted file mode 100644 index 20780124..00000000 --- a/IOSAccessAssessment/AccessibilityFeature/AttributeEstimation/Extensions/IsExistingExtension.swift +++ /dev/null @@ -1,36 +0,0 @@ -// -// IsExistingExtension.swift -// IOSAccessAssessment -// -// Created by Himanshu on 4/30/26. -// -import PointNMapShared -import CoreLocation -import MapKit - -extension AttributeEstimationPipeline { - func processIsExistingRequest( - deviceLocation: CLLocationCoordinate2D, - mappingData: CurrentMappingData, - accessibilityFeature: MappedEditableAccessibilityFeature - ) { - /// Threshold needs to be in Map Units - let distanceThreshold = PointNMapConstants.WorkspaceConstants.fetchUpdateRadiusThresholdInMeters * MKMapPointsPerMeterAtLatitude(deviceLocation.latitude) - guard let LocationDetails = accessibilityFeature.locationDetails else { - accessibilityFeature.setIsExisting(false) - return - } - let matchedElement: (any OSWElement)? = mappingData.getMatchedFeature( - to: LocationDetails, featureClass: accessibilityFeature.accessibilityFeatureClass, - captureId: self.captureImageData?.id, - distanceThreshold: distanceThreshold - ) - guard let matchedElement = matchedElement else { - accessibilityFeature.setIsExisting(false) - return - } - let isExisting = accessibilityFeature.accessibilityFeatureClass.kind.oswPolicy.isExistingFirst - accessibilityFeature.setIsExisting(isExisting) - accessibilityFeature.setOSWElement(oswElement: matchedElement) - } -} diff --git a/IOSAccessAssessment/AccessibilityFeature/Components/AmbiguityCase.swift b/IOSAccessAssessment/AccessibilityFeature/Components/AmbiguityCase.swift new file mode 100644 index 00000000..721eecf4 --- /dev/null +++ b/IOSAccessAssessment/AccessibilityFeature/Components/AmbiguityCase.swift @@ -0,0 +1,26 @@ +// +// AmbiguityCase.swift +// IOSAccessAssessment +// +// Created by Himanshu on 8/1/26. +// + +import Foundation + +enum AmbiguityCase: String, CaseIterable, Sendable, Codable, Equatable, Hashable { + case parallel_sidewalks = "parallel_sidewalks" + case wide_plazas = "wide_plazas" + case closely_spaced_features = "closely_spaced_features" + case divided_or_offset_crossings = "divided_or_offset_crossings" + case multiple_intersection_corners = "multiple_intersection_corners" + case partial_masks = "partial_masks" + case no_ambiguity = "no_ambiguity" +} + +struct AmbiguityCasePolicy: Sendable, Codable, Equatable, Hashable { + let ambiguityCases: [AmbiguityCase] +} + +extension AmbiguityCasePolicy { + static let `default` = AmbiguityCasePolicy(ambiguityCases: AmbiguityCase.allCases) +} diff --git a/IOSAccessAssessment/AccessibilityFeature/Config/AccessibilityFeatureKindExtension.swift b/IOSAccessAssessment/AccessibilityFeature/Config/AccessibilityFeatureKindExtension.swift index 951b3f8e..1df39a9f 100644 --- a/IOSAccessAssessment/AccessibilityFeature/Config/AccessibilityFeatureKindExtension.swift +++ b/IOSAccessAssessment/AccessibilityFeature/Config/AccessibilityFeatureKindExtension.swift @@ -13,14 +13,32 @@ import PointNMapShared extension AccessibilityFeatureKind { var oswPolicy: OSWPolicy { switch self { - case .sidewalk: return OSWPolicy(oswElementClass: .Sidewalk, isExistingFirst: false) - case .building: return OSWPolicy(oswElementClass: .Building, isExistingFirst: false) - case .pole: return OSWPolicy(oswElementClass: .Pole, isExistingFirst: false) - case .trafficLight: return OSWPolicy(oswElementClass: .TrafficLight, isExistingFirst: false) - case .trafficSign: return OSWPolicy(oswElementClass: .TrafficSign, isExistingFirst: false) - case .vegetation: return OSWPolicy(oswElementClass: .Vegetation, isExistingFirst: false) - case .curbRamp: return OSWPolicy(oswElementClass: .CurbRamp, isExistingFirst: false) + case .sidewalk: return OSWPolicy(oswElementClass: .Sidewalk, isExistingFirst: true) + case .building: return OSWPolicy(oswElementClass: .Building, isExistingFirst: true) + case .pole: return OSWPolicy(oswElementClass: .Pole, isExistingFirst: true) + case .trafficLight: return OSWPolicy(oswElementClass: .TrafficLight, isExistingFirst: true) + case .trafficSign: return OSWPolicy(oswElementClass: .TrafficSign, isExistingFirst: true) + case .vegetation: return OSWPolicy(oswElementClass: .Vegetation, isExistingFirst: true) + case .curbRamp: return OSWPolicy(oswElementClass: .CurbRamp, isExistingFirst: true) default: return OSWPolicy.default } } + + var ambiguityCasePolicy: AmbiguityCasePolicy { + switch self { + case .sidewalk: + return AmbiguityCasePolicy.default + case .building: + return AmbiguityCasePolicy.default + case .pole: + return AmbiguityCasePolicy.default + case .trafficLight: + return AmbiguityCasePolicy.default + case .trafficSign: + return AmbiguityCasePolicy.default + case .curbRamp: + return AmbiguityCasePolicy.default + default: return AmbiguityCasePolicy.default + } + } } diff --git a/IOSAccessAssessment/AccessibilityFeature/Definitions/MappedEditableAccessibilityFeature.swift b/IOSAccessAssessment/AccessibilityFeature/Definitions/MappedEditableAccessibilityFeature.swift index f2db7eb4..1461050b 100644 --- a/IOSAccessAssessment/AccessibilityFeature/Definitions/MappedEditableAccessibilityFeature.swift +++ b/IOSAccessAssessment/AccessibilityFeature/Definitions/MappedEditableAccessibilityFeature.swift @@ -12,8 +12,32 @@ class MappedEditableAccessibilityFeature: EditableAccessibilityFeature { /// If isExisting is false, even if an osw element is associated, it means the feature is new. /// If isExisting is true, it means the feature corresponds to an existing real-world feature, and the oswElement (if present) represents that existing feature in OSW. var isExisting: Bool = false + var isCaptureMatched: Bool = false var oswElement: (any OSWElement)? + var nearestOSWElements: [(any OSWElement, CLLocationDistance)]? + var selectedNearestOSWElement: (any OSWElement, CLLocationDistance)? + var selectedNearestOSWElementIndex: Int? { + guard let selectedNearestOSWElement else { return nil } + return nearestOSWElements?.firstIndex(where: { $0.0.id == selectedNearestOSWElement.0.id }) + } + var isCorrectOSWElementSelected: Bool = true + + /// Corrected location-based adhoc details + var correctedLocationDetails: LocationDetails? + var correctedIsExisting: Bool = false + var correctedOSWElement: (any OSWElement)? + var correctedNearestOSWElements: [(any OSWElement, CLLocationDistance)]? + var correctedSelectedNearestOSWElement: (any OSWElement, CLLocationDistance)? + var correctedSelectedNearestOSWElementIndex: Int? { + guard let correctedSelectedNearestOSWElement else { return nil } + return correctedNearestOSWElements?.firstIndex(where: { $0.0.id == correctedSelectedNearestOSWElement.0.id }) + } + var correctedIsCorrectOSWElementSelected: Bool = true + + /// Ambiguity cases + var ambiguityCases: [AmbiguityCase] = [] + required init( id: UUID = UUID(), detectedAccessibilityFeature: DetectedAccessibilityFeature @@ -44,6 +68,15 @@ class MappedEditableAccessibilityFeature: EditableAccessibilityFeature { locationDetails: LocationDetails?, isExisting: Bool = false, oswElement: (any OSWElement)? = nil, + nearestOSWElements: [(any OSWElement, CLLocationDistance)]? = nil, + selectedNearestOSWElement: (any OSWElement, CLLocationDistance)? = nil, + isCorrectOSWElementSelected: Bool = true, + correctedLocationDetails: LocationDetails? = nil, + correctedIsExisting: Bool = false, + correctedOSWElement: (any OSWElement)? = nil, + correctedNearestOSWElements: [(any OSWElement, CLLocationDistance)]? = nil, + correctedSelectedNearestOSWElement: (any OSWElement, CLLocationDistance)? = nil, + correctedIsCorrectOSWElementSelected: Bool = true, calculatedAttributeValues: [AccessibilityFeatureAttribute: AccessibilityFeatureAttribute.Value?], attributeValues: [AccessibilityFeatureAttribute: AccessibilityFeatureAttribute.Value?], experimentalAttributeValues: [AccessibilityFeatureAttribute : AccessibilityFeatureAttribute.Value?] @@ -65,10 +98,74 @@ class MappedEditableAccessibilityFeature: EditableAccessibilityFeature { self.isExisting = isExisting } - func setOSWElement(oswElement: any OSWElement) { + func setOSWElement(oswElement: (any OSWElement)?) { self.oswElement = oswElement } + func setNearestOSWElements(nearestOSWElements: [(any OSWElement, CLLocationDistance)]) { + self.nearestOSWElements = nearestOSWElements + /// Select the first element by default if no element is selected yet + self.selectedNearestOSWElement = self.selectedNearestOSWElement ?? nearestOSWElements.first + } + + func setSelectedNearestOSWElement(selectedNearestOSWElement: (any OSWElement, CLLocationDistance)?) { + self.selectedNearestOSWElement = selectedNearestOSWElement + /// Update the oswElement to match the selected nearest OSW element +// if let selectedNearestOSWElement = selectedNearestOSWElement?.0 { +// self.oswElement = selectedNearestOSWElement +// } + } + + func setIsCorrectOSWElementSelected(_ isCorrectOSWElementSelected: Bool) { + self.isCorrectOSWElementSelected = isCorrectOSWElementSelected + } + + func setCorrectedLocationDetails(_ correctedLocationDetails: LocationDetails?) { + self.correctedLocationDetails = correctedLocationDetails + } + + func setCorrectedIsExisting(_ correctedIsExisting: Bool) { + self.correctedIsExisting = correctedIsExisting + } + + func setCorrectedOSWElement(_ correctedOSWElement: (any OSWElement)?) { + self.correctedOSWElement = correctedOSWElement + } + + func setCorrectedNearestOSWElements(_ correctedNearestOSWElements: [(any OSWElement, CLLocationDistance)]?) { + self.correctedNearestOSWElements = correctedNearestOSWElements + /// Select the first element by default if no element is selected yet + self.correctedSelectedNearestOSWElement = self.correctedSelectedNearestOSWElement ?? correctedNearestOSWElements?.first + } + + func setCorrectedSelectedNearestOSWElement(_ correctedSelectedNearestOSWElement: (any OSWElement, CLLocationDistance)?) { + self.correctedSelectedNearestOSWElement = correctedSelectedNearestOSWElement + } + + func setCorrectedIsCorrectOSWElementSelected(_ correctedIsCorrectOSWElementSelected: Bool) { + self.correctedIsCorrectOSWElementSelected = correctedIsCorrectOSWElementSelected + } + + func setAmbiguityCases(_ ambiguityCases: [AmbiguityCase]) { + self.ambiguityCases = ambiguityCases + } + + func addAmbiguityCase(_ ambiguityCase: AmbiguityCase) { + if !ambiguityCases.contains(where: { $0.rawValue == ambiguityCase.rawValue }) { + self.ambiguityCases.append(ambiguityCase) + } + } + + func removeAmbiguityCase(_ ambiguityCase: AmbiguityCase) { + self.ambiguityCases.removeAll(where: { $0.rawValue == ambiguityCase.rawValue }) + } + + public func getCorrectedLastLocationCoordinate() -> CLLocationCoordinate2D? { + guard let correctedLocationDetails else { return nil } + guard let lastCoordinate = correctedLocationDetails.locations.last?.coordinates.last else { return nil } + return lastCoordinate + } + static func == ( lhs: MappedEditableAccessibilityFeature, rhs: MappedEditableAccessibilityFeature ) -> Bool { diff --git a/IOSAccessAssessment/LocalDataset/DatasetDecoder.swift b/IOSAccessAssessment/LocalDataset/DatasetDecoder.swift index 5e2116df..71991165 100644 --- a/IOSAccessAssessment/LocalDataset/DatasetDecoder.swift +++ b/IOSAccessAssessment/LocalDataset/DatasetDecoder.swift @@ -45,6 +45,7 @@ struct DatasetCaptureData { let captureMeshData: MeshContents? let location: CLLocationCoordinate2D? let heading: CLLocationDirection? + let correctedLocation: CLLocationCoordinate2D? } /** @@ -68,6 +69,7 @@ class DatasetDecoder { // public let cameraMatrixPath: URL public let cameraTransformPath: URL public let locationPath: URL + public let correctedLocationPath: URL public let headingPath: URL public let otherDetailsPath: URL public let meshPath: URL @@ -77,6 +79,7 @@ class DatasetDecoder { private let cameraIntrinsicsDecoder: CameraIntrinsicsDecoder private let cameraTransformDecoder: CameraTransformDecoder private let locationDecoder: LocationDecoder + private let correctedLocationDecoder: LocationDecoder? private let headingDecoder: HeadingDecoder? private let otherDetailsDecoder: OtherDetailsDecoder private let meshDecoder: MeshDecoder @@ -97,6 +100,7 @@ class DatasetDecoder { // self.cameraMatrixPath = datasetDirectory.appendingPathComponent("camera_matrix.csv", isDirectory: false) self.cameraTransformPath = datasetDirectory.appendingPathComponent("camera_transform.csv", isDirectory: false) self.locationPath = datasetDirectory.appendingPathComponent("location.csv", isDirectory: false) + self.correctedLocationPath = datasetDirectory.appendingPathComponent("corrected_location.csv", isDirectory: false) self.headingPath = datasetDirectory.appendingPathComponent("heading.csv", isDirectory: false) self.otherDetailsPath = datasetDirectory.appendingPathComponent("other_details.csv", isDirectory: false) self.meshPath = datasetDirectory.appendingPathComponent("mesh", isDirectory: true) @@ -106,6 +110,12 @@ class DatasetDecoder { self.cameraIntrinsicsDecoder = try CameraIntrinsicsDecoder(path: self.cameraIntrinsicsPath) self.cameraTransformDecoder = try CameraTransformDecoder(path: self.cameraTransformPath) self.locationDecoder = try LocationDecoder(path: self.locationPath) + do { + self.correctedLocationDecoder = try LocationDecoder(path: self.correctedLocationPath) + } catch { + print("Corrected location data not found, proceeding without it.") + self.correctedLocationDecoder = nil + } do { self.headingDecoder = try HeadingDecoder(url: self.headingPath) } catch { @@ -153,6 +163,7 @@ class DatasetDecoder { guard let locationData = locationDecoder.load(index: index, frameNumber: frameNumber) else { throw DatasetDecoderError.indexDataNotFound(index) } + let correctedLocationData = correctedLocationDecoder?.load(index: index, frameNumber: frameNumber) let headingData = headingDecoder?.load(index: index, frameNumber: frameNumber) guard let otherDetailsData = otherDetailsDecoder.load(index: index, frameNumber: frameNumber) else { throw DatasetDecoderError.indexDataNotFound(index) @@ -171,11 +182,13 @@ class DatasetDecoder { ) let location = CLLocationCoordinate2D(latitude: locationData.latitude, longitude: locationData.longitude) let heading = headingData?.trueHeading + let correctedLocation = correctedLocationData != nil ? CLLocationCoordinate2D(latitude: correctedLocationData!.latitude, longitude: correctedLocationData!.longitude) : nil let datasetCaptureData = DatasetCaptureData( captureImageData: datasetCaptureBaseData, captureMeshData: meshContents, location: location, - heading: heading + heading: heading, + correctedLocation: correctedLocation ) return datasetCaptureData } diff --git a/IOSAccessAssessment/LocalDataset/DatasetEncoder.swift b/IOSAccessAssessment/LocalDataset/DatasetEncoder.swift index 9d7d036e..fbaff58f 100644 --- a/IOSAccessAssessment/LocalDataset/DatasetEncoder.swift +++ b/IOSAccessAssessment/LocalDataset/DatasetEncoder.swift @@ -45,6 +45,7 @@ class DatasetEncoder { public let cameraMatrixPath: URL public let cameraTransformPath: URL public let locationPath: URL + public let correctedLocationpath: URL public let headingPath: URL public let accessibilityFeaturePath: URL public let otherDetailsPath: URL @@ -57,6 +58,7 @@ class DatasetEncoder { private let cameraIntrinsicsEncoder: CameraIntrinsicsEncoder private let cameraTransformEncoder: CameraTransformEncoder private let locationEncoder: LocationEncoder + private let correctedLocationEncoder: LocationEncoder private let headingEncoder: HeadingEncoder private let accessibilityFeatureEncoder: AccessibilityFeatureEncoder private let otherDetailsEncoder: OtherDetailsEncoder @@ -83,6 +85,7 @@ class DatasetEncoder { self.cameraMatrixPath = datasetDirectory.appendingPathComponent("camera_matrix.csv", isDirectory: false) self.cameraTransformPath = datasetDirectory.appendingPathComponent("camera_transform.csv", isDirectory: false) self.locationPath = datasetDirectory.appendingPathComponent("location.csv", isDirectory: false) + self.correctedLocationpath = datasetDirectory.appendingPathComponent("corrected_location.csv", isDirectory: false) self.headingPath = datasetDirectory.appendingPathComponent("heading.csv", isDirectory: false) self.accessibilityFeaturePath = datasetDirectory.appendingPathComponent("features", isDirectory: true) self.otherDetailsPath = datasetDirectory.appendingPathComponent("other_details.csv", isDirectory: false) @@ -95,6 +98,7 @@ class DatasetEncoder { self.cameraIntrinsicsEncoder = try CameraIntrinsicsEncoder(url: self.cameraIntrinsicsPath) self.cameraTransformEncoder = try CameraTransformEncoder(url: self.cameraTransformPath) self.locationEncoder = try LocationEncoder(url: self.locationPath) + self.correctedLocationEncoder = try LocationEncoder(url: self.correctedLocationpath) self.headingEncoder = try HeadingEncoder(url: self.headingPath) self.accessibilityFeatureEncoder = try AccessibilityFeatureEncoder(outDirectory: self.accessibilityFeaturePath) self.otherDetailsEncoder = try OtherDetailsEncoder(url: self.otherDetailsPath) @@ -122,7 +126,8 @@ class DatasetEncoder { captureImageData: any CaptureImageDataProtocol, captureMeshData: (any CaptureMeshDataProtocol)? = nil, location: CLLocationCoordinate2D?, - heading: CLLocationDirection? + heading: CLLocationDirection?, + correctedLocation: CLLocationCoordinate2D? = nil ) throws { let otherDetailsData = OtherDetailsData( timestamp: captureImageData.timestamp, @@ -139,6 +144,7 @@ class DatasetEncoder { cameraTransform: captureImageData.cameraTransform, cameraIntrinsics: captureImageData.cameraIntrinsics, location: location, + correctedLocation: correctedLocation, heading: heading, otherDetails: otherDetailsData, meshAnchors: meshAnchors, @@ -153,6 +159,7 @@ class DatasetEncoder { segmentationLabelImage: CIImage, cameraTransform: simd_float4x4, cameraIntrinsics: simd_float3x3, location: CLLocationCoordinate2D?, + correctedLocation: CLLocationCoordinate2D? = nil, heading: CLLocationDirection?, otherDetails: OtherDetailsData?, meshAnchors: [ARMeshAnchor]? = nil, @@ -182,6 +189,12 @@ class DatasetEncoder { let locationData = LocationData(timestamp: timestamp, latitude: latitude, longitude: longitude) try self.locationEncoder.add(locationData: locationData, frameNumber: frameNumber) } + if let correctedLocation = correctedLocation { + let latitude = correctedLocation.latitude + let longitude = correctedLocation.longitude + let correctedLocationData = LocationData(timestamp: timestamp, latitude: latitude, longitude: longitude) + try self.correctedLocationEncoder.add(locationData: correctedLocationData, frameNumber: frameNumber) + } if let heading = heading { let headingData = HeadingData(timestamp: timestamp, trueHeading: heading) try self.headingEncoder.add(headingData: headingData, frameNumber: frameNumber) diff --git a/IOSAccessAssessment/Shared/Definitions/CurrentMappingData.swift b/IOSAccessAssessment/Shared/Definitions/CurrentMappingData.swift index 5f2b7c2e..c1b68948 100644 --- a/IOSAccessAssessment/Shared/Definitions/CurrentMappingData.swift +++ b/IOSAccessAssessment/Shared/Definitions/CurrentMappingData.swift @@ -201,28 +201,80 @@ class CurrentMappingData: CustomStringConvertible { return nearestFeature } + func getNearestFeatures( + to LocationDetails: LocationDetails, featureClass: AccessibilityFeatureClass, + captureId: UUID?, + distanceThreshold: CLLocationDistance = 50.0, + ) -> [(any OSWElement, CLLocationDistance)] { + guard let featureIds = featuresMap[featureClass] else { return [] } + var nearestFeatures: [(any OSWElement, CLLocationDistance)] = [] + let oswElementClass = featureClass.kind.oswPolicy.oswElementClass + let geometry = oswElementClass.geometry + + for featureId in featureIds { + guard let feature = getFeature(featureId: featureId, geometry: geometry) else { continue } + guard let featureOSMLocationDetails = self.getFeatureOSMLocationDetails( + feature: feature, geometry: geometry + ) else { continue } + guard let distance = LocationHelpers.distanceBetweenSimilarOSMLocationDetails( + srcLocationDetails: featureOSMLocationDetails, dstLocationDetails: LocationDetails + ) else { continue } + if distance < distanceThreshold { + nearestFeatures.append((feature, distance)) + } + } + + // Sort the nearest features by distance in ascending order + nearestFeatures.sort { $0.1 < $1.1 } + + if let captureId = captureId { + let captureMatchedFeatures = getCaptureMatchedFeatures( + to: LocationDetails, featureClass: featureClass, captureId: captureId + ) + for captureMatchedFeature in captureMatchedFeatures { + if nearestFeatures.contains(where: { $0.0.id == captureMatchedFeature.id }) { + continue + } + guard let captureMatchedFeatureOSMLocationDetails = self.getFeatureOSMLocationDetails( + feature: captureMatchedFeature, geometry: geometry + ) else { continue } + guard let captureMatchedFeatureDistance = LocationHelpers.distanceBetweenSimilarOSMLocationDetails( + srcLocationDetails: captureMatchedFeatureOSMLocationDetails, dstLocationDetails: LocationDetails + ) else { continue } + nearestFeatures.append((captureMatchedFeature, captureMatchedFeatureDistance)) + } + } + // Sort the nearest features by distance in ascending order + nearestFeatures.sort { $0.1 < $1.1 } + + return nearestFeatures + } + /** This function takes in OSM location details, an accessibility feature class, and a capture ID, and returns the feature of that class whose capture ID matches the given capture ID. */ - func getCaptureMatchedFeature( + func getCaptureMatchedFeatures( to LocationDetails: LocationDetails, featureClass: AccessibilityFeatureClass, captureId: UUID - ) -> (any OSWElement)? { - guard let featureIds = featuresMap[featureClass] else { return nil } - var nearestFeature: (any OSWElement)? + ) -> [any OSWElement] { + guard let featureIds = featuresMap[featureClass] else { return [] } +// var nearestFeature: (any OSWElement)? let oswElementClass = featureClass.kind.oswPolicy.oswElementClass let geometry = oswElementClass.geometry let captureIdString = captureId.uuidString + var matchedFeatures: [any OSWElement] = [] + for featureId in featureIds { guard let feature = getFeature(featureId: featureId, geometry: geometry) else { continue } guard let featureCaptureId = feature.getCaptureId() else { continue } if featureCaptureId == captureIdString { - nearestFeature = feature - break +// nearestFeature = feature +// break + matchedFeatures.append(feature) } } - return nearestFeature + return matchedFeatures } /** @@ -235,12 +287,13 @@ class CurrentMappingData: CustomStringConvertible { distanceThreshold: CLLocationDistance = 50.0 ) -> (any OSWElement)? { if let captureId = captureId { - let captureMatchedFeature = getCaptureMatchedFeature( + let captureMatchedFeatures = getCaptureMatchedFeatures( to: LocationDetails, featureClass: featureClass, captureId: captureId ) - if let captureMatchedFeature = captureMatchedFeature { - return captureMatchedFeature - } +// if let captureMatchedFeature = captureMatchedFeature { +// return captureMatchedFeature +// } + return captureMatchedFeatures.first } return getNearestFeature( to: LocationDetails, featureClass: featureClass, distanceThreshold: distanceThreshold diff --git a/IOSAccessAssessment/Shared/SharedAppConstants.swift b/IOSAccessAssessment/Shared/SharedAppConstants.swift index 51aaeca2..407629a7 100644 --- a/IOSAccessAssessment/Shared/SharedAppConstants.swift +++ b/IOSAccessAssessment/Shared/SharedAppConstants.swift @@ -45,7 +45,7 @@ struct SharedAppConstants { } struct WorkspaceConstants { - static let primaryWorkspaceIds: [String] = ["2158"] //"1940", "2151", "2158" + static let primaryWorkspaceIds: [String] = ["2170"]//, "2171", "2172"] //"1940", "2151", "2158" // ["1463"] // ["288", "349", "1411"] // "252", "322", "368", "374", "378", "381", "384", "323", "369", "156", "375", "379"] diff --git a/IOSAccessAssessment/Shared/SharedAppData.swift b/IOSAccessAssessment/Shared/SharedAppData.swift index e38e9d37..55d4e918 100644 --- a/IOSAccessAssessment/Shared/SharedAppData.swift +++ b/IOSAccessAssessment/Shared/SharedAppData.swift @@ -17,6 +17,13 @@ final class SharedAppData: ObservableObject { var currentDatasetEncoder: DatasetEncoder? var currentDatasetDecoder: DatasetDecoder? + /// For telemetry + var currentTelemetryEncoder: TelemetryEncoder = TelemetryEncoder() + lazy var deviceStateMonitor = DeviceStateMonitor( + telemetryEncoder: currentTelemetryEncoder, + sampleIntervalSeconds: 60 + ) + var currentCaptureDataRecord: CaptureData? /// A queue to hold recent capture image data. var captureDataQueue: SafeDeque @@ -28,6 +35,11 @@ final class SharedAppData: ObservableObject { init(captureDataCapacity: Int = 5) { self.captureDataCapacity = captureDataCapacity self.captureDataQueue = SafeDeque(capacity: captureDataCapacity) + + Task { + await currentTelemetryEncoder.beginMappingSession() + deviceStateMonitor.start() + } } func refreshQueue() async { @@ -41,6 +53,9 @@ final class SharedAppData: ObservableObject { self.currentCaptureDataRecord = nil self.currentMappingData = CurrentMappingData() self.currentMappedFeaturesData = CurrentMappedFeaturesData() + Task { + await currentTelemetryEncoder.endMappingSession() + } } func saveCaptureData(_ data: CaptureData) { diff --git a/IOSAccessAssessment/TDEI/Services/ChangesetService.swift b/IOSAccessAssessment/TDEI/Services/ChangesetService.swift index 76e2f67b..37e9e47b 100644 --- a/IOSAccessAssessment/TDEI/Services/ChangesetService.swift +++ b/IOSAccessAssessment/TDEI/Services/ChangesetService.swift @@ -6,6 +6,7 @@ // import Foundation +import PointNMapShared enum ChangesetDiffOperation { case create(any OSWElement) @@ -95,6 +96,7 @@ class ChangesetService { operations: [ChangesetDiffOperation], accessToken: String, environment: APIEnvironment? = nil, + telemetryEncoder: TelemetryEncoder? = nil, completion: @escaping (Result) -> Void ) { let selectedEnvironment = environment ?? EnvironmentService.shared.environment @@ -132,7 +134,20 @@ class ChangesetService { request.setValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization") request.setValue(workspaceId, forHTTPHeaderField: "X-Workspace") request.setValue("application/xml", forHTTPHeaderField: "Content-Type") - request.httpBody = osmChangeXML.data(using: .utf8) + let payloadData = osmChangeXML.data(using: .utf8) + request.httpBody = payloadData + + if let telemetryEncoder = telemetryEncoder, let payloadData = payloadData { + Task { + await PayloadSizeTracker.record( + data: payloadData, + telemetryEncoder: telemetryEncoder, + metadata: [ + "payload_type": "feature_upload" + ] + ) + } + } URLSession.shared.dataTask(with: request) { data, _, error in if let error = error { @@ -230,12 +245,14 @@ extension ChangesetService { changesetId: String, operations: [ChangesetDiffOperation], accessToken: String, - environment: APIEnvironment? = nil + environment: APIEnvironment? = nil, + telemetryEncoder: TelemetryEncoder? = nil ) async throws -> OSMChangesetUploadResponseElements { return try await withCheckedThrowingContinuation { continuation in performUpload( workspaceId: workspaceId, changesetId: changesetId, operations: operations, - accessToken: accessToken, environment: environment + accessToken: accessToken, environment: environment, + telemetryEncoder: telemetryEncoder ) { result in switch result { case .success(let uploadedElements): diff --git a/IOSAccessAssessment/TDEI/Transmission/APIChangesetUploadController.swift b/IOSAccessAssessment/TDEI/Transmission/APIChangesetUploadController.swift index b5930c26..3f149bab 100644 --- a/IOSAccessAssessment/TDEI/Transmission/APIChangesetUploadController.swift +++ b/IOSAccessAssessment/TDEI/Transmission/APIChangesetUploadController.swift @@ -36,6 +36,14 @@ enum APIChangesetUploadError: Error, LocalizedError { */ class APIChangesetUploadController: ObservableObject { + public var telemetryEncoder: TelemetryEncoder? = nil + + public func configure( + telemetryEncoder: TelemetryEncoder? = nil + ) { + self.telemetryEncoder = telemetryEncoder + } + public var idGenerator: IntIdGenerator = IntIdGenerator() public var capturedFrameIds: Set = [] @@ -82,11 +90,12 @@ class APIChangesetUploadController: ObservableObject { additionalTags: additionalTags ) let uploadOperation: ChangesetDiffOperation = .create(capturePoint) - _ = try await ChangesetService.shared.performUploadAsync( - workspaceId: inputs.workspaceId, changesetId: inputs.changesetId, - operations: [uploadOperation], - accessToken: inputs.accessToken - ) + /// MARK: Not uploading for now as in these experiments we are only focused on updates +// _ = try await ChangesetService.shared.performUploadAsync( +// workspaceId: inputs.workspaceId, changesetId: inputs.changesetId, +// operations: [uploadOperation], +// accessToken: inputs.accessToken +// ) capturedFrameIds.insert(inputs.captureData.id) } @@ -109,6 +118,41 @@ class APIChangesetUploadController: ObservableObject { APIConstants.TagKeys.enhancedAnalysisModeKey: String(enhancedAnalysisMode) ] } + + private func getAdditionalFeatureTags( + feature: MappedEditableAccessibilityFeature + ) -> [String: String] { + var additionalTags: [String: String] = [:] + if let featureLocation = feature.getLastLocationCoordinate() { + additionalTags[APIConstants.TagKeys.calculatedLatitudeKey] = String(featureLocation.latitude) + additionalTags[APIConstants.TagKeys.calculatedLongitudeKey] = String(featureLocation.longitude) + } + /// isExisting + additionalTags["is_existing"] = String(feature.isExisting) + if let selectedNearestOSWElement = feature.selectedNearestOSWElement { + additionalTags["selected_nearest_osw_element_id"] = selectedNearestOSWElement.0.id + } + if let selectedNearestOSWElementIndex = feature.selectedNearestOSWElementIndex { + additionalTags["selected_nearest_osw_element_index"] = String(selectedNearestOSWElementIndex) + } + /// isCorrectOSWElementSelected + additionalTags["is_correct_osw_element_selected"] = String(feature.isCorrectOSWElementSelected) + + /// correctedIsExisting + additionalTags["corrected_is_existing"] = String(feature.correctedIsExisting) + if let correctedSelectedNearestOSWElement = feature.correctedSelectedNearestOSWElement { + additionalTags["corrected_selected_nearest_osw_element_id"] = correctedSelectedNearestOSWElement.0.id + } + if let correctedSelectedNearestOSWElementIndex = feature.correctedSelectedNearestOSWElementIndex { + additionalTags["corrected_selected_nearest_osw_element_index"] = String(correctedSelectedNearestOSWElementIndex) + } + /// correctedIsCorrectOSWElementSelected + additionalTags["corrected_is_correct_osw_element_selected"] = String(feature.correctedIsCorrectOSWElementSelected) + + /// Ambiguity cases + additionalTags["ambiguity_cases"] = feature.ambiguityCases.map { $0.rawValue }.joined(separator: ", ") + return additionalTags + } } /** @@ -348,8 +392,14 @@ extension APIChangesetUploadController { calculatedAttributeValues = feature.calculatedAttributeValues /// Add location as additional tags as well var additionalTags = additionalTags - additionalTags[APIConstants.TagKeys.calculatedLatitudeKey] = String(featureLocation.latitude) - additionalTags[APIConstants.TagKeys.calculatedLongitudeKey] = String(featureLocation.longitude) +// additionalTags[APIConstants.TagKeys.calculatedLatitudeKey] = String(featureLocation.latitude) +// additionalTags[APIConstants.TagKeys.calculatedLongitudeKey] = String(featureLocation.longitude) + /// Add feature-based additional tags using the getAdditionalTags function + let featureAdditionalTags = getAdditionalFeatureTags(feature: feature) + /// Merge the tags + additionalTags = additionalTags.merging(featureAdditionalTags) { current, existing in + return existing + } /// If feature is of type editable accessibility feature and is existing, then use the existing id and version for the point /// to update the existing point in OSM instead of creating a new one if feature.isExisting { @@ -395,6 +445,12 @@ extension APIChangesetUploadController { var calculatedAttributeValues: [AccessibilityFeatureAttribute: AccessibilityFeatureAttribute.Value?] = [:] calculatedAttributeValues = feature.calculatedAttributeValues var additionalTags = additionalTags + /// Add feature-based additional tags using the getAdditionalTags function + let featureAdditionalTags = getAdditionalFeatureTags(feature: feature) + /// Merge the tags + additionalTags = additionalTags.merging(featureAdditionalTags) { current, existing in + return existing + } var pointDiffOperations: [ChangesetDiffOperation] = [] var pointRefs: [String] = [] if feature.isExisting { @@ -460,6 +516,12 @@ extension APIChangesetUploadController { var calculatedAttributeValues: [AccessibilityFeatureAttribute: AccessibilityFeatureAttribute.Value?] = [:] calculatedAttributeValues = feature.calculatedAttributeValues var additionalTags = additionalTags + /// Add feature-based additional tags using the getAdditionalTags function + let featureAdditionalTags = getAdditionalFeatureTags(feature: feature) + /// Merge the tags + additionalTags = additionalTags.merging(featureAdditionalTags) { current, existing in + return existing + } var pointDiffOperations: [ChangesetDiffOperation] = [] var pointRefs: [String] = [] if feature.isExisting { diff --git a/IOSAccessAssessment/View/ARCameraView.swift b/IOSAccessAssessment/View/ARCameraView.swift index af5ed3a2..6d983f33 100644 --- a/IOSAccessAssessment/View/ARCameraView.swift +++ b/IOSAccessAssessment/View/ARCameraView.swift @@ -222,12 +222,14 @@ struct ARCameraView: View { locationManager.startLocationUpdates() showAnnotationView = false segmentationPipeline.setSelectedClasses(selectedClasses) + apiChangesetUploadController.configure(telemetryEncoder: sharedAppData.currentTelemetryEncoder) do { try manager.configure( selectedClasses: selectedClasses, segmentationPipeline: segmentationPipeline, metalContext: sharedAppContext.metalContext, isEnhancedAnalysisEnabled: userStateViewModel.isEnhancedAnalysisEnabled, - cameraOutputImageCallback: cameraOutputImageCallback + cameraOutputImageCallback: cameraOutputImageCallback, + telemetryEncoder: sharedAppData.currentTelemetryEncoder ) } catch { managerConfigureStatusViewModel.update(isFailed: true, errorMessage: error.localizedDescription) diff --git a/IOSAccessAssessment/View/AnnotationView.swift b/IOSAccessAssessment/View/AnnotationView.swift index 2059a93e..6ec803e4 100644 --- a/IOSAccessAssessment/View/AnnotationView.swift +++ b/IOSAccessAssessment/View/AnnotationView.swift @@ -533,6 +533,11 @@ struct AnnotationView: View { deviceLocation: captureLocation, mappingData: sharedAppData.currentMappingData, accessibilityFeature: accessibilityFeature ) + attributeEstimationPipeline.processNearestFeaturesRequest( + deviceLocation: captureLocation, + mappingData: sharedAppData.currentMappingData, + accessibilityFeature: accessibilityFeature + ) try attributeEstimationPipeline.processAttributeRequest( accessibilityFeature: accessibilityFeature, attributes: selectedAttributesByClass[currentClass] ?? [] diff --git a/IOSAccessAssessment/View/SetupView.swift b/IOSAccessAssessment/View/SetupView.swift index 31ff02d4..2f9cf5a2 100644 --- a/IOSAccessAssessment/View/SetupView.swift +++ b/IOSAccessAssessment/View/SetupView.swift @@ -681,7 +681,7 @@ struct SetupView: View { private func initializeModels() { do { - try segmentationPipeline.configure() + try segmentationPipeline.configure(telemetryEncoder: sharedAppData.currentTelemetryEncoder) modelInitializationViewModel.update(areModelsInitialized: true, showRetryAlert: false, retryMessage: "") } catch { /// Sleep for a short duration to avoid rapid retry loops diff --git a/IOSAccessAssessment/View/SubView/AnnotationFeatureDetailView.swift b/IOSAccessAssessment/View/SubView/AnnotationFeatureDetailView.swift index a19e2039..1802c6ff 100644 --- a/IOSAccessAssessment/View/SubView/AnnotationFeatureDetailView.swift +++ b/IOSAccessAssessment/View/SubView/AnnotationFeatureDetailView.swift @@ -34,7 +34,7 @@ func AnnotationFeatureDetailView( ) -> some View { AnnotationFeatureDetailViewBase( accessibilityFeature: accessibilityFeature, title: title - ) { feature in + ) { feature, refreshTrigger in let locationFormatter = AnnotationFeatureDetailLocationFormatter() Section(header: Text(AnnotationViewConstants.Texts.featureDetailViewLocationKey)) { if let featureLocation = accessibilityFeature.getLastLocationCoordinate() { @@ -71,6 +71,16 @@ func AnnotationFeatureDetailView( .strikethrough(accessibilityFeature.oswElement == nil, pattern: .solid) Spacer() } + Divider() + HStack { + Spacer() + Text("Is Capture Matched: ") + .foregroundStyle(.secondary) + Spacer() + Text(accessibilityFeature.isCaptureMatched ? "Yes" : "No") + .foregroundStyle(.secondary) + Spacer() + } if let oswElement = accessibilityFeature.oswElement { HStack { Spacer() @@ -82,11 +92,171 @@ func AnnotationFeatureDetailView( .padding(.top, 4) .padding(.bottom, 4) } + Divider() + if let nearestOSWElements = accessibilityFeature.nearestOSWElements { + /// Create a Picker of nearest OSW elements (already sorted by their distances) where user can select the correct one. + Picker("Select the correct TDEI element", selection: Binding( + get: { accessibilityFeature.selectedNearestOSWElement?.0.id}, + set: { newValue in + if let newValue = newValue { +// accessibilityFeature.selectedNearestOSWElement = nearestOSWElements.first(where: { $0.0.id == newValue }) + accessibilityFeature.setSelectedNearestOSWElement(selectedNearestOSWElement: nearestOSWElements.first(where: { $0.0.id == newValue })) + } else { +// accessibilityFeature.selectedNearestOSWElement = nil + accessibilityFeature.setSelectedNearestOSWElement(selectedNearestOSWElement: nil) + } + refreshTrigger.wrappedValue += 1 + } + )) { + Text("None selected") + .tag(nil as String?) + ForEach(nearestOSWElements, id: \.0.id) { (oswElement, distance) in + Text("ID: \(oswElement.id), Distance: \(String(format: "%.2f", distance)) m") + .tag(Optional(oswElement.id)) + } + } + .pickerStyle(.menu) + } + Divider() + /// Create a toggle for isCorrectOSWElementSelected + Toggle(isOn: Binding( + get: { accessibilityFeature.isCorrectOSWElementSelected }, + set: { newValue in +// accessibilityFeature.isCorrectOSWElementSelected = newValue + accessibilityFeature.setIsCorrectOSWElementSelected(newValue) + } + )) { + Text("Is the selected TDEI element correct?") + } } } else { Text(AnnotationMappedFeatureDetailViewConstants.Texts.invalidTextKey) .foregroundStyle(.secondary) } + + } + } correctedLocationSection: { feature, refreshTrigger in + let locationFormatter = AnnotationFeatureDetailLocationFormatter() + Section(header: Text("Corrected Location")) { + if let correctedFeatureLocation = accessibilityFeature.getCorrectedLastLocationCoordinate() { + VStack { + HStack { + Spacer() + Text( + locationFormatter.string( + from: NSNumber(value: correctedFeatureLocation.latitude) + ) ?? AnnotationMappedFeatureDetailViewConstants.Texts.invalidTextKey + ) + .padding(.horizontal) + Text( + locationFormatter.string( + from: NSNumber(value: correctedFeatureLocation.longitude) + ) ?? AnnotationMappedFeatureDetailViewConstants.Texts.invalidTextKey + ) + .padding(.horizontal) + Spacer() + } + Divider() + HStack { + Spacer() + Toggle(isOn: Binding( + get: { accessibilityFeature.correctedIsExisting && accessibilityFeature.correctedOSWElement != nil }, + set: { newValue in + accessibilityFeature.setCorrectedIsExisting(newValue) + } + )) { + Text(AnnotationMappedFeatureDetailViewConstants.Texts.isExistingTitle) + } + .disabled(accessibilityFeature.correctedOSWElement == nil) + .foregroundStyle(accessibilityFeature.correctedOSWElement == nil ? .secondary : .primary) + .strikethrough(accessibilityFeature.correctedOSWElement == nil, pattern: .solid) + Spacer() + } + Divider() + HStack { + Spacer() + Text("Is Capture Matched: ") + .foregroundStyle(.secondary) + Spacer() + Text(accessibilityFeature.isCaptureMatched ? "Yes" : "No") + .foregroundStyle(.secondary) + Spacer() + } + if let correctedOSWElement = accessibilityFeature.correctedOSWElement { + HStack { + Spacer() + Text("TDEI Element ID: \(correctedOSWElement.id)") + .foregroundStyle(.secondary) + Spacer() + } + .padding(.horizontal) + .padding(.top, 4) + .padding(.bottom, 4) + } + Divider() + if let correctedNearestOSWElements = accessibilityFeature.correctedNearestOSWElements { + /// Create a Picker of nearest OSW elements (already sorted by their distances) where user can select the correct one. + Picker("Select the correct TDEI element", selection: Binding( + get: { accessibilityFeature.correctedSelectedNearestOSWElement?.0.id}, + set: { newValue in + if let newValue = newValue { +// accessibilityFeature.correctedSelectedNearestOSWElement = correctedNearestOSWElements.first(where: { $0.0.id == newValue }) + accessibilityFeature.setCorrectedSelectedNearestOSWElement(correctedNearestOSWElements.first(where: { $0.0.id == newValue })) + } else { +// accessibilityFeature.correctedSelectedNearestOSWElement = nil + accessibilityFeature.setCorrectedSelectedNearestOSWElement(nil) + } + refreshTrigger.wrappedValue += 1 + } + )) { + Text("None selected") + .tag(nil as String?) + ForEach(correctedNearestOSWElements, id: \.0.id) { (oswElement, distance) in + Text("ID: \(oswElement.id), Distance: \(String(format: "%.2f", distance)) m") + .tag(Optional(oswElement.id)) + } + } + .pickerStyle(.menu) + } + Divider() + /// Create a toggle for isCorrectOSWElementSelected + Toggle(isOn: Binding( + get: { accessibilityFeature.correctedIsCorrectOSWElementSelected }, + set: { newValue in +// accessibilityFeature.correctedIsCorrectOSWElementSelected = newValue + accessibilityFeature.setCorrectedIsCorrectOSWElementSelected(newValue) + } + )) { + Text("Is the selected TDEI element correct?") + } + } + } else { + Text(AnnotationMappedFeatureDetailViewConstants.Texts.invalidTextKey) + .foregroundStyle(.secondary) + } + } + } ambiguitySection: { feature, refreshTrigger in + let ambiguityCasePolicy = accessibilityFeature.accessibilityFeatureClass.kind.ambiguityCasePolicy + /// Create a multi-select list of ambiguity cases with checkboxes, where user can select multiple ambiguity cases. + Section(header: Text("Ambiguity Cases")) { + ForEach(ambiguityCasePolicy.ambiguityCases, id: \.self) { ambiguityCase in + HStack { + Toggle(isOn: Binding( + get: { accessibilityFeature.ambiguityCases.contains(ambiguityCase) }, + set: { newValue in + if newValue { +// accessibilityFeature.ambiguityCases.append(ambiguityCase) + accessibilityFeature.addAmbiguityCase(ambiguityCase) + } else { + accessibilityFeature.ambiguityCases.removeAll(where: { $0 == ambiguityCase }) + accessibilityFeature.removeAmbiguityCase(ambiguityCase) + } + } + )) { + Text(ambiguityCase.rawValue) + } + } + } } } } diff --git a/IOSAccessAssessment/View/TestMode/TestAnnotationView.swift b/IOSAccessAssessment/View/TestMode/TestAnnotationView.swift new file mode 100644 index 00000000..70ff9aee --- /dev/null +++ b/IOSAccessAssessment/View/TestMode/TestAnnotationView.swift @@ -0,0 +1,578 @@ +// +// AnnotationView.swift +// IOSAccessAssessment +// +// Created by Himanshu on 11/10/25. +// + +import SwiftUI +import TipKit +import CoreLocation +import simd +import PointNMapShared + +struct TestAnnotationView: View { + let selectedClasses: [AccessibilityFeatureClass] + let selectedAttributesByClass: [AccessibilityFeatureClass: Set] + let captureLocation: CLLocationCoordinate2D + let correctedLocation: CLLocationCoordinate2D? + let apiChangesetUploadController: APIChangesetUploadController + + @EnvironmentObject var userStateViewModel: UserStateViewModel + @EnvironmentObject var workspaceViewModel: WorkspaceViewModel + @EnvironmentObject var sharedAppData: SharedAppData + @Environment(\.dismiss) var dismiss + + @StateObject var manager: AnnotationImageManager = AnnotationImageManager() + + @StateObject var segmentationAnnontationPipeline: SegmentationAnnotationPipeline = SegmentationAnnotationPipeline() + @StateObject var attributeEstimationPipeline: AttributeEstimationPipeline = AttributeEstimationPipeline() + + @StateObject private var managerStatusViewModel = AnnotationViewStatusViewModel() + @StateObject private var apiChangesetUploadStatusViewModel = APIChangesetUploadStatusViewModel() + @State private var interfaceOrientation: UIInterfaceOrientation = .portrait // To bind one-way with manager's orientation + + @StateObject var featureClassSelectionViewModel = AnnotationFeatureClassSelectionViewModel() + @StateObject var featureSelectionViewModel = AnnotationFeatureSelectionViewModel() + @State private var isShowingAnnotationFeatureDetailView: Bool = false + +// var selectFeatureInfoTip = SelectFeatureInfoTip() + @State private var showSelectFeatureLearnMoreSheet = false + + var body: some View { + VStack { + HStack { + Spacer() + Text(AnnotationViewConstants.Texts.annotationViewTitle) + .font(.headline) + .padding() + Spacer() + } + .overlay( + HStack { + Spacer() + Button(action: { + dismiss() + }) { + Image(systemName: AnnotationViewConstants.Images.closeIcon) + .resizable() + .frame(width: 20, height: 20) + } + .padding() + } + ) + + if let currentClass = featureClassSelectionViewModel.currentClass { + mainContent(currentClass: currentClass) + } else { + loadingPageView() + } + } + .task { + await handleOnAppear() + } + .onChange(of: featureClassSelectionViewModel.currentClass) { oldClass, newClass in + handleOnClassChange() + } + /// We are using index to track change in instance, instead of the instance itself, because we want to use the index for naming the instance in the picker. + /// To use the instance directly would require AccessibilityFeature to conform to Hashable, which is possible, by just using id. + /// But while rendering the picker, we would need to create a new Array of enumerated instances, which would be less efficient. + .onChange(of: featureSelectionViewModel.currentIndex) { oldIndex, newIndex in + handleOnInstanceChange(oldIndex: oldIndex, newIndex: newIndex) + } + .sheet(isPresented: $isShowingAnnotationFeatureDetailView) { + if let currentFeature = featureSelectionViewModel.currentFeature, + let currentFeatureIndex = featureSelectionViewModel.currentIndex + { + AnnotationFeatureDetailView( + accessibilityFeature: currentFeature, + title: "\(currentFeature.accessibilityFeatureClass.name.capitalized): \(currentFeatureIndex)" + ) + .presentationDetents([.medium, .large]) + } else { + Text(AnnotationViewConstants.Texts.featureDetailNotAvailableText) + .presentationDetents([.medium, .large]) + } + } + .sheet(isPresented: $showSelectFeatureLearnMoreSheet) { + SelectFeatureLearnMoreSheetView() + .presentationDetents([.medium, .large]) + } + .alert(AnnotationViewConstants.Texts.managerStatusAlertTitleKey, isPresented: $managerStatusViewModel.isFailed, actions: { + Button(AnnotationViewConstants.Texts.managerStatusAlertDismissButtonKey) { + let shouldDismiss = managerStatusViewModel.shouldDismiss + managerStatusViewModel.update(isFailed: false, errorMessage: "") + if shouldDismiss { + dismiss() + } + } + }, message: { + Text(managerStatusViewModel.errorMessage) + }) + .alert(AnnotationViewConstants.Texts.apiChangesetUploadStatusAlertTitleKey, + isPresented: $apiChangesetUploadStatusViewModel.isFailed, actions: { + Button(AnnotationViewConstants.Texts.managerStatusAlertDismissButtonKey) { + apiChangesetUploadStatusViewModel.update(isFailed: false, errorMessage: "") + do { + try moveToNextClass() + } catch { + managerStatusViewModel.update(isFailed: true, error: error) + } + } + }, message: { + Text(apiChangesetUploadStatusViewModel.errorMessage) + }) + } + + private func loadingPageView() -> some View { + VStack { + Spacer() + Text(AnnotationViewConstants.Texts.loadingPageText) + SpinnerView() + Spacer() + } + } + + @ViewBuilder + private func orientationStack(@ViewBuilder content: () -> Content) -> some View { + manager.interfaceOrientation.isLandscape ? + AnyLayout(HStackLayout())(content) : + AnyLayout(VStackLayout())(content) + } + + @ViewBuilder + private func mainContent(currentClass: AccessibilityFeatureClass) -> some View { + let isDisabledFeatureDetailButton = featureSelectionViewModel.currentFeature == nil + orientationStack { + HostedAnnotationImageViewController(annotationImageManager: manager) + + VStack { + HStack { + Spacer() + Text("\(AnnotationViewConstants.Texts.currentClassPrefixText): \(currentClass.name)") + Spacer() + } + + HStack { + Spacer() + CustomPicker ( + label: AnnotationViewConstants.Texts.selectObjectText, + selection: $featureSelectionViewModel.currentIndex, +// isContainsAll: currentClass.kind.oswPolicy.oswElementClass != .Sidewalk + isContainsAll: currentClass.kind.isUniquePerCapture == false + ) { + ForEach(featureSelectionViewModel.instances.indices, id: \.self) { featureIndex in + Text("\(currentClass.name.capitalized): \(featureIndex)") + .tag(featureIndex as Int?) + } + } + Button(action: { + isShowingAnnotationFeatureDetailView = true + }) { + Image(systemName: AnnotationViewConstants.Images.ellipsisIcon) + } + .buttonStyle(.bordered) + .padding(.horizontal, 5) + .disabled(isDisabledFeatureDetailButton) + Spacer() + } + .frame(maxWidth: .infinity) + .padding(.horizontal, 30) + .overlay( + HStack { + Spacer() + Button(action: { + showSelectFeatureLearnMoreSheet = true + }) { + Image(systemName: AnnotationViewConstants.Images.infoIcon) + .resizable() + .frame(width: 20, height: 20) + } + .padding(.trailing, 10) + } + ) + + ProgressBar(value: 0) + + HStack { + Spacer() + annotationOptionsView(currentClass: currentClass) + Spacer() + } + .padding() + + Button(action: { + confirmAnnotation() + }) { + Text(isCurrentIndexLast() ? AnnotationViewConstants.Texts.finishText : AnnotationViewConstants.Texts.nextText) + .padding() + } + } + } + } + + private func annotationOptionsView(currentClass: AccessibilityFeatureClass) -> some View { + if let currentFeature = featureSelectionViewModel.currentFeature { + let annotationOptions: [AnnotationOption] = AnnotationOptionFeature.allCases.map { .individualOption($0) } + return VStack(spacing: 10) { + ForEach(annotationOptions, id: \.self) { option in + Button(action: { + featureSelectionViewModel.setOptionOnFeature(option: option) + }) { + Text(option.rawValue) + .font(.subheadline) + .frame(maxWidth: .infinity) + .padding() + .background(currentFeature.selectedAnnotationOption == option ? Color.blue : Color.gray) + .foregroundStyle(.white) + .cornerRadius(10) + } + } + } + } else { + let annotationOptions: [AnnotationOption] = AnnotationOptionFeatureClass.allCases.map { .classOption($0) } + return VStack(spacing: 10) { + ForEach(annotationOptions, id: \.self) { option in + Button(action: { + featureClassSelectionViewModel.setOption(option: option) + }) { + Text(option.rawValue) + .font(.subheadline) + .frame(maxWidth: .infinity) + .padding() + .background(featureClassSelectionViewModel.selectedAnnotationOption == option ? Color.blue : Color.gray) + .foregroundStyle(.white) + .cornerRadius(10) + } + } + } + } + } + + private func isCurrentIndexValid() -> Bool { + guard let currentCaptureDataRecord = sharedAppData.currentCaptureDataRecord, + let currentClassIndex = featureClassSelectionViewModel.currentIndex else { + return false + } + let segmentedClasses = currentCaptureDataRecord.imageData.captureImageDataResults.segmentedClasses + return (currentClassIndex >= 0 && currentClassIndex < segmentedClasses.count) + } + + private func isCurrentIndexLast() -> Bool { + guard let currentCaptureDataRecord = sharedAppData.currentCaptureDataRecord, + let currentClassIndex = featureClassSelectionViewModel.currentIndex else { + return false + } + let segmentedClasses = currentCaptureDataRecord.imageData.captureImageDataResults.segmentedClasses + return currentClassIndex == segmentedClasses.count - 1 + } + + private func handleOnAppear() async { + do { + guard let currentCaptureDataRecord = sharedAppData.currentCaptureDataRecord else { + throw AnnotationViewError.invalidCaptureDataRecord + } + var captureMeshData: (any CaptureMeshDataProtocol)? = nil + if userStateViewModel.isEnhancedAnalysisEnabled { + guard let captureMeshDataResults = currentCaptureDataRecord.meshData?.captureMeshDataResults else { + throw AnnotationViewError.invalidCaptureDataRecord + } + captureMeshData = CaptureImageAndMeshData( + captureImageData: CaptureImageData(currentCaptureDataRecord.imageData), + captureMeshDataResults: captureMeshDataResults + ) + } + let segmentedClasses = currentCaptureDataRecord.imageData.captureImageDataResults.segmentedClasses + try segmentationAnnontationPipeline.configure() + try attributeEstimationPipeline.configure( + captureImageData: currentCaptureDataRecord.imageData, + /// TODO: MESH PROCESSING: Enable mesh data processing + captureMeshData: captureMeshData + ) + try manager.configure( + selectedClasses: selectedClasses, segmentationAnnotationPipeline: segmentationAnnontationPipeline, + captureImageData: currentCaptureDataRecord.imageData, + captureMeshData: captureMeshData, + isEnhancedAnalysisEnabled: userStateViewModel.isEnhancedAnalysisEnabled + ) + let captureDataHistory = Array(await sharedAppData.captureDataQueue.snapshot()) + manager.setupAlignedSegmentationLabelImages(captureDataHistory: captureDataHistory) + try featureClassSelectionViewModel.setCurrent(index: 0, classes: segmentedClasses) + } catch { + managerStatusViewModel.update(isFailed: true, error: error) + } + } + + private func handleOnClassChange() { + do { + guard let currentClass = featureClassSelectionViewModel.currentClass else { + throw AnnotationViewError.invalidCaptureDataRecord + } + let accessibilityFeatures = try manager.updateFeatureClass(accessibilityFeatureClass: currentClass) + var lastEstimationError: Error? = nil +// let deviceLocationLocal = correctedLocation ?? captureLocation + try attributeEstimationPipeline.processAllFeaturesForAssignments( +// deviceLocation: deviceLocationLocal, + captureLocation: captureLocation, + correctedLocation: correctedLocation, + mappingData: sharedAppData.currentMappingData, + accessibilityFeatures: accessibilityFeatures, + ) + accessibilityFeatures.enumerated().forEach { (featureIndex, accessibilityFeature) in + do { +// try attributeEstimationPipeline.setPrerequisites(accessibilityFeature: accessibilityFeature) +// try attributeEstimationPipeline.processLocationRequestTypeBased( +// deviceLocation: captureLocation, +// accessibilityFeature: accessibilityFeature +// ) +// attributeEstimationPipeline.processIsExistingRequest( +// deviceLocation: captureLocation, +// mappingData: sharedAppData.currentMappingData, accessibilityFeature: accessibilityFeature, +// featureIndex: featureIndex +// ) +// attributeEstimationPipeline.processNearestFeaturesRequest( +// deviceLocation: captureLocation, +// mappingData: sharedAppData.currentMappingData, +// accessibilityFeature: accessibilityFeature, +// featureIndex: featureIndex +// ) + try attributeEstimationPipeline.processAttributeRequest( + accessibilityFeature: accessibilityFeature, + attributes: selectedAttributesByClass[currentClass] ?? [] + ) + attributeEstimationPipeline.clearPrerequisites() + } catch { + lastEstimationError = error + } +// if let correctedLocation { +// do { +// try attributeEstimationPipeline.processLocationRequestTypeBased( +// deviceLocation: correctedLocation, +// accessibilityFeature: accessibilityFeature, +// locationType: .correctedLocation +// ) +// attributeEstimationPipeline.processIsExistingRequest( +// deviceLocation: correctedLocation, +// mappingData: sharedAppData.currentMappingData, accessibilityFeature: accessibilityFeature, +// locationType: .correctedLocation, +// featureIndex: featureIndex +// ) +// attributeEstimationPipeline.processNearestFeaturesRequest( +// deviceLocation: correctedLocation, +// mappingData: sharedAppData.currentMappingData, +// accessibilityFeature: accessibilityFeature, +// locationType: .correctedLocation, +// featureIndex: featureIndex +// ) +// attributeEstimationPipeline.clearPrerequisites() +// } catch { +// lastEstimationError = error +// } +// } + } + featureClassSelectionViewModel.setOption(option: .classOption(.default)) + try featureSelectionViewModel.setInstances(accessibilityFeatures, currentClass: currentClass) + if let lastEstimationError { + throw AnnotationViewError.attributeEstimationFailed(lastEstimationError) + } + } catch AnnotationViewError.attributeEstimationFailed(let error) { + managerStatusViewModel.update( + isFailed: true, error: AnnotationViewError.attributeEstimationFailed(error), shouldDismiss: false + ) + } catch { + managerStatusViewModel.update(isFailed: true, error: error, shouldDismiss: false) + } + } + + private func handleOnInstanceChange(oldIndex: Int?, newIndex: Int?) { + do { + try featureSelectionViewModel.setIndex(index: featureSelectionViewModel.currentIndex) + } catch { + managerStatusViewModel.update(isFailed: true, error: error) + } + do { + guard let currentClass = featureClassSelectionViewModel.currentClass else { + throw AnnotationViewError.invalidCaptureDataRecord + } + var accessibilityFeatures: [MappedEditableAccessibilityFeature] + var featureSelectedStatus: [UUID: Bool] = [:] + var updateFeatureResults: AnnotationImageFeatureUpdateResults? = nil + if let currentFeature = featureSelectionViewModel.currentFeature { + accessibilityFeatures = [currentFeature] + featureSelectedStatus[currentFeature.id] = true /// Selected and highlighted + if let oldIndex = oldIndex, oldIndex != featureSelectionViewModel.currentIndex, + oldIndex >= 0, oldIndex < featureSelectionViewModel.instances.count { + let oldFeature = featureSelectionViewModel.instances[oldIndex] + accessibilityFeatures.append(oldFeature) + featureSelectedStatus[oldFeature.id] = false /// Selected, but not highlighted + } + /// MARK: Temporary code for visualization. Incurs significant performance overhead. + /// TODO: Check what happens when we use the cache-based methods instead + if currentClass.kind.attributes.contains(where: { + $0 == .width || $0 == .runningSlope || $0 == .crossSlope || $0 == .surfaceIntegrity + }) { + let worldPoints = try attributeEstimationPipeline.getWorldPoints(accessibilityFeature: currentFeature) + let plane = try attributeEstimationPipeline.calculateAlignedPlane( + accessibilityFeature: currentFeature, worldPoints: worldPoints + ) + let projectedPlane = try attributeEstimationPipeline.calculateProjectedPlane( + accessibilityFeature: currentFeature, plane: plane + ) + let damageDetectionResults = try attributeEstimationPipeline.getDamageDetectionResults( + accessibilityFeature: currentFeature + ) + updateFeatureResults = AnnotationImageFeatureUpdateResults( + plane: plane, projectedPlane: projectedPlane, + damageDetectionResults: damageDetectionResults + ) + } + } else { + accessibilityFeatures = featureSelectionViewModel.instances + featureSelectedStatus = featureSelectionViewModel.instances.reduce(into: [:]) { dict, feature in + dict[feature.id] = false /// Selected, but not highlighted + } + } +// let isSelected = featureSelectionViewModel.currentFeature != nil + try manager.updateFeature( + accessibilityFeatureClass: currentClass, + accessibilityFeatures: accessibilityFeatures, + featureSelectedStatus: featureSelectedStatus, + updateFeatureResults: updateFeatureResults + ) + } catch { + managerStatusViewModel.update(isFailed: true, error: error, shouldDismiss: false) + } + } + + private func confirmAnnotation() { + Task { + do { + let apiChangesetUploadResults = try await uploadFeatures() + if let apiChangesetUploadResults, apiChangesetUploadResults.failedFeatureUploads > 0 { + throw AnnotationViewError.apiChangesetUploadFailed(apiChangesetUploadResults) + } + try moveToNextClass() + } catch AnnotationViewError.classIndexOutofBounds { + managerStatusViewModel.update(isFailed: true, error: AnnotationViewError.classIndexOutofBounds) + } catch AnnotationViewError.apiChangesetUploadFailed(let results) { + apiChangesetUploadStatusViewModel.update(apiChangesetUploadResults: results) + } catch { + apiChangesetUploadStatusViewModel.update( + isFailed: true, + errorMessage: AnnotationViewConstants.Texts.apiChangesetUploadStatusAlertGenericMessageKey + ) + } + } + } + + /// Fetches the latest map data for the current location and workspace, and updates the sharedAppData's currentMappingData. + /// This ensures that the most up-to-date map data is used for the next class's annotation, in case there were any changes from the previous upload. + /// May be used as a fail-safe if API changeset upload fails, to ensure that the user is working with the latest map data. If the upload is successful, the map data is already updated in the sharedAppData within the uploadFeatures function, so this would just be an extra fetch that may not be necessary. + private func refreshMap() async throws { + guard let workspaceId = workspaceViewModel.workspaceId else { + throw ARCameraViewError.workspaceConfigurationFailed + } + guard let accessToken = userStateViewModel.getAccessToken() else { + throw ARCameraViewError.authenticationError + } + let mapData = try await WorkspaceService.shared.fetchMapData( + workspaceId: workspaceId, + location: captureLocation, + radius: SharedAppConstants.WorkspaceConstants.fetchRadiusInMeters, + accessToken: accessToken, + environment: userStateViewModel.selectedEnvironment + ) + sharedAppData.currentMappingData.replace( + osmMapDataResponse: mapData, + accessibilityFeatureClasses: selectedClasses + ) + } + + private func moveToNextClass() throws { + if isCurrentIndexLast() { + self.dismiss() + return + } + /// Move to next class + guard let currentCaptureDataRecord = sharedAppData.currentCaptureDataRecord, + let currentClassIndex = featureClassSelectionViewModel.currentIndex else { + throw AnnotationViewError.invalidCaptureDataRecord + } + let segmentedClasses = currentCaptureDataRecord.imageData.captureImageDataResults.segmentedClasses + try featureClassSelectionViewModel.setCurrent(index: currentClassIndex + 1, classes: segmentedClasses) + } + + private func uploadFeatures() async throws -> APIChangesetUploadResults? { + guard let currentCaptureDataRecord = sharedAppData.currentCaptureDataRecord else { + throw AnnotationViewError.invalidCaptureDataRecord + } + guard let workspaceId = workspaceViewModel.workspaceId, + let changesetId = workspaceViewModel.changesetId else { + throw AnnotationViewError.workspaceConfigurationFailed + } + guard let accessToken = userStateViewModel.getAccessToken() else { + throw AnnotationViewError.authenticationError + } + guard let accessibilityFeatureClass = featureClassSelectionViewModel.currentClass else { + throw AnnotationViewError.classIndexOutofBounds + } + guard featureClassSelectionViewModel.selectedAnnotationOption != .classOption(.discard) else { + return nil + } + let featuresToUpload: [MappedEditableAccessibilityFeature] = featureSelectionViewModel.instances.filter { feature in + feature.selectedAnnotationOption != .individualOption(.discard) && + feature.accessibilityFeatureClass == accessibilityFeatureClass + } + guard !featuresToUpload.isEmpty else { + return nil + } + let apiChangesetUploadInputs = APIChangesetUploadInputs( + workspaceId: workspaceId, + changesetId: changesetId, + accessibilityFeatureClass: accessibilityFeatureClass, + captureData: currentCaptureDataRecord, + captureLocation: captureLocation, + accessToken: accessToken, + environment: userStateViewModel.selectedEnvironment + ) + let apiChangesetUploadResults = try await apiChangesetUploadController.uploadFeatures( + accessibilityFeatures: featuresToUpload, + currentMappedFeaturesData: sharedAppData.currentMappedFeaturesData, + inputs: apiChangesetUploadInputs + ) + guard let mappedAccessibilityFeatures = apiChangesetUploadResults.accessibilityFeatures, + let mappedElements = apiChangesetUploadResults.oswElements else { + throw AnnotationViewError.apiChangesetUploadFailed(apiChangesetUploadResults) + } + sharedAppData.currentMappedFeaturesData.updateFeatures(mappedAccessibilityFeatures, for: accessibilityFeatureClass) + sharedAppData.currentMappingData.updateFeatures(mappedElements, for: accessibilityFeatureClass) + + addFeaturesToCurrentDataset( + captureImageData: currentCaptureDataRecord.imageData, + featuresToUpload: featuresToUpload, mappedAccessibilityFeatures: mappedAccessibilityFeatures + ) + + sharedAppData.isUploadReady = true + return apiChangesetUploadResults + } + + private func addFeaturesToCurrentDataset( + captureImageData: any CaptureImageDataProtocol, + featuresToUpload: [any AccessibilityFeatureProtocol], + mappedAccessibilityFeatures: [any AccessibilityFeatureProtocol] + ) { + Task { + do { + try sharedAppData.currentDatasetEncoder?.addFeatures( + features: featuresToUpload, frameNumber: captureImageData.id, timestamp: captureImageData.timestamp + ) + try sharedAppData.currentDatasetEncoder?.addFeatures( + features: mappedAccessibilityFeatures, frameNumber: captureImageData.id, timestamp: captureImageData.timestamp + ) + } catch { + print("Error adding feature data to dataset encoder: \(error)") + } + } + } +} + diff --git a/IOSAccessAssessment/View/TestMode/TestCameraView.swift b/IOSAccessAssessment/View/TestMode/TestCameraView.swift index 1f127487..5c450c75 100644 --- a/IOSAccessAssessment/View/TestMode/TestCameraView.swift +++ b/IOSAccessAssessment/View/TestMode/TestCameraView.swift @@ -14,7 +14,7 @@ import PointNMapShared */ enum TestCameraViewConstants { enum Texts { - static let contentViewTitle = "Test: Capture" + static let contentViewTitle = "Capture" /// Change index buttons static let previousButtonText = "Previous" @@ -58,6 +58,7 @@ enum TestCameraViewError: Error, LocalizedError { class LocationManagerPlaceholder: NSObject, ObservableObject { @Published var currentLocation: CLLocation? @Published var currentHeading: CLHeading? + @Published var correctedLocation: CLLocation? override init() { super.init() @@ -94,6 +95,12 @@ class LocationManagerPlaceholder: NSObject, ObservableObject { } } + func locationManager(didUpdateCorrectedLocation correctedLocation: CLLocation) { + Task { @MainActor in + self.correctedLocation = correctedLocation + } + } + func stopLocationUpdates() {} } @@ -153,9 +160,11 @@ struct TestCameraView: View { Button { self.currentIndex = max(self.currentIndex - 1, 0) } label: { + /// White color image Image(systemName: TestCameraViewConstants.Images.previousIcon) .resizable() .frame(width: 30, height: 30) + .foregroundColor(.white) } .padding(.leading, 20) .padding(.bottom, 20) @@ -164,13 +173,13 @@ struct TestCameraView: View { Button { cameraCapture() } label: { -// Image(systemName: ARCameraViewConstants.Images.cameraIcon) -// .resizable() -// .frame(width: 60, height: 60) - Text("\(self.currentIndex)") + Image(systemName: ARCameraViewConstants.Images.cameraIcon) + .resizable() .frame(width: 60, height: 60) - // .foregroundColor(.white) - .border(Color.black, width: 2) +// Text("\(self.currentIndex)") +// .frame(width: 60, height: 60) +// // .foregroundColor(.white) +// .border(Color.black, width: 2) } .padding(.bottom, 20) Spacer() @@ -181,6 +190,7 @@ struct TestCameraView: View { Image(systemName: TestCameraViewConstants.Images.nextIcon) .resizable() .frame(width: 30, height: 30) + .foregroundColor(.white) } .padding(.trailing, 20) .padding(.bottom, 20) @@ -229,6 +239,10 @@ struct TestCameraView: View { heading.setValue(captureHeading, forKey: "trueHeading") self.locationManager.locationManager(didUpdateHeading: heading) } + if let correctedLocation = datasetCaptureData.correctedLocation { + let correctedCLLocation = CLLocation(latitude: correctedLocation.latitude, longitude: correctedLocation.longitude) + self.locationManager.locationManager(didUpdateCorrectedLocation: correctedCLLocation) + } try manager.configure( selectedClasses: selectedClasses, segmentationPipeline: segmentationPipeline, metalContext: sharedAppContext.metalContext, @@ -239,7 +253,7 @@ struct TestCameraView: View { /// For easier testing cameraHintDefaultText = datasetCaptureData.captureImageData.id.uuidString - setHintText(datasetCaptureData.captureImageData.id.uuidString) +// setHintText(datasetCaptureData.captureImageData.id.uuidString) } catch { managerConfigureStatusViewModel.update(isFailed: true, errorMessage: error.localizedDescription) } @@ -268,9 +282,10 @@ struct TestCameraView: View { }) .fullScreenCover(isPresented: $showAnnotationView) { if let captureLocation = locationManager.currentLocation?.coordinate { - AnnotationView( + TestAnnotationView( selectedClasses: selectedClasses, selectedAttributesByClass: selectedAttributesByClass, captureLocation: captureLocation, + correctedLocation: locationManager.correctedLocation?.coordinate, apiChangesetUploadController: apiChangesetUploadController ) } else { @@ -314,12 +329,16 @@ struct TestCameraView: View { heading.setValue(captureHeading, forKey: "trueHeading") self.locationManager.locationManager(didUpdateHeading: heading) } + if let correctedLocation = datasetCaptureData.correctedLocation { + let correctedCLLocation = CLLocation(latitude: correctedLocation.latitude, longitude: correctedLocation.longitude) + self.locationManager.locationManager(didUpdateCorrectedLocation: correctedCLLocation) + } manager.handleSessionUpdate(datasetCaptureData: datasetCaptureData) /// For easier testing cameraHintDefaultText = datasetCaptureData.captureImageData.id.uuidString - setHintText(datasetCaptureData.captureImageData.id.uuidString) +// setHintText(datasetCaptureData.captureImageData.id.uuidString) } catch { managerConfigureStatusViewModel.update(isFailed: true, errorMessage: error.localizedDescription) } @@ -433,14 +452,16 @@ struct TestCameraView: View { } private func handleLocationUpdate(oldLocation: CLLocation?, newLocation: CLLocation?) { - var shouldUpdateMap = oldLocation == nil && newLocation != nil - if let oldLocation, let newLocation { - let distance = oldLocation.distance(from: newLocation) - shouldUpdateMap = distance > SharedAppConstants.WorkspaceConstants.fetchUpdateRadiusThresholdInMeters - } - if !shouldUpdateMap { - return - } +// var shouldUpdateMap = oldLocation == nil && newLocation != nil +// if let oldLocation, let newLocation { +// let distance = oldLocation.distance(from: newLocation) +// shouldUpdateMap = distance > SharedAppConstants.WorkspaceConstants.fetchUpdateRadiusThresholdInMeters +// } + // Always trigger map update in test view + var shouldUpdateMap = true +// if !shouldUpdateMap { +// return +// } Task { do { mappingDataStatusViewModel.update(isInProgress: true)